Add AudioTrackRecorder for audio component of MediaStream recording.

content/renderer/media/audio_track_recorder.cc

+// Copyright 2015 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "base/bind.h"
+#include "base/stl_util.h"
+#include "content/renderer/media/audio_track_recorder.h"
+#include "media/base/bind_to_current_loop.h"
+
+#include "third_party/opus/src/include/opus.h"
+
+namespace content {
+
+namespace {
+
+enum {
+  // TODO(ajose): This likely shouldn't be hardcoded.
+  DEFAULT_FRAMES_PER_SECOND = 100,
+
+  // This is the recommended value, according to documentation in
+  // third_party/opus/src/include/opus.h, so that the Opus encoder does not
+  // degrade the audio due to memory constraints.
+  OPUS_MAX_PAYLOAD_SIZE = 4000,
+
+  // Support for max sampling rate of 48KHz, 2 channels, 100 ms duration.
+  MAX_SAMPLES_x_CHANNELS_PER_FRAME = 48 * 2 * 100,
+};
+
+}  // anonymous namespace
+
+// Nested class encapsulating opus-related encoding details.
+// AudioEncoder is created and destroyed on ATR's main thread (usually the
+// main render thread) but otherwise should operate entirely on
+// |encoder_thread_|, which is owned by AudioTrackRecorder.
+// Be sure to delete |encoder_thread_| before deleting the AudioEncoder using
+// it.
+class AudioTrackRecorder::AudioEncoder
+    : public base::RefCountedThreadSafe<AudioEncoder> {
+ public:
+  explicit AudioEncoder(const OnEncodedAudioCB& on_encoded_audio_cb)
+      : initialized_(false),
+        on_encoded_audio_cb_(on_encoded_audio_cb),
+        opus_encoder_(nullptr) {}
+
+  bool IsInitialized() { return initialized_; }

[mcasas, 2015/10/26 18:56:25]

Since IsInitialized() is only used for DCHECK()s,
there's no need for |initialized_|, you can relay 
to e.g. |audio_params_.IsValid()|, WDYT?
And it should be private.

[ajose, 2015/10/26 20:26:46]

I kinda like explicit initialization (doesn't hide state in AudioParameters,
which might be confusing to future maintainers) but will change if you prefer.

+
+  void OnSetFormat(const media::AudioParameters& params);
+
+  void EncodeAudio(scoped_ptr<media::AudioBus> audio_bus,
+                   const base::TimeTicks& capture_time);
+
+ private:
+  friend class base::RefCountedThreadSafe<AudioEncoder>;
+
+  ~AudioEncoder();
+
+  bool InitOpus(const media::AudioParameters& params);
+  void DestroyOpus();
+
+  void TransferSamplesIntoBuffer(const media::AudioBus* audio_bus,
+                                 int source_offset,
+                                 int buffer_fill_offset,
+                                 int num_samples);
+  bool EncodeFromFilledBuffer(std::string* out);
+
+  static bool IsValidFrameDuration(base::TimeDelta duration);
+
+  bool initialized_;
+
+  int samples_per_frame_;
+  const OnEncodedAudioCB on_encoded_audio_cb_;
+
+  base::ThreadChecker encoder_thread_checker_;
+
+  // In the case where a call to EncodeAudio() cannot completely fill the
+  // buffer, this points to the position at which to populate data in a later
+  // call.
+  int buffer_fill_end_;
+
+  media::AudioParameters audio_params_;
+
+  // Buffer for passing AudioBus data to OpusEncoder.
+  scoped_ptr<float[]> buffer_;
+
+  OpusEncoder* opus_encoder_;
+
+  DISALLOW_COPY_AND_ASSIGN(AudioEncoder);
+};
+
+AudioTrackRecorder::AudioEncoder::~AudioEncoder() {

[mcasas, 2015/10/26 18:56:25]

Order of method definition should follow the declaration order [1], 
please reshuffle.

[1]
http://google-styleguide.googlecode.com/svn/trunk/cppguide.html#Declaration_O...

[ajose, 2015/10/26 20:26:46]

Done.

+  // We don't DCHECK that we're on the encoder thread here, as it should have
+  // already been deleted at this point.
+  DestroyOpus();
+}
+
+void AudioTrackRecorder::AudioEncoder::OnSetFormat(
+    const media::AudioParameters& params) {
+  // AudioEncoder is constructed on the thread that ATR lives on, but should
+  // operate only on the encoder thread after that. Reset
+  // |encoder_thread_checker_| here to reflect this.
+  encoder_thread_checker_.DetachFromThread();

[mcasas, 2015/10/26 18:56:25]

Hmm this would reset the checker every time OnSetFormat() is called.
Is it guaranteed that would happen only once, and/or that we'd get
an OnSetFormat() before EncodeAudio()? I'd say, move this 
DetachFromThread() to AudioEncode(). First time CalledOnValidThread()
is called, it will peg itself to the current thread, and that'd would
also catch (potential) OnSetFormat() calls from other threads.
In those lines, perhaps also add here an:
DCHECK(!IsInitialized());

[ajose, 2015/10/26 20:26:46]

Done.

+  DCHECK(encoder_thread_checker_.CalledOnValidThread());
+  if (!InitOpus(params)) {
+    DLOG(ERROR) << "Couldn't initialize opus.";

[mcasas, 2015/10/26 18:56:25]

No need for more DLOG()s here, InitOpus() is informational enough.
OnSetFormat() is returning void but InitOpus() returns a boolean.
Is there no way to signal an initialization error upwards (to
MediaRecorderHandler I presume)?

[ajose, 2015/10/26 20:26:46]

Hmm, tricky because ATR::OnSetFormat() is calling AudioEncoder::OnSetFormat()
asynchronously, and also ATR::OnSetFormat() is called by
media::MediaStreamAudioSinkOwner which we don't control in MediaRecorderHandler.
Any ideas?

+  }
+}
+
+bool AudioTrackRecorder::AudioEncoder::InitOpus(
+    const media::AudioParameters& params) {
+  DCHECK(encoder_thread_checker_.CalledOnValidThread());
+  if (audio_params_.Equals(params))
+    return true;
+
+  if (!params.IsValid()) {
+    DLOG(ERROR) << "Invalid audio params: " << params.AsHumanReadableString();
+    return false;
+  }
+
+  if (params.bits_per_sample() != 16) {
+    DLOG(ERROR) << "Invalid bits per sample: " << params.bits_per_sample();
+    return false;
+  }
+
+  samples_per_frame_ = params.sample_rate() / DEFAULT_FRAMES_PER_SECOND;
+  if (samples_per_frame_ <= 0 ||
+      params.sample_rate() % samples_per_frame_ != 0 ||
+      samples_per_frame_ * params.channels() >
+          MAX_SAMPLES_x_CHANNELS_PER_FRAME) {
+    DLOG(ERROR) << "Invalid |samples_per_frame_|: " << samples_per_frame_;
+    return false;
+  }
+
+  const base::TimeDelta frame_duration = base::TimeDelta::FromMicroseconds(
+      base::Time::kMicrosecondsPerSecond * samples_per_frame_ /
+      params.sample_rate());
+  if (frame_duration == base::TimeDelta() ||
+      !IsValidFrameDuration(frame_duration)) {
+    DLOG(ERROR) << "Invalid |frame_duration|: " << frame_duration;
+    return false;
+  }
+
+  // Initialize AudioBus buffer for OpusEncoder.
+  buffer_fill_end_ = 0;
+  buffer_.reset(new float[params.channels() * samples_per_frame_]);
+
+  // Check for and destroy previous OpusEncoder, if necessary.
+  DestroyOpus();
+  // Initialize OpusEncoder.
+  int opus_result;
+  opus_encoder_ = opus_encoder_create(params.sample_rate(), params.channels(),
+                                      OPUS_APPLICATION_AUDIO, &opus_result);
+  if (opus_result < 0) {
+    DLOG(ERROR) << "Couldn't init opus encoder: " << opus_strerror(opus_result);
+    return false;
+  }
+
+  // Note: As of 2013-10-31, the encoder in "auto bitrate" mode would use a
+  // variable bitrate up to 102kbps for 2-channel, 48 kHz audio and a 10 ms
+  // frame size. The opus library authors may, of course, adjust this in
+  // later versions.
+  if (opus_encoder_ctl(opus_encoder_, OPUS_SET_BITRATE(OPUS_AUTO)) != OPUS_OK) {
+    DLOG(ERROR) << "Failed to set opus bitrate.";
+    return false;
+  }
+
+  audio_params_ = params;
+  initialized_ = true;
+  return true;
+}
+
+void AudioTrackRecorder::AudioEncoder::DestroyOpus() {
+  // We don't DCHECK that we're on the encoder thread here, as this could be
+  // called from the dtor (main thread) or from OnSetFormat (render thread);
+  if (opus_encoder_) {
+    opus_encoder_destroy(opus_encoder_);
+    opus_encoder_ = nullptr;
+  }
+}
+
+void AudioTrackRecorder::AudioEncoder::EncodeAudio(
+    scoped_ptr<media::AudioBus> audio_bus,
+    const base::TimeTicks& capture_time) {
+  DCHECK(encoder_thread_checker_.CalledOnValidThread());
+  DCHECK(IsInitialized());
+  DCHECK_EQ(audio_bus->channels(), audio_params_.channels());
+  DCHECK_EQ(audio_bus->frames(), audio_params_.frames_per_buffer());
+
+  // Encode all audio in |audio_bus| into zero or more frames.
+  int src_pos = 0;
+  while (src_pos < audio_bus->frames()) {
+    const int num_samples_to_xfer = std::min(
+        samples_per_frame_ - buffer_fill_end_, audio_bus->frames() - src_pos);
+    TransferSamplesIntoBuffer(audio_bus.get(), src_pos, buffer_fill_end_,

[mcasas, 2015/10/26 18:56:25]

AudioBus has some method called ToInterleaved() [1] which
could be handy here (or not).

[1]
https://code.google.com/p/chromium/codesearch#chromium/src/media/base/audio_b...

[ajose, 2015/10/26 20:26:46]

I'll look into it, added a bug.

+                              num_samples_to_xfer);
+    src_pos += num_samples_to_xfer;
+    buffer_fill_end_ += num_samples_to_xfer;
+
+    if (buffer_fill_end_ < samples_per_frame_)
+      break;
+
+    scoped_ptr<std::string> encoded_data(new std::string());
+    if (EncodeFromFilledBuffer(encoded_data.get())) {
+      on_encoded_audio_cb_.Run(audio_params_, encoded_data.Pass(),
+                               capture_time);
+    }
+
+    // Reset the internal buffer for the next frame.
+    buffer_fill_end_ = 0;
+  }
+}
+
+void AudioTrackRecorder::AudioEncoder::TransferSamplesIntoBuffer(
+    const media::AudioBus* audio_bus,
+    int source_offset,
+    int buffer_fill_offset,
+    int num_samples) {
+  DCHECK(encoder_thread_checker_.CalledOnValidThread());
+  DCHECK(IsInitialized());
+  // Opus requires channel-interleaved samples in a single array.
+  for (int ch = 0; ch < audio_bus->channels(); ++ch) {
+    const float* src = audio_bus->channel(ch) + source_offset;
+    const float* const src_end = src + num_samples;
+    float* dest =
+        buffer_.get() + buffer_fill_offset * audio_params_.channels() + ch;
+    for (; src < src_end; ++src, dest += audio_params_.channels())
+      *dest = *src;
+  }
+}
+
+bool AudioTrackRecorder::AudioEncoder::EncodeFromFilledBuffer(
+    std::string* out) {
+  DCHECK(encoder_thread_checker_.CalledOnValidThread());
+  DCHECK(IsInitialized());
+
+  out->resize(OPUS_MAX_PAYLOAD_SIZE);
+  const opus_int32 result = opus_encode_float(
+      opus_encoder_, buffer_.get(), samples_per_frame_,
+      reinterpret_cast<uint8*>(string_as_array(out)), OPUS_MAX_PAYLOAD_SIZE);
+  if (result > 1) {
+    out->resize(result);

[mcasas, 2015/10/26 18:56:25]

This is not your fault, but are we allocating an array of
|OPUS_MAX_PAYLOAD_SIZE| and, after encoding, 
resizing it to the appropriate size, for every single frame?
That doesn't sound very efficient :(

[ajose, 2015/10/26 20:26:46]

I'll look into it, added a bug.

+    return true;
+  }
+  // If |result| in {0,1}, do nothing; the documentation says that a return
+  // value of zero or one means the packet does not need to be transmitted.
+  // Otherwise, we have an error.
+  DLOG_IF(ERROR, result < 0) << __FUNCTION__
+                             << " failed: " << opus_strerror(result);
+  return false;
+}
+
+// static
+bool AudioTrackRecorder::AudioEncoder::IsValidFrameDuration(

[mcasas, 2015/10/26 18:56:25]

Move this method to an anonymous namespace and out of the 
class (smaller footprint).

[ajose, 2015/10/26 20:26:46]

Done.

+    base::TimeDelta duration) {
+  // See https://tools.ietf.org/html/rfc6716#section-2.1.4
+  return duration == base::TimeDelta::FromMicroseconds(2500) ||
+         duration == base::TimeDelta::FromMilliseconds(5) ||
+         duration == base::TimeDelta::FromMilliseconds(10) ||
+         duration == base::TimeDelta::FromMilliseconds(20) ||
+         duration == base::TimeDelta::FromMilliseconds(40) ||
+         duration == base::TimeDelta::FromMilliseconds(60);
+}
+
+AudioTrackRecorder::AudioTrackRecorder(
+    const blink::WebMediaStreamTrack& track,
+    const OnEncodedAudioCB& on_encoded_audio_cb)
+    : track_(track),
+      encoder_(new AudioEncoder(media::BindToCurrentLoop(on_encoded_audio_cb))),
+      encoder_thread_(new base::Thread("AudioEncoderThread")) {
+  DCHECK(main_render_thread_checker_.CalledOnValidThread());
+  DCHECK(!track_.isNull());
+  DCHECK(track_.extraData());
+
+  // Start the |encoder_thread_|. From this point on, |encoder_| should work
+  // only on |encoder_thread_|, as enforced by DCHECKs.
+  DCHECK(!encoder_thread_->IsRunning());
+  encoder_thread_->Start();
+
+  // Connect the source provider to the track as a sink.
+  MediaStreamAudioSink::AddToAudioTrack(this, track_);
+}
+
+AudioTrackRecorder::~AudioTrackRecorder() {
+  DCHECK(main_render_thread_checker_.CalledOnValidThread());
+  MediaStreamAudioSink::RemoveFromAudioTrack(this, track_);
+  track_.reset();

[mcasas, 2015/10/26 18:56:25]

I don't think you need to reset() it explicitly since is anyway
going out of scope [1]. With that, you could make |track_| const.
(This could apply to VideoTrackRecorder too)

[1]
https://code.google.com/p/chromium/codesearch#chromium/src/third_party/WebKit...

[ajose, 2015/10/26 20:26:46]

Done.

+}
+
+void AudioTrackRecorder::OnData(const media::AudioBus& audio_bus,
+                                base::TimeTicks capture_time) {
+  DCHECK(encoder_thread_->IsRunning());
+  DCHECK(capture_thread_checker_.CalledOnValidThread());
+  DCHECK(!capture_time.is_null());
+
+  // TODO(ajose): When will audio_bus be deleted?
+  scoped_ptr<media::AudioBus> audio_data =
+      media::AudioBus::Create(audio_bus.channels(), audio_bus.frames());
+  audio_bus.CopyTo(audio_data.get());
+
+  encoder_thread_->task_runner()->PostTask(
+      FROM_HERE, base::Bind(&AudioEncoder::EncodeAudio, encoder_,
+                            base::Passed(&audio_data), capture_time));
+}
+
+void AudioTrackRecorder::OnSetFormat(const media::AudioParameters& params) {
+  DCHECK(encoder_thread_->IsRunning());
+  // If the source is restarted, might have changed to another capture thread.
+  capture_thread_checker_.DetachFromThread();
+  DCHECK(capture_thread_checker_.CalledOnValidThread());
+
+  encoder_thread_->task_runner()->PostTask(
+      FROM_HERE, base::Bind(&AudioEncoder::OnSetFormat, encoder_, params));
+}
+
+}  // namespace content