Add AudioTrackRecorder for audio component of MediaStream recording.

content/renderer/media/audio_track_recorder.cc

Index: content/renderer/media/audio_track_recorder.cc
diff --git a/content/renderer/media/audio_track_recorder.cc b/content/renderer/media/audio_track_recorder.cc
new file mode 100644
index 0000000000000000000000000000000000000000..eba7251e267d7bfca0bd3915146933036e610a84
--- /dev/null
+++ b/content/renderer/media/audio_track_recorder.cc
@@ -0,0 +1,297 @@
+// Copyright 2015 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "base/bind.h"
+#include "base/stl_util.h"
+#include "base/threading/thread.h"
+#include "content/renderer/media/audio_track_recorder.h"
+
+#include "third_party/opus/src/include/opus.h"
+
+namespace content {
+
+namespace {
+
+// TODO(ajose): This likely shouldn't be hardcoded.
+const int kDefaultFramesPerSecond = 100;
+const int kDefaultAudioEncoderBitrate = 0;  // let opus choose
+// This is the recommended value, according to documentation in
+// third_party/opus/src/include/opus.h, so that the Opus encoder does not
+// degrade the audio due to memory constraints.
+//
+// Note: Whereas other RTP implementations do not, the cast library is
+// perfectly capable of transporting larger than MTU-sized audio frames.
+static const int kOpusMaxPayloadSize = 4000;
+
+// TODO(ajose): Removed code that might be useful for ensuring A/V sync.
+// See cast/sender/AudioEncoder.
+
+}  // anonymous namespace
+
+class AudioTrackRecorder::AudioEncoder
+    : public base::RefCountedThreadSafe<AudioEncoder> {
+ public:
+  static void ShutdownEncoder(scoped_ptr<base::Thread> encoding_thread) {
+    DCHECK(encoding_thread->IsRunning());
+    encoding_thread->Stop();
+  }
+
+  AudioEncoder(const OnEncodedAudioCB& on_encoded_audio_cb)

[mcasas, 2015/10/19 20:02:08]

Do not inline large methods such as this ctor and dtor below.

[ajose, 2015/10/20 03:21:11]

Done.

+      : initialized_(false),
+        on_encoded_audio_cb_(on_encoded_audio_cb),
+        encoding_thread_(new base::Thread("AudioEncodingThread")),
+        main_task_runner_(base::MessageLoop::current()->task_runner()) {
+    DCHECK(!encoding_thread_->IsRunning());
+    encoding_thread_->Start();
+  }
+
+  void OnSetFormat(const media::AudioParameters& params) {
+    DCHECK(params.IsValid());
+    InitOpus(params);
+  }
+
+  void InsertAudio(scoped_ptr<media::AudioBus> audio_bus,
+                   const base::TimeTicks& recorded_time);
+
+  bool IsInitialized() { return initialized_; }

[mcasas, 2015/10/19 20:02:08]

You can also turn |audio_params_| into a scoped_ptr()
and redirect this method to testing it.

[ajose, 2015/10/20 03:21:11]

Done.

+
+ private:
+  friend class base::RefCountedThreadSafe<AudioEncoder>;
+
+  ~AudioEncoder() {
+    main_task_runner_->PostTask(FROM_HERE,
+                                base::Bind(&AudioEncoder::ShutdownEncoder,
+                                           base::Passed(&encoding_thread_)));
+  }
+
+  void InitOpus(const media::AudioParameters& params);
+
+  void EncodeAudio(scoped_ptr<media::AudioBus> audio_bus,
+                   const base::TimeTicks& recorded_time);
+
+  void TransferSamplesIntoBuffer(const media::AudioBus* audio_bus,
+                                 int source_offset,
+                                 int buffer_fill_offset,
+                                 int num_samples);
+  bool EncodeFromFilledBuffer(std::string* out);
+
+  static bool IsValidFrameDuration(base::TimeDelta duration);
+
+  bool initialized_;
+
+  int samples_per_frame_;
+  const OnEncodedAudioCB on_encoded_audio_cb_;
+
+  // In the case where a call to EncodeAudio() cannot completely fill the
+  // buffer, this points to the position at which to populate data in a later
+  // call.
+  int buffer_fill_end_;
+
+  // Do actual opus encoding on a separate thread.
+  scoped_ptr<base::Thread> encoding_thread_;

[mcasas, 2015/10/19 20:02:08]

Maybe miu@ will have something to say about the necessity
of this thread down the line, for the time being I'll skip it. I'd
say that perhaps the extra delay+jitter of the thread jump 
offsets the benefits of not loading the IO thread.

+
+  // Used to shutdown properly on the same thread we were created on.
+  const scoped_refptr<base::SingleThreadTaskRunner> main_task_runner_;
+
+  // Task runner where frames to encode and reply callbacks must happen.
+  scoped_refptr<base::SingleThreadTaskRunner> origin_task_runner_;
+
+  media::AudioParameters audio_params_;
+
+  // OpusEncoder-related.
+  scoped_ptr<uint8[]> encoder_memory_;
+  OpusEncoder* opus_encoder_;
+  scoped_ptr<float[]> buffer_;
+
+  DISALLOW_COPY_AND_ASSIGN(AudioEncoder);
+};
+
+void AudioTrackRecorder::AudioEncoder::InsertAudio(
+    scoped_ptr<media::AudioBus> audio_bus,
+    const base::TimeTicks& recorded_time) {
+  if (!origin_task_runner_.get())
+    origin_task_runner_ = base::MessageLoop::current()->task_runner();
+  DCHECK(origin_task_runner_->BelongsToCurrentThread());
+
+  DCHECK(audio_bus.get());
+  DCHECK(initialized_);
+
+  encoding_thread_->task_runner()->PostTask(
+      FROM_HERE, base::Bind(&AudioEncoder::EncodeAudio, this,
+                            base::Passed(&audio_bus), recorded_time));
+}
+
+void AudioTrackRecorder::AudioEncoder::InitOpus(
+    const media::AudioParameters& params) {
+  int sampling_rate = params.sample_rate();
+  int bitrate = kDefaultAudioEncoderBitrate;
+  int num_channels = params.channels();
+  samples_per_frame_ = sampling_rate / kDefaultFramesPerSecond;
+  base::TimeDelta frame_duration = base::TimeDelta::FromMicroseconds(
+      base::Time::kMicrosecondsPerSecond * samples_per_frame_ / sampling_rate);
+
+  // Initialize  things that OpusEncoder needs.
+  buffer_fill_end_ = 0;
+  encoder_memory_.reset(new uint8[opus_encoder_get_size(num_channels)]);
+  opus_encoder_ = reinterpret_cast<OpusEncoder*>(encoder_memory_.get());

[mcasas, 2015/10/19 20:02:08]

Wow this is black magic!
Seriously though, do we actually need |encoder_memory_|?
can't we allocate and cast to |opus_encoder_|, or would that
miss one (necessary) destructor?

[ajose, 2015/10/20 03:21:11]

Switched to opus_encoder_create

+  buffer_.reset(new float[num_channels * samples_per_frame_]);
+
+  // Support for max sampling rate of 48KHz, 2 channels, 100 ms duration.
+  const int kMaxSamplesTimesChannelsPerFrame = 48 * 2 * 100;
+  if (num_channels <= 0 || samples_per_frame_ <= 0 ||
+      frame_duration == base::TimeDelta() ||
+      samples_per_frame_ * num_channels > kMaxSamplesTimesChannelsPerFrame ||
+      sampling_rate % samples_per_frame_ != 0 ||
+      !IsValidFrameDuration(frame_duration)) {

[mcasas, 2015/10/19 20:02:08]

I'd move the check for IsValidFrameDuration() to right after
initializing it in l.131 (and make |frame_duration| const).

[ajose, 2015/10/20 03:21:12]

Done.

+    DVLOG(1) << __FUNCTION__ << ": bad inputs.";

[mcasas, 2015/10/19 20:02:09]

Make this msg more meaningful or remove it.

[ajose, 2015/10/20 03:21:11]

Done.

+    return;
+  }
+
+  if (opus_encoder_init(opus_encoder_, sampling_rate, num_channels,
+                        OPUS_APPLICATION_AUDIO) != OPUS_OK) {
+    DVLOG(1) << __FUNCTION__ << ": couldn't initialize opus encoder.";

[mcasas, 2015/10/19 20:02:08]

What about caching the result of opus_encoder_init() and
showing some info error using opus_strerror() (see below).

[ajose, 2015/10/20 03:21:11]

Done.

+    return;
+  }
+
+  if (bitrate <= 0) {
+    // Note: As of 2013-10-31, the encoder in "auto bitrate" mode would use a
+    // variable bitrate up to 102kbps for 2-channel, 48 kHz audio and a 10 ms
+    // frame size.  The opus library authors may, of course, adjust this in
+    // later versions.
+    bitrate = OPUS_AUTO;

[mcasas, 2015/10/19 20:02:08]

|bitrate| is initalized to kDefaultAudioEncoderBitrate, so I fail
to see the usefulness of this if(). Suggest some const initing
(with the comment) in the line of:

const int bitrate = (w00t) ? OPUS_AUTO : kDefaultAudioEncoderBitrate;

[ajose, 2015/10/20 03:21:11]

Was considering letting user set bitrate but will probably just let opus always
choose for now.

+  }
+
+  audio_params_ = params;
+  initialized_ = true;
+
+  CHECK_EQ(opus_encoder_ctl(opus_encoder_, OPUS_SET_BITRATE(bitrate)), OPUS_OK);
+}
+
+void AudioTrackRecorder::AudioEncoder::EncodeAudio(
+    scoped_ptr<media::AudioBus> audio_bus,
+    const base::TimeTicks& recorded_time) {
+  DCHECK(encoding_thread_->task_runner()->BelongsToCurrentThread());
+  DCHECK(initialized_);
+  DCHECK(!recorded_time.is_null());
+
+  // Encode all audio in |audio_bus| into zero or more frames.
+  int src_pos = 0;
+  while (src_pos < audio_bus->frames()) {
+    const int num_samples_to_xfer = std::min(
+        samples_per_frame_ - buffer_fill_end_, audio_bus->frames() - src_pos);
+    DCHECK_EQ(audio_bus->channels(), audio_params_.channels());

[mcasas, 2015/10/19 20:02:08]

If this doesn't change as |src_pos| moves along, move it
out of the loop.

[ajose, 2015/10/20 03:21:11]

Done.

+    TransferSamplesIntoBuffer(audio_bus.get(), src_pos, buffer_fill_end_,
+                              num_samples_to_xfer);
+    src_pos += num_samples_to_xfer;
+    buffer_fill_end_ += num_samples_to_xfer;
+
+    if (buffer_fill_end_ < samples_per_frame_)
+      break;
+
+    scoped_ptr<std::string> encoded_data(new std::string());
+    if (EncodeFromFilledBuffer(encoded_data.get())) {
+      origin_task_runner_->PostTask(
+          FROM_HERE, base::Bind(on_encoded_audio_cb_, audio_params_,
+                                base::Passed(&encoded_data), recorded_time));
+    }
+
+    // Reset the internal buffer for the next frame.
+    buffer_fill_end_ = 0;
+  }
+}
+
+void AudioTrackRecorder::AudioEncoder::TransferSamplesIntoBuffer(
+    const media::AudioBus* audio_bus,
+    int source_offset,
+    int buffer_fill_offset,
+    int num_samples) {
+  // Opus requires channel-interleaved samples in a single array.
+  for (int ch = 0; ch < audio_bus->channels(); ++ch) {
+    const float* src = audio_bus->channel(ch) + source_offset;
+    const float* const src_end = src + num_samples;
+    float* dest =
+        buffer_.get() + buffer_fill_offset * audio_params_.channels() + ch;
+    for (; src < src_end; ++src, dest += audio_params_.channels())
+      *dest = *src;
+  }
+}
+
+bool AudioTrackRecorder::AudioEncoder::EncodeFromFilledBuffer(
+    std::string* out) {
+  out->resize(kOpusMaxPayloadSize);
+  const opus_int32 result = opus_encode_float(
+      opus_encoder_, buffer_.get(), samples_per_frame_,
+      reinterpret_cast<uint8*>(string_as_array(out)), kOpusMaxPayloadSize);
+  if (result > 1) {
+    out->resize(result);
+    return true;
+  } else if (result < 0) {
+    LOG(ERROR) << __FUNCTION__
+               << ": Error code from opus_encode_float(): " << result;

[mcasas, 2015/10/19 20:02:08]

Suggestion: Use opus_strerror() [1]

[1]
https://code.google.com/p/chromium/codesearch#chromium/src/third_party/opus/s...

[ajose, 2015/10/20 03:21:11]

Nice

+    return false;
+  } else {
+    // Do nothing: The documentation says that a return value of zero or
+    // one byte means the packet does not need to be transmitted.

[mcasas, 2015/10/19 20:02:08]

nit: remove "byte"

[ajose, 2015/10/20 03:21:12]

Done.

+    return false;
+  }
+}
+
+// static
+bool AudioTrackRecorder::AudioEncoder::IsValidFrameDuration(
+    base::TimeDelta duration) {
+  // See https://tools.ietf.org/html/rfc6716#section-2.1.4
+  return duration == base::TimeDelta::FromMicroseconds(2500) ||
+         duration == base::TimeDelta::FromMilliseconds(5) ||
+         duration == base::TimeDelta::FromMilliseconds(10) ||
+         duration == base::TimeDelta::FromMilliseconds(20) ||
+         duration == base::TimeDelta::FromMilliseconds(40) ||
+         duration == base::TimeDelta::FromMilliseconds(60);
+}
+
+AudioTrackRecorder::AudioTrackRecorder(
+    const blink::WebMediaStreamTrack& track,
+    const OnEncodedAudioCB& on_encoded_audio_cb)
+    : track_(track),
+      encoder_(new AudioEncoder(on_encoded_audio_cb)),
+      on_encoded_audio_cb_(on_encoded_audio_cb) {
+  DCHECK(main_render_thread_checker_.CalledOnValidThread());
+  DCHECK(!track_.isNull());
+  DCHECK(track_.extraData());
+  // Connect the source provider to the track as a sink.
+  MediaStreamAudioSink::AddToAudioTrack(this, track_);
+}
+
+AudioTrackRecorder::~AudioTrackRecorder() {
+  DCHECK(main_render_thread_checker_.CalledOnValidThread());
+  MediaStreamAudioSink::RemoveFromAudioTrack(this, track_);
+  track_.reset();
+}
+
+void AudioTrackRecorder::OnData(const media::AudioBus& audio_bus,
+                                base::TimeTicks estimated_capture_time) {
+  DCHECK(encoder_->IsInitialized());

[mcasas, 2015/10/19 20:02:08]

Thread? Or a comment about it for the method.

[ajose, 2015/10/20 03:21:11]

Looking into this.

+  DCHECK_EQ(audio_bus.channels(), audio_params_.channels());
+  DCHECK_EQ(audio_bus.frames(), audio_params_.frames_per_buffer());
+  DCHECK(!estimated_capture_time.is_null());
+
+  // TODO(ajose): When will audio_bus be deleted?
+  scoped_ptr<media::AudioBus> audio_data =
+      media::AudioBus::Create(audio_params_);
+  audio_bus.CopyTo(audio_data.get());
+  encoder_->InsertAudio(audio_data.Pass(), estimated_capture_time);

[mcasas, 2015/10/19 20:02:08]

Is a bit confusing that we copy the data from |audio_bus|
at this stage, since we define InsertAudio(). Shouldn't we
do that inside that method instead? IOW I would see this 
method as a simple trampoline.

[ajose, 2015/10/20 03:21:11]

Done.

+}
+
+void AudioTrackRecorder::OnSetFormat(const media::AudioParameters& params) {
+  DCHECK(params.IsValid());
+  DCHECK_EQ(params.bits_per_sample(), 16);
+
+  if (audio_params_.Equals(params))
+    return;
+
+  // TODO(ajose): consider only storing params in ATR _or_ encoder, not both.

[mcasas, 2015/10/19 20:02:08]

I was thinking the same, and by preference I'd say inside ATR::AE,
no need to cache it up here.

[ajose, 2015/10/20 03:21:11]

Done.

+  audio_params_ = params;
+  encoder_->OnSetFormat(params);
+}
+
+}  // namespace content