H264 HW encode using VideoToolbox

content/common/gpu/media/vt_video_encode_accelerator_mac.cc

+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "content/common/gpu/media/vt_video_encode_accelerator_mac.h"
+
+#include "base/thread_task_runner_handle.h"
+#include "media/base/mac/coremedia_glue.h"
+#include "media/base/mac/corevideo_glue.h"
+#include "media/base/mac/video_frame_mac.h"
+
+namespace content {
+
+namespace {
+
+// Subjectively chosen.
+// TODO(emircan): Check if we can find the actual system capabilities via
+// creating VTCompressionSessions with varying requirements.
+// See crbug.com/584784.
+const size_t kNumInputBuffers = 4;
+const size_t kMaxFrameRateNumerator = 30;
+const size_t kMaxFrameRateDenominator = 1;
+const size_t kMaxResolutionWidth = 4096;
+const size_t kMaxResolutionHeight = 2160;
+const size_t kBitsPerByte = 8;
+
+}  // namespace
+
+struct VTVideoEncodeAccelerator::InProgressFrameEncode {
+  InProgressFrameEncode(base::TimeDelta rtp_timestamp, base::TimeTicks ref_time)
+      : timestamp(rtp_timestamp), reference_time(ref_time) {}
+  const base::TimeDelta timestamp;
+  const base::TimeTicks reference_time;
+
+ private:
+  DISALLOW_IMPLICIT_CONSTRUCTORS(InProgressFrameEncode);
+};
+
+struct VTVideoEncodeAccelerator::EncodeOutput {
+  EncodeOutput(VTEncodeInfoFlags info_flags, CMSampleBufferRef sbuf)
+      : info(info_flags), sample_buffer(sbuf) {}
+  const VTEncodeInfoFlags info;
+  const CMSampleBufferRef sample_buffer;
+
+ private:
+  DISALLOW_IMPLICIT_CONSTRUCTORS(EncodeOutput);
+};
+
+struct VTVideoEncodeAccelerator::BitstreamBufferRef {
+  BitstreamBufferRef(int32_t id,
+                     scoped_ptr<base::SharedMemory> shm,
+                     size_t size)
+      : id(id), shm(std::move(shm)), size(size) {}
+  const int32_t id;
+  const scoped_ptr<base::SharedMemory> shm;
+  const size_t size;
+
+ private:
+  DISALLOW_IMPLICIT_CONSTRUCTORS(BitstreamBufferRef);
+};
+
+VTVideoEncodeAccelerator::VTVideoEncodeAccelerator()
+    : set_data_rate_limit_(true),
+      client_task_runner_(base::ThreadTaskRunnerHandle::Get()) {
+}
+
+VTVideoEncodeAccelerator::~VTVideoEncodeAccelerator() {
+  DCHECK(thread_checker_.CalledOnValidThread());
+}
+
+media::VideoEncodeAccelerator::SupportedProfiles
+VTVideoEncodeAccelerator::GetSupportedProfiles() {
+  DVLOG(3) << __FUNCTION__;
+  DCHECK(thread_checker_.CalledOnValidThread());
+
+  SupportedProfiles profiles;
+  SupportedProfile profile;
+  profile.profile = media::H264PROFILE_BASELINE;
+  profile.max_framerate_numerator = kMaxFrameRateNumerator;
+  profile.max_framerate_denominator = kMaxFrameRateDenominator;
+  profile.max_resolution = gfx::Size(kMaxResolutionWidth, kMaxResolutionHeight);
+  profiles.push_back(profile);
+  return profiles;
+}
+
+bool VTVideoEncodeAccelerator::Initialize(
+    media::VideoPixelFormat format,
+    const gfx::Size& input_visible_size,
+    media::VideoCodecProfile output_profile,
+    uint32_t initial_bitrate,
+    Client* client) {
+  DVLOG(3) << __FUNCTION__
+           << ": input_format=" << media::VideoPixelFormatToString(format)
+           << ", input_visible_size=" << input_visible_size.ToString()
+           << ", output_profile=" << output_profile
+           << ", initial_bitrate=" << initial_bitrate;
+  DCHECK(thread_checker_.CalledOnValidThread());
+  DCHECK(client);
+
+  if (media::PIXEL_FORMAT_I420 != format) {
+    DLOG(ERROR) << "Input format not supported= "
+                << media::VideoPixelFormatToString(format);
+    return false;
+  }
+  if (media::H264PROFILE_BASELINE != output_profile) {
+    DLOG(ERROR) << "Output profile not supported= "
+                << output_profile;
+    return false;
+  }
+
+  videotoolbox_glue_ = VideoToolboxGlue::Get();
+  if (!videotoolbox_glue_) {
+    DLOG(ERROR) << "Failed creating VideoToolbox glue";
+    return false;
+  }
+
+  client_ptr_factory_.reset(new base::WeakPtrFactory<Client>(client));
+  client_ = client_ptr_factory_->GetWeakPtr();
+  input_visible_size_ = input_visible_size;
+  frame_rate_ = kMaxFrameRateNumerator / kMaxFrameRateDenominator;
+  target_bitrate_ = initial_bitrate;
+  bitstream_buffer_size_ = target_bitrate_ / kBitsPerByte;
+
+  if (!ResetCompressionSession()) {
+    DLOG(ERROR) << "Failed creating compression session";
+    return false;
+  }
+
+  client_->RequireBitstreamBuffers(kNumInputBuffers, input_visible_size_,
+                                   bitstream_buffer_size_);
+  return true;
+}
+
+void VTVideoEncodeAccelerator::Encode(
+    const scoped_refptr<media::VideoFrame>& frame,
+    bool force_keyframe) {
+  DVLOG(3) << __FUNCTION__;
+  DCHECK(thread_checker_.CalledOnValidThread());
+  DCHECK(compression_session_);
+  DCHECK(frame);
+
+  base::TimeTicks ref_time;
+  if (!frame->metadata()->GetTimeTicks(
+          media::VideoFrameMetadata::REFERENCE_TIME, &ref_time)) {
+    ref_time = base::TimeTicks::Now();
+  }
+  auto timestamp_cm = CoreMediaGlue::CMTimeMake(
+      frame->timestamp().InMicroseconds(), USEC_PER_SEC);
+  // Wrap information we'll need after the frame is encoded in a heap object.
+  // We'll get the pointer back from the VideoToolbox completion callback.
+  scoped_ptr<InProgressFrameEncode> request(new InProgressFrameEncode(

[Pawel Osciak, 2016/02/18 11:16:14]

Nit: please move above l.163.

[emircan, 2016/03/02 05:28:13]

Done.

+      frame->timestamp(), ref_time));
+
+  // TODO(emircan): See if we can eliminate a copy here by using
+  // CVPixelBufferPool for the allocation of incoming VideoFrames.
+  base::ScopedCFTypeRef<CVPixelBufferRef> pixel_buffer =
+      media::WrapVideoFrameInCVPixelBuffer(*frame);
+  base::ScopedCFTypeRef<CFDictionaryRef> frame_props =
+      media::video_toolbox::DictionaryWithKeyValue(
+          videotoolbox_glue_->kVTEncodeFrameOptionKey_ForceKeyFrame(),
+          force_keyframe ? kCFBooleanTrue : kCFBooleanFalse);
+
+  OSStatus status = videotoolbox_glue_->VTCompressionSessionEncodeFrame(
+      compression_session_, pixel_buffer, timestamp_cm,
+      CoreMediaGlue::CMTime{0, 0, 0, 0}, frame_props,
+      reinterpret_cast<void*>(request.release()), nullptr);
+  if (status != noErr) {
+    DLOG(ERROR) << " VTCompressionSessionEncodeFrame failed: " << status;
+    client_->NotifyError(kPlatformFailureError);
+  }
+}
+
+void VTVideoEncodeAccelerator::UseOutputBitstreamBuffer(
+    const media::BitstreamBuffer& buffer) {
+  DVLOG(3) << __FUNCTION__ << ": buffer size=" << buffer.size();
+  DCHECK(thread_checker_.CalledOnValidThread());
+  if (buffer.size() < bitstream_buffer_size_) {
+    DLOG(ERROR) << "Output BitstreamBuffer isn't big enough: " << buffer.size()
+                << " vs. " << bitstream_buffer_size_;
+    client_->NotifyError(kInvalidArgumentError);
+    return;
+  }
+
+  scoped_ptr<base::SharedMemory> shm(
+      new base::SharedMemory(buffer.handle(), false));
+  if (!shm->Map(buffer.size())) {
+    DLOG(ERROR) << "Failed mapping shared memory.";
+    client_->NotifyError(kPlatformFailureError);
+    return;
+  }
+
+  scoped_ptr<BitstreamBufferRef> buffer_ref(
+      new BitstreamBufferRef(buffer.id(), std::move(shm), buffer.size()));
+
+  // If there is already EncodeOutput waiting, copy its output first.
+  if (!encoder_output_queue_.empty()) {
+    scoped_ptr<VTVideoEncodeAccelerator::EncodeOutput> encode_output =
+        std::move(encoder_output_queue_.front());
+    encoder_output_queue_.pop_front();
+    ReturnBitstreamBuffer(encode_output->info, encode_output->sample_buffer,
+                          std::move(buffer_ref));
+    return;
+  }
+
+  bitstream_buffer_queue_.push_back(std::move(buffer_ref));
+}
+
+void VTVideoEncodeAccelerator::RequestEncodingParametersChange(
+    uint32_t bitrate,
+    uint32_t framerate) {
+  DVLOG(3) << __FUNCTION__ << ": bitrate=" << bitrate
+           << ": framerate=" << framerate;
+  DCHECK(thread_checker_.CalledOnValidThread());
+
+  frame_rate_ = framerate > 1 ? framerate : 1;
+  target_bitrate_ = bitrate > 1 ? bitrate : 1;
+
+  if (!compression_session_) {
+    client_->NotifyError(kPlatformFailureError);
+    return;
+  }
+
+  media::video_toolbox::SessionPropertySetter session_property_setter(
+      compression_session_, videotoolbox_glue_);
+
+  // TODO(emircan): See crbug.com/425352.
+  bool rv = session_property_setter.SetSessionProperty(
+      videotoolbox_glue_->kVTCompressionPropertyKey_AverageBitRate(),
+      target_bitrate_);
+  rv &= session_property_setter.SetSessionProperty(
+      videotoolbox_glue_->kVTCompressionPropertyKey_ExpectedFrameRate(),
+      frame_rate_);
+  DLOG_IF(ERROR, !rv) << "Couldn't change session encoding parameters.";
+
+  // Some HW encoder platforms are missing this parameter. Try once to set it.
+  if (set_data_rate_limit_) {
+    const int data_rate_limits[] = {target_bitrate_ / kBitsPerByte, 1};
+    set_data_rate_limit_ &= session_property_setter.SetSessionProperty(

[Pawel Osciak, 2016/02/18 11:16:14]

I believe this would have the same effect?

if (s_d_r_l_) {
  s_d_r_l = SSP();
}

[emircan, 2016/03/02 05:28:13]

Done.

+        videotoolbox_glue_->kVTCompressionPropertyKey_DataRateLimits(),
+        media::video_toolbox::ArrayWithIntegers(data_rate_limits,
+                                                arraysize(data_rate_limits)));
+  }
+}
+
+void VTVideoEncodeAccelerator::Destroy() {
+  DVLOG(3) << __FUNCTION__;
+  DCHECK(thread_checker_.CalledOnValidThread());
+
+  DestroyCompressionSession();
+  delete this;
+}
+
+// static
+void VTVideoEncodeAccelerator::CompressionCallback(void* encoder_opaque,
+                                                   void* request_opaque,
+                                                   OSStatus status,
+                                                   VTEncodeInfoFlags info,
+                                                   CMSampleBufferRef sbuf) {
+  // This function may be called asynchronously, on a different thread from the
+  // one that calls VTCompressionSessionEncodeFrame.
+  DVLOG(3) << __FUNCTION__;
+
+  auto encoder = reinterpret_cast<VTVideoEncodeAccelerator*>(encoder_opaque);
+  DCHECK(encoder);
+
+  // Release InProgressFrameEncode, since we don't have support to return
+  // timestamps at this point.
+  scoped_ptr<InProgressFrameEncode> request(
+      reinterpret_cast<InProgressFrameEncode*>(request_opaque));
+  request.reset();
+
+  if (status != noErr) {
+    DLOG(ERROR) << " encode failed: " << status;
+    encoder->client_task_runner_->PostTask(

[Pawel Osciak, 2016/02/18 11:16:14]

Do we need to do anything in this method at all? Could we just post to the task
and do error checks there?

[emircan, 2016/03/02 05:28:13]

Done.

+        FROM_HERE, base::Bind(&Client::NotifyError, encoder->client_,
+                              kPlatformFailureError));
+    return;
+  }
+
+  // CFRetain is required to hold onto CMSampleBufferRef when posting task
+  // between threads. The object should be released later using CFRelease.
+  CFRetain(sbuf);
+  // This method is NOT called on |client_task_runner_|, so we still need to
+  // post a task back to it to reach |client_|.
+  encoder->client_task_runner_->PostTask(
+      FROM_HERE,
+      base::Bind(&VTVideoEncodeAccelerator::CompressionCallbackTask,
+                 base::Unretained(encoder), info, sbuf));

[Pawel Osciak, 2016/02/18 11:16:14]

Please add a comment why it's safe to do Unretained.

[emircan, 2016/03/02 05:28:13]

Actually, Unretained looks wrong to me now. If this function is called during
the destruction sequence, |enocder| might be freed by the time task is posted. I
will change it to WeakPtr and invalidate WeakPtr in the beginning of destruction
sequence.

+}
+
+void VTVideoEncodeAccelerator::CompressionCallbackTask(VTEncodeInfoFlags info,
+                                                       CMSampleBufferRef sbuf) {
+  DVLOG(3) << __FUNCTION__;
+  DCHECK(thread_checker_.CalledOnValidThread());
+
+  // If there isn't any BitstreamBuffer to copy into, add it to a queue for

[Pawel Osciak, 2016/02/18 11:16:14]

Could we perhaps have a TryReturnBitstreamBuffer() which would do this also,
instead of duplicating this code here and in UseOutputBitstreamBuffer() please?

[emircan, 2016/03/02 05:28:13]

Here we are checking if bitstream_buffer_queue_ is empty. Inside
UseOutputBitstreamBuffer() , we check if encoder_output_queue_ is empty. The
common code is already in ReturnBitstreamBuffer() and I don't know if any
further refactor can be done.

+  // later use.
+  if (bitstream_buffer_queue_.empty()) {
+    scoped_ptr<EncodeOutput> encode_output(new EncodeOutput(info, sbuf));
+    encoder_output_queue_.push_back(std::move(encode_output));
+    return;
+  }
+
+  scoped_ptr<VTVideoEncodeAccelerator::BitstreamBufferRef> buffer_ref =
+      std::move(bitstream_buffer_queue_.front());
+  bitstream_buffer_queue_.pop_front();
+  ReturnBitstreamBuffer(info, sbuf, std::move(buffer_ref));
+}
+
+void VTVideoEncodeAccelerator::ReturnBitstreamBuffer(
+    VTEncodeInfoFlags info,
+    CMSampleBufferRef sbuf,
+    scoped_ptr<VTVideoEncodeAccelerator::BitstreamBufferRef> buffer_ref) {
+  DVLOG(3) << __FUNCTION__;
+  DCHECK(thread_checker_.CalledOnValidThread());
+
+  if (info & VideoToolboxGlue::kVTEncodeInfo_FrameDropped) {
+    DVLOG(2) << " frame dropped";
+    CFRelease(sbuf);
+    client_->BitstreamBufferReady(buffer_ref->id, 0, false);
+    return;
+  }
+
+  auto sample_attachments = static_cast<CFDictionaryRef>(CFArrayGetValueAtIndex(
+      CoreMediaGlue::CMSampleBufferGetSampleAttachmentsArray(sbuf, true), 0));
+  const bool keyframe =
+      !CFDictionaryContainsKey(sample_attachments,
+                               CoreMediaGlue::kCMSampleAttachmentKey_NotSync());
+
+  size_t used_buffer_size = 0;
+  const bool copy_rv = media::video_toolbox::CopySampleBufferToAnnexBBuffer(
+      sbuf, keyframe, buffer_ref->size,
+      reinterpret_cast<char*>(buffer_ref->shm->memory()), &used_buffer_size);
+  CFRelease(sbuf);
+  if (!copy_rv) {
+    DLOG(ERROR) << "Cannot copy output from SampleBuffer to AnnexBBuffer.";
+    used_buffer_size = 0;
+  }
+
+  client_->BitstreamBufferReady(buffer_ref->id, used_buffer_size, keyframe);
+}
+
+bool VTVideoEncodeAccelerator::ResetCompressionSession() {
+  DCHECK(thread_checker_.CalledOnValidThread());
+
+  DestroyCompressionSession();
+
+  // Keep these in-sync with those in ConfigureCompressionSession().

[Pawel Osciak, 2016/02/18 11:16:14]

These keys below seem different from the ones in CCS()...? What does it mean to
keep them in sync?

[emircan, 2016/03/02 05:28:13]

Thanks for the notice, I am removing the comment since it no longer applies.

+  CFTypeRef attributes_keys[] = {
+    kCVPixelBufferOpenGLCompatibilityKey,
+    kCVPixelBufferIOSurfacePropertiesKey,
+    kCVPixelBufferPixelFormatTypeKey
+  };
+  const int format[] = {
+      CoreVideoGlue::kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange};
+  CFTypeRef attributes_values[] = {
+      kCFBooleanTrue,
+      media::video_toolbox::DictionaryWithKeysAndValues(nullptr, nullptr, 0)
+          .release(),
+      media::video_toolbox::ArrayWithIntegers(format, arraysize(format))
+          .release()};
+  const base::ScopedCFTypeRef<CFDictionaryRef> attributes =
+      media::video_toolbox::DictionaryWithKeysAndValues(
+          attributes_keys, attributes_values, arraysize(attributes_keys));
+  for (auto& v : attributes_values)
+    CFRelease(v);
+
+  bool session_rv = CreateCompressionSession(attributes, true);
+  if (!session_rv) {
+    // Try creating session again without forcing HW encode.

[Pawel Osciak, 2016/02/18 11:16:14]

Are we interested in doing SW encode here at all? It would be preferable to do
it outside of GPU process then...

[emircan, 2016/03/02 05:28:13]

Eventually, we want to compare the performance of Videotoolbox SW to our
OpenH264 SW when they both land completely. Then, we can modify this logic.
WDYT?

+    DestroyCompressionSession();
+    session_rv = CreateCompressionSession(attributes, false);
+    if (!session_rv) {
+      DestroyCompressionSession();
+      return false;
+    }
+  }
+
+  const bool configure_rv = ConfigureCompressionSession();
+  RequestEncodingParametersChange(target_bitrate_, frame_rate_);
+  return configure_rv;
+}
+
+bool VTVideoEncodeAccelerator::CreateCompressionSession(
+    base::ScopedCFTypeRef<CFDictionaryRef> attributes,
+    bool require_hw_encoding) {
+  DCHECK(thread_checker_.CalledOnValidThread());
+
+  std::vector<CFTypeRef> encoder_keys;
+  std::vector<CFTypeRef> encoder_values;
+  encoder_keys.push_back(videotoolbox_glue_
+        ->kVTVideoEncoderSpecification_EnableHardwareAcceleratedVideoEncoder());
+  encoder_values.push_back(kCFBooleanTrue);
+
+  if (require_hw_encoding) {
+    encoder_keys.push_back(
+        videotoolbox_glue_
+      ->kVTVideoEncoderSpecification_RequireHardwareAcceleratedVideoEncoder());
+    encoder_values.push_back(kCFBooleanTrue);
+  }
+  base::ScopedCFTypeRef<CFDictionaryRef> encoder_spec =
+      media::video_toolbox::DictionaryWithKeysAndValues(
+          encoder_keys.data(), encoder_values.data(), encoder_keys.size());
+
+  // Create the compression session.
+  OSStatus status = videotoolbox_glue_->VTCompressionSessionCreate(
+      kCFAllocatorDefault,
+      input_visible_size_.width(),
+      input_visible_size_.height(),
+      CoreMediaGlue::kCMVideoCodecType_H264,
+      encoder_spec,
+      attributes,
+      nullptr /* compressedDataAllocator */,
+      &VTVideoEncodeAccelerator::CompressionCallback,
+      reinterpret_cast<void*>(this),

[Pawel Osciak, 2016/02/18 11:16:14]

Can we in any way guarantee CompressionCallback caller doesn't outlive this?

[emircan, 2016/03/02 05:28:13]

I followed the logic used by cast that you can find below. I will copy their
comments as well to make it clear.
https://code.google.com/p/chromium/codesearch#chromium/src/media/cast/sender/...

+      compression_session_.InitializeInto());
+  if (status != noErr) {
+    DLOG(ERROR) << " VTCompressionSessionCreate failed: " << status;
+    return false;
+  }
+  DVLOG(3) << " VTCompressionSession created with HW encode: "
+           << require_hw_encoding;
+  return true;
+}
+
+bool VTVideoEncodeAccelerator::ConfigureCompressionSession() {
+  DCHECK(thread_checker_.CalledOnValidThread());
+  DCHECK(compression_session_);
+
+  media::video_toolbox::SessionPropertySetter session_property_setter(
+      compression_session_, videotoolbox_glue_);
+  bool rv = true;
+  rv &= session_property_setter.SetSessionProperty(
+      videotoolbox_glue_->kVTCompressionPropertyKey_ProfileLevel(),
+      videotoolbox_glue_->kVTProfileLevel_H264_Baseline_AutoLevel());
+  rv &= session_property_setter.SetSessionProperty(
+      videotoolbox_glue_->kVTCompressionPropertyKey_RealTime(), true);
+  rv &= session_property_setter.SetSessionProperty(
+      videotoolbox_glue_->kVTCompressionPropertyKey_AllowFrameReordering(),
+      false);
+  DLOG_IF(ERROR, !rv) << " SetSessionProperty failed.";
+  return rv;
+}
+
+void VTVideoEncodeAccelerator::DestroyCompressionSession() {
+  DCHECK(thread_checker_.CalledOnValidThread());
+
+  if (compression_session_) {
+    videotoolbox_glue_->VTCompressionSessionInvalidate(compression_session_);
+    compression_session_.reset();
+  }
+}
+
+} // namespace content