H264 HW encode using VideoToolbox

content/common/gpu/media/vt_video_encode_accelerator_mac.h

-// Copyright 2014 The Chromium Authors. All rights reserved.
+// Copyright 2016 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
-#ifndef MEDIA_CAST_SENDER_H264_VT_ENCODER_H_
-#define MEDIA_CAST_SENDER_H264_VT_ENCODER_H_
-
-#include <stdint.h>
+#ifndef CONTENT_COMMON_GPU_MEDIA_VT_VIDEO_ENCODE_ACCELERATOR_MAC_H_
+#define CONTENT_COMMON_GPU_MEDIA_VT_VIDEO_ENCODE_ACCELERATOR_MAC_H_
 
 #include "base/mac/scoped_cftyperef.h"
-#include "base/macros.h"
-#include "base/power_monitor/power_observer.h"
-#include "base/threading/thread_checker.h"
+#include "content/common/content_export.h"
 #include "media/base/mac/videotoolbox_glue.h"
-#include "media/cast/sender/size_adaptable_video_encoder_base.h"
-#include "media/cast/sender/video_encoder.h"
+#include "media/base/mac/videotoolbox_helpers.h"
+#include "media/video/video_encode_accelerator.h"
 
-namespace media {
-namespace cast {
+namespace content {
 
-// VideoToolbox implementation of the media::cast::VideoEncoder interface.
-// VideoToolbox makes no guarantees that it is thread safe, so this object is
-// pinned to the thread on which it is constructed. Supports changing frame
-// sizes directly. Implements the base::PowerObserver interface to reset the
-// compression session when the host process is suspended.
-class H264VideoToolboxEncoder : public VideoEncoder,
-                                public base::PowerObserver {
-  typedef CoreMediaGlue::CMSampleBufferRef CMSampleBufferRef;
-  typedef VideoToolboxGlue::VTCompressionSessionRef VTCompressionSessionRef;
-  typedef VideoToolboxGlue::VTEncodeInfoFlags VTEncodeInfoFlags;
+// VideoToolbox.framework implementation of the VideoEncodeAccelerator
+// interface for MacOSX. VideoToolbox makes no guarantees that it is thread
+// safe, so this object is pinned to the thread on which it is constructed.

[Pawel Osciak, 2016/02/18 11:16:14]

We should always avoid doing things on GPU Child thread unless we have no other
choice (e.g. need GL context), because this is a performance-critical thread.

[emircan, 2016/03/02 05:28:13]

Ok, I am adding an encoder thread. I was trying to make this patch a starting
point behind flag, but it is already big enough at this point :) Adding the
third thread that this class can be on would hurt.

+class CONTENT_EXPORT VTVideoEncodeAccelerator
+    : public media::VideoEncodeAccelerator {
+ public:
+  VTVideoEncodeAccelerator();
+  ~VTVideoEncodeAccelerator() override;
 
- public:
-  // Returns true if the current platform and system configuration supports
-  // using H264VideoToolboxEncoder with the given |video_config|.
-  static bool IsSupported(const VideoSenderConfig& video_config);
-
-  H264VideoToolboxEncoder(
-      const scoped_refptr<CastEnvironment>& cast_environment,
-      const VideoSenderConfig& video_config,
-      const StatusChangeCallback& status_change_cb);
-  ~H264VideoToolboxEncoder() final;
-
-  // media::cast::VideoEncoder implementation
-  bool EncodeVideoFrame(
-      const scoped_refptr<media::VideoFrame>& video_frame,
-      const base::TimeTicks& reference_time,
-      const FrameEncodedCallback& frame_encoded_callback) final;
-  void SetBitRate(int new_bit_rate) final;
-  void GenerateKeyFrame() final;
-  scoped_ptr<VideoFrameFactory> CreateVideoFrameFactory() final;
-  void EmitFrames() final;
-
-  // base::PowerObserver
-  void OnSuspend() final;
-  void OnResume() final;
+  // media::VideoEncodeAccelerator implementation.
+  media::VideoEncodeAccelerator::SupportedProfiles GetSupportedProfiles()
+      override;
+  bool Initialize(media::VideoPixelFormat format,
+                  const gfx::Size& input_visible_size,
+                  media::VideoCodecProfile output_profile,
+                  uint32_t initial_bitrate,
+                  Client* client) override;
+  void Encode(const scoped_refptr<media::VideoFrame>& frame,
+              bool force_keyframe) override;
+  void UseOutputBitstreamBuffer(const media::BitstreamBuffer& buffer) override;
+  void RequestEncodingParametersChange(uint32_t bitrate,
+                                       uint32_t framerate) override;
+  void Destroy() override;
 
  private:
-  // VideoFrameFactory tied to the VideoToolbox encoder.
-  class VideoFrameFactoryImpl;
+  using CMSampleBufferRef = CoreMediaGlue::CMSampleBufferRef;
+  using VTCompressionSessionRef = VideoToolboxGlue::VTCompressionSessionRef;
+  using VTEncodeInfoFlags = VideoToolboxGlue::VTEncodeInfoFlags;
 
-  // Reset the encoder's compression session by destroying the existing one
-  // using DestroyCompressionSession() and creating a new one. The new session
-  // is configured using ConfigureCompressionSession().
-  void ResetCompressionSession();
+  // Holds the associated data of a video frame being processed.
+  struct InProgressFrameEncode;
 
-  // Configure the current compression session using current encoder settings.
-  void ConfigureCompressionSession();
+  // Holds output buffers coming from the encoder.
+  struct EncodeOutput;
 
-  // Destroy the current compression session if any. Blocks until all pending
-  // frames have been flushed out (similar to EmitFrames without doing any
-  // encoding work).
-  void DestroyCompressionSession();
-
-  // Update the encoder's target frame size by resetting the compression
-  // session. This will also update the video frame factory.
-  void UpdateFrameSize(const gfx::Size& size_needed);
-
-  // Set a compression session property.
-  bool SetSessionProperty(CFStringRef key, int32_t value);
-  bool SetSessionProperty(CFStringRef key, bool value);
-  bool SetSessionProperty(CFStringRef key, CFStringRef value);
+  // Holds output buffers coming from the client ready to be filled.
+  struct BitstreamBufferRef;
 
   // Compression session callback function to handle compressed frames.
   static void CompressionCallback(void* encoder_opaque,
                                   void* request_opaque,
                                   OSStatus status,
                                   VTEncodeInfoFlags info,
                                   CMSampleBufferRef sbuf);
+  void CompressionCallbackTask(VTEncodeInfoFlags info, CMSampleBufferRef sbuf);
 
-  // The cast environment (contains worker threads & more).
-  const scoped_refptr<CastEnvironment> cast_environment_;
+  // Copy CMSampleBuffer into a BitstreamBuffer and return it to the |client_|.
+  void ReturnBitstreamBuffer(
+      VTEncodeInfoFlags info,
+      CMSampleBufferRef sbuf,
+      scoped_ptr<VTVideoEncodeAccelerator::BitstreamBufferRef> buffer_ref);
+
+  // Reset the encoder's compression session by destroying the existing one
+  // using DestroyCompressionSession() and creating a new one. The new session
+  // is configured using ConfigureCompressionSession().
+  bool ResetCompressionSession();
+
+  // Create a compression session, with HW encoder enforced if
+  // |require_hw_encoding| is set.
+  bool CreateCompressionSession(
+      base::ScopedCFTypeRef<CFDictionaryRef> attributes,
+      bool require_hw_encoding);
+
+  // Configure the current compression session using current encoder settings.
+  bool ConfigureCompressionSession();
+
+  // Destroy the current compression session if any. Blocks until all pending

[Pawel Osciak, 2016/02/18 11:16:14]

This sounds especially concerning if blocking GPU Child.

[emircan, 2016/03/02 05:28:13]

I will post this on the |encoder_thread| during dtor.

+  // frames have been flushed out (similar to EmitFrames without doing any
+  // encoding work).
+  void DestroyCompressionSession();
 
   // VideoToolboxGlue provides access to VideoToolbox at runtime.
-  const VideoToolboxGlue* const videotoolbox_glue_;
+  const VideoToolboxGlue* videotoolbox_glue_;
+  base::ScopedCFTypeRef<VTCompressionSessionRef> compression_session_;
 
-  // VideoSenderConfig copy so we can create compression sessions on demand.
-  // This is needed to recover from backgrounding and other events that can
-  // invalidate compression sessions.
-  const VideoSenderConfig video_config_;
+  gfx::Size input_visible_size_;
+  size_t bitstream_buffer_size_;
+  int32_t frame_rate_;
+  int32_t target_bitrate_;
+  bool set_data_rate_limit_;

[Pawel Osciak, 2016/02/18 11:16:14]

Please document this field.

[emircan, 2016/03/02 05:28:13]

Done.

 
-  // Frame size of the current compression session. Can be changed by submitting
-  // a frame of a different size, which will cause a compression session reset.
-  gfx::Size frame_size_;
+  // Bitstream buffers ready to be used to return encoded output as a FIFO.
+  std::deque<scoped_ptr<BitstreamBufferRef>> bitstream_buffer_queue_;
 
-  // Callback used to report initialization status and runtime errors.
-  const StatusChangeCallback status_change_cb_;
+  // EncodeOutput needs to be copied into a BitstreamBufferRef as a FIFO.
+  std::deque<scoped_ptr<EncodeOutput>> encoder_output_queue_;
+
+  // Our original calling task runner for the child thread.
+  const scoped_refptr<base::SingleThreadTaskRunner> client_task_runner_;
+
+  // To expose client callbacks from VideoEncodeAccelerator.
+  // NOTE: all calls to this object *MUST* be executed on
+  // |client_task_runner_|.
+  scoped_ptr<base::WeakPtrFactory<Client> > client_ptr_factory_;
+  base::WeakPtr<Client> client_;
 
   // Thread checker to enforce that this object is used on a specific thread.
   base::ThreadChecker thread_checker_;
 
-  // The compression session.
-  base::ScopedCFTypeRef<VTCompressionSessionRef> compression_session_;
-
-  // Video frame factory tied to the encoder.
-  scoped_refptr<VideoFrameFactoryImpl> video_frame_factory_;
-
-  // The ID of the last frame that was emitted.
-  uint32_t last_frame_id_;
-
-  // Force next frame to be a keyframe.
-  bool encode_next_frame_as_keyframe_;
-
-  // Power suspension state.
-  bool power_suspended_;
-
-  // NOTE: Weak pointers must be invalidated before all other member variables.
-  base::WeakPtrFactory<H264VideoToolboxEncoder> weak_factory_;
-
-  DISALLOW_COPY_AND_ASSIGN(H264VideoToolboxEncoder);
+  DISALLOW_COPY_AND_ASSIGN(VTVideoEncodeAccelerator);
 };
 
-}  // namespace cast
-}  // namespace media
+} // namespace content
 
-#endif  // MEDIA_CAST_SENDER_H264_VT_ENCODER_H_
+#endif  // CONTENT_COMMON_GPU_MEDIA_VT_VIDEO_ENCODE_ACCELERATOR_MAC_H_