VDA implementation for Exynos, using V4L2

content/common/gpu/media/exynos_video_decode_accelerator.h

+// Copyright (c) 2012 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+//
+// This file contains an implementation of VideoDecoderAccelerator
+// that utilizes hardware video decoder present on the Exynos SoC.
+
+#ifndef CONTENT_COMMON_GPU_MEDIA_EXYNOS_VIDEO_DECODE_ACCELERATOR_H_
+#define CONTENT_COMMON_GPU_MEDIA_EXYNOS_VIDEO_DECODE_ACCELERATOR_H_
+
+#include <deque>
+#include <vector>
+
+#include "base/callback_forward.h"
+#include "base/file_util.h"
+#include "base/memory/linked_ptr.h"
+#include "base/memory/scoped_ptr.h"
+#include "base/threading/thread.h"
+#include "content/common/content_export.h"
+#include "media/base/video_decoder_config.h"
+#include "media/video/video_decode_accelerator.h"
+#include "third_party/angle/include/EGL/egl.h"
+#include "third_party/angle/include/EGL/eglext.h"
+#include "ui/gfx/size.h"
+
+namespace base {
+class MessageLoopProxy;
+}
+
+namespace gfx {
+class GLContext;
+}
+
+namespace content {
+
+// This class handls Exynos video acceleration directly through the V4L2 devices

[Pawel Osciak, 2012/11/06 19:26:16]

s/handls/handles

[sheu, 2012/11/08 21:16:30]

Done.

+// exported by the Multi Format Codec and GScaler hardware blocks.
+//
+// The threading model of this class is driven by the fact that it needs to
+// interface two fundamentally different event queues -- the one Chromium
+// provides through MessageLoop, and the one driven by the V4L2 devices which
+// is waited on with epoll().  There are three threads involved in this class:
+//
+// * The child thread, which is the main GPU process thread which calls the
+//   media::VideoDecodeAccelerator entry points.  Calls from this thread
+//   generally do not block (with the exception of Initialize() and Destroy()).
+//   They post tasks to the decoder_thread_, which actually services the task
+//   and calls back when complete through the
+//   media::VideoDecodeAccelerator::Client interface.
+// * The decoder_thread_, owned by this class.  It services API tasks as well as
+//   device events.  Pretty much all state modification is done on this thread.
+// * The device_poll_thread_, owned by this class.  All it does is epoll() on
+//   the V4L2 device and notify the decoder_thread_ when something interesting

[Pawel Osciak, 2012/11/06 19:26:16]

s/device/devices ?

[sheu, 2012/11/08 21:16:30]

Done.

+//   happens.
+//
+// Note that this class has no locks!  Everything's serviced on the
+// decoder_thread_, so there are no synchronization issues.
+// ... well, there are, but it's a matter of getting messages posted in the
+// right order, not fiddling with locks.
+class CONTENT_EXPORT ExynosVideoDecodeAccelerator :
+    public media::VideoDecodeAccelerator {
+ public:
+  ExynosVideoDecodeAccelerator(
+      gfx::GLContext* gl_context,
+      Client* client,
+      const base::Callback<bool(void)>& make_context_current);
+  virtual ~ExynosVideoDecodeAccelerator();
+
+  // media::VideoDecodeAccelerator implementation.
+  // Note: Initialize() and Destroy() are synchronous.
+  virtual bool Initialize(media::VideoCodecProfile profile) OVERRIDE;
+  virtual void Decode(const media::BitstreamBuffer& bitstream_buffer) OVERRIDE;
+  virtual void AssignPictureBuffers(
+      const std::vector<media::PictureBuffer>& buffers) OVERRIDE;
+  virtual void ReusePictureBuffer(int32 picture_buffer_id) OVERRIDE;
+  virtual void Flush() OVERRIDE;
+  virtual void Reset() OVERRIDE;
+  virtual void Destroy() OVERRIDE;
+
+  // Do any necessary initialization before the sandbox is enabled.
+  static void PreSandboxInitialization();
+
+  // Lazily initialize static data after sandbox is enabled.  Return false on

[Pawel Osciak, 2012/11/06 19:26:16]

Any reason why you seem to like leaving two spaces between sentences?

[sheu, 2012/11/08 21:16:30]

I picked up the habit in grade school.  Is this a style thing?

There seems to be a lively debate about this:
http://en.wikipedia.org/wiki/Sentence_spacing

+  // init failure.
+  static bool PostSandboxInitialization();
+
+ private:
+  // These are rather subjectively tuned.
+  enum {
+    kMfcInputBufferCount = 4,
+    kMfcOutputBufferExtraCount = 2,

[Pawel Osciak, 2012/11/06 19:26:16]

Document what "Extra" is?

[sheu, 2012/11/08 21:16:30]

Done.

+    kMfcInputBufferMaxSize = 1024 * 1024,
+    kGscInputBufferCount = 4,
+    kGscOutputBufferCount = 6,
+  };
+
+  // Internal state of the decoder.
+  enum State {
+    kUninitialized, // Initialize() not yet called.
+    kInitialized,   // Initialize() called.  Ready to start decoding.
+    kDecoding,      // DecodeBufferInitial() successful; decoding frames.
+    kResetting,     // Presently resetting.
+    kAfterReset,    // After Reset(), ready to start decoding again.
+    kError,         // Error in kDecoding state.
+  };
+
+  // Record for decoder input buffers.
+  struct BitstreamBufferRecord {
+    BitstreamBufferRecord(base::SharedMemory* shm, size_t size, int32 input_id);
+    ~BitstreamBufferRecord();
+    const scoped_ptr<base::SharedMemory> shm;
+    const size_t size;

[Pawel Osciak, 2012/11/06 19:26:16]

Could we use SharedMemory.size()?

[sheu, 2012/11/08 21:16:30]

SharedMemory doesn't have a size().  Shall I add one?

+    const int32 input_id;
+  };
+
+  // Record for MFC input buffers.
+  struct MfcInputRecord {
+    MfcInputRecord();
+    ~MfcInputRecord();
+    bool at_device;        // held by device.
+    void* offset;          // mmap() offset.
+    size_t length;         // mmap() length.
+    off_t bytes_used;      // bytes filled in the mmap() segment.
+    int32 input_id;        // triggering input_id as given to Decode().
+  };
+
+  // Record for MFC output buffers.
+  struct MfcOutputRecord {
+    MfcOutputRecord();
+    ~MfcOutputRecord();
+    bool at_device;        // held by device.
+    size_t bytes_used[2];  // bytes used in each dmabuf.

[Pawel Osciak, 2012/11/06 19:26:16]

Perhaps consider have kNumOutputPlanes = 2 constant?

[sheu, 2012/11/08 21:16:30]

Just added a comment here.  Otherwise I'd be looping (for int i = 0; i <
kNumOutputPlanes) in more places than I care to do...

[Pawel Osciak, 2012/11/13 18:10:39]

Not sure how would that differ from looping over i<2, but not really that
important I guess...

+    void* offset[2];       // mmap() offset for each plane.
+    size_t length[2];      // mmap() length for each plane.
+    int32 input_id;        // triggering input_id as given to Decode().
+  };
+
+  // Record for GSC input buffers.
+  struct GscInputRecord {
+    GscInputRecord();
+    ~GscInputRecord();
+    bool at_device;        // held by device.
+    int mfc_output;        // MFC output buffer index to recycle when this input
+                           // is complete
+  };
+
+  // Record for GSC output buffers.
+  struct GscOutputRecord {
+    GscOutputRecord();
+    ~GscOutputRecord();
+    bool at_device;        // held by device.
+    bool at_client;        // held by client.
+    int fd;                // file descriptor from backing EGLImage.
+    EGLImageKHR egl_image; // backing EGLImage.
+    EGLSyncKHR egl_sync;   // sync the compositor's use of the EGLImage.
+    int32 picture_id;      // picture ID as returned to PictureReady().
+  };
+
+  // Auto-destruction reference for an array of EGLImage (for message-passing)
+  struct EGLImageKHRArrayRef {
+    EGLImageKHRArrayRef(EGLDisplay egl_display, EGLImageKHR egl_images[],
+                        int egl_image_fds[], int egl_images_count);
+    ~EGLImageKHRArrayRef();
+    EGLDisplay const egl_display;
+    const scoped_array<EGLImageKHR> egl_images;
+    const scoped_array<int> egl_image_fds;
+    const int egl_images_count;
+  };
+
+  // Auto-destruction reference for EGLSync (for message-passing)
+  struct EGLSyncKHRRef {
+    EGLSyncKHRRef(EGLDisplay egl_display, EGLSyncKHR egl_sync);
+    ~EGLSyncKHRRef();
+    EGLDisplay const egl_display;
+    EGLSyncKHR egl_sync;
+  };
+
+  //
+  // Decoding tasks, to be run on decode_thread_.
+  //
+
+  // Enqueue a BitstreamBuffer to decode.  This will enqueue a
+  // DecodeBufferTask() to actually decode the buffer.
+  void DecodeTask(scoped_ptr<BitstreamBufferRecord> bitstream_record);
+
+  // Decode from the queued BitstreamBuffers.  Calls DecodeBufferInitial()
+  // or DecodeBufferContinue().
+  void DecodeBufferTask();
+  // Return true if an input buffer is consumed.  It may not be an error
+  // condition not to consume a buffer.
+  bool DecodeBufferInitial(const void* data, size_t size);
+  bool DecodeBufferContinue(const void* data, size_t size);
+  // Helpers: accumulate and flush data for one decoded frame.
+  bool AppendToInputFrame(const void* data, size_t size);
+  bool FlushInputFrame();
+
+  // Process an AssignPictureBuffers() API call.  After this, the
+  // device_poll_thread_ can be started safely, since we have all our
+  // buffers.
+  void AssignPictureBuffersTask(scoped_ptr<EGLImageKHRArrayRef> egl_images_ref);
+
+  // Service I/O on the V4L2 devices.
+  void ServiceDeviceTask();
+  // Handle the various device queues.
+  void EnqueueMfc();
+  void DequeueMfc();
+  void EnqueueGsc();
+  void DequeueGsc();
+
+  // Process a ReusePictureBuffer() API call.  The API call create an EGLSync
+  // object on the main (GPU process) thread; we will record this object so we
+  // can wait on it before reusing the buffer.
+  void ReusePictureBufferTask(int32 picture_buffer_id,
+                              scoped_ptr<EGLSyncKHRRef> egl_sync_ref);
+
+  // Flush() task.  Child thread should not submit any more buffers until it
+  // receives the NotifyFlushDone callback.
+  void FlushTask();
+
+  // Reset() task.  This task will schedule a ResetDoneTask() that will send
+  // the NotifyResetDone callback, then set the decoder state to kResetting so
+  // that all intervening tasks will drain.
+  void ResetTask();
+  void ResetDoneTask();
+
+  // Device destruction task.
+  void DestroyTask();
+
+  // Attempt to start/stop DevicePollLoop() running on device_monitor_thread_.
+  // Returns false only on actual error: note that StartDevicePoll() may return
+  // true without actually starting the DevicePollLoop(), if no error is
+  // encountered.
+  bool StartDevicePoll();
+  bool StopDevicePoll();
+
+  //
+  // Device tasks, to be run on device_poll_thread_.
+  //
+
+  // The device task.
+  void DevicePollLoop();
+
+  //
+  // Safe from any thread (uses PostTask() to child thread).
+  //
+
+  // Error notification.
+  void NotifyError(Error error);
+
+  //
+  // Other utility functions.  Called on decoder_thread_, unless
+  // decoder_thread_ is not yet started, in which case the child thread can call
+  // these (e.g. in Initialize() or Destroy()).
+  //
+
+  // Create the buffers we need.
+  bool CreateMfcInputBuffers();
+  bool CreateMfcOutputBuffers();
+  bool CreateGscInputBuffers();
+  bool CreateGscOutputBuffers();
+
+  // Destroy these buffers.
+  void DestroyMfcInputBuffers();
+  void DestroyMfcOutputBuffers();
+  void DestroyGscInputBuffers();
+  void DestroyGscOutputBuffers();
+
+  // Our original calling message loop for the child thread.
+  scoped_refptr<base::MessageLoopProxy> child_message_loop_proxy_;
+
+  // WeakPtr<> pointing to |this| for use in posting tasks from the decoder or
+  // device worker threads back to the child thread.  Because the worker threads
+  // are members of this class, any task running on those threads is guaranteed
+  // that this object is still alive.  As a result, tasks posted from the child
+  // thread to the decoder or device thread should use base::Unretained(this),
+  // and tasks posted the other way should use |weak_this_|.
+  base::WeakPtr<ExynosVideoDecodeAccelerator> weak_this_;
+
+  // To expose client callbacks from VideoDecodeAccelerator.
+  // NOTE: all calls to these objects *MUST* be executed on
+  // child_message_loop_proxy_.
+  base::WeakPtrFactory<Client> client_ptr_factory_;
+  base::WeakPtr<Client> client_;
+
+  //
+  // Decoder state, owned and operated by decoder_thread_.
+  // Before decoder_thread_ has started, the decoder state is managed by
+  // the child (main) thread.  After decoder_thread_ has started, the decoder
+  // thread should be the only one managing these.
+  //
+
+  // This thread services tasks posted from the VDA API entry points by the
+  // child thread and device service callbacks posted from the device thread.
+  base::Thread decoder_thread_;
+  // Decoder state machine state.
+  State decoder_state_;
+  // Bitstream buffer we're presently reading.
+  scoped_ptr<BitstreamBufferRecord> decoder_current_bitstream_buffer_;
+  // MFC input buffer we're presently filling.
+  int decoder_current_input_buffer_;
+  // We track the number of buffer decode tasks we have scheduled, since each
+  // task execution should complete one buffer.  If we fall behind (due to
+  // resource backpressure, etc.), we'll have to schedule more to catch up.
+  int decoder_decode_buffer_tasks_scheduled_;
+  // Buffers in the pipe, five-by-five.
+  int decoder_frames_inflight_;
+  // Buffers held by the client.
+  int decoder_frames_at_client_;
+  // Are we waiting for a flush notification?
+  bool decoder_flush_notify_requested_;
+  // Input queue for decoder_thread_: BitstreamBuffers in.
+  std::deque<linked_ptr<BitstreamBufferRecord> > decoder_input_queue_;
+
+  //
+  // Hardware state and associated queues.  Since decoder_thread_ services
+  // the hardware, decoder_thread_ owns these too.
+  //
+
+  // Completed decode buffers, waiting for MFC.
+  std::deque<int> mfc_input_ready_queue_;
+
+  // MFC decode device.
+  int mfc_fd_;
+  file_util::ScopedFD mfc_fd_closer_;
+
+  // MFC input buffer state.
+  bool mfc_input_streamon_;
+  int mfc_input_buffer_count_;
+  // MFC input buffes enqueued to device, total.
+  int mfc_input_buffer_queued_count_;
+  // Input buffers ready to use, as a FIFO since we don't care about ordering.
+  std::vector<int> mfc_free_input_buffers_;
+  // Mapping of int index to MFC input buffer record.
+  std::vector<MfcInputRecord> mfc_input_buffer_map_;
+
+  // MFC output buffer state.
+  bool mfc_output_streamon_;
+  int mfc_output_buffer_count_;
+  // Output buffers ready to use, as a FIFO since we don't care about ordering.
+  std::vector<int> mfc_free_output_buffers_;
+  // Mapping of int index to MFC output buffer record.
+  std::vector<MfcOutputRecord> mfc_output_buffer_map_;
+  // Required size of MFC output buffers.  Two sizes for two planes.
+  size_t mfc_output_buffer_size_[2];
+  uint32 mfc_output_buffer_pixelformat_;
+
+  // Completed MFC outputs, waiting for GSC.
+  std::deque<int> mfc_output_gsc_input_queue_;
+
+  // GSC decode device.
+  int gsc_fd_;
+  file_util::ScopedFD gsc_fd_closer_;
+
+  // GSC input buffer state.

[Pawel Osciak, 2012/11/06 19:26:16]

s/buffer/queue

[sheu, 2012/11/08 21:16:30]

I'm referring to the input buffers that the HW read/writes from not the queues
of buffers that this is managing.

+  bool gsc_input_streamon_;
+  int gsc_input_buffer_count_;
+  // Input buffers ready to use, as a FIFO since we don't care about ordering.
+  std::vector<int> gsc_free_input_buffers_;
+  // Mapping of int index to GSC input buffer record.
+  std::vector<GscInputRecord> gsc_input_buffer_map_;
+
+  // GSC output buffer state.
+  bool gsc_output_streamon_;
+  int gsc_output_buffer_count_;
+  // GSC output buffers enqueued to device and ready, but not yet in use.
+  int gsc_output_buffer_prepared_count_;

[Pawel Osciak, 2012/11/06 19:26:16]

Could you document better what "queued and not in use" means?

[sheu, 2012/11/17 03:25:08]

Done.

+  // GSC output buffers enqueued to device, total.
+  int gsc_output_buffer_queued_count_;
+  // Output buffers ready to use.  We need a LIFO here.

[Pawel Osciak, 2012/11/06 19:26:16]

Why LIFO?

[sheu, 2012/11/08 21:16:30]

Doh.  I even discussed this with fischman@.  Should be "FIFO"; fixed.

+  std::deque<int> gsc_free_output_buffers_;
+  // Mapping of int index to GSC output buffer record.
+  std::vector<GscOutputRecord> gsc_output_buffer_map_;
+
+  // Output picture size.
+  gfx::Size frame_buffer_size_;
+
+  //
+  // The device polling thread handles notifications of V4L2 device changes.
+  //
+
+  // The thread.
+  base::Thread device_poll_thread_;
+
+  //
+  // Other state, held by the child (main) thread.
+  //
+
+  // GL state
+  gfx::GLContext* gl_context_;
+  // Make our context current before running any EGL entry points.
+  base::Callback<bool(void)> make_context_current_;
+
+  // EGL state
+  EGLContext egl_context_;
+  EGLDisplay egl_display_;
+
+  // The codec we'll be decoding for.
+  media::VideoCodecProfile video_profile_;
+
+  DISALLOW_COPY_AND_ASSIGN(ExynosVideoDecodeAccelerator);
+};
+
+}  // namespace content
+
+#endif  // CONTENT_COMMON_GPU_MEDIA_EXYNOS_VIDEO_DECODE_ACCELERATOR_H_