Add ExynosVideoEncodeAccelerator

content/common/gpu/media/exynos_video_encode_accelerator.h

+// Copyright (c) 2013 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef CONTENT_COMMON_GPU_MEDIA_EXYNOS_VIDEO_ENCODE_ACCELERATOR_H_

[Ami GONE FROM CHROMIUM, 2013/06/18 01:09:25]

TODO+crbug to add a vea_test.

+#define CONTENT_COMMON_GPU_MEDIA_EXYNOS_VIDEO_ENCODE_ACCELERATOR_H_
+
+#include <list>
+#include <vector>
+
+#include "base/callback_forward.h"
+#include "base/memory/linked_ptr.h"
+#include "base/memory/weak_ptr.h"
+#include "base/threading/thread.h"
+#include "content/common/gpu/gl_surface_encoder.h"
+#include "media/base/video_decoder_config.h"
+#include "media/video/video_encode_accelerator.h"
+#include "ui/gfx/size.h"
+#include "ui/gl/gl_bindings.h"
+
+namespace base {
+
+class MessageLoopProxy;
+
+}  // namespace base
+
+namespace gfx {
+
+class GLSurface;
+
+}  // namespace gfx
+
+namespace media {
+
+class BitstreamBuffer;
+
+}  // namespace media
+
+namespace content {
+
+// This class handles Exynos video encode acceleration by interfacing with the
+// V4L2 devices exported by the Multi Format Codec and GScaler hardware blocks
+// on the Exynos platform.

[Ami GONE FROM CHROMIUM, 2013/06/18 01:09:25]

Can this be V4L2VEA instead of ExynosVEA?
(what exynos-specifics make it unlikely to work on other v4l2 impls?)

[Ami GONE FROM CHROMIUM, 2013/06/18 01:09:25]

sanity check: no dlopen/dlsym's necessary for encode & scrape functionality?

+//
+// The threading model of this class is the same as the

[Ami GONE FROM CHROMIUM, 2013/06/18 01:09:25]

Much like EVDA IWBN to have the poll thread replaced with a TYPE_IO encoder
thread
(https://code.google.com/p/chromium/codesearch#chromium/src/content/common/gpu...)

+// ExynosVideoDecodeAccelerator (from which class this was designed).  The one
+// notable difference is in the thread "ownership" of the GSC free input buffers
+// queue (gsc_free_input_buffers_).  As this queue needs to be queried and
+// dequeued from the child thread (to obtain a texture to copy the backbuffer
+// to in Encode()), it is owned by the child thread.
+class ExynosVideoEncodeAccelerator : public media::VideoEncodeAccelerator {
+ public:
+  ExynosVideoEncodeAccelerator(
+      EGLDisplay egl_display,
+      media::VideoEncodeAccelerator::Client* client,
+      const base::Callback<bool(void)>& make_context_current,
+      bool encode_from_backbuffer);
+  ~ExynosVideoEncodeAccelerator();
+
+  // media::VideoEncodeAccelerator implementation.
+  virtual void Initialize(media::VideoCodecProfile profile,
+                          const gfx::Size& input_resolution,

[Pawel Osciak, 2013/06/17 23:21:56]

Do you know if the codec supports in-flight resolution changes?

+                          const gfx::Size& output_resolution,
+                          int32 initial_bitrate) OVERRIDE;
+  virtual void Encode(
+      const scoped_refptr<media::VideoFrame>& frame,
+      int32 frame_id,
+      bool force_keyframe) OVERRIDE;
+  virtual void UseBitstreamBuffer(
+      const media::BitstreamBuffer& buffer) OVERRIDE;
+  virtual void RequestEncodingParameterChange(int32 bitrate) OVERRIDE;
+  virtual void Destroy() OVERRIDE;
+
+ private:
+  // Auto-destroy reference for BitstreamBuffer, for tracking buffers passed as
+  // input to UseBitstreamBuffer.
+  struct BitstreamBufferRef;
+
+  // Record for GSC input buffers.

[Ami GONE FROM CHROMIUM, 2013/06/18 01:09:25]

:'( for GSC staying in our lives.  I take it there is no parallel for  coming
here?
(i.e. have the GPU do the CSC instead of having gscaler do it)

+  struct GscInputRecord {
+    GscInputRecord();
+    bool at_device;
+    int frame_id;
+    EGLSyncKHR egl_sync;
+    EGLImageKHR egl_image;
+    GLuint texture_id;
+    void* address;
+    size_t length;
+  };
+
+  // Record for GSC output buffers.
+  struct GscOutputRecord {
+    GscOutputRecord();
+    bool at_device;
+    int mfc_input;
+    void* address[3];
+    size_t length[3];
+    size_t bytes_used[3];
+  };
+
+  // Record for MFC input buffers.
+  struct MfcInputRecord {
+    MfcInputRecord();
+    bool at_device;
+    int fd[2];
+  };
+
+  // Record for MFC output buffers.
+  struct MfcOutputRecord {
+    MfcOutputRecord();
+    bool at_device;
+    void* address;
+    size_t length;
+    size_t bytes_used;
+  };
+
+  enum {
+    // These are rather subjectively tuned.
+    kGscInputBufferCount = 2,
+    kGscOutputBufferCount = 2,
+    kMfcOutputBufferCount = 3,
+    // MFC hardware does not report required output buffer size correctly.
+    // Use maximum theoretical size to avoid hanging the hardware.

[Ami GONE FROM CHROMIUM, 2013/06/18 01:09:25]

TODO/crbug pointer

[Ami GONE FROM CHROMIUM, 2013/06/18 01:09:25]

What theory makes 2MB the "max theoretical size"?

+    kMfcOutputBufferSize = (2 * 1024 * 1024),
+  };
+
+  // Internal state of the encoder.
+  enum State {
+    kUninitialized,      // Initialize() not yet called.
+    kInitialized,        // Initialize() returned true; ready to start decoding.

[Ami GONE FROM CHROMIUM, 2013/06/18 01:09:25]

"decoding"

+    kEncoding,           // Encoding frames.
+    kError,              // Error in kEncoding state.
+  };
+
+  // File descriptors we need to poll.

[Ami GONE FROM CHROMIUM, 2013/06/18 01:09:25]

Reword to avoid implying that the actual FDs are 0 & 1.

+  enum PollFds {

[Ami GONE FROM CHROMIUM, 2013/06/18 01:09:25]

can live in the .cc file

+    kPollGsc = (1 << 0),
+    kPollMfc = (1 << 1),
+  };
+
+  //
+  // Child thread utility functions, supporting API entry points.
+  //
+
+  // Return a buffer to gsc_free_input_buffers_.
+  void ReturnFreeGscInputBuffer(int index);
+
+  //
+  // Decoding tasks, to be run on encode_thread_.

[Pawel Osciak, 2013/06/17 23:21:56]

s/encode_thread_/encoder_thread_/

[Ami GONE FROM CHROMIUM, 2013/06/18 01:09:25]

lol Decoding...encode...

+  //
+
+  // Encode a GSC input buffer.
+  void EncodeTask(int gsc_input_index);
+
+  // Copy a media::VideoFrame to the GSC input buffer, then encode it.
+  void CopyFrameAndEncodeTask(const scoped_refptr<media::VideoFrame>& frame,
+                              int gsc_input_index);
+
+  // Add a BitstreamBuffer to out queue of buffers ready to be used for client
+  // output.
+  void UseBitstreamBufferTask(scoped_ptr<BitstreamBufferRef> buffer_ref);
+
+  // Device destruction task.
+  void DestroyTask();
+
+  // Service I/O on the V4L2 devices.  This task should only be scheduled from
+  // DevicePollTask().
+  void ServiceDeviceTask();
+  // Handle the various device queues.
+  void EnqueueGsc();
+  void DequeueGsc();
+  void EnqueueMfc();
+  void DequeueMfc();
+  // Enqueue a buffer on the corresponding queue.
+  bool EnqueueGscInputRecord();

[Ami GONE FROM CHROMIUM, 2013/06/18 01:09:25]

here and below, doco that a false return means fatal error?
(and handle it as such?  I think you have a bunch of silent "return"s that might
be burying errors)
WARN_UNUSED_RESULT might be useful, too.

+  bool EnqueueGscOutputRecord();
+  bool EnqueueMfcInputRecord();
+  bool EnqueueMfcOutputRecord();
+  // Return completed buffers to the client.

[Pawel Osciak, 2013/06/17 23:21:56]

Doco which buffers (bitstream buffers)?

+  void ReturnCompleteBuffers();
+
+  // Attempt to start/stop device_poll_thread_.
+  bool StartDevicePoll();
+  bool StopDevicePoll();
+  // Set/clear the device poll interrupt (using device_ooll_interrupt_fd_).

[Ami GONE FROM CHROMIUM, 2013/06/18 01:09:25]

typo: ooll

+  bool SetDevicePollInterrupt();
+  bool ClearDevicePollInterrupt();
+
+  //
+  // Device tasks, to be run on device_poll_thread_.
+  //
+
+  // The device task.
+  void DevicePollTask(unsigned int poll_fds);
+
+  //
+  // Safe from any thread.
+  //
+
+  // Error notification (using PostTask() to child thread, if necessary).
+  void NotifyError(Error error);
+
+  // Set the encoder_thread_ state (using PostTask to encoder thread, if
+  // necessary).
+  void SetEncoderState(State state);
+
+  //
+  // Other utility functions.  Called on encoder_thread_, unless
+  // encoder_thread_ is not yet started, in which case the child thread can call
+  // these (e.g. in Initialize() or Destroy()).
+  //
+
+  // Create the buffers we need.
+  bool CreateGscInputBuffers();
+  bool CreateGscOutputBuffers();
+  bool SetMfcFormats();

[Ami GONE FROM CHROMIUM, 2013/06/18 01:09:25]

Not really "creat[ing] the buffers we need".

+  bool CreateMfcInputBuffers();
+  bool CreateMfcOutputBuffers();
+
+  // Set encoding parameters.
+  void SetBitrate(int32 bitrate);
+
+  // Destroy these buffers.
+  void DestroyGscInputBuffers();
+  void DestroyGscOutputBuffers();
+  void DestroyMfcInputBuffers();
+  void DestroyMfcOutputBuffers();
+
+  // Our original calling message loop for the child thread.
+  const scoped_refptr<base::MessageLoopProxy> child_message_loop_proxy_;
+
+  // WeakPtr<> pointing to |this| for use in posting tasks from the encoder or
+  // device worker threads back to the child thread.  Because the worker threads
+  // are members of this class, any task running on those threads is guaranteed
+  // that this object is still alive.  As a result, tasks posted from the child
+  // thread to the encoder or device thread should use base::Unretained(this),
+  // and tasks posted the other way should use |weak_this_|.
+  base::WeakPtrFactory<ExynosVideoEncodeAccelerator> weak_this_ptr_factory_;
+  base::WeakPtr<ExynosVideoEncodeAccelerator> weak_this_;
+
+  // To expose client callbacks from VideoEncodeAccelerator.
+  // NOTE: all calls to these objects *MUST* be executed on
+  // child_message_loop_proxy_.
+  base::WeakPtrFactory<Client> client_ptr_factory_;
+  base::WeakPtr<Client> client_;
+
+  //
+  // Encoder state, owned and operated by encoder_thread_.
+  // Before encoder_thread_ has started, the encoder state is managed by
+  // the child (main) thread.  After encoder_thread_ has started, the encoder
+  // thread should be the only one managing these.
+  //
+
+  // This thread services tasks posted from the VEA API entry points by the
+  // child thread and device service callbacks posted from the device thread.
+  base::Thread encoder_thread_;
+  // Encoder state machine state.

[Ami GONE FROM CHROMIUM, 2013/06/18 01:09:25]

There's no explicit "machine"  so drop "machine state"?

+  State encoder_state_;
+  // The visible/allocated sizes of the input frame.
+  gfx::Size input_visible_size_;

[Ami GONE FROM CHROMIUM, 2013/06/18 01:09:25]

"visible" is usually a Rect, not a Size, and it can vary frame by frame.  You
sure it makes sense to track in members like this?

+  gfx::Size input_allocated_size_;
+  // The visible/allocated sizes of the color-converted frame.
+  gfx::Size converted_visible_size_;

[Pawel Osciak, 2013/06/17 23:21:56]

Perhaps I'm misunderstanding the intentions here, but even if GSC and MFC have
different size requirements and if it also depends on format, wouldn't it be
better performance-wise to scale it once to a common size for all initially, on
first conversion/scale operation and use one size for converted_visible and
output_visible sizes?

+  gfx::Size converted_allocated_size_;
+  // The visible/allocated sizes of the output frame.
+  gfx::Size output_visible_size_;
+  // The required byte size of output BitstreamBuffers.
+  size_t output_buffer_byte_size_;
+
+  // Bitstream buffers ready to be used to return encoded output.
+  std::vector<linked_ptr<BitstreamBufferRef> > encoder_bitstream_buffers_;
+
+  // Buffers ready to be encoded.

[Ami GONE FROM CHROMIUM, 2013/06/18 01:09:25]

Here and elsewhere, doco what the int is an index into.

+  std::list<int> encoder_input_queue_;
+
+  // Encoded buffers ready to return to the client.
+  std::list<int> encoder_output_queue_;
+
+  // Flag if we are encoding the output (and using MFC).

[Pawel Osciak, 2013/06/17 23:21:56]

In what scenario we'd not want to encode? Is this for testing/debug only?

+  bool do_output_encoding_;
+
+  // Flag if we are encoding from the backbuffer (instead of a CPU allocation).
+  const bool do_encode_from_backbuffer_;
+
+  // GSC color conversion device.
+  int gsc_fd_;
+  // GSC input buffer state.
+  bool gsc_input_streamon_;
+  // GSC input buffers enqueued to device.
+  int gsc_input_buffer_queued_count_;
+  // Mapping of int index to GSC input buffer record.
+  std::vector<GscInputRecord> gsc_input_buffer_map_;

[Ami GONE FROM CHROMIUM, 2013/06/18 01:09:25]

s/_map_/s_/ and similarly update comment since this isn't a map?

(ditto for {gsc,mfc}_{output,input}_buffer_map_)

+
+  // GSC output buffer state.
+  bool gsc_output_streamon_;
+  // GSC output buffers enqueued to device.
+  int gsc_output_buffer_queued_count_;
+  // Output buffers ready to use, as a LIFO since we don't care about ordering.
+  std::vector<int> gsc_free_output_buffers_;
+  // Mapping of int index to GSC output buffer record.
+  std::vector<GscOutputRecord> gsc_output_buffer_map_;
+
+  // Completed GSC outputs, waiting for GSC.

[Ami GONE FROM CHROMIUM, 2013/06/18 01:09:25]

lolwat?

+  std::list<int> gsc_output_mfc_input_queue_;

[Pawel Osciak, 2013/06/17 23:21:56]

s/waiting for GSC/waiting for MFC/ ?

+
+  // MFC video encoding device.
+  int mfc_fd_;
+
+  // MFC input buffer state.
+  bool mfc_input_streamon_;
+  // MFC input buffers enqueued to device.
+  int mfc_input_buffer_queued_count_;
+  // Input buffers ready to use, as a LIFO since we don't care about ordering.
+  std::vector<int> mfc_free_input_buffers_;
+  // Mapping of int index to MFC input buffer record.
+  std::vector<MfcInputRecord> mfc_input_buffer_map_;
+
+  // MFC output buffer state.
+  bool mfc_output_streamon_;
+  // MFC output buffers enqueued to device.
+  int mfc_output_buffer_queued_count_;
+  // Output buffers ready to use, as a LIFO since we don't care about ordering.
+  std::vector<int> mfc_free_output_buffers_;
+  // Mapping of int index to MFC output buffer record.
+  std::vector<MfcOutputRecord> mfc_output_buffer_map_;
+
+  //
+  // The device polling thread handles notifications of V4L2 device changes.
+  //
+
+  // The thread.
+  base::Thread device_poll_thread_;
+  // eventfd fd to signal device poll thread when its poll() should be
+  // interrupted.
+  int device_poll_interrupt_fd_;
+
+  //
+  // Other state, held by child (main) thread.
+  //
+
+  // Make our context current before running any EGL entry points.
+  base::Callback<bool(void)> make_context_current_;
+
+  // Input buffers ready to use, as a LIFO since we don't care about ordering.
+  std::vector<int> gsc_free_input_buffers_;

[Ami GONE FROM CHROMIUM, 2013/06/18 01:09:25]

This is both read & written on both the child thread & the encoder thread!

+
+  // EGL state
+  const EGLDisplay egl_display_;
+
+  // The codec we'll be decoding for.

[Ami GONE FROM CHROMIUM, 2013/06/18 01:09:25]

encoding

+  media::VideoCodecProfile video_profile_;
+};
+
+}  // namespace content
+
+#endif  // CONTENT_COMMON_GPU_MEDIA_EXYNOS_VIDEO_ENCODE_ACCELERATOR_H_