AndroidVDA by using Android's MediaCodec API.

content/common/gpu/media/android_video_decode_accelerator.cc

+// Copyright (c) 2013 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "content/common/gpu/media/android_video_decode_accelerator.h"
+
+#include <jni.h>
+
+#include "base/android/jni_android.h"
+#include "base/android/scoped_java_ref.h"
+#include "base/bind.h"
+#include "base/logging.h"
+#include "base/message_loop.h"
+#include "content/common/android/surface_callback.h"
+#include "content/common/gpu/gpu_channel.h"
+#include "content/common/gpu/media/gles2_external_texture_copier.h"
+#include "media/base/bitstream_buffer.h"
+#include "media/video/picture.h"
+#include "third_party/angle/include/GLES2/gl2.h"
+#include "third_party/angle/include/GLES2/gl2ext.h"
+
+using base::android::MethodID;
+using base::android::ScopedJavaLocalRef;
+
+namespace content {
+
+// XXX: drop the below before submitting.
+#define LOG_LINE() LOG(INFO) << __FUNCTION__
+
+#undef DCHECK
+#define DCHECK CHECK

[Ami GONE FROM CHROMIUM, 2013/02/05 20:24:33]

Oh my! :)
You might enjoy
https://code.google.com/searchframe#OAMlx_jo-ck/src/base/base_switches.cc&exa...

[dwkang1, 2013/02/07 14:01:36]

haha. It was part of XXX at L27. :) Thanks for letting me know the flag. Looks
very useful!

+
+// Helper macros for dealing with failure.  If |result| evaluates false, emit
+// |log| to ERROR, register |error| with the decoder, and return.
+#define RETURN_ON_FAILURE(result, log, error)                      \
+  do {                                                             \
+    if (!(result)) {                                               \
+      DLOG(ERROR) << log;                                          \
+      MessageLoop::current()->PostTask(FROM_HERE, base::Bind(      \
+          &AndroidVideoDecodeAccelerator::NotifyError,             \
+          base::AsWeakPtr(this), error));                          \
+      state_ = ERROR;                                              \
+      return;                                                      \
+    }                                                              \
+  } while (0)
+
+enum { kNumPictureBuffers = 4 };
+
+// static
+const base::TimeDelta AndroidVideoDecodeAccelerator::kDecodePollDelay =
+    base::TimeDelta::FromMilliseconds(10);
+
+AndroidVideoDecodeAccelerator::AndroidVideoDecodeAccelerator(
+    media::VideoDecodeAccelerator::Client* client,
+    const base::Callback<bool(void)>& make_context_current)
+    : client_(client),
+      make_context_current_(make_context_current),
+      codec_(media::MediaCodecBridge::VIDEO_H264),
+      state_(NO_ERROR),
+      surface_texture_id_(-1),
+      picturebuffers_requested_(false),
+      io_task_is_posted_(false),
+      decoder_met_eos_(false),
+      num_bytes_used_in_the_pending_buffer_(0) {
+  LOG_LINE();
+}
+
+AndroidVideoDecodeAccelerator::~AndroidVideoDecodeAccelerator() {
+  LOG_LINE();
+  DCHECK(thread_checker_.CalledOnValidThread());
+}
+
+bool AndroidVideoDecodeAccelerator::Initialize(
+    media::VideoCodecProfile profile) {
+  LOG_LINE();
+  DCHECK(media_codec_ == NULL);

[qinmin, 2013/02/05 07:43:13]

!media_codec_

[dwkang1, 2013/02/05 11:49:36]

Done.

+  DCHECK(thread_checker_.CalledOnValidThread());
+
+  if (profile == media::VP8PROFILE_MAIN) {
+    codec_ = media::MediaCodecBridge::VIDEO_VP8;
+  } else if (profile >= media::H264PROFILE_MIN &&
+      profile <= media::H264PROFILE_MAX) {
+    codec_ = media::MediaCodecBridge::VIDEO_H264;
+  } else {
+    LOG(ERROR) << "Unsupported profile: " << profile;
+    return false;
+  }
+
+  if (!make_context_current_.Run()) {
+    LOG(ERROR) << "Failed to make this decoder's GL context current.";
+    return false;
+  }
+  // XXX: apply the scheme for GL access. http://crbug.com/169433
+  glGenTextures(1, &surface_texture_id_);
+  glActiveTexture(GL_TEXTURE0);
+  glBindTexture(GL_TEXTURE_EXTERNAL_OES, surface_texture_id_);
+
+  glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
+  glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
+  glTexParameteri(GL_TEXTURE_EXTERNAL_OES,
+                  GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
+  glTexParameteri(GL_TEXTURE_EXTERNAL_OES,
+                  GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
+
+  surface_texture_ = new SurfaceTextureBridge(surface_texture_id_);
+
+  ConfigureMediaCodec();
+
+  MessageLoop::current()->PostTask(FROM_HERE, base::Bind(
+      &AndroidVideoDecodeAccelerator::NotifyInitializeDone,
+      base::AsWeakPtr(this)));
+  return true;
+}
+
+void AndroidVideoDecodeAccelerator::DoIOTask() {
+  io_task_is_posted_ = false;
+  if (state_ == ERROR) {
+    return;
+  }
+
+  DequeueOutput();
+  QueueInput();
+
+  if (!pending_bitstream_buffers_.empty() ||
+      !free_picture_ids_.empty()) {
+    io_task_is_posted_ = true;
+    MessageLoop::current()->PostDelayedTask(
+        FROM_HERE,
+        base::Bind(
+            &AndroidVideoDecodeAccelerator::DoIOTask, base::AsWeakPtr(this)),
+            kDecodePollDelay);
+  }
+}
+
+void AndroidVideoDecodeAccelerator::QueueInput() {
+  if (pending_bitstream_buffers_.empty()) {

[qinmin, 2013/02/05 07:43:13]

no {

[dwkang1, 2013/02/05 08:08:21]

afaik, it is okay to have { if it is consistent in a file in under google3. does
chrome have different rule?

+    return;
+  }
+  int input_buf_index = media_codec_->DequeueInputBuffer(base::TimeDelta());

[Ami GONE FROM CHROMIUM, 2013/02/05 20:24:33]

kNoWait or somesuch?  Otherwise reading this callsite it's not obvious what the
TimeDelta parameter is about.

[dwkang1, 2013/02/07 14:01:36]

Good idea.

+  if (input_buf_index < 0) {
+    DCHECK_EQ(input_buf_index, media::MediaCodecBridge::INFO_TRY_AGAIN_LATER);
+    return;
+  }
+  media::BitstreamBuffer& bitstream_buffer =
+      pending_bitstream_buffers_.front();
+
+  int flags = 0;
+  if (bitstream_buffer.id() == -1) {

[qinmin, 2013/02/05 07:43:13]

ditto

[dwkang1, 2013/02/05 11:49:36]

Done.

+    flags |= media::MediaCodecBridge::BUFFER_FLAG_END_OF_STREAM;
+  }
+  if (bitstream_buffer.size() > 0) {
+    scoped_ptr<base::SharedMemory> shm(
+        new base::SharedMemory(bitstream_buffer.handle(), true));
+
+    RETURN_ON_FAILURE(shm->Map(bitstream_buffer.size()),
+                      "Failed to SharedMemory::Map()",
+                      UNREADABLE_INPUT);
+
+
+    const size_t offset = num_bytes_used_in_the_pending_buffer_;
+    num_bytes_used_in_the_pending_buffer_ +=
+        media_codec_->PutToInputBuffer(
+            input_buf_index,
+            static_cast<const uint8*>(shm->memory()) + offset,
+            bitstream_buffer.size() - offset);
+    CHECK_LE(num_bytes_used_in_the_pending_buffer_, bitstream_buffer.size());
+  }
+
+  if (num_bytes_used_in_the_pending_buffer_ == bitstream_buffer.size()) {
+    num_bytes_used_in_the_pending_buffer_ = 0;
+    pending_bitstream_buffers_.pop();
+  }
+
+  // Abuse the presentation time argument to propagate the bitstream
+  // buffer ID to the output, so we can report it back to the client in
+  // PictureReady().
+  base::TimeDelta timestamp =
+      base::TimeDelta::FromMicroseconds(bitstream_buffer.id());
+  media_codec_->QueueInputBuffer(
+      input_buf_index, 0, bitstream_buffer.size(), timestamp, flags);
+
+  if (bitstream_buffer.id() != -1) {

[Ami GONE FROM CHROMIUM, 2013/02/05 20:24:33]

Yes.
Correct.
But GVD is not the only client of VDA - PPAPI is another, and you can't know
what plugin authors are doing with this, so you should obey the contract of the
API.
No.  I mean "once it is guaranteed that no more decoded output will be returned
from the contents". 
Maybe it's not possible to implement this given MediaCodec's API?
Usually the first few buffers contain SPS/PPS and other stream initialization
data.

+    MessageLoop::current()->PostTask(FROM_HERE, base::Bind(
+        &AndroidVideoDecodeAccelerator::NotifyEndOfBitstreamBuffer,

[Ami GONE FROM CHROMIUM, 2013/02/05 20:24:33]

This is still wrong - 
https://chromiumcodereview.appspot.com/11973010/diff/22001/content/common/gpu...

[dwkang1, 2013/02/07 14:01:36]

Actually, I implemented the scheme you mentioned (keeping bitstream ids in input
buffers and calling NEOBB when the input buffer id is returned again). However,
it didn't work because the # of input buffers was more than 4 and
kMaxInFlightDecodes in gpu_video_decoder was also 4. gpu_video_decoder didn't
provider bitstream buffer more than four. :(

BTW, can I ask the background of having "once it is guaranteed that no more
decoded output will be returned
from the contents" semantics? It looks fine to me because the bitstream is
copied when it is passed to the input buffer. Was there any specific use case?

[Ami GONE FROM CHROMIUM, 2013/02/07 19:40:16]

The rationale was for knowing how long metadata needed to be maintained.
Looking at the .h and the impls, I guess we don't promise the guarantee I
phrased above, so I think this is actually ok.

+        base::AsWeakPtr(this), bitstream_buffer.id()));
+  }
+}
+
+void AndroidVideoDecodeAccelerator::DequeueOutput() {
+  if (picturebuffers_requested_ && output_picture_buffers_.empty()) {

[qinmin, 2013/02/05 07:43:13]

ditto

[dwkang1, 2013/02/05 11:49:36]

Done.

+    return;
+  }
+  if (!output_picture_buffers_.empty() && free_picture_ids_.empty()) {
+    // Don't have any picture buffer to send. Need to wait more.
+    return;
+  }
+
+  int32 flag = 0;
+  base::TimeDelta timestamp;
+  int32 buf_index = 0;
+  do {
+    int32 offset = 0;
+    int32 size = 0;
+    buf_index = media_codec_->DequeueOutputBuffer(
+        base::TimeDelta(), &offset, &size, timestamp, &flag);
+    switch (buf_index) {
+      case media::MediaCodecBridge::INFO_TRY_AGAIN_LATER:
+        return;
+
+      case media::MediaCodecBridge::INFO_OUTPUT_FORMAT_CHANGED: {
+        int32 unused_color_format, width, height;
+        media_codec_->GetOutputFormat(&unused_color_format, &width, &height);
+
+        if (!picturebuffers_requested_) {
+          picturebuffers_requested_ = true;
+          size_ = gfx::Size(width, height);
+          texture_copier_.reset(new Gles2ExternalTextureCopier());
+          texture_copier_->Init(width, height);
+          MessageLoop::current()->PostTask(FROM_HERE, base::Bind(
+              &AndroidVideoDecodeAccelerator::RequestPictureBuffers,
+              base::AsWeakPtr(this)));
+
+        } else {
+          // TODO(dwkang): support the dynamic resolution change.
+          RETURN_ON_FAILURE(size_ == gfx::Size(width, height),

[Ami GONE FROM CHROMIUM, 2013/02/05 20:24:33]

So, what if this == is true?  Ignore changes in color format?  Can something
else have changed other than colorformat/width/height?

[dwkang1, 2013/02/07 14:01:36]

In the decoder case, only width and height matter. So, I removed color format.
http://developer.android.com/reference/android/media/MediaFormat.html

+                            "Dynamic resolution change is not supported.",
+                            PLATFORM_FAILURE);
+        }
+        return;
+      }
+
+      case media::MediaCodecBridge::INFO_OUTPUT_BUFFERS_CHANGED:
+        media_codec_->GetOutputBuffers();
+        break;
+    }
+  } while (buf_index < 0);
+
+  if (flag & media::MediaCodecBridge::BUFFER_FLAG_END_OF_STREAM) {
+    MessageLoop::current()->PostTask(FROM_HERE, base::Bind(
+        &AndroidVideoDecodeAccelerator::NotifyFlushDone, base::AsWeakPtr(this)));

[Ami GONE FROM CHROMIUM, 2013/02/05 20:24:33]

80-cols

[dwkang1, 2013/02/07 14:01:36]

Done.

+  }
+
+  media_codec_->ReleaseOutputBuffer(buf_index, true);
+
+  int64 bitstream_buffer_id = timestamp.InMicroseconds();
+  if (bitstream_buffer_id != -1) {

[qinmin, 2013/02/05 07:43:13]

ditto here and below

[dwkang1, 2013/02/05 11:49:36]

Done.

+    SendCurrentSurfaceToClient(static_cast<int32>(bitstream_buffer_id));
+  } else {
+    decoder_met_eos_ = true;
+  }
+}
+
+void AndroidVideoDecodeAccelerator::SendCurrentSurfaceToClient(
+    int32 bitstream_id) {
+  LOG_LINE();
+  DCHECK(thread_checker_.CalledOnValidThread());
+  DCHECK_NE(bitstream_id, -1);
+  DCHECK(!free_picture_ids_.empty());
+
+  int32 picture_buffer_id = free_picture_ids_.front();
+  free_picture_ids_.pop();
+
+  RETURN_ON_FAILURE(make_context_current_.Run(),

[Ami GONE FROM CHROMIUM, 2013/02/05 20:24:33]

Do this before the pop above? 

[dwkang1, 2013/02/07 14:01:36]

Done.

+                    "Failed to make this decoder's GL context current.",
+                    PLATFORM_FAILURE);
+
+  float transfrom_matrix[16];
+  surface_texture_->UpdateTexImage();
+  surface_texture_->GetTransformMatrix(transfrom_matrix);
+
+  OutputBufferMap::const_iterator i =
+      output_picture_buffers_.find(picture_buffer_id);
+  RETURN_ON_FAILURE(i != output_picture_buffers_.end(),
+                    "Can't find a PictureBuffer for " << picture_buffer_id,
+                    PLATFORM_FAILURE);
+  uint32 picture_buffer_texture_id = i->second.texture_id();
+
+  // Here, we copy |surface_texture_id_| to the picture buffer instead of
+  // setting new texture to |surface_texture_| by calling attachToGLContext()
+  // because:
+  // 1. Once we call detachFrameGLContext(), it deletes the texture previous
+  //    attached.
+  // 2. SurfaceTexture requires us to apply a transform matrix when we show
+  //    the texture.
+  texture_copier_->Copy(
+      surface_texture_id_, picture_buffer_texture_id, transfrom_matrix);
+
+  MessageLoop::current()->PostTask(FROM_HERE, base::Bind(
+      &AndroidVideoDecodeAccelerator::NotifyPictureReady,
+      base::AsWeakPtr(this), media::Picture(picture_buffer_id, bitstream_id)));
+}
+
+void AndroidVideoDecodeAccelerator::Decode(
+    const media::BitstreamBuffer& bitstream_buffer) {
+  LOG_LINE();
+  DCHECK(thread_checker_.CalledOnValidThread());
+  pending_bitstream_buffers_.push(bitstream_buffer);
+
+  if (!io_task_is_posted_) {

[qinmin, 2013/02/05 07:43:13]

ditto

[dwkang1, 2013/02/05 11:49:36]

Done.

+    DoIOTask();
+  }
+}
+
+void AndroidVideoDecodeAccelerator::AssignPictureBuffers(

[Ami GONE FROM CHROMIUM, 2013/02/05 20:24:33]

Does this need to DoIOTask as well?  (not sure, it's getting late ;))

[dwkang1, 2013/02/07 14:01:36]

Yes, it does.

+    const std::vector<media::PictureBuffer>& buffers) {
+  LOG_LINE();
+  DCHECK(thread_checker_.CalledOnValidThread());
+  DCHECK(output_picture_buffers_.empty());
+
+  for (size_t i = 0; i < buffers.size(); ++i) {
+    output_picture_buffers_.insert(std::make_pair(buffers[i].id(), buffers[i]));
+    free_picture_ids_.push(buffers[i].id());
+  }
+
+  RETURN_ON_FAILURE(output_picture_buffers_.size() == kNumPictureBuffers,
+                    "Invalid picture buffers were passed.",
+                    INVALID_ARGUMENT);
+}
+
+void AndroidVideoDecodeAccelerator::ReusePictureBuffer(
+    int32 picture_buffer_id) {
+  DCHECK(thread_checker_.CalledOnValidThread());
+  free_picture_ids_.push(picture_buffer_id);
+
+  if (!io_task_is_posted_) {

[qinmin, 2013/02/05 07:43:13]

ditto

[dwkang1, 2013/02/05 11:49:36]

Done.

+    DoIOTask();
+  }
+}
+
+void AndroidVideoDecodeAccelerator::Flush() {
+  LOG_LINE();
+  DCHECK(thread_checker_.CalledOnValidThread());
+
+  Decode(media::BitstreamBuffer(-1, base::SharedMemoryHandle(), 0));
+}
+
+void AndroidVideoDecodeAccelerator::ConfigureMediaCodec() {
+  DCHECK(surface_texture_.get());
+
+  media_codec_.reset(new media::MediaCodecBridge(codec_));
+
+  JNIEnv* env = base::android::AttachCurrentThread();
+  CHECK(env);
+  ScopedJavaLocalRef<jclass> cls(
+      base::android::GetClass(env, "android/view/Surface"));
+  jmethodID constructor = MethodID::Get<MethodID::TYPE_INSTANCE>(
+      env, cls.obj(), "<init>", "(Landroid/graphics/SurfaceTexture;)V");
+  ScopedJavaLocalRef<jobject> j_surface(
+      env, env->NewObject(
+          cls.obj(), constructor,
+          surface_texture_->j_surface_texture().obj()));
+
+  // VDA does not pass the container indicated resolution in the initialization
+  // phase. Here, we set 1080p by default.
+  // TODO(dwkang): find out a way to remove the following hard-coded value.
+  media_codec_->StartVideo(
+      codec_, gfx::Size(1920, 1080), j_surface.obj());

[qinmin, 2013/02/05 07:43:13]

why 1080p? most android devices are 720p at most

[dwkang1, 2013/02/05 11:49:36]

Changed to 720p.

[Ami GONE FROM CHROMIUM, 2013/02/05 20:24:33]

to ensure we don't hide bugs, can we miminize this (16x16, maybe?)?

[dwkang1, 2013/02/07 14:01:36]

I tried from 16x16 ~ 256x256 because 16 ~ 128 showed show some broken area in
the texture. 256 was fine, but now I think using predefined value is not a good
idea. It seems to me that getting the container indicated resolution via vda
could be a resolution. Do you see any problem with this approach?

[Ami GONE FROM CHROMIUM, 2013/02/07 19:40:16]

THe only problem with container-specified resolution is that it requires
plumbing through the APIs (which span many layers), or else parsing SPS/PPS like
the MacVDA does, both of which are grody.  Also then you fall victim to
containers that lie.

+  content::ReleaseSurface(j_surface.obj());
+
+  media_codec_->GetInputBuffers();
+  media_codec_->GetOutputBuffers();
+}
+
+void AndroidVideoDecodeAccelerator::Reset() {
+  LOG_LINE();
+  DCHECK(thread_checker_.CalledOnValidThread());
+
+  while(!pending_bitstream_buffers_.empty()) {
+    media::BitstreamBuffer& bitstream_buffer =
+        pending_bitstream_buffers_.front();
+    pending_bitstream_buffers_.pop();
+
+    if (bitstream_buffer.id() != -1) {
+      MessageLoop::current()->PostTask(FROM_HERE, base::Bind(
+          &AndroidVideoDecodeAccelerator::NotifyEndOfBitstreamBuffer,
+          base::AsWeakPtr(this), bitstream_buffer.id()));
+    }
+  }
+
+  if (!decoder_met_eos_) {
+    media_codec_->Flush();
+  } else {
+    // MediaCodec should be usable after meeting EOS, but it is not on some
+    // devices. To avoid the case, we recreate a new one.

[Ami GONE FROM CHROMIUM, 2013/02/05 20:24:33]

This sort of comment should really have a pointer to a filed bug (probably on
the android bug tracker)

[dwkang1, 2013/02/07 14:01:36]

bug id is added.

+    media_codec_->Stop();
+    ConfigureMediaCodec();
+  }
+  decoder_met_eos_ = false;
+  num_bytes_used_in_the_pending_buffer_ = 0;
+  state_ = NO_ERROR;
+
+  MessageLoop::current()->PostTask(FROM_HERE, base::Bind(
+      &AndroidVideoDecodeAccelerator::NotifyResetDone, base::AsWeakPtr(this)));
+}
+
+void AndroidVideoDecodeAccelerator::Destroy() {
+  LOG_LINE();
+  DCHECK(thread_checker_.CalledOnValidThread());
+  delete this;

[Ami GONE FROM CHROMIUM, 2013/02/05 20:24:33]

Do you want to stop the codec first?

[dwkang1, 2013/02/07 14:01:36]

Done.

+}
+
+void AndroidVideoDecodeAccelerator::NotifyInitializeDone() {
+  client_->NotifyInitializeDone();
+}
+
+void AndroidVideoDecodeAccelerator::RequestPictureBuffers() {
+  client_->ProvidePictureBuffers(kNumPictureBuffers, size_, GL_TEXTURE_2D);
+}
+
+void AndroidVideoDecodeAccelerator::NotifyPictureReady(
+    const media::Picture& picture) {
+  client_->PictureReady(picture);
+}
+
+void AndroidVideoDecodeAccelerator::NotifyEndOfBitstreamBuffer(
+    int input_buffer_id) {
+  client_->NotifyEndOfBitstreamBuffer(input_buffer_id);
+}
+
+void AndroidVideoDecodeAccelerator::NotifyFlushDone() {
+  client_->NotifyFlushDone();
+}
+
+void AndroidVideoDecodeAccelerator::NotifyResetDone() {
+  client_->NotifyResetDone();
+}
+
+void AndroidVideoDecodeAccelerator::NotifyError(
+    media::VideoDecodeAccelerator::Error error) {
+  client_->NotifyError(error);
+}
+
+}  // namespace content