Memory copy the VideoFrame to match the requirement for HW encoders.

media/cast/sender/external_video_encoder.cc

 // Copyright 2014 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "media/cast/sender/external_video_encoder.h"
 
 #include <cmath>
 #include <utility>
 
 #include "base/bind.h"
 #include "base/debug/crash_logging.h"
 #include "base/debug/dump_without_crashing.h"
 #include "base/format_macros.h"
 #include "base/logging.h"
 #include "base/macros.h"
-#include "base/memory/scoped_vector.h"
 #include "base/memory/shared_memory.h"
 #include "base/message_loop/message_loop.h"
 #include "base/metrics/histogram.h"
 #include "base/strings/stringprintf.h"
 #include "build/build_config.h"
+#include "media/base/bind_to_current_loop.h"
 #include "media/base/video_frame.h"
 #include "media/base/video_types.h"
 #include "media/base/video_util.h"
 #include "media/cast/cast_config.h"
 #include "media/cast/common/rtp_time.h"
 #include "media/cast/logging/logging_defines.h"
 #include "media/cast/net/cast_transport_config.h"
 #include "media/cast/sender/vp8_quantizer_parser.h"
 #include "media/filters/h264_parser.h"
 
 namespace {
 
 enum { MAX_H264_QUANTIZER = 51 };
-static const size_t kOutputBufferCount = 3;
+
+// Number of buffers for encoded bit stream.
+constexpr size_t kOutputBufferCount = 3;
+
+// Maximum number of extra input buffers for encoder. The input buffers are only
+// used when copy is needed to match the required coded size.
+constexpr size_t kExtraInputBufferCount = 2;
 
 }  // namespace
 
 namespace media {
 namespace cast {
 
 // Container for the associated data of a video frame being processed.
 struct InProgressFrameEncode {
   // The source content to encode.
   const scoped_refptr<VideoFrame> video_frame;
@@ skipping 43 matching lines @@
         max_frame_rate_(max_frame_rate),
         status_change_cb_(status_change_cb),
         create_video_encode_memory_cb_(create_video_encode_memory_cb),
         video_encode_accelerator_(std::move(vea)),
         encoder_active_(false),
         next_frame_id_(FrameId::first()),
         key_frame_encountered_(false),
         codec_profile_(media::VIDEO_CODEC_PROFILE_UNKNOWN),
         key_frame_quantizer_parsable_(false),
         requested_bit_rate_(-1),
-        has_seen_zero_length_encoded_frame_(false) {}
+        has_seen_zero_length_encoded_frame_(false),
+        max_allowed_input_buffers_(0),
+        allocate_input_buffer_in_progress_(false) {}
 
   base::SingleThreadTaskRunner* task_runner() const {
     return task_runner_.get();
   }
 
   void Initialize(const gfx::Size& frame_size,
                   VideoCodecProfile codec_profile,
                   int start_bit_rate,
                   FrameId first_frame_id) {
     DCHECK(task_runner_->RunsTasksOnCurrentThread());
@@ skipping 17 matching lines @@
   }
 
   void SetBitRate(int bit_rate) {
     DCHECK(task_runner_->RunsTasksOnCurrentThread());
 
     requested_bit_rate_ = bit_rate;
     video_encode_accelerator_->RequestEncodingParametersChange(bit_rate,
                                                                max_frame_rate_);
   }
 
+  // The destruction call back of the copied video frame to free its use of
+  // the input buffer.
+  void ReturnInputBufferToPool(int index) {
+    DCHECK(task_runner_->RunsTasksOnCurrentThread());
+    DCHECK_GE(index, 0);
+    DCHECK_LT(index, static_cast<int>(input_buffers_.size()));
+    free_input_buffer_index_.push_back(index);
+  }
+
   void EncodeVideoFrame(
       const scoped_refptr<media::VideoFrame>& video_frame,
       const base::TimeTicks& reference_time,
       bool key_frame_requested,
       const VideoEncoder::FrameEncodedCallback& frame_encoded_callback) {
     DCHECK(task_runner_->RunsTasksOnCurrentThread());
 
     if (!encoder_active_)
       return;
 
     in_progress_frame_encodes_.push_back(InProgressFrameEncode(
         video_frame, reference_time, frame_encoded_callback,
         requested_bit_rate_));
 
+    scoped_refptr<media::VideoFrame> frame = video_frame;
+    if (video_frame->coded_size() != frame_coded_size_) {
+      DCHECK_GE(frame_coded_size_.width(), video_frame->visible_rect().width());
+      DCHECK_GE(frame_coded_size_.height(),
+                video_frame->visible_rect().height());
+
+      if (free_input_buffer_index_.empty()) {
+        if (!allocate_input_buffer_in_progress_ &&
+            input_buffers_.size() < max_allowed_input_buffers_) {
+          allocate_input_buffer_in_progress_ = true;
+          create_video_encode_memory_cb_.Run(
+              media::VideoFrame::AllocationSize(media::PIXEL_FORMAT_I420,
+                                                frame_coded_size_),
+              base::Bind(&VEAClientImpl::OnCreateInputSharedMemory, this));
+        }
+        ExitEncodingWithErrors();
+        return;
+      }
+
+      int index = free_input_buffer_index_.back();
+      base::SharedMemory* input_buffer = input_buffers_[index].get();
+      frame = VideoFrame::WrapExternalSharedMemory(
+          video_frame->format(), frame_coded_size_, video_frame->visible_rect(),
+          video_frame->visible_rect().size(),
+          static_cast<uint8_t*>(input_buffer->memory()),
+          input_buffer->mapped_size(), input_buffer->handle(), 0,
+          video_frame->timestamp());
+      if (!frame || !media::I420CopyWithPadding(*video_frame, frame.get())) {
+        LOG(DFATAL) << "Error: ExternalVideoEncoder: copy failed.";
+        ExitEncodingWithErrors();
+        return;
+      }
+
+      frame->AddDestructionObserver(media::BindToCurrentLoop(base::Bind(
+          &ExternalVideoEncoder::VEAClientImpl::ReturnInputBufferToPool, this,
+          index)));
+      free_input_buffer_index_.pop_back();
+    }
+
     // BitstreamBufferReady will be called once the encoder is done.
-    video_encode_accelerator_->Encode(video_frame, key_frame_requested);
+    video_encode_accelerator_->Encode(frame, key_frame_requested);
   }
 
  protected:
   void NotifyError(VideoEncodeAccelerator::Error error) final {
     DCHECK(task_runner_->RunsTasksOnCurrentThread());
 
     DCHECK(error != VideoEncodeAccelerator::kInvalidArgumentError &&
            error != VideoEncodeAccelerator::kIllegalStateError);
 
     encoder_active_ = false;
 
     cast_environment_->PostTask(
         CastEnvironment::MAIN,
         FROM_HERE,
         base::Bind(status_change_cb_, STATUS_CODEC_RUNTIME_ERROR));
 
     // TODO(miu): Force-flush all |in_progress_frame_encodes_| immediately so
     // pending frames do not become stuck, freezing VideoSender.
   }
 
   // Called to allocate the input and output buffers.
   void RequireBitstreamBuffers(unsigned int input_count,
                                const gfx::Size& input_coded_size,
                                size_t output_buffer_size) final {
     DCHECK(task_runner_->RunsTasksOnCurrentThread());
 
-    // TODO(miu): Investigate why we are ignoring |input_count| (4) and instead
-    // using |kOutputBufferCount| (3) here.
+    frame_coded_size_ = input_coded_size;
+
+    max_allowed_input_buffers_ = input_count + kExtraInputBufferCount;
+
     for (size_t j = 0; j < kOutputBufferCount; ++j) {
       create_video_encode_memory_cb_.Run(
           output_buffer_size,
           base::Bind(&VEAClientImpl::OnCreateSharedMemory, this));
     }
   }
 
   // Encoder has encoded a frame and it's available in one of the output
   // buffers.  Package the result in a media::cast::EncodedFrame and post it
   // to the Cast MAIN thread via the supplied callback.
   void BitstreamBufferReady(int32_t bitstream_buffer_id,
                             size_t payload_size,
                             bool key_frame) final {
     DCHECK(task_runner_->RunsTasksOnCurrentThread());
     if (bitstream_buffer_id < 0 ||
         bitstream_buffer_id >= static_cast<int32_t>(output_buffers_.size())) {
       NOTREACHED();
       VLOG(1) << "BitstreamBufferReady(): invalid bitstream_buffer_id="
               << bitstream_buffer_id;
       NotifyError(media::VideoEncodeAccelerator::kPlatformFailureError);
       return;
     }
-    base::SharedMemory* output_buffer = output_buffers_[bitstream_buffer_id];
+    base::SharedMemory* output_buffer =
+        output_buffers_[bitstream_buffer_id].get();
     if (payload_size > output_buffer->mapped_size()) {
       NOTREACHED();
       VLOG(1) << "BitstreamBufferReady(): invalid payload_size = "
               << payload_size;
       NotifyError(media::VideoEncodeAccelerator::kPlatformFailureError);
       return;
     }
     if (key_frame)
       key_frame_encountered_ = true;
     if (!key_frame_encountered_) {
@@ skipping 147 matching lines @@
   }
 
   // Note: This method can be called on any thread.
   void OnCreateSharedMemory(std::unique_ptr<base::SharedMemory> memory) {
     task_runner_->PostTask(FROM_HERE,
                            base::Bind(&VEAClientImpl::OnReceivedSharedMemory,
                                       this,
                                       base::Passed(&memory)));
   }
 
+  void OnCreateInputSharedMemory(std::unique_ptr<base::SharedMemory> memory) {
+    task_runner_->PostTask(
+        FROM_HERE, base::Bind(&VEAClientImpl::OnReceivedInputSharedMemory, this,
+                              base::Passed(&memory)));
+  }
+
   void OnReceivedSharedMemory(std::unique_ptr<base::SharedMemory> memory) {
     DCHECK(task_runner_->RunsTasksOnCurrentThread());
 
     output_buffers_.push_back(std::move(memory));
 
     // Wait until all requested buffers are received.
     if (output_buffers_.size() < kOutputBufferCount)
       return;
 
     // Immediately provide all output buffers to the VEA.
     for (size_t i = 0; i < output_buffers_.size(); ++i) {
       video_encode_accelerator_->UseOutputBitstreamBuffer(
           media::BitstreamBuffer(static_cast<int32_t>(i),
                                  output_buffers_[i]->handle(),
                                  output_buffers_[i]->mapped_size()));
     }
   }
 
+  void OnReceivedInputSharedMemory(std::unique_ptr<base::SharedMemory> memory) {
+    DCHECK(task_runner_->RunsTasksOnCurrentThread());
+
+    if (memory.get()) {
+      input_buffers_.push_back(std::move(memory));
+      free_input_buffer_index_.push_back(input_buffers_.size() - 1);
+    }
+    allocate_input_buffer_in_progress_ = false;
+  }
+
+  // This is called when copy errors occur in encoding process when there is
+  // need to copy the VideoFrames to match the required coded size for encoder.
+  void ExitEncodingWithErrors() {
+    DCHECK(task_runner_->RunsTasksOnCurrentThread());
+
+    std::unique_ptr<SenderEncodedFrame> no_result(nullptr);
+    cast_environment_->PostTask(
+        CastEnvironment::MAIN, FROM_HERE,
+        base::Bind(in_progress_frame_encodes_.back().frame_encoded_callback,
+                   base::Passed(&no_result)));
+    in_progress_frame_encodes_.pop_back();
+  }
+
   // Parse H264 SPS, PPS, and Slice header, and return the averaged frame
   // quantizer in the range of [0, 51], or -1 on parse error.
   double GetH264FrameQuantizer(const uint8_t* encoded_data, off_t size) {
     DCHECK(encoded_data);
     if (!size)
       return -1;
     h264_parser_.SetStream(encoded_data, size);
     double total_quantizer = 0;
     int num_slices = 0;
 
@@ skipping 54 matching lines @@
   std::unique_ptr<media::VideoEncodeAccelerator> video_encode_accelerator_;
   bool encoder_active_;
   FrameId next_frame_id_;
   bool key_frame_encountered_;
   std::string stream_header_;
   VideoCodecProfile codec_profile_;
   bool key_frame_quantizer_parsable_;
   H264Parser h264_parser_;
 
   // Shared memory buffers for output with the VideoAccelerator.
-  ScopedVector<base::SharedMemory> output_buffers_;
+  std::vector<std::unique_ptr<base::SharedMemory>> output_buffers_;
+
+  // Shared memory buffers for input video frames with the VideoAccelerator.
+  // These buffers will be allocated only when copy is needed to match the
+  // required coded size for encoder. They are allocated on-demand, up to
+  // |max_allowed_input_buffers_|.
+  std::vector<std::unique_ptr<base::SharedMemory>> input_buffers_;
+
+  // Available input buffer index. These buffers are used in FILO order.
+  std::vector<int> free_input_buffer_index_;
 
   // FIFO list.
   std::list<InProgressFrameEncode> in_progress_frame_encodes_;
 
   // The requested encode bit rate for the next frame.
   int requested_bit_rate_;
 
   // Used to compute utilization metrics for each frame.
   QuantizerEstimator quantizer_estimator_;
 
   // Set to true once a frame with zero-length encoded data has been
   // encountered.
   // TODO(miu): Remove after discovering cause.  http://crbug.com/519022
   bool has_seen_zero_length_encoded_frame_;
 
+  // The coded size of the video frame required by Encoder. This size is
+  // obtained from VEA through |RequireBitstreamBuffers()|.
+  gfx::Size frame_coded_size_;
+
+  // The maximum number of input buffers. These buffers are used to copy
+  // VideoFrames in order to match the required coded size for encoder.
+  size_t max_allowed_input_buffers_;
+
+  // Set to true when the allocation of an input buffer is in progress, and
+  // reset to false after the allocated buffer is received.
+  bool allocate_input_buffer_in_progress_;
+
   DISALLOW_COPY_AND_ASSIGN(VEAClientImpl);
 };
 
 // static
 bool ExternalVideoEncoder::IsSupported(const VideoSenderConfig& video_config) {
   if (video_config.codec != CODEC_VIDEO_VP8 &&
       video_config.codec != CODEC_VIDEO_H264)
     return false;
 
   // TODO(miu): "Layering hooks" are needed to be able to query outside of
@@ skipping 295 matching lines @@
   const double kEntropyAtMaxQuantizer = 7.5;
   const double slope =
       (MAX_VP8_QUANTIZER - MIN_VP8_QUANTIZER) / kEntropyAtMaxQuantizer;
   const double quantizer = std::min<double>(
       MAX_VP8_QUANTIZER, MIN_VP8_QUANTIZER + slope * shannon_entropy);
   return quantizer;
 }
 
 }  //  namespace cast
 }  //  namespace media