Support configuration changes in VTVideoDecodeAccelerator.

content/common/gpu/media/vt_video_decode_accelerator.cc

 // Copyright 2014 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include <CoreVideo/CoreVideo.h>
 #include <OpenGL/CGLIOSurface.h>
 #include <OpenGL/gl.h>
 
 #include "base/bind.h"
 #include "base/callback_helpers.h"
@@ skipping 128 matching lines @@
       nalu_data_ptrs.size(),      // parameter_set_count
       &nalu_data_ptrs.front(),    // &parameter_set_pointers
       &nalu_data_sizes.front(),   // &parameter_set_sizes
       kNALUHeaderLength,          // nal_unit_header_length
       format_.InitializeInto());
   if (status) {
     NOTIFY_STATUS("CMVideoFormatDescriptionCreateFromH264ParameterSets()",
                   status);
     return false;
   }
-  CMVideoDimensions coded_dimensions =
-      CMVideoFormatDescriptionGetDimensions(format_);
+
+  // If the session is compatible, there's nothing to do.
+  if (session_ &&
+      VTDecompressionSessionCanAcceptFormatDescription(session_, format_)) {
+      return true;
+  }
 
   // Prepare VideoToolbox configuration dictionaries.
   base::ScopedCFTypeRef<CFMutableDictionaryRef> decoder_config(
       CFDictionaryCreateMutable(
           kCFAllocatorDefault,
           1,  // capacity
           &kCFTypeDictionaryKeyCallBacks,
           &kCFTypeDictionaryValueCallBacks));
 
   CFDictionarySetValue(
       decoder_config,
       // kVTVideoDecoderSpecification_EnableHardwareAcceleratedVideoDecoder
       CFSTR("EnableHardwareAcceleratedVideoDecoder"),
       kCFBooleanTrue);
 
   base::ScopedCFTypeRef<CFMutableDictionaryRef> image_config(
       CFDictionaryCreateMutable(
           kCFAllocatorDefault,
           4,  // capacity
           &kCFTypeDictionaryKeyCallBacks,
           &kCFTypeDictionaryValueCallBacks));
 
+  CMVideoDimensions coded_dimensions =
+      CMVideoFormatDescriptionGetDimensions(format_);
 #define CFINT(i) CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &i)
   // TODO(sandersd): RGBA option for 4:4:4 video.
   int32_t pixel_format = kCVPixelFormatType_422YpCbCr8;
   base::ScopedCFTypeRef<CFNumberRef> cf_pixel_format(CFINT(pixel_format));
   base::ScopedCFTypeRef<CFNumberRef> cf_width(CFINT(coded_dimensions.width));
   base::ScopedCFTypeRef<CFNumberRef> cf_height(CFINT(coded_dimensions.height));
 #undef CFINT
   CFDictionarySetValue(
       image_config, kCVPixelBufferPixelFormatTypeKey, cf_pixel_format);
   CFDictionarySetValue(image_config, kCVPixelBufferWidthKey, cf_width);
   CFDictionarySetValue(image_config, kCVPixelBufferHeightKey, cf_height);
   CFDictionarySetValue(
       image_config, kCVPixelBufferOpenGLCompatibilityKey, kCFBooleanTrue);
 
-  // TODO(sandersd): Check if the session is already compatible.
+  // TODO(sandersd): Does the old session need to be flushed first?
   session_.reset();
   status = VTDecompressionSessionCreate(
       kCFAllocatorDefault,
       format_,              // video_format_description
       decoder_config,       // video_decoder_specification
       image_config,         // destination_image_buffer_attributes
       &callback_,           // output_callback
       session_.InitializeInto());
   if (status) {
     NOTIFY_STATUS("VTDecompressionSessionCreate()", status);
     return false;
   }
 
-  // If the size has changed, trigger a request for new picture buffers.
-  // TODO(sandersd): Move to SendPictures(), and use this just as a hint for an
-  // upcoming size change.
-  gfx::Size new_coded_size(coded_dimensions.width, coded_dimensions.height);
-  if (coded_size_ != new_coded_size) {
-    coded_size_ = new_coded_size;
-    gpu_task_runner_->PostTask(FROM_HERE, base::Bind(
-        &VTVideoDecodeAccelerator::SizeChangedTask,
-        weak_this_factory_.GetWeakPtr(),
-        coded_size_));;
-  }
-
   return true;
 }
 
 void VTVideoDecodeAccelerator::Decode(const media::BitstreamBuffer& bitstream) {
   DCHECK(CalledOnValidThread());
   // Not actually a requirement of the VDA API, but we're lazy and use negative
   // values as flags internally. Revisit that if this actually happens.
   if (bitstream.id() < 0) {
     LOG(ERROR) << "Negative bitstream ID";
     NotifyError(INVALID_ARGUMENT);
@@ skipping 26 matching lines @@
     return;
   }
   const uint8_t* buf = static_cast<uint8_t*>(memory.memory());
 
   // NALUs are stored with Annex B format in the bitstream buffer (start codes),
   // but VideoToolbox expects AVCC/MPEG-4 format (length headers), so we must
   // rewrite the data.
   //
   // 1. Locate relevant NALUs and compute the size of the translated data.
   //    Also record any parameter sets for VideoToolbox initialization.
+  bool config_changed = false;
   size_t data_size = 0;
   std::vector<media::H264NALU> nalus;
-  std::vector<const uint8_t*> config_nalu_data_ptrs;
-  std::vector<size_t> config_nalu_data_sizes;
   parser_.SetStream(buf, size);
   media::H264NALU nalu;
   while (true) {
     media::H264Parser::Result result = parser_.AdvanceToNextNALU(&nalu);
     if (result == media::H264Parser::kEOStream)
       break;
     if (result != media::H264Parser::kOk) {
       LOG(ERROR) << "Failed to find H.264 NALU";
       NotifyError(PLATFORM_FAILURE);
       return;
     }
-    // TODO(sandersd): Check that these are only at the start.
-    if (nalu.nal_unit_type == media::H264NALU::kSPS ||
-        nalu.nal_unit_type == media::H264NALU::kPPS ||
-        nalu.nal_unit_type == media::H264NALU::kSPSExt) {
-      DVLOG(2) << "Parameter set " << nalu.nal_unit_type;
-      config_nalu_data_ptrs.push_back(nalu.data);
-      config_nalu_data_sizes.push_back(nalu.size);
-    } else {
-      nalus.push_back(nalu);
-      data_size += kNALUHeaderLength + nalu.size;
+    // TODO(sandersd): Strict ordering rules.
+    switch (nalu.nal_unit_type) {
+      case media::H264NALU::kSPS:
+        last_sps_.assign(nalu.data, nalu.data + nalu.size);
+        last_spsext_.clear();
+        config_changed = true;
+        break;
+      case media::H264NALU::kSPSExt:
+        // TODO(sandersd): Check that the previous NALU was an SPS.
+        last_spsext_.assign(nalu.data, nalu.data + nalu.size);
+        config_changed = true;
+        break;
+      case media::H264NALU::kPPS:
+        last_pps_.assign(nalu.data, nalu.data + nalu.size);
+        config_changed = true;
+        break;
+      default:
+        nalus.push_back(nalu);
+        data_size += kNALUHeaderLength + nalu.size;
+        break;
     }
   }
 
   // 2. Initialize VideoToolbox.
-  // TODO(sandersd): Reinitialize when there are new parameter sets.
-  if (!session_) {
-    // If configuring fails, ConfigureDecoder() already called NotifyError().
-    if (!ConfigureDecoder(config_nalu_data_ptrs, config_nalu_data_sizes))
+  // TODO(sandersd): Check if the new configuration is identical before
+  // reconfiguring.
+  if (config_changed) {
+    if (last_sps_.size() == 0 || last_pps_.size() == 0) {
+      LOG(ERROR) << "Invalid configuration data";
+      NotifyError(INVALID_ARGUMENT);
+      return;
+    }
+    // TODO(sandersd): Check that the SPS and PPS IDs match.
+    std::vector<const uint8_t*> nalu_data_ptrs;
+    std::vector<size_t> nalu_data_sizes;
+    nalu_data_ptrs.push_back(&last_sps_.front());
+    nalu_data_sizes.push_back(last_sps_.size());
+    if (last_spsext_.size() != 0) {
+      nalu_data_ptrs.push_back(&last_spsext_.front());
+      nalu_data_sizes.push_back(last_spsext_.size());
+    }
+    nalu_data_ptrs.push_back(&last_pps_.front());
+    nalu_data_sizes.push_back(last_pps_.size());
+
+    // If ConfigureDecoder() fails, it already called NotifyError().
+    if (!ConfigureDecoder(nalu_data_ptrs, nalu_data_sizes))
       return;
   }
 
   // If there are no non-configuration units, immediately return an empty
   // (ie. dropped) frame. It is an error to create a MemoryBlock with zero
   // size.
   if (!data_size)
     return;
 
+  // If the session is not configured, fail.
+  if (!session_) {
+    LOG(ERROR) << "Image slice without configuration data";
+    NotifyError(INVALID_ARGUMENT);
+    return;
+  }
+
   // 3. Allocate a memory-backed CMBlockBuffer for the translated data.
+  // TODO(sandersd): Check that the slice's PPS matches the current PPS.
   base::ScopedCFTypeRef<CMBlockBufferRef> data;
   OSStatus status = CMBlockBufferCreateWithMemoryBlock(
       kCFAllocatorDefault,
       NULL,                 // &memory_block
       data_size,            // block_length
       kCFAllocatorDefault,  // block_allocator
       NULL,                 // &custom_block_source
       0,                    // offset_to_data
       data_size,            // data_length
       0,                    // flags
@@ skipping 68 matching lines @@
 
 // This method may be called on any VideoToolbox thread.
 void VTVideoDecodeAccelerator::Output(
     int32_t bitstream_id,
     OSStatus status,
     CVImageBufferRef image_buffer) {
   if (status) {
     // TODO(sandersd): Handle dropped frames.
     NOTIFY_STATUS("Decoding", status);
     image_buffer = NULL;
-  } else  if (CFGetTypeID(image_buffer) != CVPixelBufferGetTypeID()) {
+  } else if (CFGetTypeID(image_buffer) != CVPixelBufferGetTypeID()) {
     LOG(ERROR) << "Decoded frame is not a CVPixelBuffer";
     NotifyError(PLATFORM_FAILURE);
     image_buffer = NULL;
   } else {
     CFRetain(image_buffer);
   }
   gpu_task_runner_->PostTask(FROM_HERE, base::Bind(
       &VTVideoDecodeAccelerator::OutputTask,
       weak_this_factory_.GetWeakPtr(),
       DecodedFrame(bitstream_id, image_buffer)));
 }
 
 void VTVideoDecodeAccelerator::OutputTask(DecodedFrame frame) {
   DCHECK(CalledOnValidThread());
   decoded_frames_.push(frame);
   ProcessDecodedFrames();
 }
 
-void VTVideoDecodeAccelerator::SizeChangedTask(gfx::Size coded_size) {
-  DCHECK(CalledOnValidThread());
-  texture_size_ = coded_size;
-  // TODO(sandersd): Dismiss existing picture buffers.
-  client_->ProvidePictureBuffers(
-      kNumPictureBuffers, texture_size_, GL_TEXTURE_RECTANGLE_ARB);
-}
-
 void VTVideoDecodeAccelerator::AssignPictureBuffers(
     const std::vector<media::PictureBuffer>& pictures) {
   DCHECK(CalledOnValidThread());
 
   for (size_t i = 0; i < pictures.size(); i++) {
     DCHECK(!texture_ids_.count(pictures[i].id()));
-    available_picture_ids_.push(pictures[i].id());
+    assigned_picture_ids_.insert(pictures[i].id());
+    available_picture_ids_.push_back(pictures[i].id());
     texture_ids_[pictures[i].id()] = pictures[i].texture_id();
   }
 
   // Pictures are not marked as uncleared until after this method returns, and
   // they will be broken if they are used before that happens. So, schedule
   // future work after that happens.
   gpu_task_runner_->PostTask(FROM_HERE, base::Bind(
       &VTVideoDecodeAccelerator::ProcessDecodedFrames,
       weak_this_factory_.GetWeakPtr()));
 }
 
 void VTVideoDecodeAccelerator::ReusePictureBuffer(int32_t picture_id) {
   DCHECK(CalledOnValidThread());
   DCHECK_EQ(CFGetRetainCount(picture_bindings_[picture_id]), 1);
   picture_bindings_.erase(picture_id);
-  available_picture_ids_.push(picture_id);
-  ProcessDecodedFrames();
+  // Don't put the picture back in the available list if has been dismissed.
+  if (assigned_picture_ids_.count(picture_id) != 0) {
+    available_picture_ids_.push_back(picture_id);
+    ProcessDecodedFrames();
+  }
 }
 
 void VTVideoDecodeAccelerator::CompleteAction(Action action) {
   DCHECK(CalledOnValidThread());
+
   switch (action) {
     case ACTION_FLUSH:
       client_->NotifyFlushDone();
       break;
     case ACTION_RESET:
       client_->NotifyResetDone();
       break;
     case ACTION_DESTROY:
       delete this;
       break;
@@ skipping 64 matching lines @@
     // until more frames are decoded.
     if (last_sent_bitstream_id != next_action_bitstream_id)
       return;
 
     // Complete all actions pending for this |bitstream_id|, then loop to see
     // if progress can be made on the next action.
     CompleteActions(next_action_bitstream_id);
   }
 }
 
+int32_t VTVideoDecodeAccelerator::ProcessDroppedFrames(
+    int32_t last_sent_bitstream_id,
+    int32_t up_to_bitstream_id) {
+  DCHECK(CalledOnValidThread());
+  // Drop frames as long as there is a frame, we have not reached the next
+  // action, and the next frame has no image.
+  while (!decoded_frames_.empty() &&
+         last_sent_bitstream_id != up_to_bitstream_id &&
+         decoded_frames_.front().image_buffer.get() == NULL) {
+    const DecodedFrame& frame = decoded_frames_.front();
+    DCHECK_EQ(pending_bitstream_ids_.front(), frame.bitstream_id);
+    client_->NotifyEndOfBitstreamBuffer(frame.bitstream_id);
+    last_sent_bitstream_id = frame.bitstream_id;
+    decoded_frames_.pop();
+    pending_bitstream_ids_.pop();
+  }
+  return last_sent_bitstream_id;
+}
+
+// TODO(sandersd): If GpuVideoDecoder didn't specifically check the size of
+// textures, this would be unnecessary, as the size is actually a property of
+// the texture binding, not the texture. We rebind every frame, so the size
+// passed to ProvidePictureBuffers() is meaningless.
+void VTVideoDecodeAccelerator::ProcessSizeChange() {
+  DCHECK(CalledOnValidThread());
+  DCHECK(!decoded_frames_.empty());
+
+  // Find the size of the next image.
+  const DecodedFrame& frame = decoded_frames_.front();
+  CVImageBufferRef image_buffer = frame.image_buffer.get();
+  size_t width = CVPixelBufferGetWidth(image_buffer);
+  size_t height = CVPixelBufferGetHeight(image_buffer);
+  gfx::Size image_size(width, height);
+
+  if (picture_size_ != image_size) {
+    // Dismiss all assigned picture buffers.
+    for (int32_t picture_id : assigned_picture_ids_)
+      client_->DismissPictureBuffer(picture_id);
+    assigned_picture_ids_.clear();
+    available_picture_ids_.clear();
+
+    // Request new pictures.
+    client_->ProvidePictureBuffers(
+        kNumPictureBuffers, image_size, GL_TEXTURE_RECTANGLE_ARB);
+    picture_size_ = image_size;
+  }
+}
+
 int32_t VTVideoDecodeAccelerator::SendPictures(int32_t up_to_bitstream_id) {
   DCHECK(CalledOnValidThread());
   DCHECK(!decoded_frames_.empty());
 
   // TODO(sandersd): Store the actual last sent bitstream ID?
   int32_t last_sent_bitstream_id = -1;
 
+  last_sent_bitstream_id =
+      ProcessDroppedFrames(last_sent_bitstream_id, up_to_bitstream_id);
+  if (last_sent_bitstream_id == up_to_bitstream_id || decoded_frames_.empty())
+    return last_sent_bitstream_id;
+
+  ProcessSizeChange();
   if (available_picture_ids_.empty())
     return last_sent_bitstream_id;
 
   if (!make_context_current_.Run()) {
     LOG(ERROR) << "Failed to make GL context current";
     NotifyError(PLATFORM_FAILURE);
     return last_sent_bitstream_id;
   }
 
   glEnable(GL_TEXTURE_RECTANGLE_ARB);
-  while (!available_picture_ids_.empty() &&
-         !decoded_frames_.empty() &&
-         last_sent_bitstream_id != up_to_bitstream_id &&
-         !has_error_) {
-    // We don't pop |frame| until it is consumed, which won't happen if an
-    // error occurs. Conveniently, this also removes some refcounting.
+  while (!available_picture_ids_.empty() && !has_error_) {
+    DCHECK_NE(last_sent_bitstream_id, up_to_bitstream_id);
+    DCHECK(!decoded_frames_.empty());

[Pawel Osciak, 2014/11/07 10:09:38]

Can this ever fail given l612 and l668?

[sandersd (OOO until July 31), 2014/11/07 19:01:11]

No, I just make it explicit because the next line assumes it.

+
+    // We don't pop |frame| or |picture_id| until they are consumed, which may
+    // not happen if an error occurs. Conveniently, this also removes some
+    // refcounting.
     const DecodedFrame& frame = decoded_frames_.front();
     DCHECK_EQ(pending_bitstream_ids_.front(), frame.bitstream_id);
-
-    // Likewise, |picture_id| won't be popped if |image_buffer| is NULL or an
-    // error occurs.
-    // TODO(sandersd): Don't block waiting for a |picture_id| when
-    // |image_buffer| is NULL.
-    int32_t picture_id = available_picture_ids_.front();
+    int32_t picture_id = available_picture_ids_.back();
 
     CVImageBufferRef image_buffer = frame.image_buffer.get();
-    if (image_buffer) {
-      IOSurfaceRef surface = CVPixelBufferGetIOSurface(image_buffer);
+    IOSurfaceRef surface = CVPixelBufferGetIOSurface(image_buffer);
 
-      gfx::ScopedTextureBinder
-          texture_binder(GL_TEXTURE_RECTANGLE_ARB, texture_ids_[picture_id]);
-      CGLError status = CGLTexImageIOSurface2D(
-          cgl_context_,                 // ctx
-          GL_TEXTURE_RECTANGLE_ARB,     // target
-          GL_RGB,                       // internal_format
-          texture_size_.width(),        // width
-          texture_size_.height(),       // height
-          GL_YCBCR_422_APPLE,           // format
-          GL_UNSIGNED_SHORT_8_8_APPLE,  // type
-          surface,                      // io_surface
-          0);                           // plane
-      if (status != kCGLNoError) {
-        NOTIFY_STATUS("CGLTexImageIOSurface2D()", status);
-        break;
-      }
-
-      picture_bindings_[picture_id] = frame.image_buffer;
-      client_->PictureReady(media::Picture(
-          picture_id, frame.bitstream_id, gfx::Rect(texture_size_)));
-      available_picture_ids_.pop();
+    gfx::ScopedTextureBinder
+        texture_binder(GL_TEXTURE_RECTANGLE_ARB, texture_ids_[picture_id]);
+    CGLError status = CGLTexImageIOSurface2D(
+        cgl_context_,                 // ctx
+        GL_TEXTURE_RECTANGLE_ARB,     // target
+        GL_RGB,                       // internal_format
+        picture_size_.width(),        // width
+        picture_size_.height(),       // height
+        GL_YCBCR_422_APPLE,           // format
+        GL_UNSIGNED_SHORT_8_8_APPLE,  // type
+        surface,                      // io_surface
+        0);                           // plane
+    if (status != kCGLNoError) {
+      NOTIFY_STATUS("CGLTexImageIOSurface2D()", status);
+      break;
     }
 
+    picture_bindings_[picture_id] = frame.image_buffer;
+    client_->PictureReady(media::Picture(
+        picture_id, frame.bitstream_id, gfx::Rect(picture_size_)));
+    available_picture_ids_.pop_back();
     client_->NotifyEndOfBitstreamBuffer(frame.bitstream_id);
     last_sent_bitstream_id = frame.bitstream_id;
     decoded_frames_.pop();
     pending_bitstream_ids_.pop();
+
+    last_sent_bitstream_id =
+        ProcessDroppedFrames(last_sent_bitstream_id, up_to_bitstream_id);
+    if (last_sent_bitstream_id == up_to_bitstream_id || decoded_frames_.empty())
+      break;
+
+    ProcessSizeChange();
   }
   glDisable(GL_TEXTURE_RECTANGLE_ARB);
 
   return last_sent_bitstream_id;
 }
 
 void VTVideoDecodeAccelerator::FlushTask() {
   DCHECK(decoder_thread_.message_loop_proxy()->BelongsToCurrentThread());
   OSStatus status = VTDecompressionSessionFinishDelayedFrames(session_);
   if (status)
@@ skipping 59 matching lines @@
     pending_bitstream_ids_.pop();
   }
   QueueAction(ACTION_DESTROY);
 }
 
 bool VTVideoDecodeAccelerator::CanDecodeOnIOThread() {
   return false;
 }
 
 }  // namespace content