Implement screen capturer for MediaStream API.

content/browser/renderer_host/media/screen_capturer.cc

+// Copyright (c) 2012 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "content/browser/renderer_host/media/screen_capturer.h"
+
+#include "base/bind.h"
+#include "base/location.h"
+#include "base/logging.h"
+#include "base/sequenced_task_runner.h"
+#include "base/synchronization/lock.h"
+#include "remoting/capturer/capture_data.h"
+#include "remoting/capturer/mouse_cursor_shape.h"
+#include "third_party/skia/include/core/SkCanvas.h"
+#include "third_party/skia/include/core/SkDevice.h"
+
+namespace content {
+
+namespace {
+const int kBytesPerPixel = 4;
+}  // namespace
+
+class ScreenCapturer::Core
+    : public base::RefCountedThreadSafe<Core>,
+      public remoting::VideoFrameCapturer::Delegate {
+ public:
+  Core(scoped_refptr<base::SequencedTaskRunner> task_runner);
+
+  // Helper used in tests to supply a fake capturer.
+  void set_test_frame_capturer(
+      scoped_ptr<remoting::VideoFrameCapturer> capturer) {
+    video_frame_capturer_ = capturer.Pass();
+  }
+
+  void Allocate(int width, int height,
+                int frame_rate,
+                EventHandler* event_handler);
+  void Start();
+  void Stop();
+  void DeAllocate();
+
+  // VideoFrameCapturer::Delegate interface. Called by |video_frame_capturer_|
+  // on the |task_runner_|.
+  virtual void OnCaptureCompleted(
+      scoped_refptr<remoting::CaptureData> capture_data) OVERRIDE;
+  virtual void OnCursorShapeChanged(
+      scoped_ptr<remoting::MouseCursorShape> cursor_shape) OVERRIDE;
+
+ private:
+  friend class base::RefCountedThreadSafe<Core>;
+  virtual ~Core();
+
+  // Helper methods that run on the |task_runner_|. Posted from the
+  // corresponding public methods.
+  void DoAllocate(int frame_rate);
+  void DoStart();
+  void DoStop();
+  void DoDeAllocate();
+
+  // Method that is scheduled on |task_runner_| to be called on regular interval
+  // to capture the screen.
+  void OnCaptureTimer();
+
+  // Captures a single frame.
+  void DoCapture();
+
+  // Task runner used for screen capturing operations.
+  scoped_refptr<base::SequencedTaskRunner> task_runner_;
+
+  // |event_handler_lock_| must be locked whenever |event_handler_| is used.
+  // It's necessary because DeAllocate() needs to reset it on the calling thread
+  // to ensure that the event handler is not called once DeAllocate() returns.
+  base::Lock event_handler_lock_;
+  EventHandler* event_handler_;
+
+  // Frame rate specified in Allocate().
+  int frame_rate_;
+
+  scoped_ptr<remoting::VideoFrameCapturer> video_frame_capturer_;
+
+  // After Allocate() is called we need to call OnFrameInfo() method of the
+  // |event_handler_| to specify the size of the frames this capturer will
+  // produce. In order to get screen size from |video_frame_capturer_| we need
+  // to capture at least one frame. Once screen size is known it's stored in
+  // |frame_size_|.
+  bool waiting_frame_size_;
+  SkISize frame_size_;
+  SkBitmap resized_bitmap_;
+
+  // Set to true between DoStart() and DoStop().
+  bool started_;
+
+  // Set to true when we have delayed OnCaptureTimer() task posted on
+  // |task_runner_|.
+  bool capture_task_posted_;

[Wez, 2013/01/15 06:13:27]

nit: capture_pending_

[Sergey Ulanov, 2013/01/15 19:29:18]

IMO capture_task_poster_ better reflects what this flag is used for.

+
+  // Set to true when waiting for |video_frame_capturer_| to capture current
+  // frame.
+  bool capture_pending_;

[Wez, 2013/01/15 06:13:27]

nit: capture_in_progress_

[Sergey Ulanov, 2013/01/15 19:29:18]

Done.

+
+  DISALLOW_COPY_AND_ASSIGN(Core);
+};
+
+ScreenCapturer::Core::Core(
+    scoped_refptr<base::SequencedTaskRunner> task_runner)
+    : task_runner_(task_runner),
+      event_handler_(NULL),
+      waiting_frame_size_(false),
+      started_(false),
+      capture_task_posted_(false),
+      capture_pending_(false) {
+}
+
+ScreenCapturer::Core::~Core() {
+}
+
+void ScreenCapturer::Core::Allocate(int width, int height,
+                                    int frame_rate,
+                                    EventHandler* event_handler) {
+  DCHECK_GT(width, 0);
+  DCHECK_GT(height, 0);
+  DCHECK_GT(frame_rate, 0);
+
+  {
+    base::AutoLock auto_lock(event_handler_lock_);
+    event_handler_ = event_handler;
+  }
+
+  task_runner_->PostTask(
+      FROM_HERE, base::Bind(&Core::DoAllocate, this, frame_rate));
+}
+
+void ScreenCapturer::Core::Start() {
+  task_runner_->PostTask(
+      FROM_HERE, base::Bind(&Core::DoStart, this));
+}
+
+void ScreenCapturer::Core::Stop() {
+  task_runner_->PostTask(
+      FROM_HERE, base::Bind(&Core::DoStop, this));
+}
+
+void ScreenCapturer::Core::DeAllocate() {
+  {
+    base::AutoLock auto_lock(event_handler_lock_);

[Wez, 2013/01/15 06:13:27]

This will block if the capture thread is mid-capture at the time it's called,
which I don't think is what you want - captures could take e.g. 500ms on some
systems.

[Sergey Ulanov, 2013/01/15 19:29:18]

The lock is not acquired during screen capture - it's only acquired when calling
the event handler. In either case there is no way to implement
media::VideoCaptureDevice interface without some kind of synchronization, and
this lock here is better than thread_.Stop() I had in the first version of this
CL.

[Wez, 2013/01/16 21:26:45]

Agreed, but I think we should try to resolve this if possible as part of the
capturer interface refactoring work.

+    event_handler_ = NULL;
+  }
+  task_runner_->PostTask(FROM_HERE, base::Bind(&Core::DoDeAllocate, this));
+}
+
+void ScreenCapturer::Core::OnCaptureCompleted(
+    scoped_refptr<remoting::CaptureData> capture_data) {
+  DCHECK(task_runner_->RunsTasksOnCurrentThread());
+  DCHECK(capture_pending_);
+  DCHECK(!capture_data->size().isEmpty());
+  capture_pending_ = false;
+
+
+  if (waiting_frame_size_) {
+    frame_size_ = capture_data->size();
+    waiting_frame_size_ = false;
+
+    // Inform the EventHandler of the video frame dimensions and format.
+    media::VideoCaptureCapability caps;
+    caps.width = frame_size_.width();
+    caps.height = frame_size_.height();
+    caps.frame_rate = frame_rate_;
+    caps.color = media::VideoCaptureCapability::kARGB;
+    caps.expected_capture_delay =
+        base::Time::kMillisecondsPerSecond / frame_rate_;
+    caps.interlaced = false;
+
+    base::AutoLock auto_lock(event_handler_lock_);
+    if (event_handler_)
+      event_handler_->OnFrameInfo(caps);
+  }
+
+  if (!started_)
+    return;
+  size_t buffer_size =
+      frame_size_.width() * frame_size_.height() *
+      remoting::CaptureData::kBytesPerPixel;
+
+  if (capture_data->size() == frame_size_) {
+    // If the captured frame matches the requested size, we don't need to
+    // resize it.
+    resized_bitmap_.reset();
+
+    base::AutoLock auto_lock(event_handler_lock_);
+    if (event_handler_) {
+      event_handler_->OnIncomingCapturedFrame(
+          capture_data->data(), buffer_size, base::Time::Now());
+    }
+    return;
+  }
+
+  // In case screen size has changed we need to resize the image. Resized image
+  // is stored to |resized_bitmap_|. Only regions of the screen that are
+  // changing are copied.
+
+  SkRegion dirty_region = capture_data->dirty_region();
+
+  if (resized_bitmap_.width() != frame_size_.width() ||
+      resized_bitmap_.height() != frame_size_.height()) {
+    resized_bitmap_.setConfig(SkBitmap::kARGB_8888_Config,
+                              frame_size_.width(), frame_size_.height());
+    resized_bitmap_.setIsOpaque(true);
+    resized_bitmap_.allocPixels();
+    dirty_region.setRect(SkIRect::MakeSize(frame_size_));
+  }
+
+  float scale_x = static_cast<float>(frame_size_.width()) /
+      capture_data->size().width();
+  float scale_y = static_cast<float>(frame_size_.height()) /
+      capture_data->size().height();
+  float scale;
+  float x, y;
+  // Center the image in case aspect ratio is different.
+  if (scale_x > scale_y) {
+    scale = scale_y;
+    x = (scale_x - scale_y) / scale * frame_size_.width() / 2.0;
+    y = 0.f;
+  } else {
+    scale = scale_x;
+    x = 0.f;
+    y = (scale_y - scale_x) / scale * frame_size_.height() / 2.0;
+  }
+
+  // Create skia device and canvas that draw to |resized_bitmap_|.
+  SkDevice device(resized_bitmap_);
+  SkCanvas canvas(&device);
+  canvas.scale(scale, scale);
+
+  int source_stride = capture_data->stride();
+  for (SkRegion::Iterator i(dirty_region); !i.done(); i.next()) {
+    SkBitmap source_bitmap;
+    source_bitmap.setConfig(SkBitmap::kARGB_8888_Config,
+                            i.rect().width(), i.rect().height(),
+                            source_stride);
+    source_bitmap.setIsOpaque(true);
+    source_bitmap.setPixels(
+        capture_data->data() + i.rect().top() * source_stride +
+        i.rect().left() * remoting::CaptureData::kBytesPerPixel);
+    canvas.drawBitmap(source_bitmap, i.rect().left() + x / scale,
+                      i.rect().top() + y / scale, NULL);
+  }
+
+  base::AutoLock auto_lock(event_handler_lock_);
+  if (event_handler_) {
+    event_handler_->OnIncomingCapturedFrame(
+        reinterpret_cast<uint8*>(resized_bitmap_.getPixels()), buffer_size,
+        base::Time::Now());
+  }
+}
+
+void ScreenCapturer::Core::OnCursorShapeChanged(
+    scoped_ptr<remoting::MouseCursorShape> cursor_shape) {
+  DCHECK(task_runner_->RunsTasksOnCurrentThread());
+}
+
+void ScreenCapturer::Core::DoAllocate(int frame_rate) {
+  DCHECK(task_runner_->RunsTasksOnCurrentThread());
+
+  frame_rate_ = frame_rate;
+
+  // Create and start frame capturer.
+  if (!video_frame_capturer_)
+    video_frame_capturer_ = remoting::VideoFrameCapturer::Create();
+  if (video_frame_capturer_)
+    video_frame_capturer_->Start(this);
+
+  // Capture first frame, so that we can call OnFrameInfo() callback.
+  waiting_frame_size_ = true;
+  DoCapture();
+}
+
+void ScreenCapturer::Core::DoStart() {
+  DCHECK(task_runner_->RunsTasksOnCurrentThread());
+  started_ = true;
+  if (!capture_task_posted_) {
+    capture_task_posted_ = true;
+    task_runner_->PostDelayedTask(
+        FROM_HERE, base::Bind(&Core::OnCaptureTimer, this),
+        base::TimeDelta::FromSeconds(1) / frame_rate_);
+  }
+}
+
+void ScreenCapturer::Core::DoStop() {
+  DCHECK(task_runner_->RunsTasksOnCurrentThread());
+  started_ = false;
+  if (video_frame_capturer_)
+    video_frame_capturer_->Stop();
+  resized_bitmap_.reset();
+}
+
+void ScreenCapturer::Core::DoDeAllocate() {
+  DCHECK(task_runner_->RunsTasksOnCurrentThread());
+  DoStop();
+  video_frame_capturer_.reset();
+  waiting_frame_size_ = false;
+}
+
+void ScreenCapturer::Core::OnCaptureTimer() {
+  DCHECK(capture_task_posted_);
+  capture_task_posted_ = false;
+
+  if (started_) {
+    // Schedule a task for the next frame.
+    capture_task_posted_ = true;
+    task_runner_->PostDelayedTask(
+        FROM_HERE, base::Bind(&Core::OnCaptureTimer, this),
+        base::TimeDelta::FromSeconds(1) / frame_rate_);
+  } else if (!waiting_frame_size_) {
+    return;
+  }
+
+  DoCapture();
+}
+
+void ScreenCapturer::Core::DoCapture() {
+  DCHECK(!capture_pending_);
+
+  capture_pending_ = true;
+  video_frame_capturer_->CaptureFrame();
+
+  // Currently remoting::VideoFrameCapturer always calls OnCaptureCompleted()
+  // and we rely on it.

[Wez, 2013/01/15 06:13:27]

This comment is a little weird; the current interface defines that things work
that way, so you're not relying on anything un-defined - lose this and just keep
the TODO?

[Sergey Ulanov, 2013/01/15 19:29:18]

We do rely on the fact that OnCaptureCompleted() is called from CaptureFrame()
when running this code in a thread pool. Problem is that if
video_frame_capturer_ calls that callback later then there may be other task
running on a different thread of the thread pool. VideoFrameCapturer doesn't
know about  SequentialTaskRunner that this class uses, so it would call
OnCaptureCompleted() outside of the sequence.

[Wez, 2013/01/16 21:26:45]

So what the comment should say is that we assume that
VideoFrameCapturer::CaptureFrame() will call-back OnCaptureCompleted() before it
returns.

[Sergey Ulanov, 2013/01/17 02:21:17]

Done.

+  //
+  // TODO(sergeyu): Fix remoting::VideoFrameCapturer to return video frame
+  // synchronously instead of using Delegate interface.
+  DCHECK(!capture_pending_);
+}
+
+ScreenCapturer::ScreenCapturer(
+    scoped_refptr<base::SequencedTaskRunner> task_runner)
+    : core_(new Core(task_runner)) {
+  name_.device_name = "Screen";
+}
+
+ScreenCapturer::~ScreenCapturer() {
+  core_->DeAllocate();

[Wez, 2013/01/15 06:13:27]

nit: Call this->DeAllocate() here in case the impl changes.

[Sergey Ulanov, 2013/01/15 19:29:18]

Done.

+}
+
+void ScreenCapturer::set_test_frame_capturer(
+  scoped_ptr<remoting::VideoFrameCapturer> capturer) {
+  core_->set_test_frame_capturer(capturer.Pass());
+}
+
+void ScreenCapturer::Allocate(int width, int height,
+                              int frame_rate,
+                              EventHandler* event_handler) {
+  core_->Allocate(width, height, frame_rate, event_handler);

[Wez, 2013/01/15 06:13:27]

Can you PostTask() from here directly to ScreenCapturer::Core::Allocate() and
avoid the need for DoAllocate()?

[Sergey Ulanov, 2013/01/15 19:29:18]

DeAllocate() needs to reset event_handler_ on the calling thread, and I think
it's better to keep other methods consistent. Also ScreenCapturer doesn't store
reference to task_runner_.

+}
+
+void ScreenCapturer::Start() {
+  core_->Start();
+}
+
+void ScreenCapturer::Stop() {
+  core_->Stop();
+}
+
+void ScreenCapturer::DeAllocate() {
+  core_->DeAllocate();
+}
+
+const media::VideoCaptureDevice::Name& ScreenCapturer::device_name() {
+  return name_;
+}
+
+}  // namespace content