create a separate WebMediaPlayer for URL derived from media stream

webkit/media/webmediaplayer_ms.cc

+// Copyright (c) 2012 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "webkit/media/webmediaplayer_ms.h"
+
+#include <limits>
+
+#include "base/bind.h"
+#include "base/callback.h"
+#include "base/metrics/histogram.h"
+#include "media/base/media_log.h"
+#include "media/base/video_frame.h"
+#include "third_party/WebKit/Source/WebKit/chromium/public/WebFrame.h"
+#include "third_party/WebKit/Source/WebKit/chromium/public/WebMediaPlayerClient.h"
+#include "third_party/WebKit/Source/WebKit/chromium/public/WebVideoFrame.h"
+#include "third_party/WebKit/Source/WebKit/chromium/public/WebView.h"
+#include "third_party/WebKit/Source/WebKit/chromium/public/platform/WebRect.h"
+#include "third_party/WebKit/Source/WebKit/chromium/public/platform/WebSize.h"
+#include "third_party/WebKit/Source/WebKit/chromium/public/platform/WebURL.h"
+#include "v8/include/v8.h"
+#include "webkit/media/media_stream_client.h"
+#include "webkit/media/video_frame_provider.h"
+#include "webkit/media/webmediaplayer_delegate.h"
+#include "webkit/media/webmediaplayer_util.h"
+#include "webkit/media/webvideoframe_impl.h"
+
+using WebKit::WebCanvas;
+using WebKit::WebMediaPlayer;
+using WebKit::WebRect;
+using WebKit::WebSize;
+
+namespace {
+
+// Amount of extra memory used by each player instance reported to V8.
+// It is not exact number -- first, it differs on different platforms,
+// and second, it is very hard to calculate. Instead, use some arbitrary
+// value that will cause garbage collection from time to time. We don't want
+// it to happen on every allocation, but don't want 5k players to sit in memory
+// either. Looks that chosen constant achieves both goals, at least for audio
+// objects. (Do not worry about video objects yet, JS programs do not create
+// thousands of them...)
+const int kPlayerExtraMemory = 1024 * 1024;
+
+}  // namespace
+
+namespace webkit_media {
+
+WebMediaPlayerMS::WebMediaPlayerMS(
+    WebKit::WebFrame* frame,
+    WebKit::WebMediaPlayerClient* client,
+    base::WeakPtr<WebMediaPlayerDelegate> delegate,
+    MediaStreamClient* media_stream_client,
+    media::MediaLog* media_log)
+    : frame_(frame),
+      network_state_(WebMediaPlayer::NetworkStateEmpty),
+      ready_state_(WebMediaPlayer::ReadyStateHaveNothing),
+      buffered_(static_cast<size_t>(1)),
+      main_loop_(MessageLoop::current()),

[scherkus (not reviewing), 2012/09/07 11:44:03]

You don't need this member variable as you're only using it for DCHECK purposes
and not posting tasks.

If you want to DCHECK single-threadedness use base::ThreadChecker (which
optimizes to nothing in release) or just remove all the DCHECK()s

[wjia(left Chromium), 2012/09/13 01:22:07]

Done.

+      client_(client),
+      delegate_(delegate),
+      media_stream_client_(media_stream_client),
+      video_frame_provider_started_(false),
+      paused_(true),
+      pending_repaint_(false),
+      got_first_frame_(false),
+      total_frame_count_(0),
+      dropped_frame_count_(0),
+      media_log_(media_log),
+      accelerated_compositing_reported_(false),
+      incremented_externally_allocated_memory_(false) {
+  DVLOG(1) << "WebMediaPlayerMS::ctor";
+  DCHECK(media_stream_client);
+  media_log_->AddEvent(

[scherkus (not reviewing), 2012/09/07 11:44:03]

this will end up on chrome://media-internals but I can't guarantee you'll find
the data very useful :)

was it your intent to use it? (it's purely optional and you can remove if you
want)

[wjia(left Chromium), 2012/09/13 01:22:07]

I'd keep it for now.

+      media_log_->CreateEvent(media::MediaLogEvent::WEBMEDIAPLAYER_CREATED));
+
+  // Let V8 know we started new thread if we did not did it yet.
+  // Made separate task to avoid deletion of player currently being created.
+  // Also, delaying GC until after player starts gets rid of starting lag --
+  // collection happens in parallel with playing.
+  //
+  // TODO(enal): remove when we get rid of per-audio-stream thread.
+  MessageLoop::current()->PostTask(
+      FROM_HERE,
+      base::Bind(&WebMediaPlayerMS::IncrementExternallyAllocatedMemory,
+                 AsWeakPtr()));
+
+  // Also we want to be notified of |main_loop_| destruction.
+  main_loop_->AddDestructionObserver(this);

[scherkus (not reviewing), 2012/09/07 11:44:03]

I don't think you need a destruction observer as you don't have any teardown
code that requires the render loop.

Looking at the code you have here... in the worst case RTCVideoRender will fail
to post a task to execute the paint CB but since its using MLProxy it's OK.

If so, then you can also inline Destroy() into the dtor

[wjia(left Chromium), 2012/09/13 01:22:07]

Done.

+}
+
+WebMediaPlayerMS::~WebMediaPlayerMS() {
+  DVLOG(1) << "WebMediaPlayerMS::dtor";
+  DCHECK_EQ(main_loop_, MessageLoop::current());
+  Destroy();
+  media_log_->AddEvent(
+      media_log_->CreateEvent(media::MediaLogEvent::WEBMEDIAPLAYER_DESTROYED));
+
+  if (delegate_)
+    delegate_->PlayerGone(this);
+
+  // Finally tell the |main_loop_| we don't want to be notified of destruction
+  // event.
+  if (main_loop_) {
+    main_loop_->RemoveDestructionObserver(this);
+  }
+}
+
+void WebMediaPlayerMS::load(const WebKit::WebURL& url, CORSMode cors_mode) {
+  DVLOG(1) << "WebMediaPlayerMS::load";
+  DCHECK_EQ(main_loop_, MessageLoop::current());
+
+  GURL gurl(url);
+
+  setVolume(GetClient()->volume());
+  SetNetworkState(WebMediaPlayer::NetworkStateLoading);
+  SetReadyState(WebMediaPlayer::ReadyStateHaveNothing);
+  media_log_->AddEvent(media_log_->CreateLoadEvent(url.spec()));
+
+  // Check if this url is media stream.
+  video_frame_provider_ = media_stream_client_->GetVideoFrameProvider(
+      url,
+      base::Bind(&WebMediaPlayerMS::OnSourceError, AsWeakPtr()),
+      base::Bind(&WebMediaPlayerMS::Repaint, AsWeakPtr()));
+  if (video_frame_provider_) {
+    SetNetworkState(WebMediaPlayer::NetworkStateLoaded);
+    GetClient()->sourceOpened();
+    GetClient()->setOpaque(true);
+    SetReadyState(WebMediaPlayer::ReadyStateHaveMetadata);
+    SetReadyState(WebMediaPlayer::ReadyStateHaveEnoughData);
+    RepaintInternal();
+  } else {
+    SetNetworkState(WebMediaPlayer::NetworkStateNetworkError);
+  }
+}
+
+void WebMediaPlayerMS::cancelLoad() {
+  DCHECK_EQ(main_loop_, MessageLoop::current());
+}
+
+void WebMediaPlayerMS::play() {
+  DVLOG(1) << "WebMediaPlayerMS::play";
+  DCHECK_EQ(main_loop_, MessageLoop::current());
+
+  paused_ = false;
+  if (video_frame_provider_) {
+    if (video_frame_provider_started_) {
+      video_frame_provider_->Play();
+    } else {
+      video_frame_provider_started_ = true;
+      video_frame_provider_->Start();
+    }
+  }
+  // TODO: add audio.

[scherkus (not reviewing), 2012/09/07 11:44:03]

TODO(wjia)?

is there a bug filed for WebRTC audio?

[wjia(left Chromium), 2012/09/13 01:22:07]

Done. I'd use the same bug for adding WebRTC audio into WMPMS.

+
+  media_log_->AddEvent(media_log_->CreateEvent(media::MediaLogEvent::PLAY));
+
+  if (delegate_)
+    delegate_->DidPlay(this);
+}
+
+void WebMediaPlayerMS::pause() {
+  DVLOG(1) << "WebMediaPlayerMS::pause";
+  DCHECK_EQ(main_loop_, MessageLoop::current());
+
+  if (video_frame_provider_)
+    video_frame_provider_->Pause();
+  // TODO: add audio.

[scherkus (not reviewing), 2012/09/07 11:44:03]

ditto

[wjia(left Chromium), 2012/09/13 01:22:07]

Done.

+  paused_ = true;
+
+  media_log_->AddEvent(media_log_->CreateEvent(media::MediaLogEvent::PAUSE));
+
+  if (delegate_)
+    delegate_->DidPause(this);
+}
+
+bool WebMediaPlayerMS::supportsFullscreen() const {
+  DCHECK_EQ(main_loop_, MessageLoop::current());
+  return true;
+}
+
+bool WebMediaPlayerMS::supportsSave() const {
+  DCHECK_EQ(main_loop_, MessageLoop::current());
+  return false;
+}
+
+void WebMediaPlayerMS::seek(float seconds) {
+  DCHECK_EQ(main_loop_, MessageLoop::current());
+}
+
+void WebMediaPlayerMS::setEndTime(float seconds) {
+  DCHECK_EQ(main_loop_, MessageLoop::current());
+}
+
+void WebMediaPlayerMS::setRate(float rate) {
+  DCHECK_EQ(main_loop_, MessageLoop::current());
+}
+
+void WebMediaPlayerMS::setVolume(float volume) {
+  DCHECK_EQ(main_loop_, MessageLoop::current());
+  // TODO: set audio volume.

[scherkus (not reviewing), 2012/09/07 11:44:03]

TODO(wjia)?
NOTIMPLEMENTED?

[wjia(left Chromium), 2012/09/13 01:22:07]

Done.

+}
+
+void WebMediaPlayerMS::setVisible(bool visible) {
+  DCHECK_EQ(main_loop_, MessageLoop::current());
+}
+
+void WebMediaPlayerMS::setPreload(WebMediaPlayer::Preload preload) {
+  DCHECK_EQ(main_loop_, MessageLoop::current());
+}
+
+bool WebMediaPlayerMS::totalBytesKnown() {
+  DCHECK_EQ(main_loop_, MessageLoop::current());
+  return false;
+}
+
+bool WebMediaPlayerMS::hasVideo() const {
+  DCHECK_EQ(main_loop_, MessageLoop::current());
+  return !!video_frame_provider_;

[scherkus (not reviewing), 2012/09/07 11:44:03]

use != NULL

[wjia(left Chromium), 2012/09/13 01:22:07]

Done.

+}
+
+bool WebMediaPlayerMS::hasAudio() const {
+  DCHECK_EQ(main_loop_, MessageLoop::current());
+  // TODO: add audio support.

[scherkus (not reviewing), 2012/09/07 11:44:03]

TODO(wjia)?

[wjia(left Chromium), 2012/09/13 01:22:07]

Done.

+  return false;
+}
+
+WebKit::WebSize WebMediaPlayerMS::naturalSize() const {
+  DCHECK_EQ(main_loop_, MessageLoop::current());
+
+  gfx::Size size;
+  if (current_frame_)
+    size = current_frame_->natural_size();
+  DVLOG(1) << "WebMediaPlayerMS::naturalSize, " << size.ToString();
+  return WebKit::WebSize(size);
+}
+
+bool WebMediaPlayerMS::paused() const {
+  DCHECK_EQ(main_loop_, MessageLoop::current());
+  return paused_;
+}
+
+bool WebMediaPlayerMS::seeking() const {
+  DCHECK_EQ(main_loop_, MessageLoop::current());
+  return false;
+}
+
+float WebMediaPlayerMS::duration() const {
+  DCHECK_EQ(main_loop_, MessageLoop::current());
+  return std::numeric_limits<float>::infinity();
+}
+
+float WebMediaPlayerMS::currentTime() const {
+  DCHECK_EQ(main_loop_, MessageLoop::current());
+  base::TimeDelta current_time;
+  if (current_frame_.get()) {

[scherkus (not reviewing), 2012/09/07 11:44:03]

how about:

if (current_frame_.get())
  return current_frame_->GetTimestamp().InSecondsF();
return 0.0f;

[wjia(left Chromium), 2012/09/13 01:22:07]

Done.

+    current_time = current_frame_->GetTimestamp();
+  }
+  return static_cast<float>(current_time.InSecondsF());
+}
+
+int WebMediaPlayerMS::dataRate() const {
+  DCHECK_EQ(main_loop_, MessageLoop::current());
+  return 0;
+}
+
+WebMediaPlayer::NetworkState WebMediaPlayerMS::networkState() const {
+  DCHECK_EQ(main_loop_, MessageLoop::current());
+  DVLOG(1) << "WebMediaPlayerMS::networkState, state:" << network_state_;
+  return network_state_;
+}
+
+WebMediaPlayer::ReadyState WebMediaPlayerMS::readyState() const {
+  DCHECK_EQ(main_loop_, MessageLoop::current());
+  DVLOG(1) << "WebMediaPlayerMS::readyState, state:" << ready_state_;
+  return ready_state_;
+}
+
+const WebKit::WebTimeRanges& WebMediaPlayerMS::buffered() {
+  DCHECK_EQ(main_loop_, MessageLoop::current());
+  return buffered_;

[scherkus (not reviewing), 2012/09/07 11:44:03]

you don't need this variable -- just alloc + return a new empty range here
instead

[wjia(left Chromium), 2012/09/13 01:22:07]

This variable has to be here. If a WebTimeRanges is allocated and passed to
client, nobody will free it.

[scherkus (not reviewing), 2012/09/13 10:22:43]

ah didn't see the const-ref return value

+}
+
+float WebMediaPlayerMS::maxTimeSeekable() const {
+  DCHECK_EQ(main_loop_, MessageLoop::current());
+  return 0.0f;
+}
+
+bool WebMediaPlayerMS::didLoadingProgress() const {
+  DCHECK_EQ(main_loop_, MessageLoop::current());
+  return false;
+}
+
+unsigned long long WebMediaPlayerMS::totalBytes() const {
+  DCHECK_EQ(main_loop_, MessageLoop::current());
+  return 0;
+}
+
+void WebMediaPlayerMS::setSize(const WebSize& size) {
+  DCHECK_EQ(main_loop_, MessageLoop::current());
+  // Don't need to do anything as we use the dimensions passed in via paint().
+}
+
+void WebMediaPlayerMS::paint(WebCanvas* canvas,
+                             const WebRect& rect,
+                             uint8_t alpha) {
+  DVLOG(1) << "WebMediaPlayerMS::paint";
+  DCHECK_EQ(main_loop_, MessageLoop::current());
+
+  if (!accelerated_compositing_reported_) {
+    accelerated_compositing_reported_ = true;
+    // Normally paint() is only called in non-accelerated rendering, but there
+    // are exceptions such as webgl where compositing is used in the WebView but
+    // video frames are still rendered to a canvas.
+    UMA_HISTOGRAM_BOOLEAN(
+        "Media.AcceleratedCompositingActive",
+        frame_->view()->isAcceleratedCompositingActive());
+  }
+
+  video_renderer_.Paint(current_frame_, canvas, rect, alpha);
+}
+
+bool WebMediaPlayerMS::hasSingleSecurityOrigin() const {
+  DCHECK_EQ(main_loop_, MessageLoop::current());
+  return true;
+}
+
+bool WebMediaPlayerMS::didPassCORSAccessCheck() const {
+  DCHECK_EQ(main_loop_, MessageLoop::current());
+  return true;
+}
+
+WebMediaPlayer::MovieLoadType WebMediaPlayerMS::movieLoadType() const {
+  DCHECK_EQ(main_loop_, MessageLoop::current());
+  return WebMediaPlayer::MovieLoadTypeUnknown;
+}
+
+float WebMediaPlayerMS::mediaTimeForTimeValue(float timeValue) const {
+  return ConvertSecondsToTimestamp(timeValue).InSecondsF();
+}
+
+unsigned WebMediaPlayerMS::decodedFrameCount() const {
+  DCHECK_EQ(main_loop_, MessageLoop::current());
+  DVLOG(1) << "WebMediaPlayerMS::decodedFrameCount, " << total_frame_count_;
+  return total_frame_count_;
+}
+
+unsigned WebMediaPlayerMS::droppedFrameCount() const {
+  DCHECK_EQ(main_loop_, MessageLoop::current());
+  DVLOG(1) << "WebMediaPlayerMS::droppedFrameCount, " << dropped_frame_count_;
+  return dropped_frame_count_;
+}
+
+unsigned WebMediaPlayerMS::audioDecodedByteCount() const {
+  DCHECK_EQ(main_loop_, MessageLoop::current());
+  NOTIMPLEMENTED();
+  return 0;
+}
+
+unsigned WebMediaPlayerMS::videoDecodedByteCount() const {
+  DCHECK_EQ(main_loop_, MessageLoop::current());
+  NOTIMPLEMENTED();
+  return 0;
+}
+
+WebKit::WebVideoFrame* WebMediaPlayerMS::getCurrentFrame() {
+  DVLOG(1) << "WebMediaPlayerMS::getCurrentFrame";
+  DCHECK_EQ(main_loop_, MessageLoop::current());
+  DCHECK(!pending_repaint_);
+  pending_repaint_ = true;

[scherkus (not reviewing), 2012/09/07 11:44:03]

this seems odd -- what exactly is pending_repaint_ used for and what is it
tracking?

it looks like you're using it for both making sure getCurrentFrame() isn't
called too many times and for tracking dropped frames -- which one is it?

[wjia(left Chromium), 2012/09/13 01:22:07]

|pending_repaint_| is used to ensure client->repaint() will not be called
between getCurrentFrame() and putCurrentFrame(). I have seen some blinking
effect when it's done, e.g., client calls getCurrentFrame() when it tries to
repaint due to some reason other than the signal from WMPMS. Before client calls
putCurrentFrame(), a new frame comes from VideoFrameProvider and
client->repaint() is called. This results in embedded get/putCurrentFrame()
calls and blinking effect.

VideoFrameProvider always passes frames to WMPMS. When a frame is available, but
client->repaint() is not called, that frame is treated as dropped.

+  if (current_frame_.get())
+    return new webkit_media::WebVideoFrameImpl(current_frame_);
+  return NULL;
+}
+
+void WebMediaPlayerMS::putCurrentFrame(
+    WebKit::WebVideoFrame* web_video_frame) {
+  DVLOG(1) << "WebMediaPlayerMS::putCurrentFrame";
+  DCHECK_EQ(main_loop_, MessageLoop::current());
+  DCHECK(pending_repaint_);
+  pending_repaint_ = false;
+  if (!accelerated_compositing_reported_) {
+    accelerated_compositing_reported_ = true;
+    UMA_HISTOGRAM_BOOLEAN("Media.AcceleratedCompositingActive",
+                          frame_->view()->isAcceleratedCompositingActive());
+  }
+  if (web_video_frame) {
+    delete web_video_frame;
+  }
+}
+
+void WebMediaPlayerMS::WillDestroyCurrentMessageLoop() {
+  Destroy();
+  main_loop_ = NULL;
+}
+
+void WebMediaPlayerMS::Repaint(const scoped_refptr<media::VideoFrame>& frame) {
+  DVLOG(1) << "WebMediaPlayerMS::Repaint with frame";
+  DCHECK_EQ(main_loop_, MessageLoop::current());
+  ++total_frame_count_;
+  if (!got_first_frame_) {
+    got_first_frame_ = true;
+    start_time_ = frame->GetTimestamp();
+  }
+  current_frame_ = frame;
+  current_frame_->SetTimestamp(frame->GetTimestamp() - start_time_);
+  if (pending_repaint_) {

[scherkus (not reviewing), 2012/09/07 11:44:03]

considering you only set this in the HW decode path I don't think the stats
you're attempting to collect are accurate

in order to determine whether you really dropped a frame you have to track
whether current_frame_ was ever painted at all via software or hardware

let's remove the stats stuff for now and implement it in a later CL

[wjia(left Chromium), 2012/09/13 01:22:07]

Removed dropped_frame_count_ for now. Is it possible both get/putCurrentFrame
and paint() could be called? For example, <video> display uses
get/putCurrentFrame(), while the same video is copied to <canvas>.

[scherkus (not reviewing), 2012/09/13 10:22:43]

I wouldn't say the two are mutually exclusive -- for example, if the GPU process
crashes and we fall back into software rendering we'll switch back to using
paint() calls

+    ++dropped_frame_count_;
+  } else {
+    GetClient()->repaint();
+  }
+}
+
+void WebMediaPlayerMS::RepaintInternal() {
+  DVLOG(1) << "WebMediaPlayerMS::Repaint";
+  DCHECK_EQ(main_loop_, MessageLoop::current());
+  if (!pending_repaint_) {
+    GetClient()->repaint();
+  }
+}
+
+void WebMediaPlayerMS::OnSourceError() {
+  DVLOG(1) << "WebMediaPlayerMS::OnSourceError";
+  DCHECK_EQ(main_loop_, MessageLoop::current());
+  SetNetworkState(WebMediaPlayer::NetworkStateFormatError);
+  RepaintInternal();
+}
+
+void WebMediaPlayerMS::SetNetworkState(WebMediaPlayer::NetworkState state) {
+  DCHECK_EQ(main_loop_, MessageLoop::current());
+  network_state_ = state;
+  // Always notify to ensure client has the latest value.
+  GetClient()->networkStateChanged();
+}
+
+void WebMediaPlayerMS::SetReadyState(WebMediaPlayer::ReadyState state) {
+  DCHECK_EQ(main_loop_, MessageLoop::current());
+  ready_state_ = state;
+  // Always notify to ensure client has the latest value.
+  GetClient()->readyStateChanged();
+}
+
+void WebMediaPlayerMS::Destroy() {
+  DCHECK_EQ(main_loop_, MessageLoop::current());
+
+  if (video_frame_provider_) {
+    video_frame_provider_->Stop();

[scherkus (not reviewing), 2012/09/07 11:44:03]

considering Stop() is synchronous how about replacing Stop() with destruction?

[wjia(left Chromium), 2012/09/13 01:22:07]

That won't work for LocalVideoCapture, since Start() will increase ref count.
WMPMS can only decrease the ref count, instead of triggering the destruction of
LVC.

[scherkus (not reviewing), 2012/09/13 10:22:43]

Let's see if I can untangle this a bit...

VideoFrameProvider is refcounted because...
LocalVideoCapture calls AddRef() in Start() because...
vc_manager_->AddDevice() is called and VCD::EventHandler is not refcounted

It seems what you really want is a way for LVC to stop / unregister itself so
that its guaranteed to be never called again via its VCD::EventHandler
interface. Relying on OnRemoved() so you can call balance refcounts via
Release() is a perilous path to go down.

Can this be handled via VideoCaptureHandlerProxy instead of requiring LVC to be
refcounted?

[wjia(left Chromium), 2012/09/19 03:22:06]

Please see my comments in local_video_capture.cc about the plan to refactor
VideoCapture::EventHandler.

+  }
+
+  // Let V8 know we are not using extra resources anymore.
+  if (incremented_externally_allocated_memory_) {
+    v8::V8::AdjustAmountOfExternalAllocatedMemory(-kPlayerExtraMemory);
+    incremented_externally_allocated_memory_ = false;
+  }
+}
+
+WebKit::WebMediaPlayerClient* WebMediaPlayerMS::GetClient() {
+  DCHECK_EQ(main_loop_, MessageLoop::current());
+  DCHECK(client_);
+  return client_;
+}
+
+void WebMediaPlayerMS::IncrementExternallyAllocatedMemory() {
+  DCHECK_EQ(main_loop_, MessageLoop::current());
+  incremented_externally_allocated_memory_ = true;
+  v8::V8::AdjustAmountOfExternalAllocatedMemory(kPlayerExtraMemory);
+}
+
+}  // namespace webkit_media