Upstream implementation for embedded video for WebMediaPlayerAndroid

webkit/media/android/webmediaplayer_android.cc

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "webkit/media/android/webmediaplayer_android.h"
 
 #include <string>
 
 #include "base/bind.h"
 #include "base/command_line.h"
 #include "base/file_path.h"
 #include "base/logging.h"
 #include "base/utf_string_conversions.h"
 #include "media/base/android/media_player_bridge.h"
 #include "net/base/mime_util.h"
 #include "third_party/WebKit/Source/WebKit/chromium/public/WebMediaPlayerClient.h"
 #include "third_party/WebKit/Source/WebKit/chromium/public/WebView.h"
 #include "third_party/WebKit/Source/WebKit/chromium/public/platform/WebCookieJar.h"
 #include "third_party/WebKit/Source/WebKit/chromium/public/platform/WebRect.h"
+#include "webkit/media/android/stream_texture_factory_android.h"
+#include "webkit/media/android/webmediaplayer_manager_android.h"
 #include "webkit/media/android/webmediaplayer_proxy_android.h"
 #include "webkit/media/webmediaplayer_util.h"
 #include "webkit/media/webvideoframe_impl.h"
 
 using WebKit::WebCanvas;
 using WebKit::WebMediaPlayerClient;
 using WebKit::WebMediaPlayer;
 using WebKit::WebRect;
 using WebKit::WebSize;
 using WebKit::WebTimeRanges;
 using WebKit::WebURL;
 using WebKit::WebVideoFrame;
 using media::MediaPlayerBridge;
 using media::VideoFrame;
 using webkit_media::WebVideoFrameImpl;
 
+// TODO(qinmin): Figure out where we should define this more appropriately
+#define GL_TEXTURE_EXTERNAL_OES 0x8D65
+
 namespace webkit_media {
 
 // Because we create the media player lazily on android, the duration of the
 // media is initially unknown to us. This makes the user unable to perform
 // seek. To solve this problem, we use a temporary duration of 100 seconds when
 // the duration is unknown. And we scale the seek position later when duration
 // is available.
 // TODO(qinmin): create a thread and use android MediaMetadataRetriever
 // class to extract the duration.
 static const float kTemporaryDuration = 100.0f;
 
 bool WebMediaPlayerAndroid::incognito_mode_ = false;
 
 WebMediaPlayerAndroid::WebMediaPlayerAndroid(
     WebMediaPlayerClient* client,
-    WebKit::WebCookieJar* cookie_jar)
+    WebKit::WebCookieJar* cookie_jar,
+    webkit_media::WebMediaPlayerManagerAndroid* manager,
+    webkit_media::StreamTextureFactory* factory)
     : client_(client),
       buffered_(1u),
       video_frame_(new WebVideoFrameImpl(VideoFrame::CreateEmptyFrame())),
       proxy_(new WebMediaPlayerProxyAndroid(base::MessageLoopProxy::current(),
                                             AsWeakPtr())),
       prepared_(false),
       duration_(0),
       pending_seek_(0),
       seeking_(false),
       playback_completed_(false),
       buffered_bytes_(0),
       cookie_jar_(cookie_jar),
+      manager_(manager),
       pending_play_event_(false),
       network_state_(WebMediaPlayer::NetworkStateEmpty),
-      ready_state_(WebMediaPlayer::ReadyStateHaveNothing) {
-  video_frame_.reset(new WebVideoFrameImpl(VideoFrame::CreateEmptyFrame()));
+      ready_state_(WebMediaPlayer::ReadyStateHaveNothing),
+      texture_id_(0),
+      stream_id_(0),
+      needs_establish_peer_(true),
+      stream_texture_proxy_initialized_(false),
+      stream_texture_factory_(factory) {
+  player_id_ = manager_->RegisterMediaPlayer(this);
+  if (stream_texture_factory_.get())

[scherkus (not reviewing), 2012/05/24 21:36:32]

so right now this is NULL but what about after you upstream and no longer pass
in NULL? will these checks go away?

[qinmin, 2012/05/25 01:04:21]

Yes, eventually when StreamTextureProxy implementation got upstreamed, I will
remove this check.

On 2012/05/24 21:36:32, scherkus wrote:

+    stream_texture_proxy_.reset(stream_texture_factory_->CreateProxy());
 }
 
 WebMediaPlayerAndroid::~WebMediaPlayerAndroid() {
   if (media_player_.get()) {
     media_player_->Stop();
+    media_player_->SetVideoSurface(0);

[scherkus (not reviewing), 2012/05/24 21:36:32]

should this also be NULL?

OOC do these methods need to be called prior to destruction? what would happen
if we didn't?

[qinmin, 2012/05/25 01:04:21]

Changed to NULL. If we don't call it, the surface will not be released
immediately until JVM decides to do a GC on the java MediaPlayer. And that is
causing a lot of OOM problems.

On 2012/05/24 21:36:32, scherkus wrote:

   }
+
+  manager_->UnRegisterMediaPlayer(player_id_);
 }
 
 void WebMediaPlayerAndroid::InitIncognito(bool incognito_mode) {
   incognito_mode_ = incognito_mode;
 }
 
 void WebMediaPlayerAndroid::load(const WebURL& url) {
   url_ = url;
 
   UpdateNetworkState(WebMediaPlayer::NetworkStateLoading);
@@ skipping 338 matching lines @@
   network_state_ = state;
   client_->networkStateChanged();
 }
 
 void WebMediaPlayerAndroid::UpdateReadyState(
     WebMediaPlayer::ReadyState state) {
   ready_state_ = state;
   client_->readyStateChanged();
 }
 
+void WebMediaPlayerAndroid::ReleaseMediaResources() {
+  // Pause the media player first.
+  pause();
+  client_->playbackStateChanged();
+
+  if (media_player_.get()) {
+    // Save the current media player status.
+    pending_seek_ = currentTime();
+    duration_ = duration();
+
+    // Disconnecting the Surface from the producer side allows
+    // all buffers but the one current (to the client) to be freed.
+    // We intentionally keep the current texture so we have something
+    // to show in the video layer.
+    // At the same time we happily avoid recreating the SurfaceTexture to

[scherkus (not reviewing), 2012/05/24 21:36:32]

s/SurfaceTexture/Surface

[qinmin, 2012/05/25 01:04:21]

Done.

+    // not have to deal with rewiring any proxies receiving notifications.
+    media_player_->SetVideoSurface(NULL);

[scherkus (not reviewing), 2012/05/24 21:36:32]

we delete the media player on the next line -- what would happen if we didn't
have this line of code?

[qinmin, 2012/05/25 01:04:21]

Just as the reason above, deleting media_player_ will not immediately GC the
java mediaplayer. And surface is taking a lot of memories.

On 2012/05/24 21:36:32, scherkus wrote:

+    media_player_.reset(NULL);

[scherkus (not reviewing), 2012/05/24 21:36:32]

nit: s/reset(NULL)/reset()/

[qinmin, 2012/05/25 01:04:21]

Done.

+    needs_establish_peer_ = true;

[scherkus (not reviewing), 2012/05/24 21:36:32]

requiring a peer seems roughly correlated with having a new media player -- does
it make sense to be able to query the media player object (via some methods)
that it requires peering?

for example, is it true that peering is requiring if the media player's video
surface is NULL?

[qinmin, 2012/05/25 01:04:21]

need_establish_peer_ does not always correlate to a new media player. The
situation could happen when user enters fullscreen mode. In that case, we will
release the embedded surface, and create an android surfaceView, and pass that
surfaceView to the mediaplayer.
When user quit the fullscreen mode, we will request the embedded video surface
again. That will make us to set needs_establish_peer_ to true again. The
mediaplayer will not get recreated during the embed->fullscreen->embed process. 

Also there is a delay between EstablishPeer() and SetVideoSurface() due to IPC.
So in PlayInternal(), if we call EstablishPeer(), and then do a pause, and play
again, the mediaplayer might not hold the surface when the 2nd PlayInternal got
called.


On 2012/05/24 21:36:32, scherkus wrote:

+  }
+  prepared_ = false;
+}
+
 void WebMediaPlayerAndroid::SetVideoSurface(jobject j_surface) {
   if (media_player_.get())
     media_player_->SetVideoSurface(j_surface);
 }
 
 void WebMediaPlayerAndroid::InitializeMediaPlayer() {
   CHECK(!media_player_.get());
   prepared_ = false;
   media_player_.reset(new MediaPlayerBridge());
   media_player_->SetStayAwakeWhilePlaying();
 
   std::string cookies;
   if (cookie_jar_ != NULL) {
     WebURL url(url_);
     cookies = UTF16ToUTF8(cookie_jar_->cookies(url, url));
   }
   media_player_->SetDataSource(url_.spec(), cookies, incognito_mode_);
 
   media_player_->Prepare(
       base::Bind(&WebMediaPlayerProxyAndroid::MediaInfoCallback, proxy_),
       base::Bind(&WebMediaPlayerProxyAndroid::MediaErrorCallback, proxy_),
       base::Bind(&WebMediaPlayerProxyAndroid::VideoSizeChangedCallback, proxy_),
       base::Bind(&WebMediaPlayerProxyAndroid::BufferingUpdateCallback, proxy_),
       base::Bind(&WebMediaPlayerProxyAndroid::MediaPreparedCallback, proxy_));
 }
 
 void WebMediaPlayerAndroid::PlayInternal() {
   CHECK(prepared_);
 
+  if (hasVideo() && stream_texture_factory_.get()) {
+    if (!stream_id_)
+      CreateStreamTexture();
+
+    if (needs_establish_peer_) {
+      stream_texture_factory_->EstablishPeer(stream_id_, player_id_);
+      needs_establish_peer_ = false;
+    }
+  }
+
   if (paused())
     media_player_->Start(base::Bind(
         &WebMediaPlayerProxyAndroid::PlaybackCompleteCallback, proxy_));
 }
 
 void WebMediaPlayerAndroid::PauseInternal() {
   CHECK(prepared_);
   media_player_->Pause();
 }
 
 void WebMediaPlayerAndroid::SeekInternal(float seconds) {
   CHECK(prepared_);
   seeking_ = true;
   media_player_->SeekTo(ConvertSecondsToTimestamp(seconds), base::Bind(
       &WebMediaPlayerProxyAndroid::SeekCompleteCallback, proxy_));
 }
 
+void WebMediaPlayerAndroid::CreateStreamTexture() {
+  DCHECK(!stream_id_);
+  DCHECK(!texture_id_);
+  stream_id_ = stream_texture_factory_->CreateStreamTexture(&texture_id_);

[scherkus (not reviewing), 2012/05/24 21:36:32]

isn't this NULL right now?

is CreateStreamTexture() called anywhere else than PlayInternal()?

if not you may want to consider inlining into PlayInternal() because right now
(for example) you have DCHECKs checking for !stream_id_ ... but you have an if
statement that handles that

[qinmin, 2012/05/25 01:04:21]

This function will not get called if stream_texture_factory_ is NULL.

On 2012/05/24 21:36:32, scherkus wrote:

+  if (texture_id_)
+    video_frame_.reset(new WebVideoFrameImpl(VideoFrame::WrapNativeTexture(
+        texture_id_,
+        GL_TEXTURE_EXTERNAL_OES,
+        texture_size_.width,
+        texture_size_.height,
+        base::TimeDelta(),
+        base::TimeDelta(),
+        base::Bind(&WebMediaPlayerAndroid::DestroyStreamTexture,
+                   base::Unretained(this)))));
+}
+
+void WebMediaPlayerAndroid::DestroyStreamTexture() {
+  DCHECK(stream_id_);
+  DCHECK(texture_id_);
+  stream_texture_factory_->DestroyStreamTexture(texture_id_);
+  texture_id_ = 0;
+  stream_id_ = 0;
+}
+
 WebVideoFrame* WebMediaPlayerAndroid::getCurrentFrame() {
+  if (!stream_texture_proxy_initialized_ && stream_id_) {

[scherkus (not reviewing), 2012/05/24 21:36:32]

would stream_texture_proxy_initialized_ be better served as an attribute of
StreamTextureProxy itself?

it seems strange to track the state of an object externally

[qinmin, 2012/05/25 01:04:21]

Done.

+    stream_texture_proxy_->Initialize(
+        stream_id_, video_frame_->width(), video_frame_->height());
+    stream_texture_proxy_initialized_ = true;
+  }
+
   return video_frame_.get();
 }
 
 void WebMediaPlayerAndroid::putCurrentFrame(
     WebVideoFrame* web_video_frame) {
 }
 
+// This gets called both on impl and main thread.

[scherkus (not reviewing), 2012/05/24 21:36:32]

s/impl/compositor

[scherkus (not reviewing), 2012/05/24 21:36:32]

who is calling it on the main thread and for what purpose?

[qinmin, 2012/05/25 01:04:21]

Done.

[qinmin, 2012/05/25 01:04:21]

It is called in WebMediaPlayerClientImpl::~WebMediaPlayerClientImpl() to clean
up things. This change has been upstreamed to webkit.

On 2012/05/24 21:36:32, scherkus wrote:

+void WebMediaPlayerAndroid::setStreamTextureClient(
+    WebKit::WebStreamTextureClient* client) {
+  if (stream_texture_proxy_.get())
+    stream_texture_proxy_->SetClient(client);
+}
+
 }  // namespace webkit_media