Render inband text tracks in the media pipeline

media/filters/chunk_demuxer.cc

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "media/filters/chunk_demuxer.h"
 
 #include <algorithm>
 #include <deque>
 #include <limits>
 
 #include "base/bind.h"
 #include "base/callback_helpers.h"
 #include "base/location.h"
 #include "base/message_loop/message_loop_proxy.h"
 #include "media/base/audio_decoder_config.h"
 #include "media/base/bind_to_loop.h"
 #include "media/base/stream_parser_buffer.h"
 #include "media/base/video_decoder_config.h"
 #include "media/filters/stream_parser_factory.h"
-#include "media/webm/webm_webvtt_parser.h"
 
 using base::TimeDelta;
 
 namespace media {
 
 // Contains state belonging to a source id.
 class SourceState {
  public:
   // Callback signature used to create ChunkDemuxerStreams.
   typedef base::Callback<ChunkDemuxerStream*(
       DemuxerStream::Type)> CreateDemuxerStreamCB;
 
   // Callback signature used to notify ChunkDemuxer of timestamps
   // that may cause the duration to be updated.
   typedef base::Callback<void(
       TimeDelta, ChunkDemuxerStream*)> IncreaseDurationCB;
 
   SourceState(scoped_ptr<StreamParser> stream_parser, const LogCB& log_cb,
               const CreateDemuxerStreamCB& create_demuxer_stream_cb,
               const IncreaseDurationCB& increase_duration_cb);
 
+  // Called when a new text track has been parsed, so the demuxer can alert
+  // the demuxer host.
+  typedef base::Callback<void(ChunkDemuxerStream* stream,
+                              TextKind kind,
+                              const std::string& name,
+                              const std::string& language)> NewTextTrackCB;
+
   void Init(const StreamParser::InitCB& init_cb,
             bool allow_audio,
             bool allow_video,
-            const StreamParser::NewTextBuffersCB& text_cb,
             const StreamParser::NeedKeyCB& need_key_cb,
-            const AddTextTrackCB& add_text_track_cb);
+            const NewTextTrackCB& new_text_track_cb);
 
   // Appends new data to the StreamParser.
   // Returns true if the data was successfully appended. Returns false if an
   // error occurred.
   bool Append(const uint8* data, size_t length);
 
   // Aborts the current append sequence and resets the parser.
   void Abort();
 
   // Sets |timestamp_offset_| if possible.
@@ skipping 25 matching lines @@
 
   // Called by the |stream_parser_| when new buffers have been parsed. It
   // applies |timestamp_offset_| to all buffers in |audio_buffers| and
   // |video_buffers| and then calls Append() on |audio_| and/or
   // |video_| with the modified buffers.
   // Returns true on a successful call. Returns false if an error occured while
   // processing the buffers.
   bool OnNewBuffers(const StreamParser::BufferQueue& audio_buffers,
                     const StreamParser::BufferQueue& video_buffers);
 
+  void OnNewTextTrack(int text_track_num,
+                      TextKind kind,
+                      const std::string& name,
+                      const std::string& language);
+
   // Called by the |stream_parser_| when new text buffers have been parsed. It
-  // applies |timestamp_offset_| to all buffers in |buffers| and then calls
-  // |new_buffers_cb| with the modified buffers.
+  // applies |timestamp_offset_| to all buffers in |buffers| and then appends
+  // the (modified) buffers to the demuxer stream associated with
+  // the track having |text_track_number|.
   // Returns true on a successful call. Returns false if an error occured while
   // processing the buffers.
-  bool OnTextBuffers(const StreamParser::NewTextBuffersCB& new_buffers_cb,
-                     TextTrack* text_track,
+  bool OnTextBuffers(int text_track_number,
                      const StreamParser::BufferQueue& buffers);
 
   // Helper function that adds |timestamp_offset_| to each buffer in |buffers|.
   void AdjustBufferTimestamps(const StreamParser::BufferQueue& buffers);
 
   // Filters out buffers that are outside of the append window
   // [|append_window_start_|, |append_window_end_|).
   // |needs_keyframe| is a pointer to the |xxx_need_keyframe_| flag
   // associated with the |buffers|. Its state is read an updated as
   // this method filters |buffers|.
@@ skipping 25 matching lines @@
 
   // The object used to parse appended data.
   scoped_ptr<StreamParser> stream_parser_;
 
   ChunkDemuxerStream* audio_;
   bool audio_needs_keyframe_;
 
   ChunkDemuxerStream* video_;
   bool video_needs_keyframe_;
 
+  NewTextTrackCB new_text_track_cb_;
+
+  typedef std::map<int, ChunkDemuxerStream*> TextStreamMap;
+  TextStreamMap text_stream_map_;
+
   LogCB log_cb_;
 
   DISALLOW_COPY_AND_ASSIGN(SourceState);
 };
 
 class ChunkDemuxerStream : public DemuxerStream {
  public:
   typedef std::deque<scoped_refptr<StreamParserBuffer> > BufferQueue;
 
   explicit ChunkDemuxerStream(Type type);
@@ skipping 31 matching lines @@
 
   // Signal to the stream that buffers handed in through subsequent calls to
   // Append() belong to a media segment that starts at |start_timestamp|.
   void OnNewMediaSegment(TimeDelta start_timestamp);
 
   // Called when midstream config updates occur.
   // Returns true if the new config is accepted.
   // Returns false if the new config should trigger an error.
   bool UpdateAudioConfig(const AudioDecoderConfig& config, const LogCB& log_cb);
   bool UpdateVideoConfig(const VideoDecoderConfig& config, const LogCB& log_cb);
+  void UpdateTextConfig(const LogCB& log_cb);
 
   void MarkEndOfStream();
   void UnmarkEndOfStream();
 
   // DemuxerStream methods.
   virtual void Read(const ReadCB& read_cb) OVERRIDE;
   virtual Type type() OVERRIDE;
   virtual void EnableBitstreamConverter() OVERRIDE;
   virtual AudioDecoderConfig audio_decoder_config() OVERRIDE;
   virtual VideoDecoderConfig video_decoder_config() OVERRIDE;
@@ skipping 48 matching lines @@
       video_(NULL),
       video_needs_keyframe_(true),
       log_cb_(log_cb) {
   DCHECK(!create_demuxer_stream_cb_.is_null());
   DCHECK(!increase_duration_cb_.is_null());
 }
 
 void SourceState::Init(const StreamParser::InitCB& init_cb,
                        bool allow_audio,
                        bool allow_video,
-                       const StreamParser::NewTextBuffersCB& text_cb,
                        const StreamParser::NeedKeyCB& need_key_cb,
-                       const AddTextTrackCB& add_text_track_cb) {
+                       const NewTextTrackCB& new_text_track_cb) {
   StreamParser::NewBuffersCB audio_cb;
+  new_text_track_cb_ = new_text_track_cb;
 
   stream_parser_->Init(init_cb,
                        base::Bind(&SourceState::OnNewConfigs,
                                   base::Unretained(this),
                                   allow_audio,
                                   allow_video),
                        base::Bind(&SourceState::OnNewBuffers,
                                   base::Unretained(this)),
                        base::Bind(&SourceState::OnTextBuffers,
-                                  base::Unretained(this), text_cb),
+                                  base::Unretained(this)),
                        need_key_cb,
-                       add_text_track_cb,
+                       base::Bind(&SourceState::OnNewTextTrack,
+                                  base::Unretained(this)),
                        base::Bind(&SourceState::OnNewMediaSegment,
                                   base::Unretained(this)),
                        base::Bind(&SourceState::OnEndOfMediaSegment,
                                   base::Unretained(this)),
                        log_cb_);
 }
 
 bool SourceState::SetTimestampOffset(TimeDelta timestamp_offset) {
   if (!can_update_offset_)
     return false;
@@ skipping 135 matching lines @@
       segment_timestamp = filtered_video.front()->GetDecodeTimestamp();
     }
 
     new_media_segment_ = false;
 
     if (audio_)
       audio_->OnNewMediaSegment(segment_timestamp);
 
     if (video_)
       video_->OnNewMediaSegment(segment_timestamp);
+
+    for (TextStreamMap::iterator itr = text_stream_map_.begin();
+         itr != text_stream_map_.end(); ++itr) {
+      itr->second->OnNewMediaSegment(segment_timestamp);
+    }
   }
 
   if (!filtered_audio.empty()) {
     if (!audio_ || !audio_->Append(filtered_audio))
       return false;
     increase_duration_cb_.Run(filtered_audio.back()->timestamp(), audio_);
   }
 
   if (!filtered_video.empty()) {
     if (!video_ || !video_->Append(filtered_video))
       return false;
     increase_duration_cb_.Run(filtered_video.back()->timestamp(), video_);
   }
 
   return true;
 }
 
 bool SourceState::OnTextBuffers(
-    const StreamParser::NewTextBuffersCB& new_buffers_cb,
-    TextTrack* text_track,
+    int text_track_number,
     const StreamParser::BufferQueue& buffers) {
-  if (new_buffers_cb.is_null())
-    return false;
+  if (buffers.empty())

[acolwell GONE FROM CHROMIUM, 2013/10/14 20:42:24]

nit: DCHECK(!buffers.empty()) instead.

[Matthew Heaney (Chromium), 2013/10/17 05:46:44]

Done.

[Matthew Heaney (Chromium), 2013/10/17 05:46:44]

Done.

+    return true;
+
+  TextStreamMap::iterator itr = text_stream_map_.find(text_track_number);
+  if (itr == text_stream_map_.end())
+    return true;

[acolwell GONE FROM CHROMIUM, 2013/10/14 20:42:24]

nit: I think this should return false. Doesn't this mean that we have received
cues for a track that we don't know about? This situation seems like it should
trigger an error.

[Matthew Heaney (Chromium), 2013/10/17 05:46:44]

Done.

 
   AdjustBufferTimestamps(buffers);
 
-  return new_buffers_cb.Run(text_track, buffers);
+  return itr->second->Append(buffers);
+}
+
+void SourceState::OnNewTextTrack(int text_track_number,

[acolwell GONE FROM CHROMIUM, 2013/10/14 20:42:24]

I think this may need to be folded into OnNewConfigs(). You need a global
picture of all the text tracks provided in the headers so that you can determine
whether the track numbers, kind, and language change. If there is one text track
then the track number is allowed to change, but the kind and language are not.
If there are more than one text track then the kind, language, and track numbers
must all be the same. The number of tracks must also stay constant throughout
the life of a SourceState object.

You might want to consider creating a TextTrackConfig object similar to the
AudioDecoderConfig that contains language, kind, and codec (i.e., WebVTT) so
that it is easy to transport these values together and enables Matches() and
IsValid() operations.

[Matthew Heaney (Chromium), 2013/10/17 05:46:44]

I did an incomplete implementation.  You can check it out and make sure I'm
headed in the right direction.

+                                 TextKind kind,
+                                 const std::string& name,
+                                 const std::string& language) {
+  ChunkDemuxerStream* const text_stream =
+      create_demuxer_stream_cb_.Run(DemuxerStream::TEXT);
+
+  // Interpret NULL as meaning that inband text tracks are disabled.
+  if (text_stream) {
+    text_stream->UpdateTextConfig(log_cb_);
+    text_stream_map_[text_track_number] = text_stream;
+    new_text_track_cb_.Run(text_stream, kind, name, language);
+  }
 }
 
 void SourceState::FilterWithAppendWindow(
     const StreamParser::BufferQueue& buffers, bool* needs_keyframe,
     StreamParser::BufferQueue* filtered_buffers) {
   DCHECK(needs_keyframe);
   DCHECK(filtered_buffers);
 
   // This loop implements steps 1.9, 1.10, & 1.11 of the "Coded frame
   // processing loop" in the Media Source Extensions spec.
@@ skipping 170 matching lines @@
 
   if (!stream_) {
     DCHECK_EQ(state_, UNINITIALIZED);
     stream_.reset(new SourceBufferStream(config, log_cb));
     return true;
   }
 
   return stream_->UpdateVideoConfig(config);
 }
 
+void ChunkDemuxerStream::UpdateTextConfig(const LogCB& log_cb) {
+  DCHECK_EQ(type_, TEXT);
+  base::AutoLock auto_lock(lock_);
+
+  if (!stream_) {
+    DCHECK_EQ(state_, UNINITIALIZED);
+    stream_.reset(new SourceBufferStream(log_cb));
+  }

[acolwell GONE FROM CHROMIUM, 2013/10/14 20:42:24]

This needs to verify that kind and language haven't changed just like
UpdateVideoConfig() and UpdateAudioConfig() verify that the codec and parameters
like sample rate can't change from config to config.

[Matthew Heaney (Chromium), 2013/10/17 05:46:44]

See if I got OnNewConfigs right, then I can adjust this as necessary.

I am a little unclear about going from no streams (as when the source state
object is first created) to non-zero streams (e.g. the very first parse of a
webm stream). It's one thing to change the configs, but I'm talking about
first-time initialization.

+}
+
 void ChunkDemuxerStream::MarkEndOfStream() {
   base::AutoLock auto_lock(lock_);
   stream_->MarkEndOfStream();
 }
 
 void ChunkDemuxerStream::UnmarkEndOfStream() {
   base::AutoLock auto_lock(lock_);
   stream_->UnmarkEndOfStream();
 }
 
@@ skipping 75 matching lines @@
       status = DemuxerStream::kOk;
       buffer = StreamParserBuffer::CreateEOSBuffer();
       break;
   }
 
   base::ResetAndReturn(&read_cb_).Run(status, buffer);
 }
 
 ChunkDemuxer::ChunkDemuxer(const base::Closure& open_cb,
                            const NeedKeyCB& need_key_cb,
-                           const AddTextTrackCB& add_text_track_cb,
+                           bool enable_text,
                            const LogCB& log_cb)
     : state_(WAITING_FOR_INIT),
       cancel_next_seek_(false),
       host_(NULL),
       open_cb_(open_cb),
       need_key_cb_(need_key_cb),
-      add_text_track_cb_(add_text_track_cb),
+      enable_text_(enable_text),
       log_cb_(log_cb),
       duration_(kNoTimestamp()),
       user_specified_duration_(-1) {
   DCHECK(!open_cb_.is_null());
   DCHECK(!need_key_cb_.is_null());
 }
 
 void ChunkDemuxer::Initialize(DemuxerHost* host, const PipelineStatusCB& cb) {
   DVLOG(1) << "Init()";
 
@@ skipping 56 matching lines @@
 
 // Demuxer implementation.
 DemuxerStream* ChunkDemuxer::GetStream(DemuxerStream::Type type) {
   base::AutoLock auto_lock(lock_);
   if (type == DemuxerStream::VIDEO)
     return video_.get();
 
   if (type == DemuxerStream::AUDIO)
     return audio_.get();
 
-  return NULL;
+  if (!enable_text_ || text_stream_set_.empty())

[acolwell GONE FROM CHROMIUM, 2013/10/14 20:42:24]

nit: I'd prefer to not return text tracks through this interface since it
doesn't allow all tracks to be accessed and it represents the old way. Please
add a DCHECK for the TEXT type and make sure the DemuxerStream interface has a
comment indicating that |type| is not allowed to be TEXT. 

[Matthew Heaney (Chromium), 2013/10/17 05:46:44]

Done.

+    return NULL;
+
+  TextStreamSet::iterator itr = text_stream_set_.begin();
+  return *itr;
 }
 
 TimeDelta ChunkDemuxer::GetStartTime() const {
   return TimeDelta();
 }
 
 void ChunkDemuxer::StartWaitingForSeek(TimeDelta seek_time) {
   DVLOG(1) << "StartWaitingForSeek()";
   base::AutoLock auto_lock(lock_);
   DCHECK(state_ == INITIALIZED || state_ == ENDED || state_ == SHUTDOWN ||
@@ skipping 61 matching lines @@
       new SourceState(stream_parser.Pass(), log_cb_,
                       base::Bind(&ChunkDemuxer::CreateDemuxerStream,
                                  base::Unretained(this)),
                       base::Bind(&ChunkDemuxer::IncreaseDurationIfNecessary,
                                  base::Unretained(this))));
 
   source_state->Init(
       base::Bind(&ChunkDemuxer::OnSourceInitDone, base::Unretained(this)),
       has_audio,
       has_video,
-      base::Bind(&ChunkDemuxer::OnTextBuffers, base::Unretained(this)),
       need_key_cb_,
-      add_text_track_cb_);
+      base::Bind(&ChunkDemuxer::OnNewTextTrack, base::Unretained(this)));
 
   source_state_map_[id] = source_state.release();
   return kOk;
 }
 
 void ChunkDemuxer::RemoveId(const std::string& id) {
   base::AutoLock auto_lock(lock_);
   CHECK(IsValidId(id));
 
   delete source_state_map_[id];
   source_state_map_.erase(id);
 
   if (source_id_audio_ == id) {
     if (audio_)
       audio_->Shutdown();
     source_id_audio_.clear();
   }
 
   if (source_id_video_ == id) {
     if (video_)
       video_->Shutdown();
     source_id_video_.clear();
   }
+
+  // TODO(matthewjheaney): delete the text streams associated with this id

[acolwell GONE FROM CHROMIUM, 2013/10/14 20:42:24]

This needs to be addressed in this CL. If you let the SourceState object own the
text DemuxerStreams instead of text_stream_set_, this should be pretty straight
forward.

You will also need a way to notify pipeline that these streams are being removed
from the presentation. Adding the following signature to DemuxerHost and adding
the necessary plumbing to Pipeline & TextRenderer should work.

void RemoveTextStream(DemuxerStream* text_stream)

[Matthew Heaney (Chromium), 2013/10/17 05:46:44]

I did a partial implementation.  You can see if I'm headed in the right
direction.

 }
 
 Ranges<TimeDelta> ChunkDemuxer::GetBufferedRanges(const std::string& id) const {
   base::AutoLock auto_lock(lock_);
   DCHECK(!id.empty());
   DCHECK(IsValidId(id));
   DCHECK(id == source_id_audio_ || id == source_id_video_);
 
   if (id == source_id_audio_ && id != source_id_video_) {
     // Only include ranges that have been buffered in |audio_|
@@ skipping 296 matching lines @@
   state_ = new_state;
 }
 
 ChunkDemuxer::~ChunkDemuxer() {
   DCHECK_NE(state_, INITIALIZED);
   for (SourceStateMap::iterator it = source_state_map_.begin();
        it != source_state_map_.end(); ++it) {
     delete it->second;
   }
   source_state_map_.clear();
+
+  TextStreamSet::iterator it = text_stream_set_.begin();

[acolwell GONE FROM CHROMIUM, 2013/10/14 20:42:24]

This would go away if you let SourceState own the text DemuxerStreams.

[Matthew Heaney (Chromium), 2013/10/17 05:46:44]

Done.

+  while (it != text_stream_set_.end()) {
+    ChunkDemuxerStream* text_stream = *it;
+    text_stream_set_.erase(it++);
+    delete text_stream;
+  }
 }
 
 void ChunkDemuxer::ReportError_Locked(PipelineStatus error) {
   DVLOG(1) << "ReportError_Locked(" << error << ")";
   lock_.AssertAcquired();
   DCHECK_NE(error, PIPELINE_OK);
 
   ChangeState_Locked(PARSE_ERROR);
 
   PipelineStatusCB cb;
@@ skipping 70 matching lines @@
         return NULL;
       audio_.reset(new ChunkDemuxerStream(DemuxerStream::AUDIO));
       return audio_.get();
       break;
     case DemuxerStream::VIDEO:
       if (video_)
         return NULL;
       video_.reset(new ChunkDemuxerStream(DemuxerStream::VIDEO));
       return video_.get();
       break;
+    case DemuxerStream::TEXT: {
+      // TODO(matthewjheaney): we need to destroy this text stream
+      ChunkDemuxerStream* text_stream =
+          new ChunkDemuxerStream(DemuxerStream::TEXT);
+      text_stream_set_.insert(text_stream);
+      return text_stream;
+      break;
+    }
     case DemuxerStream::UNKNOWN:
     case DemuxerStream::NUM_TYPES:
       NOTREACHED();
       return NULL;
   }
   NOTREACHED();
   return NULL;
 }
 
-bool ChunkDemuxer::OnTextBuffers(
-  TextTrack* text_track,
-  const StreamParser::BufferQueue& buffers) {
+void ChunkDemuxer::OnNewTextTrack(ChunkDemuxerStream* text_stream,
+                                  TextKind kind,
+                                  const std::string& name,
+                                  const std::string& language) {
   lock_.AssertAcquired();
   DCHECK_NE(state_, SHUTDOWN);
-
-  // TODO(matthewjheaney): IncreaseDurationIfNecessary
-
-  for (StreamParser::BufferQueue::const_iterator itr = buffers.begin();
-       itr != buffers.end(); ++itr) {
-    const StreamParserBuffer* const buffer = itr->get();
-    const TimeDelta start = buffer->timestamp();
-    const TimeDelta end = start + buffer->duration();
-
-    std::string id, settings, content;
-
-    WebMWebVTTParser::Parse(buffer->data(),
-                            buffer->data_size(),
-                            &id, &settings, &content);
-
-    text_track->addWebVTTCue(start, end, id, content, settings);
-  }
-
-  return true;
+  DCHECK(text_stream_set_.find(text_stream) != text_stream_set_.end());
+  host_->AddTextStream(text_stream, kind, name, language);
 }
 
 bool ChunkDemuxer::IsValidId(const std::string& source_id) const {
   lock_.AssertAcquired();
   return source_state_map_.count(source_id) > 0u;
 }
 
 void ChunkDemuxer::UpdateDuration(TimeDelta new_duration) {
   DCHECK(duration_ != new_duration);
   user_specified_duration_ = -1;
@@ skipping 33 matching lines @@
     return video_->GetBufferedRanges(duration_);
   return ComputeIntersection();
 }
 
 void ChunkDemuxer::StartReturningData() {
   if (audio_)
     audio_->StartReturningData();
 
   if (video_)
     video_->StartReturningData();
+
+  for (TextStreamSet::iterator itr = text_stream_set_.begin();

[acolwell GONE FROM CHROMIUM, 2013/10/14 20:42:24]

nit: Consider iterating over the SourceState objects instead and have them each
iterate over their DemuxerStreams. You'd likely be able to move the calls on
audio_ & video_ into SourceState as well.

[Matthew Heaney (Chromium), 2013/10/17 05:46:44]

Done.

+       itr != text_stream_set_.end(); ++itr) {
+    (*itr)->StartReturningData();
+  }
 }
 
 void ChunkDemuxer::AbortPendingReads() {
   if (audio_)
     audio_->AbortReads();
 
   if (video_)
     video_->AbortReads();
+
+  for (TextStreamSet::iterator itr = text_stream_set_.begin();

[acolwell GONE FROM CHROMIUM, 2013/10/14 20:42:24]

ditto for this & the other 2 methods below.

[Matthew Heaney (Chromium), 2013/10/17 05:46:44]

Done.

+       itr != text_stream_set_.end(); ++itr) {
+    (*itr)->AbortReads();
+  }
 }
 
 void ChunkDemuxer::SeekAllSources(TimeDelta seek_time) {
   if (audio_)
     audio_->Seek(seek_time);
 
   if (video_)
     video_->Seek(seek_time);
+
+  for (TextStreamSet::iterator itr = text_stream_set_.begin();
+       itr != text_stream_set_.end(); ++itr) {
+    (*itr)->Seek(seek_time);
+  }
 }
 
 void ChunkDemuxer::CompletePendingReadsIfPossible() {
   if (audio_)
     audio_->CompletePendingReadIfPossible();
 
   if (video_)
     video_->CompletePendingReadIfPossible();
+
+  for (TextStreamSet::iterator itr = text_stream_set_.begin();
+       itr != text_stream_set_.end(); ++itr) {
+    (*itr)->CompletePendingReadIfPossible();
+  }
 }
 
 }  // namespace media