Implement ISO BMFF support in Media Source.

media/mp4/mp4_stream_parser.cc

+// Copyright (c) 2012 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "media/mp4/mp4_stream_parser.h"
+
+#include "base/callback.h"
+#include "base/logging.h"
+#include "base/time.h"
+#include "media/base/stream_parser_buffer.h"
+#include "media/filters/ffmpeg_glue.h"
+#include "media/mp4/box_definitions.h"
+#include "media/mp4/box_reader.h"
+#include "media/mp4/rcheck.h"
+#include "openssl/aes.h"
+
+namespace media {
+
+using mp4::BoxReader;
+
+OffsetByteQueue::OffsetByteQueue() : buf_(NULL), size_(0), head_(0) {}
+OffsetByteQueue::~OffsetByteQueue() {};
+
+void OffsetByteQueue::Reset() {
+  queue_.Reset();
+  buf_ = NULL;
+  size_ = 0;
+  head_ = 0;
+}
+
+void OffsetByteQueue::Push(const uint8* buf, size_t size) {
+  queue_.Push(buf, size);
+  Sync();
+  DVLOG(4) << "Buffer pushed. head=" << head() << " tail=" << tail();
+}
+
+void OffsetByteQueue::Peek(const uint8** buf, size_t* size) {
+  *buf = buf_;
+  *size = size_;
+}
+
+void OffsetByteQueue::Pop(size_t count) {
+  queue_.Pop(count);
+  head_ += count;
+  Sync();
+}
+
+void OffsetByteQueue::PeekAt(size_t offset, const uint8** buf, size_t* size) {
+  DCHECK(offset >= head());
+  if (offset >= head() && offset < tail()) {
+    *buf = &buf_[offset - head()];
+    *size = tail() - offset;
+  } else {

[acolwell GONE FROM CHROMIUM, 2012/06/06 17:32:39]

early return and drop else

[strobe_, 2012/06/07 16:33:54]

Done.

+    *buf = NULL;
+    *size = 0;
+  }
+}
+
+bool OffsetByteQueue::Trim(size_t max_offset) {
+  if (max_offset < head_) return true;
+  if (max_offset > tail()) {
+    Pop(size_);
+    return false;
+  }
+  Pop(max_offset - head_);
+  return true;
+}
+
+void OffsetByteQueue::Sync() {
+  int size;
+  queue_.Peek(&buf_, &size);
+  size_ = size;
+}
+
+MP4StreamParser::MP4StreamParser()
+    : state_(kWaitingForInit),
+      moof_head_(0),
+      mdat_tail_(0),
+      has_audio_(false),
+      has_video_(false) {}

[acolwell GONE FROM CHROMIUM, 2012/06/06 17:32:39]

} on next line.

[strobe_, 2012/06/07 16:33:54]

Done.

+
+MP4StreamParser::~MP4StreamParser() {}
+
+void MP4StreamParser::Init(const InitCB& init_cb,
+                           const NewConfigCB& config_cb,
+                           const NewBuffersCB& audio_cb,
+                           const NewBuffersCB& video_cb,
+                           const KeyNeededCB& key_needed_cb) {
+  DCHECK_EQ(state_, kWaitingForInit);
+  DCHECK(init_cb_.is_null());
+  DCHECK(!init_cb.is_null());
+  DCHECK(!config_cb.is_null());
+  DCHECK(!audio_cb.is_null() || !video_cb.is_null());
+  DCHECK(!key_needed_cb.is_null());
+
+  // TODO(strobe): Codec searches fail unless the FFmpegGlue singleton is
+  // initialized. Find the appropriate place to put this initialization.
+  FFmpegGlue::GetInstance();
+
+  ChangeState(kParsingBoxes);
+  init_cb_ = init_cb;
+  config_cb_ = config_cb;
+  audio_cb_ = audio_cb;
+  video_cb_ = video_cb;
+  key_needed_cb_ = key_needed_cb;
+}
+
+void MP4StreamParser::Flush() {
+  DCHECK_NE(state_, kWaitingForInit);
+
+  queue_.Reset();
+  moof_head_ = 0;
+  mdat_tail_ = 0;
+
+  if (state_ != kParsingBoxes || kEmittingSamples)

[acolwell GONE FROM CHROMIUM, 2012/06/06 17:32:39]

why is this here? It looks wrong and isn't doing anything.

[strobe_, 2012/06/07 16:33:54]

Done.

+    return;
+}
+
+bool MP4StreamParser::Parse(const uint8* buf, int size) {
+  DCHECK_NE(state_, kWaitingForInit);
+
+  if (state_ == kError)
+    return false;
+
+  queue_.Push(buf, size);
+
+  BufferQueue audio_buffers;
+  BufferQueue video_buffers;
+
+  bool result, err = false;
+
+  do {
+    if (state_ == kParsingBoxes) {
+      if (mdat_tail_ > queue_.head()) {
+        result = queue_.Trim(mdat_tail_);
+      } else {
+        result = ParseBox(&err);
+        DCHECK(!err); // XXX
+      }
+    } else {
+      DCHECK_EQ(kEmittingSamples, state_);
+      result = EnqueueSample(&audio_buffers, &video_buffers, &err);
+      if (result) {
+        size_t max_clear = runs_.GetMaxClearOffset() + moof_head_;
+        DCHECK(max_clear <= queue_.tail());
+        err = !(ReadMDATsUntil(max_clear) && queue_.Trim(max_clear));
+      }
+    }
+  } while (result && !err);
+
+  if (err) {
+    DLOG(ERROR) << "Unknown error while parsing MP4";
+    // XXX flush byte queue
+    ChangeState(kError);
+    return false;
+  }
+
+  if (!audio_buffers.empty()) DCHECK(audio_cb_.Run(audio_buffers));

[acolwell GONE FROM CHROMIUM, 2012/06/06 17:32:39]

You don't want DCHECK here. If the callback returns false you want to return
false from this function.

if (!audio_buffers.empty() && 
    (audio_cb_.is_null() || !audio_cb_.Run(audio_buffers)))
  return false;

This also makes sure you don't accidentally return audio data when you aren't
expected to. This applies to video below as well.

[strobe_, 2012/06/07 16:33:54]

Done.

+  if (!video_buffers.empty()) DCHECK(video_cb_.Run(video_buffers));
+  return true;
+}
+
+bool MP4StreamParser::ParseBox(bool* err) {
+  const uint8* buf;
+  size_t size;
+  queue_.Peek(&buf, &size);
+  if (!size) return false;
+
+  scoped_ptr<BoxReader> r(BoxReader::ReadTopLevelBox(buf, size, err));

[acolwell GONE FROM CHROMIUM, 2012/06/06 17:32:39]

r -> reader

[strobe_, 2012/06/07 16:33:54]

Done.

+  if (r.get() == NULL) return false;
+
+  if (r->type() == FOURCC_MOOV) {
+    *err = !ParseMoov(r.get());
+  } else if (r->type() == FOURCC_MOOF) {
+    moof_head_ = queue_.head();
+    *err = !ParseMoof(r.get());
+
+    // Set up first mdat offset for ParseMDATsUntil()
+    mdat_tail_ = queue_.head() + r->size();
+  } else {
+    DVLOG(2) << "Skipping unrecognized top-level box: "
+             << mp4::FourCCToString(r->type());
+  }
+
+  queue_.Pop(r->size());
+  return !(*err);
+}
+
+
+bool MP4StreamParser::ParseMoov(BoxReader* r) {
+  // TODO(strobe): respect edit lists
+  if (moov_.get()) {
+    DLOG(FATAL) << "TODO: Reinitialization not yet supported.";
+    return false;
+  }
+
+  moov_.reset(new mp4::Movie);
+
+  RCHECK(mp4::Parse(r, moov_.get()));
+
+  has_audio_ = false;
+  has_video_ = false;
+
+  // Select first video and audio track
+  AudioDecoderConfig ac;

[acolwell GONE FROM CHROMIUM, 2012/06/06 17:32:39]

ac -> audio_config
vc -> video_config

[strobe_, 2012/06/07 16:33:54]

Done.

+  VideoDecoderConfig vc;
+
+  for (auto track = moov_->tracks.begin();
+       track != moov_->tracks.end(); ++track) {
+    // TODO(strobe): We only take the first of each type of track for now
+    const mp4::SampleDescription& samp_descr =
+        track->media.information.sample_table.description;
+    if (track->media.handler.type == mp4::kAudio && !ac.IsValidConfig()) {
+      RCHECK(!samp_descr.audio_entries.empty());
+      const mp4::AudioSampleEntry& entry = samp_descr.audio_entries[0];
+
+      // TODO(strobe): this does not conform to specs for protected audio in the
+      // test files provided by Widevine. Need clarification on the scheme and
+      // whether we expect to deviate from it.
+      //RCHECK(entry.format == FOURCC_MP4A ||
+      //       (entry.format == FOURCC_ENCA &&
+      //        entry.sinf.format.format == FOURCC_MP4A));
+
+      const ChannelLayout layout =
+          mp4::ConvertAACChannelCountToChannelLayout(entry.channelcount);
+      ac.Initialize(kCodecAAC, entry.samplesize, layout,
+                    entry.samplerate, NULL, 0, false);
+      has_audio_ = true;
+      audio_track_id_ = track->header.track_id;
+    }
+    if (track->media.handler.type == mp4::kVideo && !vc.IsValidConfig()) {
+      RCHECK(!samp_descr.video_entries.empty());
+      const mp4::VideoSampleEntry& entry = samp_descr.video_entries[0];
+
+      //RCHECK(entry.format == FOURCC_AVC1 ||
+      //       (entry.format == FOURCC_ENCV &&
+      //        entry.sinf.format.format == FOURCC_AVC1));
+
+      // TODO(strobe): Profile level recovery
+      // TODO(strobe): Crop box (if needed)
+      // TODO(strobe): Rectangle semantics
+      // TODO(strobe): PAR (if needed)
+      // TODO(strobe): Ensure framerate is not used
+      vc.Initialize(kCodecH264, H264PROFILE_MAIN,  VideoFrame::YV12,
+                    gfx::Size(entry.width, entry.height),
+                    gfx::Rect(0, 0, entry.width, entry.height),
+                    1000, track->media.header.timescale,
+                    1, 1,
+                    NULL, 0, false);
+      has_video_ = true;
+      video_track_id_ = track->header.track_id;
+
+      size_of_nalu_length_ = entry.avcc.length_size;
+
+    }
+  }
+
+  RCHECK(config_cb_.Run(ac, vc));
+
+  base::TimeDelta duration;
+  if (moov_->extends.header.fragment_duration > 0) {
+    duration = mp4::TimeDeltaFromFrac(moov_->extends.header.fragment_duration,
+                                      moov_->header.timescale);
+  } else if (moov_->header.duration > 0) {
+    duration = mp4::TimeDeltaFromFrac(moov_->header.duration,
+                                      moov_->header.timescale);
+  } else {
+    duration = kInfiniteDuration();
+  }
+
+  init_cb_.Run(true, duration);
+  return true;
+}
+
+bool MP4StreamParser::ParseMoof(BoxReader* r) {
+  mp4::MovieFragment moof;
+  RCHECK(mp4::Parse(r, &moof));
+  RCHECK(runs_.Init(*moov_, moof));
+  ChangeState(kEmittingSamples);
+  return true;
+}
+
+bool MP4StreamParser::EnqueueSample(BufferQueue* audio_buffers,
+                                    BufferQueue* video_buffers,
+                                    bool* err) {
+  DCHECK(!(*err));
+
+  const uint8* buf;
+  size_t size;
+  queue_.Peek(&buf, &size);
+  if (!size) return false;
+  if (runs_.RunValid()) {
+    bool audio = has_audio_ && audio_track_id_ == runs_.track_id();
+    bool video = has_video_ && video_track_id_ == runs_.track_id();
+
+    if (!audio && !video) runs_.AdvanceRun();
+
+    if (runs_.NeedsCENC()) {
+      queue_.PeekAt(runs_.cenc_offset() + moof_head_, &buf, &size);
+      return runs_.CacheCENC(buf, size);
+    } else if (runs_.SampleValid()) {

[acolwell GONE FROM CHROMIUM, 2012/06/06 17:32:39]

remove else

[strobe_, 2012/06/07 16:33:54]

Done.

+      queue_.PeekAt(runs_.offset() + moof_head_, &buf, &size);
+      if (size < runs_.size()) return false;
+
+      std::vector<uint8> frame_buf(buf, buf + runs_.size());
+      if (video) {
+        // TODO(strobe): reinject SPS/PPS at every keyframe (or whatever the
+        // spec says to do).
+        mp4::ConvertAVCCToAnnexB(size_of_nalu_length_, &frame_buf);
+      }
+
+      scoped_refptr<StreamParserBuffer> stream_buf =
+        StreamParserBuffer::CopyFrom(&frame_buf[0], frame_buf.size(),
+                                     runs_.is_keyframe());
+
+      stream_buf->SetDuration(runs_.duration());
+      stream_buf->SetTimestamp(runs_.dts());
+      // TODO(strobe): verify that Chrome's various media backends perform
+      // B-frame reordering by zipping the un-reordered frame timestamp stream
+      // with the reordered frame data stream, such that we only need to pass
+      // DTS to the source buffer. (Doing so will always carry a risk for
+      // bizarre videos that do things like have a DTS stream that isn't a
+      // strict reordering of the CTS stream, but I'm in favor of ignoring that
+      // case.)
+
+      DVLOG(3) << "Pushing frame: aud=" << audio
+               << ", dur=" << runs_.duration().InMilliseconds()
+               << ", dts=" << runs_.dts().InMilliseconds()
+               << ", size=" << runs_.size();
+
+      if (audio) {
+        audio_buffers->push_back(stream_buf);
+      } else {
+        video_buffers->push_back(stream_buf);
+      }
+      runs_.AdvanceSample();

[acolwell GONE FROM CHROMIUM, 2012/06/06 17:32:39]

looks like you can remove the else below and then just have a single
runs_.AdvanceSample()

[strobe_, 2012/06/07 16:33:54]

Done.

+    } else {
+      runs_.AdvanceRun();
+    }
+  } else {
+    ChangeState(kParsingBoxes);

[acolwell GONE FROM CHROMIUM, 2012/06/06 17:32:39]

reverse condition in if above, change state and return early. Then you can
de-indent the large block above.

[strobe_, 2012/06/07 16:33:54]

Done.

+  }
+  return true;
+}
+
+bool MP4StreamParser::ReadMDATsUntil(const size_t tgt_offset) {
+  DCHECK(tgt_offset <= queue_.tail());
+
+  while (mdat_tail_ < tgt_offset) {
+    const uint8* buf;
+    size_t size;
+    queue_.PeekAt(mdat_tail_, &buf, &size);
+
+    FourCC type;
+    size_t box_sz;
+    bool err;
+    if (!BoxReader::StartTopLevelBox(buf, size, &type, &box_sz, &err))
+      return false;
+
+    if (type != FOURCC_MDAT)

[acolwell GONE FROM CHROMIUM, 2012/06/06 17:32:39]

Add {} since the body spans multiple lines.

[strobe_, 2012/06/07 16:33:54]

Done.

+      DLOG(WARNING) << "Unexpected type while parsing MDATs: "
+                    << mp4::FourCCToString(type);
+
+    mdat_tail_ += box_sz;
+  }
+
+  return true;
+}
+
+void MP4StreamParser::ChangeState(State new_state) {
+  DVLOG(2) << "Changing state: " << new_state;
+  state_ = new_state;
+}
+
+}  // namespace media