Bug 1293576: [MSE] P1. Always process the earliest frames first when in sequence mode. r?gerald draft
authorJean-Yves Avenard <jyavenard@mozilla.com>
Tue, 09 Aug 2016 23:11:36 +1000
changeset 398979 604247b6c4a49122c64a2684c0964524843be953
parent 398978 525505cb05a26b558243a27d29379b167363d1b0
child 398980 e43df1921b2dec90e0387843311add1aa228969e
push id25694
push userbmo:jyavenard@mozilla.com
push dateWed, 10 Aug 2016 06:24:12 +0000
reviewersgerald
bugs1293576
milestone51.0a1
Bug 1293576: [MSE] P1. Always process the earliest frames first when in sequence mode. r?gerald MozReview-Commit-ID: 2b3EyYCtNai
dom/media/mediasource/TrackBuffersManager.cpp
dom/media/mediasource/TrackBuffersManager.h
--- a/dom/media/mediasource/TrackBuffersManager.cpp
+++ b/dom/media/mediasource/TrackBuffersManager.cpp
@@ -1214,44 +1214,60 @@ TrackBuffersManager::OnAudioDemuxComplet
 
 void
 TrackBuffersManager::CompleteCodedFrameProcessing()
 {
   MOZ_ASSERT(OnTaskQueue());
 
   // 1. For each coded frame in the media segment run the following steps:
   // Coded Frame Processing steps 1.1 to 1.21.
-  ProcessFrames(mVideoTracks.mQueuedSamples, mVideoTracks);
-  mVideoTracks.mQueuedSamples.Clear();
+
+  if (mSourceBufferAttributes->GetAppendMode() == SourceBufferAppendMode::Sequence &&
+      mVideoTracks.mQueuedSamples.Length() && mAudioTracks.mQueuedSamples.Length()) {
+    // When we are in sequence mode, the order in which we process the frames is
+    // important as it determines the future value of timestampOffset.
+    // So we process the earliest sample first. See bug 1293576.
+    TimeInterval videoInterval =
+      PresentationInterval(mVideoTracks.mQueuedSamples);
+    TimeInterval audioInterval =
+      PresentationInterval(mAudioTracks.mQueuedSamples);
+    if (audioInterval.mStart < videoInterval.mStart) {
+      ProcessFrames(mAudioTracks.mQueuedSamples, mAudioTracks);
+      ProcessFrames(mVideoTracks.mQueuedSamples, mVideoTracks);
+    } else {
+      ProcessFrames(mVideoTracks.mQueuedSamples, mVideoTracks);
+      ProcessFrames(mAudioTracks.mQueuedSamples, mAudioTracks);
+    }
+  } else {
+    ProcessFrames(mVideoTracks.mQueuedSamples, mVideoTracks);
+    ProcessFrames(mAudioTracks.mQueuedSamples, mAudioTracks);
+  }
 
 #if defined(DEBUG)
   if (HasVideo()) {
     const auto& track = mVideoTracks.mBuffers.LastElement();
     MOZ_ASSERT(track.IsEmpty() || track[0]->mKeyframe);
     for (uint32_t i = 1; i < track.Length(); i++) {
       MOZ_ASSERT((track[i-1]->mTrackInfo->GetID() == track[i]->mTrackInfo->GetID() && track[i-1]->mTimecode <= track[i]->mTimecode) ||
                  track[i]->mKeyframe);
     }
   }
-#endif
-
-  ProcessFrames(mAudioTracks.mQueuedSamples, mAudioTracks);
-  mAudioTracks.mQueuedSamples.Clear();
-
-#if defined(DEBUG)
   if (HasAudio()) {
     const auto& track = mAudioTracks.mBuffers.LastElement();
     MOZ_ASSERT(track.IsEmpty() || track[0]->mKeyframe);
     for (uint32_t i = 1; i < track.Length(); i++) {
       MOZ_ASSERT((track[i-1]->mTrackInfo->GetID() == track[i]->mTrackInfo->GetID() && track[i-1]->mTimecode <= track[i]->mTimecode) ||
                  track[i]->mKeyframe);
     }
   }
 #endif
 
+  mVideoTracks.mQueuedSamples.Clear();
+  mAudioTracks.mQueuedSamples.Clear();
+
   UpdateBufferedRanges();
 
   // Update our reported total size.
   mSizeSourceBuffer = mVideoTracks.mSizeBuffer + mAudioTracks.mSizeBuffer;
 
   // Return to step 6.4 of Segment Parser Loop algorithm
   // 4. If this SourceBuffer is full and cannot accept more media data, then set the buffer full flag to true.
   if (mSizeSourceBuffer >= EvictionThreshold()) {
@@ -1317,16 +1333,32 @@ TrackBuffersManager::CheckSequenceDiscon
     mSourceBufferAttributes->SetGroupEndTimestamp(
       mSourceBufferAttributes->GetGroupStartTimestamp());
     mVideoTracks.mNeedRandomAccessPoint = true;
     mAudioTracks.mNeedRandomAccessPoint = true;
     mSourceBufferAttributes->ResetGroupStartTimestamp();
   }
 }
 
+TimeInterval
+TrackBuffersManager::PresentationInterval(const TrackBuffer& aSamples) const
+{
+  TimeInterval presentationInterval =
+    TimeInterval(TimeUnit::FromMicroseconds(aSamples[0]->mTime),
+                 TimeUnit::FromMicroseconds(aSamples[0]->GetEndTime()));
+
+  for (uint32_t i = 1; i < aSamples.Length(); i++) {
+    auto& sample = aSamples[i];
+    presentationInterval = presentationInterval.Span(
+      TimeInterval(TimeUnit::FromMicroseconds(sample->mTime),
+                   TimeUnit::FromMicroseconds(sample->GetEndTime())));
+  }
+  return presentationInterval;
+}
+
 void
 TrackBuffersManager::ProcessFrames(TrackBuffer& aSamples, TrackData& aTrackData)
 {
   if (!aSamples.Length()) {
     return;
   }
 
   // 1. If generate timestamps flag equals true
--- a/dom/media/mediasource/TrackBuffersManager.h
+++ b/dom/media/mediasource/TrackBuffersManager.h
@@ -340,16 +340,17 @@ private:
       mNextInsertionIndex.reset();
     }
 
     void AddSizeOfResources(MediaSourceDecoder::ResourceSizes* aSizes);
   };
 
   void CheckSequenceDiscontinuity(const media::TimeUnit& aPresentationTime);
   void ProcessFrames(TrackBuffer& aSamples, TrackData& aTrackData);
+  media::TimeInterval PresentationInterval(const TrackBuffer& aSamples) const;
   bool CheckNextInsertionIndex(TrackData& aTrackData,
                                const media::TimeUnit& aSampleTime);
   void InsertFrames(TrackBuffer& aSamples,
                     const media::TimeIntervals& aIntervals,
                     TrackData& aTrackData);
   void UpdateHighestTimestamp(TrackData& aTrackData,
                               const media::TimeUnit& aHighestTime);
   // Remove all frames and their dependencies contained in aIntervals.