Bug 1201363 - Call MediaStreamVideoSink::setCurrentFrames in SourceMediaStream::AppendToTrack. r?jesup draft
authorctai <ctai@mozilla.com>
Mon, 25 Jul 2016 10:01:26 +0800
changeset 396582 3daa00cea35356358f8496ace2d910ec85074edf
parent 396581 de90a1e0ccfd27acbb93319fefb84dcef0c33fb7
child 396583 aa1b44201d8fd09a5b9c7e66a01ce9d7eeaee693
push id25058
push userbmo:ctai@mozilla.com
push dateThu, 04 Aug 2016 05:45:45 +0000
reviewersjesup
bugs1201363
milestone51.0a1
Bug 1201363 - Call MediaStreamVideoSink::setCurrentFrames in SourceMediaStream::AppendToTrack. r?jesup In this patch, we first deal with the case of MediaElement. Now we replace |PlayVideo| with |VideoFrameContainer::SetCurrentFrames| in |SourceMediaStream::AppendToTrack|. The MSG use TimeStamp::Now() for the TimeStamp of each video frame in most of case except MediaElement case. Becasue the MediaElement has its own VideoQueue, we need to calucalte the correct Timestamp based on the StartTimeStamp of this MediaStream and the elpased time of the video frame in DecodedStream. MozReview-Commit-ID: 2bm2AHkFXHu
dom/html/HTMLMediaElement.cpp
dom/media/MediaStreamGraph.cpp
dom/media/MediaStreamGraph.h
dom/media/MediaStreamGraphImpl.h
dom/media/TrackUnionStream.h
dom/media/VideoFrameContainer.cpp
dom/media/VideoFrameContainer.h
dom/media/VideoSegment.cpp
dom/media/VideoSegment.h
dom/media/mediasink/DecodedStream.cpp
--- a/dom/html/HTMLMediaElement.cpp
+++ b/dom/html/HTMLMediaElement.cpp
@@ -3622,36 +3622,36 @@ void HTMLMediaElement::UpdateSrcMediaStr
 
     mWatchManager.Watch(*mMediaStreamListener,
         &HTMLMediaElement::UpdateReadyStateInternal);
 
     stream->AddAudioOutput(this);
     SetVolumeInternal();
 
     VideoFrameContainer* container = GetVideoFrameContainer();
-    if (container) {
-      stream->AddVideoOutput(container);
+    if (mSelectedVideoStreamTrack && container) {
+      mSelectedVideoStreamTrack->AddVideoOutput(container);
     }
     VideoTrack* videoTrack = VideoTracks()->GetSelectedTrack();
     if (videoTrack) {
       VideoStreamTrack* videoStreamTrack = videoTrack->GetVideoStreamTrack();
       if (videoStreamTrack && container) {
         videoStreamTrack->AddVideoOutput(container);
       }
     }
   } else {
     if (stream) {
       mSrcStreamPausedCurrentTime = CurrentTime();
 
       stream->RemoveListener(mMediaStreamListener);
 
       stream->RemoveAudioOutput(this);
       VideoFrameContainer* container = GetVideoFrameContainer();
-      if (container) {
-        stream->RemoveVideoOutput(container);
+      if (mSelectedVideoStreamTrack && container) {
+        mSelectedVideoStreamTrack->RemoveVideoOutput(container);
       }
       VideoTrack* videoTrack = VideoTracks()->GetSelectedTrack();
       if (videoTrack) {
         VideoStreamTrack* videoStreamTrack = videoTrack->GetVideoStreamTrack();
         if (videoStreamTrack && container) {
           videoStreamTrack->RemoveVideoOutput(container);
         }
       }
@@ -3786,16 +3786,19 @@ void HTMLMediaElement::ConstructMediaTra
     // must be selected.
     int index = firstEnabledVideo >= 0 ? firstEnabledVideo : 0;
     (*VideoTracks())[index]->SetEnabledInternal(true, MediaTrack::FIRE_NO_EVENTS);
     VideoTrack* track = (*VideoTracks())[index];
     VideoStreamTrack* streamTrack = track->GetVideoStreamTrack();
     mMediaStreamSizeListener = new StreamSizeListener(this);
     streamTrack->AddDirectListener(mMediaStreamSizeListener);
     mSelectedVideoStreamTrack = streamTrack;
+    if (GetVideoFrameContainer()) {
+      mSelectedVideoStreamTrack->AddVideoOutput(GetVideoFrameContainer());
+    }
   }
 }
 
 void
 HTMLMediaElement::NotifyMediaStreamTrackAdded(const RefPtr<MediaStreamTrack>& aTrack)
 {
   MOZ_ASSERT(aTrack);
 
@@ -3818,16 +3821,20 @@ HTMLMediaElement::NotifyMediaStreamTrack
     // New MediaStreamTrack added, set the new added video track as selected
     // video track when there is no selected track.
     if (selectedIndex == -1) {
       MOZ_ASSERT(!mSelectedVideoStreamTrack);
       videoTrack->SetEnabledInternal(true, MediaTrack::FIRE_NO_EVENTS);
       mMediaStreamSizeListener = new StreamSizeListener(this);
       t->AddDirectListener(mMediaStreamSizeListener);
       mSelectedVideoStreamTrack = t;
+      VideoFrameContainer* container = GetVideoFrameContainer();
+      if (mSrcStreamIsPlaying && container) {
+        mSelectedVideoStreamTrack->AddVideoOutput(container);
+      }
     }
 
   }
 }
 
 void
 HTMLMediaElement::NotifyMediaStreamTrackRemoved(const RefPtr<MediaStreamTrack>& aTrack)
 {
@@ -3847,16 +3854,20 @@ HTMLMediaElement::NotifyMediaStreamTrack
     // If the removed media stream track is selected video track and there are
     // still video tracks, change the selected video track to the first
     // remaining track.
     if (aTrack == mSelectedVideoStreamTrack) {
       // The mMediaStreamSizeListener might already reset to nullptr.
       if (mMediaStreamSizeListener) {
         mSelectedVideoStreamTrack->RemoveDirectListener(mMediaStreamSizeListener);
       }
+      VideoFrameContainer* container = GetVideoFrameContainer();
+      if (mSrcStreamIsPlaying && container) {
+        mSelectedVideoStreamTrack->RemoveVideoOutput(container);
+      }
       mSelectedVideoStreamTrack = nullptr;
       MOZ_ASSERT(mSrcStream);
       nsTArray<RefPtr<VideoStreamTrack>> tracks;
       mSrcStream->GetVideoTracks(tracks);
 
       for (const RefPtr<VideoStreamTrack>& track : tracks) {
         if (track->Ended()) {
           continue;
@@ -3870,16 +3881,19 @@ HTMLMediaElement::NotifyMediaStreamTrack
         MediaTrack* videoTrack = VideoTracks()->GetTrackById(trackId);
         MOZ_ASSERT(videoTrack);
 
         videoTrack->SetEnabledInternal(true, MediaTrack::FIRE_NO_EVENTS);
         if (mMediaStreamSizeListener) {
           track->AddDirectListener(mMediaStreamSizeListener);
         }
         mSelectedVideoStreamTrack = track;
+        if (container) {
+          mSelectedVideoStreamTrack->AddVideoOutput(container);
+        }
         return;
       }
 
       // There is no enabled video track existing, clean the
       // mMediaStreamSizeListener.
       if (mMediaStreamSizeListener) {
         mMediaStreamSizeListener->Forget();
         mMediaStreamSizeListener = nullptr;
--- a/dom/media/MediaStreamGraph.cpp
+++ b/dom/media/MediaStreamGraph.cpp
@@ -104,16 +104,25 @@ MediaStreamGraphImpl::FinishStream(Media
 
   SetStreamOrderDirty();
 }
 
 void
 MediaStreamGraphImpl::AddStreamGraphThread(MediaStream* aStream)
 {
   aStream->mTracksStartTime = mProcessedTime;
+
+  if (aStream->AsSourceStream()) {
+    SourceMediaStream* source = aStream->AsSourceStream();
+    TimeStamp currentTimeStamp = CurrentDriver()->GetCurrentTimeStamp();
+    TimeStamp processedTimeStamp = currentTimeStamp +
+      TimeDuration::FromSeconds(MediaTimeToSeconds(mProcessedTime - IterationEnd()));
+    source->SetStreamTracksStartTimeStamp(processedTimeStamp);
+  }
+
   if (aStream->IsSuspended()) {
     mSuspendedStreams.AppendElement(aStream);
     STREAM_LOG(LogLevel::Debug, ("Adding media stream %p to the graph, in the suspended stream array", aStream));
   } else {
     mStreams.AppendElement(aStream);
     STREAM_LOG(LogLevel::Debug, ("Adding media stream %p to graph %p, count %lu", aStream, this, mStreams.Length()));
     LIFECYCLE_LOG("Adding media stream %p to graph %p, count %lu", aStream, this, mStreams.Length());
   }
@@ -923,214 +932,16 @@ MediaStreamGraphImpl::PlayAudio(MediaStr
     // Need unique id for stream & track - and we want it to match the inserter
     output.WriteTo(LATENCY_STREAM_ID(aStream, track->GetID()),
                                      mMixer, AudioChannelCount(),
                                      mSampleRate);
   }
   return ticksWritten;
 }
 
-static void
-SetImageToBlackPixel(PlanarYCbCrImage* aImage)
-{
-  uint8_t blackPixel[] = { 0x10, 0x80, 0x80 };
-
-  PlanarYCbCrData data;
-  data.mYChannel = blackPixel;
-  data.mCbChannel = blackPixel + 1;
-  data.mCrChannel = blackPixel + 2;
-  data.mYStride = data.mCbCrStride = 1;
-  data.mPicSize = data.mYSize = data.mCbCrSize = IntSize(1, 1);
-  aImage->CopyData(data);
-}
-
-class VideoFrameContainerInvalidateRunnable : public Runnable {
-public:
-  explicit VideoFrameContainerInvalidateRunnable(VideoFrameContainer* aVideoFrameContainer)
-    : mVideoFrameContainer(aVideoFrameContainer)
-  {}
-  NS_IMETHOD Run()
-  {
-    MOZ_ASSERT(NS_IsMainThread());
-
-    mVideoFrameContainer->Invalidate();
-
-    return NS_OK;
-  }
-private:
-  RefPtr<VideoFrameContainer> mVideoFrameContainer;
-};
-
-void
-MediaStreamGraphImpl::PlayVideo(MediaStream* aStream)
-{
-  MOZ_ASSERT(mRealtime, "Should only attempt to play video in realtime mode");
-
-  if (aStream->mVideoOutputs.IsEmpty())
-    return;
-
-  TimeStamp currentTimeStamp = CurrentDriver()->GetCurrentTimeStamp();
-
-  // Collect any new frames produced in this iteration.
-  AutoTArray<ImageContainer::NonOwningImage,4> newImages;
-  PrincipalHandle lastPrincipalHandle = PRINCIPAL_HANDLE_NONE;
-  RefPtr<Image> blackImage;
-
-  MOZ_ASSERT(mProcessedTime >= aStream->mTracksStartTime, "frame position before buffer?");
-  // We only look at the non-blocking interval
-  StreamTime frameBufferTime = aStream->GraphTimeToStreamTime(mProcessedTime);
-  StreamTime bufferEndTime = aStream->GraphTimeToStreamTime(aStream->mStartBlocking);
-  StreamTime start;
-  const VideoChunk* chunk;
-  for ( ;
-       frameBufferTime < bufferEndTime;
-       frameBufferTime = start + chunk->GetDuration()) {
-    // Pick the last track that has a video chunk for the time, and
-    // schedule its frame.
-    chunk = nullptr;
-    for (StreamTracks::TrackIter tracks(aStream->GetStreamTracks(),
-                                        MediaSegment::VIDEO);
-         !tracks.IsEnded();
-         tracks.Next()) {
-      VideoSegment* segment = tracks->Get<VideoSegment>();
-      StreamTime thisStart;
-      const VideoChunk* thisChunk =
-        segment->FindChunkContaining(frameBufferTime, &thisStart);
-      if (thisChunk && thisChunk->mFrame.GetImage()) {
-        start = thisStart;
-        chunk = thisChunk;
-      }
-    }
-    if (!chunk)
-      break;
-
-    const VideoFrame* frame = &chunk->mFrame;
-    if (*frame == aStream->mLastPlayedVideoFrame) {
-      continue;
-    }
-
-    Image* image = frame->GetImage();
-    STREAM_LOG(LogLevel::Verbose,
-               ("MediaStream %p writing video frame %p (%dx%d)",
-                aStream, image, frame->GetIntrinsicSize().width,
-                frame->GetIntrinsicSize().height));
-    // Schedule this frame after the previous frame finishes, instead of at
-    // its start time.  These times only differ in the case of multiple
-    // tracks.
-    // frameBufferTime is in the non-blocking interval.
-    GraphTime frameTime = aStream->StreamTimeToGraphTime(frameBufferTime);
-    TimeStamp targetTime = currentTimeStamp +
-      TimeDuration::FromSeconds(MediaTimeToSeconds(frameTime - IterationEnd()));
-
-    if (frame->GetForceBlack()) {
-      if (!blackImage) {
-        // Fixme: PlayVideo will be replaced in latter changeset
-        // "Call MediaStreamVideoSink::setCurrentFrames in SourceMediaStream::AppendToTrack."
-        // of this bug.
-        // This is a temp workaround to pass the build and test.
-        if (!aStream->mVideoOutputs[0].mListener->AsVideoFrameContainer()) {
-          return;
-        }
-        blackImage = aStream->mVideoOutputs[0].mListener->AsVideoFrameContainer()->
-          GetImageContainer()->CreatePlanarYCbCrImage();
-        if (blackImage) {
-          // Sets the image to a single black pixel, which will be scaled to
-          // fill the rendered size.
-          SetImageToBlackPixel(blackImage->AsPlanarYCbCrImage());
-        }
-      }
-      if (blackImage) {
-        image = blackImage;
-      }
-    }
-    newImages.AppendElement(ImageContainer::NonOwningImage(image, targetTime));
-
-    lastPrincipalHandle = chunk->GetPrincipalHandle();
-
-    aStream->mLastPlayedVideoFrame = *frame;
-  }
-
-  if (!aStream->mLastPlayedVideoFrame.GetImage())
-    return;
-
-  AutoTArray<ImageContainer::NonOwningImage,4> images;
-  bool haveMultipleImages = false;
-
-  for (const TrackBound<MediaStreamVideoSink>& sink : aStream->mVideoOutputs) {
-    VideoFrameContainer* output = sink.mListener->AsVideoFrameContainer();
-    if (!output) {
-      continue;
-    }
-
-    bool principalHandleChanged =
-      lastPrincipalHandle != PRINCIPAL_HANDLE_NONE &&
-      lastPrincipalHandle != output->GetLastPrincipalHandle();
-
-    // Find previous frames that may still be valid.
-    AutoTArray<ImageContainer::OwningImage,4> previousImages;
-    output->GetImageContainer()->GetCurrentImages(&previousImages);
-    uint32_t j = previousImages.Length();
-    if (j) {
-      // Re-use the most recent frame before currentTimeStamp and subsequent,
-      // always keeping at least one frame.
-      do {
-        --j;
-      } while (j > 0 && previousImages[j].mTimeStamp > currentTimeStamp);
-    }
-    if (previousImages.Length() - j + newImages.Length() > 1) {
-      haveMultipleImages = true;
-    }
-
-    // Don't update if there are no changes.
-    if (j == 0 && newImages.IsEmpty())
-      continue;
-
-    for ( ; j < previousImages.Length(); ++j) {
-      const auto& image = previousImages[j];
-      // Cope with potential clock skew with AudioCallbackDriver.
-      if (newImages.Length() && image.mTimeStamp > newImages[0].mTimeStamp) {
-        STREAM_LOG(LogLevel::Warning,
-                   ("Dropping %u video frames due to clock skew",
-                    unsigned(previousImages.Length() - j)));
-        break;
-      }
-
-      images.AppendElement(ImageContainer::
-                           NonOwningImage(image.mImage,
-                                          image.mTimeStamp, image.mFrameID));
-    }
-
-    // Add the frames from this iteration.
-    for (auto& image : newImages) {
-      image.mFrameID = output->NewFrameID();
-      images.AppendElement(image);
-    }
-
-    if (principalHandleChanged) {
-      output->UpdatePrincipalHandleForFrameID(lastPrincipalHandle,
-                                              newImages.LastElement().mFrameID);
-    }
-
-    output->SetCurrentFrames(aStream->mLastPlayedVideoFrame.GetIntrinsicSize(),
-                             images);
-
-    nsCOMPtr<nsIRunnable> event =
-      new VideoFrameContainerInvalidateRunnable(output);
-    DispatchToMainThreadAfterStreamStateUpdate(event.forget());
-
-    images.ClearAndRetainStorage();
-  }
-
-  // If the stream has finished and the timestamps of all frames have expired
-  // then no more updates are required.
-  if (aStream->mFinished && !haveMultipleImages) {
-    aStream->mLastPlayedVideoFrame.SetNull();
-  }
-}
-
 void
 MediaStreamGraphImpl::OpenAudioInputImpl(int aID,
                                          AudioDataListener *aListener)
 {
   // Bug 1238038 Need support for multiple mics at once
   if (mInputDeviceUsers.Count() > 0 &&
       !mInputDeviceUsers.Get(aListener, nullptr)) {
     NS_ASSERTION(false, "Input from multiple mics not yet supported; bug 1238038");
@@ -1534,17 +1345,16 @@ MediaStreamGraphImpl::Process()
         StreamTime ticksPlayedForThisStream = PlayAudio(stream);
         if (!ticksPlayed) {
           ticksPlayed = ticksPlayedForThisStream;
         } else {
           MOZ_ASSERT(!ticksPlayedForThisStream || ticksPlayedForThisStream == ticksPlayed,
               "Each stream should have the same number of frame.");
         }
       }
-      PlayVideo(stream);
     }
     if (stream->mStartBlocking > mProcessedTime) {
       allBlockedForever = false;
     }
   }
 
   if (CurrentDriver()->AsAudioCallbackDriver() && ticksPlayed) {
     mMixer.FinishMixing();
@@ -2831,16 +2641,26 @@ SourceMediaStream::ResampleAudioToGraphS
       return;
     }
     aTrackData->mResampler.own(state);
     aTrackData->mResamplerChannelCount = channels;
   }
   segment->ResampleChunks(aTrackData->mResampler, aTrackData->mInputRate, GraphImpl()->GraphRate());
 }
 
+void
+SourceMediaStream::AdvanceTimeVaryingValuesToCurrentTime(GraphTime aCurrentTime,
+                                                         GraphTime aBlockedTime)
+{
+  MutexAutoLock lock(mMutex);
+  mTracksStartTime += aBlockedTime;
+  mStreamTracksStartTimeStamp += TimeDuration::FromSeconds(GraphImpl()->MediaTimeToSeconds(aBlockedTime));
+  mTracks.ForgetUpTo(aCurrentTime - mTracksStartTime);
+}
+
 bool
 SourceMediaStream::AppendToTrack(TrackID aID, MediaSegment* aSegment, MediaSegment *aRawSegment)
 {
   MutexAutoLock lock(mMutex);
   // ::EndAllTrackAndFinished() can end these before the sources notice
   bool appended = false;
   auto graph = GraphImpl();
   if (!mFinished && graph) {
@@ -2953,31 +2773,44 @@ SourceMediaStream::RemoveDirectListener(
 }
 
 void
 SourceMediaStream::AddDirectTrackListenerImpl(already_AddRefed<DirectMediaStreamTrackListener> aListener,
                                               TrackID aTrackID)
 {
   MOZ_ASSERT(IsTrackIDExplicit(aTrackID));
   TrackData* data;
-  bool found;
-  bool isAudio;
-  bool isVideo;
+  bool found = false;
+  bool isAudio = false;
+  bool isVideo = false;
   RefPtr<DirectMediaStreamTrackListener> listener = aListener;
   STREAM_LOG(LogLevel::Debug, ("Adding direct track listener %p bound to track %d to source stream %p",
              listener.get(), aTrackID, this));
 
   {
     MutexAutoLock lock(mMutex);
     data = FindDataForTrack(aTrackID);
     found = !!data;
     if (found) {
       isAudio = data->mData->GetType() == MediaSegment::AUDIO;
       isVideo = data->mData->GetType() == MediaSegment::VIDEO;
     }
+
+    // The track might be removed from mUpdateTrack but still exist in
+    // mTracks.
+    auto streamTrack = FindTrack(aTrackID);
+    bool foundTrack = !!streamTrack;
+    if (foundTrack) {
+      MediaStreamVideoSink* videoSink = listener->AsMediaStreamVideoSink();
+      // Re-send missed VideoSegment to new added MediaStreamVideoSink.
+      if (streamTrack->GetType() == MediaSegment::VIDEO && videoSink) {
+        videoSink->SetCurrentFrames(*(static_cast<VideoSegment*>(streamTrack->GetSegment())));
+      }
+    }
+
     if (found && (isAudio || isVideo)) {
       for (auto entry : mDirectTrackListeners) {
         if (entry.mListener == listener &&
             (entry.mTrackID == TRACK_ANY || entry.mTrackID == aTrackID)) {
           listener->NotifyDirectListenerInstalled(
             DirectMediaStreamTrackListener::InstallationResult::ALREADY_EXISTS);
           return;
         }
--- a/dom/media/MediaStreamGraph.h
+++ b/dom/media/MediaStreamGraph.h
@@ -538,17 +538,19 @@ public:
   void IncrementSuspendCount() { ++mSuspendedCount; }
   void DecrementSuspendCount()
   {
     NS_ASSERTION(mSuspendedCount > 0, "Suspend count underrun");
     --mSuspendedCount;
   }
 
 protected:
-  void AdvanceTimeVaryingValuesToCurrentTime(GraphTime aCurrentTime, GraphTime aBlockedTime)
+  // |AdvanceTimeVaryingValuesToCurrentTime| will be override in SourceMediaStream.
+  virtual void AdvanceTimeVaryingValuesToCurrentTime(GraphTime aCurrentTime,
+                                                     GraphTime aBlockedTime)
   {
     mTracksStartTime += aBlockedTime;
     mTracks.ForgetUpTo(aCurrentTime - mTracksStartTime);
   }
 
   void NotifyMainThreadListeners()
   {
     NS_ASSERTION(NS_IsMainThread(), "Call only on main thread");
@@ -800,16 +802,21 @@ public:
 
   /**
    * Returns true if this SourceMediaStream contains at least one audio track
    * that is in pending state.
    * This is thread safe, and takes the SourceMediaStream mutex.
    */
   bool HasPendingAudioTrack();
 
+  TimeStamp GetStreamTracksStrartTimeStamp() {
+    MutexAutoLock lock(mMutex);
+    return mStreamTracksStartTimeStamp;
+  }
+
   // XXX need a Reset API
 
   friend class MediaStreamGraphImpl;
 
 protected:
   enum TrackCommands : uint32_t;
 
   virtual ~SourceMediaStream();
@@ -864,27 +871,40 @@ protected:
    * Notify direct consumers of new data to one of the stream tracks.
    * The data doesn't have to be resampled (though it may be).  This is called
    * from AppendToTrack on the thread providing the data, and will call
    * the Listeners on this thread.
    */
   void NotifyDirectConsumers(TrackData *aTrack,
                              MediaSegment *aSegment);
 
+  virtual void
+  AdvanceTimeVaryingValuesToCurrentTime(GraphTime aCurrentTime,
+                                        GraphTime aBlockedTime) override;
+  void SetStreamTracksStartTimeStamp(const TimeStamp& aTimeStamp)
+  {
+    MutexAutoLock lock(mMutex);
+    mStreamTracksStartTimeStamp = aTimeStamp;
+  }
+
   // Only accessed on the MSG thread.  Used so to ask the MSGImpl to usecount
   // users of a specific input.
   // XXX Should really be a CubebUtils::AudioDeviceID, but they aren't
   // copyable (opaque pointers)
   RefPtr<AudioDataListener> mInputListener;
 
   // This must be acquired *before* MediaStreamGraphImpl's lock, if they are
   // held together.
   Mutex mMutex;
   // protected by mMutex
   StreamTime mUpdateKnownTracksTime;
+  // This time stamp will be updated in adding and blocked SourceMediaStream,
+  // |AddStreamGraphThread| and |AdvanceTimeVaryingValuesToCurrentTime| in
+  // particularly.
+  TimeStamp mStreamTracksStartTimeStamp;
   nsTArray<TrackData> mUpdateTracks;
   nsTArray<TrackData> mPendingTracks;
   nsTArray<RefPtr<DirectMediaStreamListener>> mDirectListeners;
   nsTArray<TrackBound<DirectMediaStreamTrackListener>> mDirectTrackListeners;
   bool mPullEnabled;
   bool mUpdateFinished;
   bool mNeedsMixing;
 };
--- a/dom/media/MediaStreamGraphImpl.h
+++ b/dom/media/MediaStreamGraphImpl.h
@@ -407,20 +407,16 @@ public:
    */
   void CreateOrDestroyAudioStreams(MediaStream* aStream);
   /**
    * Queue audio (mix of stream audio and silence for blocked intervals)
    * to the audio output stream. Returns the number of frames played.
    */
   StreamTime PlayAudio(MediaStream* aStream);
   /**
-   * Set the correct current video frame for stream aStream.
-   */
-  void PlayVideo(MediaStream* aStream);
-  /**
    * No more data will be forthcoming for aStream. The stream will end
    * at the current buffer end point. The StreamTracks's tracks must be
    * explicitly set to finished by the caller.
    */
   void OpenAudioInputImpl(int aID,
                           AudioDataListener *aListener);
   virtual nsresult OpenAudioInput(int aID,
                                   AudioDataListener *aListener) override;
--- a/dom/media/TrackUnionStream.h
+++ b/dom/media/TrackUnionStream.h
@@ -25,16 +25,18 @@ public:
   void RemoveInput(MediaInputPort* aPort) override;
   void ProcessInput(GraphTime aFrom, GraphTime aTo, uint32_t aFlags) override;
 
   void SetTrackEnabledImpl(TrackID aTrackID, bool aEnabled) override;
 
   MediaStream* GetInputStreamFor(TrackID aTrackID) override;
   TrackID GetInputTrackIDFor(TrackID aTrackID) override;
 
+  friend class MediaStreamGraphImpl;
+
 protected:
   // Only non-ended tracks are allowed to persist in this map.
   struct TrackMapEntry {
     // mEndOfConsumedInputTicks is the end of the input ticks that we've consumed.
     // 0 if we haven't consumed any yet.
     StreamTime mEndOfConsumedInputTicks;
     // mEndOfLastInputIntervalInInputStream is the timestamp for the end of the
     // previous interval which was unblocked for both the input and output
--- a/dom/media/VideoFrameContainer.cpp
+++ b/dom/media/VideoFrameContainer.cpp
@@ -34,23 +34,34 @@ VideoFrameContainer::VideoFrameContainer
 }
 
 VideoFrameContainer::~VideoFrameContainer()
 {}
 
 PrincipalHandle VideoFrameContainer::GetLastPrincipalHandle()
 {
   MutexAutoLock lock(mMutex);
+  return GetLastPrincipalHandleLocked();
+}
+
+PrincipalHandle VideoFrameContainer::GetLastPrincipalHandleLocked()
+{
   return mLastPrincipalHandle;
 }
 
 void VideoFrameContainer::UpdatePrincipalHandleForFrameID(const PrincipalHandle& aPrincipalHandle,
                                                           const ImageContainer::FrameID& aFrameID)
 {
   MutexAutoLock lock(mMutex);
+  UpdatePrincipalHandleForFrameIDLocked(aPrincipalHandle, aFrameID);
+}
+
+void VideoFrameContainer::UpdatePrincipalHandleForFrameIDLocked(const PrincipalHandle& aPrincipalHandle,
+                                                                const ImageContainer::FrameID& aFrameID)
+{
   if (mPendingPrincipalHandle == aPrincipalHandle) {
     return;
   }
   mPendingPrincipalHandle = aPrincipalHandle;
   mFrameIDForPendingPrincipalHandle = aFrameID;
 }
 
 static void
@@ -142,27 +153,27 @@ void VideoFrameContainer::SetCurrentFram
   if (newImages.IsEmpty()) {
     return;
   }
 
   AutoTArray<ImageContainer::NonOwningImage,4> images;
 
   bool principalHandleChanged =
      lastPrincipalHandle != PRINCIPAL_HANDLE_NONE &&
-     lastPrincipalHandle != GetLastPrincipalHandle();
+     lastPrincipalHandle != GetLastPrincipalHandleLocked();
 
   // Add the frames from this iteration.
   for (auto& image : newImages) {
     image.mFrameID = NewFrameID();
     images.AppendElement(image);
   }
 
   if (principalHandleChanged) {
-    UpdatePrincipalHandleForFrameID(lastPrincipalHandle,
-                                    newImages.LastElement().mFrameID);
+    UpdatePrincipalHandleForFrameIDLocked(lastPrincipalHandle,
+                                          newImages.LastElement().mFrameID);
   }
 
   SetCurrentFramesLocked(mLastPlayedVideoFrame.GetIntrinsicSize(), images);
   nsCOMPtr<nsIRunnable> event =
     new VideoFrameContainerInvalidateRunnable(this);
   NS_DispatchToMainThread(event.forget());
 
   images.ClearAndRetainStorage();
--- a/dom/media/VideoFrameContainer.h
+++ b/dom/media/VideoFrameContainer.h
@@ -43,21 +43,24 @@ public:
 
   // Call on any thread
   virtual void SetCurrentFrames(const VideoSegment& aSegment) override;
   virtual void ClearFrames() override;
   void SetCurrentFrame(const gfx::IntSize& aIntrinsicSize, Image* aImage,
                        const TimeStamp& aTargetTime);
   // Returns the last principalHandle we notified mElement about.
   PrincipalHandle GetLastPrincipalHandle();
+  PrincipalHandle GetLastPrincipalHandleLocked();
   // We will notify mElement that aPrincipalHandle has been applied when all
   // FrameIDs prior to aFrameID have been flushed out.
   // aFrameID is ignored if aPrincipalHandle already is our pending principalHandle.
   void UpdatePrincipalHandleForFrameID(const PrincipalHandle& aPrincipalHandle,
                                        const ImageContainer::FrameID& aFrameID);
+  void UpdatePrincipalHandleForFrameIDLocked(const PrincipalHandle& aPrincipalHandle,
+                                             const ImageContainer::FrameID& aFrameID);
   void SetCurrentFrames(const gfx::IntSize& aIntrinsicSize,
                         const nsTArray<ImageContainer::NonOwningImage>& aImages);
   void ClearCurrentFrame(const gfx::IntSize& aIntrinsicSize)
   {
     SetCurrentFrames(aIntrinsicSize, nsTArray<ImageContainer::NonOwningImage>());
   }
   VideoFrameContainer* AsVideoFrameContainer() override { return this; }
 
--- a/dom/media/VideoSegment.cpp
+++ b/dom/media/VideoSegment.cpp
@@ -95,19 +95,21 @@ VideoChunk::VideoChunk()
 VideoChunk::~VideoChunk()
 {}
 
 void
 VideoSegment::AppendFrame(already_AddRefed<Image>&& aImage,
                           StreamTime aDuration,
                           const IntSize& aIntrinsicSize,
                           const PrincipalHandle& aPrincipalHandle,
-                          bool aForceBlack)
+                          bool aForceBlack,
+                          TimeStamp aTimeStamp)
 {
   VideoChunk* chunk = AppendChunk(aDuration);
+  chunk->mTimeStamp = aTimeStamp;
   VideoFrame frame(aImage, aIntrinsicSize);
   frame.SetForceBlack(aForceBlack);
   frame.SetPrincipalHandle(aPrincipalHandle);
   chunk->mFrame.TakeFrom(&frame);
 }
 
 VideoSegment::VideoSegment()
   : MediaSegmentBase<VideoSegment, VideoChunk>(VIDEO)
--- a/dom/media/VideoSegment.h
+++ b/dom/media/VideoSegment.h
@@ -104,17 +104,18 @@ public:
 
   VideoSegment();
   ~VideoSegment();
 
   void AppendFrame(already_AddRefed<Image>&& aImage,
                    StreamTime aDuration,
                    const IntSize& aIntrinsicSize,
                    const PrincipalHandle& aPrincipalHandle,
-                   bool aForceBlack = false);
+                   bool aForceBlack = false,
+                   TimeStamp aTimeStamp = TimeStamp::Now());
   const VideoFrame* GetLastFrame(StreamTime* aStart = nullptr)
   {
     VideoChunk* c = GetLastChunk();
     if (!c) {
       return nullptr;
     }
     if (aStart) {
       *aStart = mDuration - c->mDuration;
--- a/dom/media/mediasink/DecodedStream.cpp
+++ b/dom/media/mediasink/DecodedStream.cpp
@@ -493,24 +493,26 @@ DecodedStream::SendAudio(double aVolume,
 }
 
 static void
 WriteVideoToMediaStream(MediaStream* aStream,
                         layers::Image* aImage,
                         int64_t aEndMicroseconds,
                         int64_t aStartMicroseconds,
                         const mozilla::gfx::IntSize& aIntrinsicSize,
+                        const TimeStamp& aTimeStamp,
                         VideoSegment* aOutput,
                         const PrincipalHandle& aPrincipalHandle)
 {
   RefPtr<layers::Image> image = aImage;
   StreamTime duration =
       aStream->MicrosecondsToStreamTimeRoundDown(aEndMicroseconds) -
       aStream->MicrosecondsToStreamTimeRoundDown(aStartMicroseconds);
-  aOutput->AppendFrame(image.forget(), duration, aIntrinsicSize, aPrincipalHandle);
+  aOutput->AppendFrame(image.forget(), duration, aIntrinsicSize,
+                       aPrincipalHandle, false, aTimeStamp);
 }
 
 static bool
 ZeroDurationAtLastChunk(VideoSegment& aInput)
 {
   // Get the last video frame's start time in VideoSegment aInput.
   // If the start time is equal to the duration of aInput, means the last video
   // frame's duration is zero.
@@ -532,38 +534,48 @@ DecodedStream::SendVideo(bool aIsSameOri
   TrackID videoTrackId = mInfo.mVideo.mTrackId;
   AutoTArray<RefPtr<MediaData>, 10> video;
   SourceMediaStream* sourceStream = mData->mStream;
 
   // It's OK to hold references to the VideoData because VideoData
   // is ref-counted.
   mVideoQueue.GetElementsAfter(mData->mNextVideoTime, &video);
 
+  // tracksStartTimeStamp might be null when the SourceMediaStream not yet
+  // be added to MediaStreamGraph.
+  TimeStamp tracksStartTimeStamp = sourceStream->GetStreamTracksStrartTimeStamp();
+  if (tracksStartTimeStamp.IsNull()) {
+    tracksStartTimeStamp = TimeStamp::Now();
+  }
+
   for (uint32_t i = 0; i < video.Length(); ++i) {
     VideoData* v = video[i]->As<VideoData>();
 
     if (mData->mNextVideoTime < v->mTime) {
       // Write last video frame to catch up. mLastVideoImage can be null here
       // which is fine, it just means there's no video.
 
       // TODO: |mLastVideoImage| should come from the last image rendered
       // by the state machine. This will avoid the black frame when capture
       // happens in the middle of playback (especially in th middle of a
       // video frame). E.g. if we have a video frame that is 30 sec long
       // and capture happens at 15 sec, we'll have to append a black frame
       // that is 15 sec long.
       WriteVideoToMediaStream(sourceStream, mData->mLastVideoImage, v->mTime,
-          mData->mNextVideoTime, mData->mLastVideoImageDisplaySize, &output,
-          aPrincipalHandle);
+          mData->mNextVideoTime, mData->mLastVideoImageDisplaySize,
+          tracksStartTimeStamp + TimeDuration::FromMicroseconds(v->mTime),
+          &output, aPrincipalHandle);
       mData->mNextVideoTime = v->mTime;
     }
 
     if (mData->mNextVideoTime < v->GetEndTime()) {
       WriteVideoToMediaStream(sourceStream, v->mImage, v->GetEndTime(),
-          mData->mNextVideoTime, v->mDisplay, &output, aPrincipalHandle);
+          mData->mNextVideoTime, v->mDisplay,
+          tracksStartTimeStamp + TimeDuration::FromMicroseconds(v->GetEndTime()),
+          &output, aPrincipalHandle);
       mData->mNextVideoTime = v->GetEndTime();
       mData->mLastVideoImage = v->mImage;
       mData->mLastVideoImageDisplaySize = v->mDisplay;
     }
   }
 
   // Check the output is not empty.
   if (output.GetLastFrame()) {
@@ -580,17 +592,19 @@ DecodedStream::SendVideo(bool aIsSameOri
 
   if (mVideoQueue.IsFinished() && !mData->mHaveSentFinishVideo) {
     if (mData->mEOSVideoCompensation) {
       VideoSegment endSegment;
       // Calculate the deviation clock time from DecodedStream.
       int64_t deviation_usec = sourceStream->StreamTimeToMicroseconds(1);
       WriteVideoToMediaStream(sourceStream, mData->mLastVideoImage,
           mData->mNextVideoTime + deviation_usec, mData->mNextVideoTime,
-          mData->mLastVideoImageDisplaySize, &endSegment, aPrincipalHandle);
+          mData->mLastVideoImageDisplaySize,
+          tracksStartTimeStamp + TimeDuration::FromMicroseconds(mData->mNextVideoTime + deviation_usec),
+          &endSegment, aPrincipalHandle);
       mData->mNextVideoTime += deviation_usec;
       MOZ_ASSERT(endSegment.GetDuration() > 0);
       if (!aIsSameOrigin) {
         endSegment.ReplaceWithDisabled();
       }
       sourceStream->AppendToTrack(videoTrackId, &endSegment);
     }
     sourceStream->EndTrack(videoTrackId);