Bug 1354457 - Record time spent paused in media encoder and adjust time stamps when muxing. r?pehrsons draft
authorBryce Van Dyk <bvandyk@mozilla.com>
Fri, 05 May 2017 15:13:11 +1200
changeset 577597 14e261387598aff5adc9953a191a9d9b993fc302
parent 577596 6f51bc4c67651e336d2676b169b1c4abb46670c4
child 577598 692a0b6cf1bb1899fa96649ceadfdb8d9a317be1
child 579215 b38b83eac64c32f24f9060866196d7dd963f8c26
child 581172 b20fb616ee3343376208679d33d38f0f8b00ce5e
child 581173 f93bdef4e72d124c69fae80770f1ae7920c9f4e9
push id58728
push userbvandyk@mozilla.com
push dateMon, 15 May 2017 02:52:45 +0000
reviewerspehrsons
bugs1354457
milestone55.0a1
Bug 1354457 - Record time spent paused in media encoder and adjust time stamps when muxing. r?pehrsons MozReview-Commit-ID: 7J6oQf7qn4u
dom/media/encoder/MediaEncoder.cpp
dom/media/encoder/MediaEncoder.h
--- a/dom/media/encoder/MediaEncoder.cpp
+++ b/dom/media/encoder/MediaEncoder.cpp
@@ -1,16 +1,17 @@
 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-*/
 /* This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this file,
  * You can obtain one at http://mozilla.org/MPL/2.0/. */
 #include "MediaEncoder.h"
 #include "MediaDecoder.h"
 #include "nsIPrincipal.h"
 #include "nsMimeTypes.h"
+#include "TimeUnits.h"
 #include "mozilla/Logging.h"
 #include "mozilla/Preferences.h"
 #include "mozilla/StaticPtr.h"
 #include "mozilla/gfx/Point.h" // IntSize
 
 #include"GeckoProfiler.h"
 #include "OggWriter.h"
 #include "OpusTrackEncoder.h"
@@ -35,16 +36,42 @@ MediaStreamVideoRecorderSink::SetCurrent
   MOZ_ASSERT(mVideoEncoder);
   // If we're suspended (paused) we don't forward frames
   if (!mSuspended) {
     mVideoEncoder->SetCurrentFrames(aSegment);
   }
 }
 
 void
+MediaEncoder::Suspend()
+{
+  MOZ_ASSERT(NS_IsMainThread());
+  mLastPauseStartTime = TimeStamp::Now();
+  mSuspended = true;
+  mVideoSink->Suspend();
+}
+
+void
+MediaEncoder::Resume()
+{
+  MOZ_ASSERT(NS_IsMainThread());
+  if (!mSuspended) {
+    return;
+  }
+  media::TimeUnit timeSpentPaused =
+    media::TimeUnit::FromTimeDuration(
+      TimeStamp::Now() - mLastPauseStartTime);
+  MOZ_ASSERT(timeSpentPaused.ToMicroseconds() >= 0);
+  MOZ_RELEASE_ASSERT(timeSpentPaused.IsValid());
+  mMicrosecondsSpentPaused += timeSpentPaused.ToMicroseconds();;
+  mSuspended = false;
+  mVideoSink->Resume();
+}
+
+void
 MediaEncoder::SetDirectConnect(bool aConnected)
 {
   mDirectConnected = aConnected;
 }
 
 void
 MediaEncoder::NotifyRealtimeData(MediaStreamGraph* aGraph,
                                  TrackID aID,
@@ -316,16 +343,39 @@ MediaEncoder::WriteEncodedDataToMuxer(Tr
   EncodedFrameContainer encodedVideoData;
   nsresult rv = aTrackEncoder->GetEncodedTrack(encodedVideoData);
   if (NS_FAILED(rv)) {
     // Encoding might be canceled.
     LOG(LogLevel::Error, ("Error! Fail to get encoded data from video encoder."));
     mState = ENCODE_ERROR;
     return rv;
   }
+
+  // Update timestamps to accommodate pauses
+  const nsTArray<RefPtr<EncodedFrame> >& encodedFrames =
+    encodedVideoData.GetEncodedFrames();
+  // Take a copy of the atomic so we don't continually access it
+  uint64_t microsecondsSpentPaused = mMicrosecondsSpentPaused;
+  for (size_t i = 0; i < encodedFrames.Length(); ++i) {
+    RefPtr<EncodedFrame> frame = encodedFrames[i];
+    if (frame->GetTimeStamp() > microsecondsSpentPaused &&
+        frame->GetTimeStamp() - microsecondsSpentPaused > mLastMuxedTimestamp) {
+      // Use the adjusted timestamp if it's after the last timestamp
+      frame->SetTimeStamp(frame->GetTimeStamp() - microsecondsSpentPaused);
+    } else {
+      // If not, we force the last time stamp. We do this so the frames are
+      // still around and in order in case the codec needs to reference them.
+      // Dropping them here may result in artifacts in playback.
+      frame->SetTimeStamp(mLastMuxedTimestamp);
+    }
+    MOZ_ASSERT(mLastMuxedTimestamp <= frame->GetTimeStamp(),
+      "Our frames should be ordered by this point!");
+    mLastMuxedTimestamp = frame->GetTimeStamp();
+  }
+
   rv = mWriter->WriteEncodedTrack(encodedVideoData,
                                   aTrackEncoder->IsEncodingComplete() ?
                                   ContainerWriter::END_OF_STREAM : 0);
   if (NS_FAILED(rv)) {
     LOG(LogLevel::Error, ("Error! Fail to write encoded video track to the media container."));
     mState = ENCODE_ERROR;
   }
   return rv;
--- a/dom/media/encoder/MediaEncoder.h
+++ b/dom/media/encoder/MediaEncoder.h
@@ -98,38 +98,29 @@ public :
     , mVideoSink(new MediaStreamVideoRecorderSink(mVideoEncoder))
     , mStartTime(TimeStamp::Now())
     , mMIMEType(aMIMEType)
     , mSizeOfBuffer(0)
     , mState(MediaEncoder::ENCODE_METADDATA)
     , mShutdown(false)
     , mDirectConnected(false)
     , mSuspended(false)
+    , mMicrosecondsSpentPaused(0)
+    , mLastMuxedTimestamp(0)
 {}
 
   ~MediaEncoder() {};
 
   /* Note - called from control code, not on MSG threads. */
-  void Suspend()
-  {
-    MOZ_ASSERT(NS_IsMainThread());
-    mSuspended = true;
-    mVideoSink->Suspend();
-  }
+  void Suspend();
 
   /**
    * Note - called from control code, not on MSG threads.
-   * Arm to collect the Duration of the next video frame and give it
-   * to the next frame, in order to avoid any possible loss of sync. */
-  void Resume()
-  {
-    MOZ_ASSERT(NS_IsMainThread());
-    mSuspended = false;
-    mVideoSink->Resume();
-  }
+   * Calculates time spent paused in order to offset frames. */
+  void Resume();
 
   /**
    * Tells us which Notify to pay attention to for media
    */
   void SetDirectConnect(bool aConnected);
 
   /**
    * Notified by the AppendToTrack in MediaStreamGraph; aRealtimeMedia is the raw
@@ -239,17 +230,28 @@ private:
   nsAutoPtr<VideoTrackEncoder> mVideoEncoder;
   RefPtr<MediaStreamVideoRecorderSink> mVideoSink;
   TimeStamp mStartTime;
   nsString mMIMEType;
   int64_t mSizeOfBuffer;
   int mState;
   bool mShutdown;
   bool mDirectConnected;
+  // Tracks if the encoder is suspended (paused). Used on the main thread and
+  // MediaRecorder's read thread.
   Atomic<bool> mSuspended;
+  // Timestamp of when the last pause happened. Should only be accessed on the
+  // main thread.
+  TimeStamp mLastPauseStartTime;
+  // Exposes the time spend paused in microseconds. Read by the main thread
+  // and MediaRecorder's read thread. Should only be written by main thread.
+  Atomic<uint64_t> mMicrosecondsSpentPaused;
+  // The timestamp of the last muxed sample. Should only be used on
+  // MediaRecorder's read thread.
+  uint64_t mLastMuxedTimestamp;
   // Get duration from create encoder, for logging purpose
   double GetEncodeTimeStamp()
   {
     TimeDuration decodeTime;
     decodeTime = TimeStamp::Now() - mStartTime;
     return decodeTime.ToMilliseconds();
   }
 };