Bug 1231848 - Encode every frame as it comes in to VP8TrackEncoder. r?jesup, r?bechen draft
authorAndreas Pehrson <pehrsons@gmail.com>
Thu, 12 Jan 2017 12:19:34 +0100
changeset 460559 8241182f2d808e13297934b0b1e01555e47686a7
parent 460558 2062563ee69c37b7202391a490ce2c113335c669
child 460644 d30ff173fd6c42443250fa7c221c639c2ff6b572
push id41416
push userbmo:pehrson@telenordigital.com
push dateFri, 13 Jan 2017 08:13:12 +0000
reviewersjesup, bechen
bugs1231848
milestone53.0a1
Bug 1231848 - Encode every frame as it comes in to VP8TrackEncoder. r?jesup, r?bechen MozReview-Commit-ID: HZOcj9bqY0A
dom/media/encoder/VP8TrackEncoder.cpp
dom/media/encoder/VP8TrackEncoder.h
--- a/dom/media/encoder/VP8TrackEncoder.cpp
+++ b/dom/media/encoder/VP8TrackEncoder.cpp
@@ -19,17 +19,16 @@
 namespace mozilla {
 
 LazyLogModule gVP8TrackEncoderLog("VP8TrackEncoder");
 #define VP8LOG(msg, ...) MOZ_LOG(gVP8TrackEncoderLog, mozilla::LogLevel::Debug, \
                                   (msg, ##__VA_ARGS__))
 // Debug logging macro with object pointer and class name.
 
 #define DEFAULT_BITRATE_BPS 2500000
-#define DEFAULT_ENCODE_FRAMERATE 30
 
 using namespace mozilla::gfx;
 using namespace mozilla::layers;
 using namespace mozilla::media;
 
 static already_AddRefed<SourceSurface>
 GetSourceSurface(already_AddRefed<Image> aImg)
 {
@@ -51,19 +50,17 @@ GetSourceSurface(already_AddRefed<Image>
   });
 
   NS_DispatchToMainThread(runnable, NS_DISPATCH_SYNC);
   return surf.forget();
 }
 
 VP8TrackEncoder::VP8TrackEncoder(TrackRate aTrackRate)
   : VideoTrackEncoder(aTrackRate)
-  , mEncodedFrameDuration(0)
   , mEncodedTimestamp(0)
-  , mRemainingTicks(0)
   , mVPXContext(new vpx_codec_ctx_t())
   , mVPXImageWrapper(new vpx_image_t())
 {
   MOZ_COUNT_CTOR(VP8TrackEncoder);
 }
 
 VP8TrackEncoder::~VP8TrackEncoder()
 {
@@ -82,18 +79,16 @@ VP8TrackEncoder::Init(int32_t aWidth, in
                       int32_t aDisplayHeight)
 {
   if (aWidth < 1 || aHeight < 1 || aDisplayWidth < 1 || aDisplayHeight < 1) {
     return NS_ERROR_FAILURE;
   }
 
   ReentrantMonitorAutoEnter mon(mReentrantMonitor);
 
-  mEncodedFrameRate = DEFAULT_ENCODE_FRAMERATE;
-  mEncodedFrameDuration = mTrackRate / mEncodedFrameRate;
   mFrameWidth = aWidth;
   mFrameHeight = aHeight;
   mDisplayWidth = aDisplayWidth;
   mDisplayHeight = aDisplayHeight;
 
   // Encoder configuration structure.
   vpx_codec_enc_cfg_t config;
   memset(&config, 0, sizeof(vpx_codec_enc_cfg_t));
@@ -142,17 +137,17 @@ VP8TrackEncoder::Init(int32_t aWidth, in
   config.rc_undershoot_pct = 100;
   config.rc_overshoot_pct = 15;
   config.rc_buf_initial_sz = 500;
   config.rc_buf_optimal_sz = 600;
   config.rc_buf_sz = 1000;
 
   config.kf_mode = VPX_KF_AUTO;
   // Ensure that we can output one I-frame per second.
-  config.kf_max_dist = mEncodedFrameRate;
+  config.kf_max_dist = 60;
 
   vpx_codec_flags_t flags = 0;
   flags |= VPX_CODEC_USE_OUTPUT_PARTITION;
   if (vpx_codec_enc_init(mVPXContext, vpx_codec_vp8_cx(), &config, flags)) {
     return NS_ERROR_FAILURE;
   }
 
   vpx_codec_control(mVPXContext, VP8E_SET_STATIC_THRESHOLD, 1);
@@ -467,201 +462,128 @@ nsresult VP8TrackEncoder::PrepareRawFram
  * the processed frame duration in mSourceSegment
  * in order to set the nextEncodeOperation for next target frame.
  */
 VP8TrackEncoder::EncodeOperation
 VP8TrackEncoder::GetNextEncodeOperation(TimeDuration aTimeElapsed,
                                         StreamTime aProcessedDuration)
 {
   int64_t durationInUsec =
-    FramesToUsecs(aProcessedDuration + mEncodedFrameDuration,
-                  mTrackRate).value();
+    FramesToUsecs(aProcessedDuration, mTrackRate).value();
   if (aTimeElapsed.ToMicroseconds() > (durationInUsec * SKIP_FRAME_RATIO)) {
     // The encoder is too slow.
     // We should skip next frame to consume the mSourceSegment.
     return SKIP_FRAME;
   } else if (aTimeElapsed.ToMicroseconds() > (durationInUsec * I_FRAME_RATIO)) {
     // The encoder is a little slow.
     // We force the encoder to encode an I-frame to accelerate.
     return ENCODE_I_FRAME;
   } else {
     return ENCODE_NORMAL_FRAME;
   }
 }
 
-StreamTime
-VP8TrackEncoder::CalculateRemainingTicks(StreamTime aDurationCopied,
-                                         StreamTime aEncodedDuration)
-{
-  return mRemainingTicks + aEncodedDuration - aDurationCopied;
-}
-
-// Try to extend the encodedDuration as long as possible if the target frame
-// has a long duration.
-StreamTime
-VP8TrackEncoder::CalculateEncodedDuration(StreamTime aDurationCopied)
-{
-  StreamTime temp64 = aDurationCopied;
-  StreamTime encodedDuration = mEncodedFrameDuration;
-  temp64 -= mRemainingTicks;
-  while (temp64 > mEncodedFrameDuration) {
-    temp64 -= mEncodedFrameDuration;
-    encodedDuration += mEncodedFrameDuration;
-  }
-  return encodedDuration;
-}
-
 /**
  * Encoding flow in GetEncodedTrack():
  * 1: Check the mInitialized state and the packet duration.
  * 2: Move the data from mRawSegment to mSourceSegment.
  * 3: Encode the video chunks in mSourceSegment in a for-loop.
- * 3.1: Pick the video chunk by mRemainingTicks.
- * 3.2: Calculate the encoding duration for the parameter of vpx_codec_encode().
- *      The encoding duration is a multiple of mEncodedFrameDuration.
- * 3.3: Setup the video chunk to mVPXImageWrapper by PrepareRawFrame().
- * 3.4: Send frame into vp8 encoder by vpx_codec_encode().
- * 3.5: Get the output frame from encoder by calling GetEncodedPartitions().
- * 3.6: Calculate the mRemainingTicks for next target frame.
- * 3.7: Set the nextEncodeOperation for the next target frame.
+ * 3.1: The duration is taken straight from the video chunk's duration.
+ * 3.2: Setup the video chunk with mVPXImageWrapper by PrepareRawFrame().
+ * 3.3: Pass frame to vp8 encoder by vpx_codec_encode().
+ * 3.4: Get the encoded frame from encoder by GetEncodedPartitions().
+ * 3.5: Set the nextEncodeOperation for the next target frame.
  *      There is a heuristic: If the frame duration we have processed in
  *      mSourceSegment is 100ms, means that we can't spend more than 100ms to
  *      encode it.
  * 4. Remove the encoded chunks in mSourceSegment after for-loop.
- *
- * Ex1: Input frame rate is 100 => input frame duration is 10ms for each.
- *     mEncodedFrameRate is 30 => output frame duration is 33ms.
- *     In this case, the frame duration in mSourceSegment will be:
- *     1st : 0~10ms
- *     2nd : 10~20ms
- *     3rd : 20~30ms
- *     4th : 30~40ms
- *     ...
- *     The VP8 encoder will take the 1st and 4th frames to encode. At beginning
- *     mRemainingTicks is 0 for 1st frame, then the mRemainingTicks is set
- *     to 23 to pick the 4th frame. (mEncodedFrameDuration - 1st frame duration)
- *
- * Ex2: Input frame rate is 25 => frame duration is 40ms for each.
- *     mEncodedFrameRate is 30 => output frame duration is 33ms.
- *     In this case, the frame duration in mSourceSegment will be:
- *     1st : 0~40ms
- *     2nd : 40~80ms
- *     3rd : 80~120ms
- *     4th : 120~160ms
- *     ...
- *     Because the input frame duration is 40ms larger than 33ms, so the first
- *     encoded frame duration will be 66ms by calling CalculateEncodedDuration.
- *     And the mRemainingTicks will be set to 26
- *     (CalculateRemainingTicks 0+66-40) in order to pick the next frame(2nd)
- *     in mSourceSegment.
  */
 nsresult
 VP8TrackEncoder::GetEncodedTrack(EncodedFrameContainer& aData)
 {
   PROFILER_LABEL("VP8TrackEncoder", "GetEncodedTrack",
     js::ProfileEntry::Category::OTHER);
   bool EOS;
   {
     // Move all the samples from mRawSegment to mSourceSegment. We only hold
     // the monitor in this block.
     ReentrantMonitorAutoEnter mon(mReentrantMonitor);
     // Wait if mEncoder is not initialized, or when not enough raw data, but is
     // not the end of stream nor is being canceled.
     while (!mCanceled && (!mInitialized ||
-           (mRawSegment.GetDuration() + mSourceSegment.GetDuration() <
-            mEncodedFrameDuration && !mEndOfStream))) {
+           (mRawSegment.GetDuration() + mSourceSegment.GetDuration() == 0 &&
+            !mEndOfStream))) {
       mon.Wait();
     }
     if (mCanceled || mEncodingComplete) {
       return NS_ERROR_FAILURE;
     }
     mSourceSegment.AppendFrom(&mRawSegment);
     EOS = mEndOfStream;
   }
 
-  VideoSegment::ChunkIterator iter(mSourceSegment);
-  StreamTime durationCopied = 0;
   StreamTime totalProcessedDuration = 0;
   TimeStamp timebase = TimeStamp::Now();
   EncodeOperation nextEncodeOperation = ENCODE_NORMAL_FRAME;
 
-  for (; !iter.IsEnded(); iter.Next()) {
+  for (VideoSegment::ChunkIterator iter(mSourceSegment);
+       !iter.IsEnded(); iter.Next()) {
     VideoChunk &chunk = *iter;
-    // Accumulate chunk's duration to durationCopied until it reaches
-    // mRemainingTicks.
-    durationCopied += chunk.GetDuration();
-    MOZ_ASSERT(mRemainingTicks <= mEncodedFrameDuration);
-    VP8LOG("durationCopied %lld mRemainingTicks %lld\n",
-           durationCopied, mRemainingTicks);
-    if (durationCopied >= mRemainingTicks) {
-      VP8LOG("nextEncodeOperation is %d\n",nextEncodeOperation);
-      // Calculate encodedDuration for this target frame.
-      StreamTime encodedDuration = CalculateEncodedDuration(durationCopied);
+    VP8LOG("nextEncodeOperation is %d for frame of duration %lld\n",
+           nextEncodeOperation, chunk.GetDuration());
 
-      // Encode frame.
-      if (nextEncodeOperation != SKIP_FRAME) {
-        nsresult rv = PrepareRawFrame(chunk);
-        NS_ENSURE_SUCCESS(rv, NS_ERROR_FAILURE);
+    // Encode frame.
+    if (nextEncodeOperation != SKIP_FRAME) {
+      nsresult rv = PrepareRawFrame(chunk);
+      NS_ENSURE_SUCCESS(rv, NS_ERROR_FAILURE);
 
-        // Encode the data with VP8 encoder
-        int flags = (nextEncodeOperation == ENCODE_NORMAL_FRAME) ?
-                    0 : VPX_EFLAG_FORCE_KF;
-        if (vpx_codec_encode(mVPXContext, mVPXImageWrapper, mEncodedTimestamp,
-                             (unsigned long)encodedDuration, flags,
-                             VPX_DL_REALTIME)) {
-          return NS_ERROR_FAILURE;
-        }
-        // Get the encoded data from VP8 encoder.
-        GetEncodedPartitions(aData);
-      } else {
-        // SKIP_FRAME
-        // Extend the duration of the last encoded data in aData
-        // because this frame will be skip.
-        RefPtr<EncodedFrame> last = nullptr;
-        last = aData.GetEncodedFrames().LastElement();
-        if (last) {
-          last->SetDuration(last->GetDuration() + encodedDuration);
-        }
+      // Encode the data with VP8 encoder
+      int flags = (nextEncodeOperation == ENCODE_NORMAL_FRAME) ?
+                  0 : VPX_EFLAG_FORCE_KF;
+      if (vpx_codec_encode(mVPXContext, mVPXImageWrapper, mEncodedTimestamp,
+                           (unsigned long)chunk.GetDuration(), flags,
+                           VPX_DL_REALTIME)) {
+        return NS_ERROR_FAILURE;
       }
-      // Move forward the mEncodedTimestamp.
-      mEncodedTimestamp += encodedDuration;
-      totalProcessedDuration += durationCopied;
-      // Calculate mRemainingTicks for next target frame.
-      mRemainingTicks = CalculateRemainingTicks(durationCopied,
-                                                encodedDuration);
-
-      // Check the remain data is enough for next target frame.
-      if (mSourceSegment.GetDuration() - totalProcessedDuration
-          >= mEncodedFrameDuration) {
-        TimeDuration elapsedTime = TimeStamp::Now() - timebase;
-        nextEncodeOperation = GetNextEncodeOperation(elapsedTime,
-                                                     totalProcessedDuration);
-        // Reset durationCopied for next iteration.
-        durationCopied = 0;
-      } else {
-        // Process done, there is no enough data left for next iteration,
-        // break the for-loop.
-        break;
+      // Get the encoded data from VP8 encoder.
+      GetEncodedPartitions(aData);
+    } else {
+      // SKIP_FRAME
+      // Extend the duration of the last encoded data in aData
+      // because this frame will be skip.
+      NS_WARNING("MediaRecorder lagging behind. Skipping a frame.");
+      RefPtr<EncodedFrame> last = aData.GetEncodedFrames().LastElement();
+      if (last) {
+        last->SetDuration(last->GetDuration() + chunk.GetDuration());
       }
     }
+
+    // Move forward the mEncodedTimestamp.
+    mEncodedTimestamp += chunk.GetDuration();
+    totalProcessedDuration += chunk.GetDuration();
+
+    // Check what to do next.
+    TimeDuration elapsedTime = TimeStamp::Now() - timebase;
+    nextEncodeOperation = GetNextEncodeOperation(elapsedTime,
+                                                 totalProcessedDuration);
   }
+
   // Remove the chunks we have processed.
-  mSourceSegment.RemoveLeading(totalProcessedDuration);
-  VP8LOG("RemoveLeading %lld\n",totalProcessedDuration);
+  mSourceSegment.Clear();
 
   // End of stream, pull the rest frames in encoder.
   if (EOS) {
     VP8LOG("mEndOfStream is true\n");
     mEncodingComplete = true;
     // Bug 1243611, keep calling vpx_codec_encode and vpx_codec_get_cx_data
     // until vpx_codec_get_cx_data return null.
 
     do {
       if (vpx_codec_encode(mVPXContext, nullptr, mEncodedTimestamp,
-                           mEncodedFrameDuration, 0, VPX_DL_REALTIME)) {
+                           0, 0, VPX_DL_REALTIME)) {
         return NS_ERROR_FAILURE;
       }
     } while(GetEncodedPartitions(aData));
   }
 
   return NS_OK ;
 }
 
--- a/dom/media/encoder/VP8TrackEncoder.h
+++ b/dom/media/encoder/VP8TrackEncoder.h
@@ -11,20 +11,19 @@
 
 namespace mozilla {
 
 typedef struct vpx_codec_ctx vpx_codec_ctx_t;
 typedef struct vpx_codec_enc_cfg vpx_codec_enc_cfg_t;
 typedef struct vpx_image vpx_image_t;
 
 /**
- * VP8TrackEncoder implements VideoTrackEncoder by using libvpx library.
- * We implement a realtime and fixed FPS encoder. In order to achieve that,
- * there is a pick target frame and drop frame encoding policy implemented in
- * GetEncodedTrack.
+ * VP8TrackEncoder implements VideoTrackEncoder by using the libvpx library.
+ * We implement a realtime and variable frame rate encoder. In order to achieve
+ * that, there is a frame-drop encoding policy implemented in GetEncodedTrack.
  */
 class VP8TrackEncoder : public VideoTrackEncoder
 {
   enum EncodeOperation {
     ENCODE_NORMAL_FRAME, // VP8 track encoder works normally.
     ENCODE_I_FRAME, // The next frame will be encoded as I-Frame.
     SKIP_FRAME, // Skip the next frame.
   };
@@ -36,55 +35,40 @@ public:
 
   nsresult GetEncodedTrack(EncodedFrameContainer& aData) final override;
 
 protected:
   nsresult Init(int32_t aWidth, int32_t aHeight,
                 int32_t aDisplayWidth, int32_t aDisplayHeight) final override;
 
 private:
-  // Calculate the target frame's encoded duration.
-  StreamTime CalculateEncodedDuration(StreamTime aDurationCopied);
-
-  // Calculate the mRemainingTicks for next target frame.
-  StreamTime CalculateRemainingTicks(StreamTime aDurationCopied,
-                                     StreamTime aEncodedDuration);
-
   // Get the EncodeOperation for next target frame.
   EncodeOperation GetNextEncodeOperation(TimeDuration aTimeElapsed,
                                          StreamTime aProcessedDuration);
 
   // Get the encoded data from encoder to aData.
   // Return value: false if the vpx_codec_get_cx_data returns null
   //               for EOS detection.
   bool GetEncodedPartitions(EncodedFrameContainer& aData);
 
   // Prepare the input data to the mVPXImageWrapper for encoding.
   nsresult PrepareRawFrame(VideoChunk &aChunk);
 
-  // Output frame rate.
-  uint32_t mEncodedFrameRate;
-  // Duration for the output frame, reciprocal to mEncodedFrameRate.
-  StreamTime mEncodedFrameDuration;
   // Encoded timestamp.
   StreamTime mEncodedTimestamp;
-  // Duration to the next encode frame.
-  StreamTime mRemainingTicks;
 
   // Muted frame, we only create it once.
   RefPtr<layers::Image> mMuteFrame;
 
   // I420 frame, for converting to I420.
   nsTArray<uint8_t> mI420Frame;
 
   /**
    * A local segment queue which takes the raw data out from mRawSegment in the
-   * call of GetEncodedTrack(). Since we implement the fixed FPS encoding
-   * policy, it needs to be global in order to store the leftover segments
-   * taken from mRawSegment.
+   * call of GetEncodedTrack().
    */
   VideoSegment mSourceSegment;
 
   // VP8 relative members.
   // Codec context structure.
   nsAutoPtr<vpx_codec_ctx_t> mVPXContext;
   // Image Descriptor.
   nsAutoPtr<vpx_image_t> mVPXImageWrapper;