Bug 1409727 - Add a mode and pref to disallow frame drops in MediaRecorder. r?SingingTree draft
authorAndreas Pehrson <pehrsons@mozilla.com>
Thu, 19 Oct 2017 14:38:07 +0200
changeset 683241 6c228eef61c05e6be718c886ceba599f465a29f4
parent 683240 e3edbc25423086e1371870b4073c5e8caf211162
child 736575 afe54f594c3c6edf2dd77e40337aef330e50abeb
push id85300
push userbmo:apehrson@mozilla.com
push dateThu, 19 Oct 2017 12:38:44 +0000
reviewersSingingTree
bugs1409727
milestone58.0a1
Bug 1409727 - Add a mode and pref to disallow frame drops in MediaRecorder. r?SingingTree MozReview-Commit-ID: CJBJURDn7gB
dom/media/encoder/MediaEncoder.cpp
dom/media/encoder/TrackEncoder.h
dom/media/encoder/VP8TrackEncoder.cpp
dom/media/encoder/VP8TrackEncoder.h
dom/media/gtest/TestVideoTrackEncoder.cpp
dom/media/gtest/TestWebMWriter.cpp
modules/libpref/init/all.js
--- a/dom/media/encoder/MediaEncoder.cpp
+++ b/dom/media/encoder/MediaEncoder.cpp
@@ -627,17 +627,21 @@ MediaEncoder::CreateEncoder(TaskQueue* a
   else if (MediaEncoder::IsWebMEncoderEnabled() &&
       (aMIMEType.EqualsLiteral(VIDEO_WEBM) ||
        (aTrackTypes & ContainerWriter::CREATE_VIDEO_TRACK))) {
     if (aTrackTypes & ContainerWriter::CREATE_AUDIO_TRACK &&
         MediaDecoder::IsOpusEnabled()) {
       audioEncoder = MakeAndAddRef<OpusTrackEncoder>(aTrackRate);
       NS_ENSURE_TRUE(audioEncoder, nullptr);
     }
-    videoEncoder = MakeAndAddRef<VP8TrackEncoder>(aTrackRate);
+    if (Preferences::GetBool("media.recorder.video.frame_drops", true)) {
+      videoEncoder = MakeAndAddRef<VP8TrackEncoder>(aTrackRate, FrameDroppingMode::ALLOW);
+    } else {
+      videoEncoder = MakeAndAddRef<VP8TrackEncoder>(aTrackRate, FrameDroppingMode::DISALLOW);
+    }
     writer = MakeUnique<WebMWriter>(aTrackTypes);
     NS_ENSURE_TRUE(writer, nullptr);
     NS_ENSURE_TRUE(videoEncoder, nullptr);
     mimeType = NS_LITERAL_STRING(VIDEO_WEBM);
   }
 #endif //MOZ_WEBM_ENCODER
   else if (MediaDecoder::IsOggEnabled() && MediaDecoder::IsOpusEnabled() &&
            (aMIMEType.EqualsLiteral(AUDIO_OGG) ||
--- a/dom/media/encoder/TrackEncoder.h
+++ b/dom/media/encoder/TrackEncoder.h
@@ -380,27 +380,33 @@ protected:
   uint32_t mAudioBitrate;
 
   // This may only be accessed on the MSG thread.
   // I.e., in the regular NotifyQueuedChanges for audio to avoid adding data
   // from that callback when the direct one is active.
   bool mDirectConnected;
 };
 
+enum class FrameDroppingMode {
+  ALLOW, // Allowed to drop frames to keep up under load
+  DISALLOW, // Must not drop any frames, even if it means we will OOM
+};
+
 class VideoTrackEncoder : public TrackEncoder
 {
 public:
-  explicit VideoTrackEncoder(TrackRate aTrackRate)
+  VideoTrackEncoder(TrackRate aTrackRate, FrameDroppingMode aFrameDroppingMode)
     : TrackEncoder(aTrackRate)
     , mFrameWidth(0)
     , mFrameHeight(0)
     , mDisplayWidth(0)
     , mDisplayHeight(0)
     , mEncodedTicks(0)
     , mVideoBitrate(0)
+    , mFrameDroppingMode(aFrameDroppingMode)
   {
     mLastChunk.mDuration = 0;
   }
 
   /**
    * Suspends encoding from aTime, i.e., all video frame with a timestamp
    * between aTime and the timestamp of the next Resume() will be dropped.
    */
@@ -546,13 +552,19 @@ protected:
 
   /**
    * The time Suspend was called on the MediaRecorder, so we can calculate the
    * duration on the next Resume().
    */
   TimeStamp mSuspendTime;
 
   uint32_t mVideoBitrate;
+
+  /**
+   * ALLOW to drop frames under load.
+   * DISALLOW to encode all frames, mainly for testing.
+   */
+  FrameDroppingMode mFrameDroppingMode;
 };
 
 } // namespace mozilla
 
 #endif
--- a/dom/media/encoder/VP8TrackEncoder.cpp
+++ b/dom/media/encoder/VP8TrackEncoder.cpp
@@ -51,18 +51,19 @@ GetSourceSurface(already_AddRefed<Image>
     surf = img->GetAsSourceSurface();
     return NS_OK;
   });
 
   NS_DispatchToMainThread(runnable, NS_DISPATCH_SYNC);
   return surf.forget();
 }
 
-VP8TrackEncoder::VP8TrackEncoder(TrackRate aTrackRate)
-  : VideoTrackEncoder(aTrackRate)
+VP8TrackEncoder::VP8TrackEncoder(TrackRate aTrackRate,
+                                 FrameDroppingMode aFrameDroppingMode)
+  : VideoTrackEncoder(aTrackRate, aFrameDroppingMode)
   , mEncodedTimestamp(0)
   , mVPXContext(new vpx_codec_ctx_t())
   , mVPXImageWrapper(new vpx_image_t())
 {
   MOZ_COUNT_CTOR(VP8TrackEncoder);
 }
 
 VP8TrackEncoder::~VP8TrackEncoder()
@@ -571,16 +572,20 @@ nsresult VP8TrackEncoder::PrepareRawFram
  * Compares the elapsed time from the beginning of GetEncodedTrack and
  * the processed frame duration in mSourceSegment
  * in order to set the nextEncodeOperation for next target frame.
  */
 VP8TrackEncoder::EncodeOperation
 VP8TrackEncoder::GetNextEncodeOperation(TimeDuration aTimeElapsed,
                                         StreamTime aProcessedDuration)
 {
+  if (mFrameDroppingMode == FrameDroppingMode::DISALLOW) {
+    return ENCODE_NORMAL_FRAME;
+  }
+
   int64_t durationInUsec =
     FramesToUsecs(aProcessedDuration, mTrackRate).value();
   if (aTimeElapsed.ToMicroseconds() > (durationInUsec * SKIP_FRAME_RATIO)) {
     // The encoder is too slow.
     // We should skip next frame to consume the mSourceSegment.
     return SKIP_FRAME;
   } else if (aTimeElapsed.ToMicroseconds() > (durationInUsec * I_FRAME_RATIO)) {
     // The encoder is a little slow.
--- a/dom/media/encoder/VP8TrackEncoder.h
+++ b/dom/media/encoder/VP8TrackEncoder.h
@@ -22,18 +22,19 @@ typedef struct vpx_image vpx_image_t;
  */
 class VP8TrackEncoder : public VideoTrackEncoder
 {
   enum EncodeOperation {
     ENCODE_NORMAL_FRAME, // VP8 track encoder works normally.
     ENCODE_I_FRAME, // The next frame will be encoded as I-Frame.
     SKIP_FRAME, // Skip the next frame.
   };
+
 public:
-  explicit VP8TrackEncoder(TrackRate aTrackRate);
+  VP8TrackEncoder(TrackRate aTrackRate, FrameDroppingMode aFrameDroppingMode);
   virtual ~VP8TrackEncoder();
 
   already_AddRefed<TrackMetadataBase> GetMetadata() final override;
 
   nsresult GetEncodedTrack(EncodedFrameContainer& aData) final override;
 
 protected:
   nsresult Init(int32_t aWidth, int32_t aHeight,
--- a/dom/media/gtest/TestVideoTrackEncoder.cpp
+++ b/dom/media/gtest/TestVideoTrackEncoder.cpp
@@ -187,17 +187,17 @@ struct InitParam {
   int  mWidth;          // frame width
   int  mHeight;         // frame height
 };
 
 class TestVP8TrackEncoder: public VP8TrackEncoder
 {
 public:
   explicit TestVP8TrackEncoder(TrackRate aTrackRate = VIDEO_TRACK_RATE)
-    : VP8TrackEncoder(aTrackRate) {}
+    : VP8TrackEncoder(aTrackRate, FrameDroppingMode::DISALLOW) {}
 
   ::testing::AssertionResult TestInit(const InitParam &aParam)
   {
     nsresult result = Init(aParam.mWidth, aParam.mHeight, aParam.mWidth, aParam.mHeight);
 
     if (((NS_FAILED(result) && aParam.mShouldSucceed)) || (NS_SUCCEEDED(result) && !aParam.mShouldSucceed))
     {
       return ::testing::AssertionFailure()
--- a/dom/media/gtest/TestWebMWriter.cpp
+++ b/dom/media/gtest/TestWebMWriter.cpp
@@ -26,17 +26,17 @@ public:
     return false;
   }
 };
 
 class WebMVP8TrackEncoder: public VP8TrackEncoder
 {
 public:
   explicit WebMVP8TrackEncoder(TrackRate aTrackRate = 90000)
-    : VP8TrackEncoder(aTrackRate) {}
+    : VP8TrackEncoder(aTrackRate, FrameDroppingMode::DISALLOW) {}
 
   bool TestVP8Creation(int32_t aWidth, int32_t aHeight, int32_t aDisplayWidth,
                        int32_t aDisplayHeight)
   {
     if (NS_SUCCEEDED(Init(aWidth, aHeight, aDisplayWidth, aDisplayHeight))) {
       return true;
     }
     return false;
--- a/modules/libpref/init/all.js
+++ b/modules/libpref/init/all.js
@@ -615,16 +615,20 @@ pref("media.webspeech.synth.enabled", fa
 #endif
 #ifdef MOZ_WEBM_ENCODER
 pref("media.encoder.webm.enabled", true);
 #endif
 
 // Whether to allow recording of AudioNodes with MediaRecorder
 pref("media.recorder.audio_node.enabled", false);
 
+// Whether MediaRecorder's video encoder should allow dropping frames in order
+// to keep up under load. Useful for tests but beware of memory consumption!
+pref("media.recorder.video.frame_drops", true);
+
 // Whether to autostart a media element with an |autoplay| attribute
 pref("media.autoplay.enabled", true);
 
 // The default number of decoded video frames that are enqueued in
 // MediaDecoderReader's mVideoQueue.
 pref("media.video-queue.default-size", 10);
 
 // The maximum number of queued frames to send to the compositor.