Bug 1409727 - Add a mode and pref to disallow frame drops in MediaRecorder. r?SingingTree
MozReview-Commit-ID: CJBJURDn7gB
--- a/dom/media/encoder/MediaEncoder.cpp
+++ b/dom/media/encoder/MediaEncoder.cpp
@@ -627,17 +627,21 @@ MediaEncoder::CreateEncoder(TaskQueue* a
else if (MediaEncoder::IsWebMEncoderEnabled() &&
(aMIMEType.EqualsLiteral(VIDEO_WEBM) ||
(aTrackTypes & ContainerWriter::CREATE_VIDEO_TRACK))) {
if (aTrackTypes & ContainerWriter::CREATE_AUDIO_TRACK &&
MediaDecoder::IsOpusEnabled()) {
audioEncoder = MakeAndAddRef<OpusTrackEncoder>(aTrackRate);
NS_ENSURE_TRUE(audioEncoder, nullptr);
}
- videoEncoder = MakeAndAddRef<VP8TrackEncoder>(aTrackRate);
+ if (Preferences::GetBool("media.recorder.video.frame_drops", true)) {
+ videoEncoder = MakeAndAddRef<VP8TrackEncoder>(aTrackRate, FrameDroppingMode::ALLOW);
+ } else {
+ videoEncoder = MakeAndAddRef<VP8TrackEncoder>(aTrackRate, FrameDroppingMode::DISALLOW);
+ }
writer = MakeUnique<WebMWriter>(aTrackTypes);
NS_ENSURE_TRUE(writer, nullptr);
NS_ENSURE_TRUE(videoEncoder, nullptr);
mimeType = NS_LITERAL_STRING(VIDEO_WEBM);
}
#endif //MOZ_WEBM_ENCODER
else if (MediaDecoder::IsOggEnabled() && MediaDecoder::IsOpusEnabled() &&
(aMIMEType.EqualsLiteral(AUDIO_OGG) ||
--- a/dom/media/encoder/TrackEncoder.h
+++ b/dom/media/encoder/TrackEncoder.h
@@ -380,27 +380,33 @@ protected:
uint32_t mAudioBitrate;
// This may only be accessed on the MSG thread.
// I.e., in the regular NotifyQueuedChanges for audio to avoid adding data
// from that callback when the direct one is active.
bool mDirectConnected;
};
+enum class FrameDroppingMode {
+ ALLOW, // Allowed to drop frames to keep up under load
+ DISALLOW, // Must not drop any frames, even if it means we will OOM
+};
+
class VideoTrackEncoder : public TrackEncoder
{
public:
- explicit VideoTrackEncoder(TrackRate aTrackRate)
+ VideoTrackEncoder(TrackRate aTrackRate, FrameDroppingMode aFrameDroppingMode)
: TrackEncoder(aTrackRate)
, mFrameWidth(0)
, mFrameHeight(0)
, mDisplayWidth(0)
, mDisplayHeight(0)
, mEncodedTicks(0)
, mVideoBitrate(0)
+ , mFrameDroppingMode(aFrameDroppingMode)
{
mLastChunk.mDuration = 0;
}
/**
* Suspends encoding from aTime, i.e., all video frame with a timestamp
* between aTime and the timestamp of the next Resume() will be dropped.
*/
@@ -546,13 +552,19 @@ protected:
/**
* The time Suspend was called on the MediaRecorder, so we can calculate the
* duration on the next Resume().
*/
TimeStamp mSuspendTime;
uint32_t mVideoBitrate;
+
+ /**
+ * ALLOW to drop frames under load.
+ * DISALLOW to encode all frames, mainly for testing.
+ */
+ FrameDroppingMode mFrameDroppingMode;
};
} // namespace mozilla
#endif
--- a/dom/media/encoder/VP8TrackEncoder.cpp
+++ b/dom/media/encoder/VP8TrackEncoder.cpp
@@ -51,18 +51,19 @@ GetSourceSurface(already_AddRefed<Image>
surf = img->GetAsSourceSurface();
return NS_OK;
});
NS_DispatchToMainThread(runnable, NS_DISPATCH_SYNC);
return surf.forget();
}
-VP8TrackEncoder::VP8TrackEncoder(TrackRate aTrackRate)
- : VideoTrackEncoder(aTrackRate)
+VP8TrackEncoder::VP8TrackEncoder(TrackRate aTrackRate,
+ FrameDroppingMode aFrameDroppingMode)
+ : VideoTrackEncoder(aTrackRate, aFrameDroppingMode)
, mEncodedTimestamp(0)
, mVPXContext(new vpx_codec_ctx_t())
, mVPXImageWrapper(new vpx_image_t())
{
MOZ_COUNT_CTOR(VP8TrackEncoder);
}
VP8TrackEncoder::~VP8TrackEncoder()
@@ -571,16 +572,20 @@ nsresult VP8TrackEncoder::PrepareRawFram
* Compares the elapsed time from the beginning of GetEncodedTrack and
* the processed frame duration in mSourceSegment
* in order to set the nextEncodeOperation for next target frame.
*/
VP8TrackEncoder::EncodeOperation
VP8TrackEncoder::GetNextEncodeOperation(TimeDuration aTimeElapsed,
StreamTime aProcessedDuration)
{
+ if (mFrameDroppingMode == FrameDroppingMode::DISALLOW) {
+ return ENCODE_NORMAL_FRAME;
+ }
+
int64_t durationInUsec =
FramesToUsecs(aProcessedDuration, mTrackRate).value();
if (aTimeElapsed.ToMicroseconds() > (durationInUsec * SKIP_FRAME_RATIO)) {
// The encoder is too slow.
// We should skip next frame to consume the mSourceSegment.
return SKIP_FRAME;
} else if (aTimeElapsed.ToMicroseconds() > (durationInUsec * I_FRAME_RATIO)) {
// The encoder is a little slow.
--- a/dom/media/encoder/VP8TrackEncoder.h
+++ b/dom/media/encoder/VP8TrackEncoder.h
@@ -22,18 +22,19 @@ typedef struct vpx_image vpx_image_t;
*/
class VP8TrackEncoder : public VideoTrackEncoder
{
enum EncodeOperation {
ENCODE_NORMAL_FRAME, // VP8 track encoder works normally.
ENCODE_I_FRAME, // The next frame will be encoded as I-Frame.
SKIP_FRAME, // Skip the next frame.
};
+
public:
- explicit VP8TrackEncoder(TrackRate aTrackRate);
+ VP8TrackEncoder(TrackRate aTrackRate, FrameDroppingMode aFrameDroppingMode);
virtual ~VP8TrackEncoder();
already_AddRefed<TrackMetadataBase> GetMetadata() final override;
nsresult GetEncodedTrack(EncodedFrameContainer& aData) final override;
protected:
nsresult Init(int32_t aWidth, int32_t aHeight,
--- a/dom/media/gtest/TestVideoTrackEncoder.cpp
+++ b/dom/media/gtest/TestVideoTrackEncoder.cpp
@@ -187,17 +187,17 @@ struct InitParam {
int mWidth; // frame width
int mHeight; // frame height
};
class TestVP8TrackEncoder: public VP8TrackEncoder
{
public:
explicit TestVP8TrackEncoder(TrackRate aTrackRate = VIDEO_TRACK_RATE)
- : VP8TrackEncoder(aTrackRate) {}
+ : VP8TrackEncoder(aTrackRate, FrameDroppingMode::DISALLOW) {}
::testing::AssertionResult TestInit(const InitParam &aParam)
{
nsresult result = Init(aParam.mWidth, aParam.mHeight, aParam.mWidth, aParam.mHeight);
if (((NS_FAILED(result) && aParam.mShouldSucceed)) || (NS_SUCCEEDED(result) && !aParam.mShouldSucceed))
{
return ::testing::AssertionFailure()
--- a/dom/media/gtest/TestWebMWriter.cpp
+++ b/dom/media/gtest/TestWebMWriter.cpp
@@ -26,17 +26,17 @@ public:
return false;
}
};
class WebMVP8TrackEncoder: public VP8TrackEncoder
{
public:
explicit WebMVP8TrackEncoder(TrackRate aTrackRate = 90000)
- : VP8TrackEncoder(aTrackRate) {}
+ : VP8TrackEncoder(aTrackRate, FrameDroppingMode::DISALLOW) {}
bool TestVP8Creation(int32_t aWidth, int32_t aHeight, int32_t aDisplayWidth,
int32_t aDisplayHeight)
{
if (NS_SUCCEEDED(Init(aWidth, aHeight, aDisplayWidth, aDisplayHeight))) {
return true;
}
return false;
--- a/modules/libpref/init/all.js
+++ b/modules/libpref/init/all.js
@@ -615,16 +615,20 @@ pref("media.webspeech.synth.enabled", fa
#endif
#ifdef MOZ_WEBM_ENCODER
pref("media.encoder.webm.enabled", true);
#endif
// Whether to allow recording of AudioNodes with MediaRecorder
pref("media.recorder.audio_node.enabled", false);
+// Whether MediaRecorder's video encoder should allow dropping frames in order
+// to keep up under load. Useful for tests but beware of memory consumption!
+pref("media.recorder.video.frame_drops", true);
+
// Whether to autostart a media element with an |autoplay| attribute
pref("media.autoplay.enabled", true);
// The default number of decoded video frames that are enqueued in
// MediaDecoderReader's mVideoQueue.
pref("media.video-queue.default-size", 10);
// The maximum number of queued frames to send to the compositor.