--- a/dom/media/encoder/VP8TrackEncoder.cpp
+++ b/dom/media/encoder/VP8TrackEncoder.cpp
@@ -19,17 +19,16 @@
namespace mozilla {
LazyLogModule gVP8TrackEncoderLog("VP8TrackEncoder");
#define VP8LOG(msg, ...) MOZ_LOG(gVP8TrackEncoderLog, mozilla::LogLevel::Debug, \
(msg, ##__VA_ARGS__))
// Debug logging macro with object pointer and class name.
#define DEFAULT_BITRATE_BPS 2500000
-#define DEFAULT_ENCODE_FRAMERATE 30
using namespace mozilla::gfx;
using namespace mozilla::layers;
using namespace mozilla::media;
static already_AddRefed<SourceSurface>
GetSourceSurface(already_AddRefed<Image> aImg)
{
@@ -51,19 +50,17 @@ GetSourceSurface(already_AddRefed<Image>
});
NS_DispatchToMainThread(runnable, NS_DISPATCH_SYNC);
return surf.forget();
}
VP8TrackEncoder::VP8TrackEncoder(TrackRate aTrackRate)
: VideoTrackEncoder(aTrackRate)
- , mEncodedFrameDuration(0)
, mEncodedTimestamp(0)
- , mRemainingTicks(0)
, mVPXContext(new vpx_codec_ctx_t())
, mVPXImageWrapper(new vpx_image_t())
{
MOZ_COUNT_CTOR(VP8TrackEncoder);
}
VP8TrackEncoder::~VP8TrackEncoder()
{
@@ -82,18 +79,16 @@ VP8TrackEncoder::Init(int32_t aWidth, in
int32_t aDisplayHeight)
{
if (aWidth < 1 || aHeight < 1 || aDisplayWidth < 1 || aDisplayHeight < 1) {
return NS_ERROR_FAILURE;
}
ReentrantMonitorAutoEnter mon(mReentrantMonitor);
- mEncodedFrameRate = DEFAULT_ENCODE_FRAMERATE;
- mEncodedFrameDuration = mTrackRate / mEncodedFrameRate;
mFrameWidth = aWidth;
mFrameHeight = aHeight;
mDisplayWidth = aDisplayWidth;
mDisplayHeight = aDisplayHeight;
// Encoder configuration structure.
vpx_codec_enc_cfg_t config;
memset(&config, 0, sizeof(vpx_codec_enc_cfg_t));
@@ -142,17 +137,17 @@ VP8TrackEncoder::Init(int32_t aWidth, in
config.rc_undershoot_pct = 100;
config.rc_overshoot_pct = 15;
config.rc_buf_initial_sz = 500;
config.rc_buf_optimal_sz = 600;
config.rc_buf_sz = 1000;
config.kf_mode = VPX_KF_AUTO;
// Ensure that we can output one I-frame per second.
- config.kf_max_dist = mEncodedFrameRate;
+ config.kf_max_dist = 60;
vpx_codec_flags_t flags = 0;
flags |= VPX_CODEC_USE_OUTPUT_PARTITION;
if (vpx_codec_enc_init(mVPXContext, vpx_codec_vp8_cx(), &config, flags)) {
return NS_ERROR_FAILURE;
}
vpx_codec_control(mVPXContext, VP8E_SET_STATIC_THRESHOLD, 1);
@@ -467,201 +462,128 @@ nsresult VP8TrackEncoder::PrepareRawFram
* the processed frame duration in mSourceSegment
* in order to set the nextEncodeOperation for next target frame.
*/
VP8TrackEncoder::EncodeOperation
VP8TrackEncoder::GetNextEncodeOperation(TimeDuration aTimeElapsed,
StreamTime aProcessedDuration)
{
int64_t durationInUsec =
- FramesToUsecs(aProcessedDuration + mEncodedFrameDuration,
- mTrackRate).value();
+ FramesToUsecs(aProcessedDuration, mTrackRate).value();
if (aTimeElapsed.ToMicroseconds() > (durationInUsec * SKIP_FRAME_RATIO)) {
// The encoder is too slow.
// We should skip next frame to consume the mSourceSegment.
return SKIP_FRAME;
} else if (aTimeElapsed.ToMicroseconds() > (durationInUsec * I_FRAME_RATIO)) {
// The encoder is a little slow.
// We force the encoder to encode an I-frame to accelerate.
return ENCODE_I_FRAME;
} else {
return ENCODE_NORMAL_FRAME;
}
}
-StreamTime
-VP8TrackEncoder::CalculateRemainingTicks(StreamTime aDurationCopied,
- StreamTime aEncodedDuration)
-{
- return mRemainingTicks + aEncodedDuration - aDurationCopied;
-}
-
-// Try to extend the encodedDuration as long as possible if the target frame
-// has a long duration.
-StreamTime
-VP8TrackEncoder::CalculateEncodedDuration(StreamTime aDurationCopied)
-{
- StreamTime temp64 = aDurationCopied;
- StreamTime encodedDuration = mEncodedFrameDuration;
- temp64 -= mRemainingTicks;
- while (temp64 > mEncodedFrameDuration) {
- temp64 -= mEncodedFrameDuration;
- encodedDuration += mEncodedFrameDuration;
- }
- return encodedDuration;
-}
-
/**
* Encoding flow in GetEncodedTrack():
* 1: Check the mInitialized state and the packet duration.
* 2: Move the data from mRawSegment to mSourceSegment.
* 3: Encode the video chunks in mSourceSegment in a for-loop.
- * 3.1: Pick the video chunk by mRemainingTicks.
- * 3.2: Calculate the encoding duration for the parameter of vpx_codec_encode().
- * The encoding duration is a multiple of mEncodedFrameDuration.
- * 3.3: Setup the video chunk to mVPXImageWrapper by PrepareRawFrame().
- * 3.4: Send frame into vp8 encoder by vpx_codec_encode().
- * 3.5: Get the output frame from encoder by calling GetEncodedPartitions().
- * 3.6: Calculate the mRemainingTicks for next target frame.
- * 3.7: Set the nextEncodeOperation for the next target frame.
+ * 3.1: The duration is taken straight from the video chunk's duration.
+ * 3.2: Setup the video chunk with mVPXImageWrapper by PrepareRawFrame().
+ * 3.3: Pass frame to vp8 encoder by vpx_codec_encode().
+ * 3.4: Get the encoded frame from encoder by GetEncodedPartitions().
+ * 3.5: Set the nextEncodeOperation for the next target frame.
* There is a heuristic: If the frame duration we have processed in
* mSourceSegment is 100ms, means that we can't spend more than 100ms to
* encode it.
* 4. Remove the encoded chunks in mSourceSegment after for-loop.
- *
- * Ex1: Input frame rate is 100 => input frame duration is 10ms for each.
- * mEncodedFrameRate is 30 => output frame duration is 33ms.
- * In this case, the frame duration in mSourceSegment will be:
- * 1st : 0~10ms
- * 2nd : 10~20ms
- * 3rd : 20~30ms
- * 4th : 30~40ms
- * ...
- * The VP8 encoder will take the 1st and 4th frames to encode. At beginning
- * mRemainingTicks is 0 for 1st frame, then the mRemainingTicks is set
- * to 23 to pick the 4th frame. (mEncodedFrameDuration - 1st frame duration)
- *
- * Ex2: Input frame rate is 25 => frame duration is 40ms for each.
- * mEncodedFrameRate is 30 => output frame duration is 33ms.
- * In this case, the frame duration in mSourceSegment will be:
- * 1st : 0~40ms
- * 2nd : 40~80ms
- * 3rd : 80~120ms
- * 4th : 120~160ms
- * ...
- * Because the input frame duration is 40ms larger than 33ms, so the first
- * encoded frame duration will be 66ms by calling CalculateEncodedDuration.
- * And the mRemainingTicks will be set to 26
- * (CalculateRemainingTicks 0+66-40) in order to pick the next frame(2nd)
- * in mSourceSegment.
*/
nsresult
VP8TrackEncoder::GetEncodedTrack(EncodedFrameContainer& aData)
{
PROFILER_LABEL("VP8TrackEncoder", "GetEncodedTrack",
js::ProfileEntry::Category::OTHER);
bool EOS;
{
// Move all the samples from mRawSegment to mSourceSegment. We only hold
// the monitor in this block.
ReentrantMonitorAutoEnter mon(mReentrantMonitor);
// Wait if mEncoder is not initialized, or when not enough raw data, but is
// not the end of stream nor is being canceled.
while (!mCanceled && (!mInitialized ||
- (mRawSegment.GetDuration() + mSourceSegment.GetDuration() <
- mEncodedFrameDuration && !mEndOfStream))) {
+ (mRawSegment.GetDuration() + mSourceSegment.GetDuration() == 0 &&
+ !mEndOfStream))) {
mon.Wait();
}
if (mCanceled || mEncodingComplete) {
return NS_ERROR_FAILURE;
}
mSourceSegment.AppendFrom(&mRawSegment);
EOS = mEndOfStream;
}
- VideoSegment::ChunkIterator iter(mSourceSegment);
- StreamTime durationCopied = 0;
StreamTime totalProcessedDuration = 0;
TimeStamp timebase = TimeStamp::Now();
EncodeOperation nextEncodeOperation = ENCODE_NORMAL_FRAME;
- for (; !iter.IsEnded(); iter.Next()) {
+ for (VideoSegment::ChunkIterator iter(mSourceSegment);
+ !iter.IsEnded(); iter.Next()) {
VideoChunk &chunk = *iter;
- // Accumulate chunk's duration to durationCopied until it reaches
- // mRemainingTicks.
- durationCopied += chunk.GetDuration();
- MOZ_ASSERT(mRemainingTicks <= mEncodedFrameDuration);
- VP8LOG("durationCopied %lld mRemainingTicks %lld\n",
- durationCopied, mRemainingTicks);
- if (durationCopied >= mRemainingTicks) {
- VP8LOG("nextEncodeOperation is %d\n",nextEncodeOperation);
- // Calculate encodedDuration for this target frame.
- StreamTime encodedDuration = CalculateEncodedDuration(durationCopied);
+ VP8LOG("nextEncodeOperation is %d for frame of duration %lld\n",
+ nextEncodeOperation, chunk.GetDuration());
- // Encode frame.
- if (nextEncodeOperation != SKIP_FRAME) {
- nsresult rv = PrepareRawFrame(chunk);
- NS_ENSURE_SUCCESS(rv, NS_ERROR_FAILURE);
+ // Encode frame.
+ if (nextEncodeOperation != SKIP_FRAME) {
+ nsresult rv = PrepareRawFrame(chunk);
+ NS_ENSURE_SUCCESS(rv, NS_ERROR_FAILURE);
- // Encode the data with VP8 encoder
- int flags = (nextEncodeOperation == ENCODE_NORMAL_FRAME) ?
- 0 : VPX_EFLAG_FORCE_KF;
- if (vpx_codec_encode(mVPXContext, mVPXImageWrapper, mEncodedTimestamp,
- (unsigned long)encodedDuration, flags,
- VPX_DL_REALTIME)) {
- return NS_ERROR_FAILURE;
- }
- // Get the encoded data from VP8 encoder.
- GetEncodedPartitions(aData);
- } else {
- // SKIP_FRAME
- // Extend the duration of the last encoded data in aData
- // because this frame will be skip.
- RefPtr<EncodedFrame> last = nullptr;
- last = aData.GetEncodedFrames().LastElement();
- if (last) {
- last->SetDuration(last->GetDuration() + encodedDuration);
- }
+ // Encode the data with VP8 encoder
+ int flags = (nextEncodeOperation == ENCODE_NORMAL_FRAME) ?
+ 0 : VPX_EFLAG_FORCE_KF;
+ if (vpx_codec_encode(mVPXContext, mVPXImageWrapper, mEncodedTimestamp,
+ (unsigned long)chunk.GetDuration(), flags,
+ VPX_DL_REALTIME)) {
+ return NS_ERROR_FAILURE;
}
- // Move forward the mEncodedTimestamp.
- mEncodedTimestamp += encodedDuration;
- totalProcessedDuration += durationCopied;
- // Calculate mRemainingTicks for next target frame.
- mRemainingTicks = CalculateRemainingTicks(durationCopied,
- encodedDuration);
-
- // Check the remain data is enough for next target frame.
- if (mSourceSegment.GetDuration() - totalProcessedDuration
- >= mEncodedFrameDuration) {
- TimeDuration elapsedTime = TimeStamp::Now() - timebase;
- nextEncodeOperation = GetNextEncodeOperation(elapsedTime,
- totalProcessedDuration);
- // Reset durationCopied for next iteration.
- durationCopied = 0;
- } else {
- // Process done, there is no enough data left for next iteration,
- // break the for-loop.
- break;
+ // Get the encoded data from VP8 encoder.
+ GetEncodedPartitions(aData);
+ } else {
+ // SKIP_FRAME
+ // Extend the duration of the last encoded data in aData
+ // because this frame will be skip.
+ NS_WARNING("MediaRecorder lagging behind. Skipping a frame.");
+ RefPtr<EncodedFrame> last = aData.GetEncodedFrames().LastElement();
+ if (last) {
+ last->SetDuration(last->GetDuration() + chunk.GetDuration());
}
}
+
+ // Move forward the mEncodedTimestamp.
+ mEncodedTimestamp += chunk.GetDuration();
+ totalProcessedDuration += chunk.GetDuration();
+
+ // Check what to do next.
+ TimeDuration elapsedTime = TimeStamp::Now() - timebase;
+ nextEncodeOperation = GetNextEncodeOperation(elapsedTime,
+ totalProcessedDuration);
}
+
// Remove the chunks we have processed.
- mSourceSegment.RemoveLeading(totalProcessedDuration);
- VP8LOG("RemoveLeading %lld\n",totalProcessedDuration);
+ mSourceSegment.Clear();
// End of stream, pull the rest frames in encoder.
if (EOS) {
VP8LOG("mEndOfStream is true\n");
mEncodingComplete = true;
// Bug 1243611, keep calling vpx_codec_encode and vpx_codec_get_cx_data
// until vpx_codec_get_cx_data return null.
do {
if (vpx_codec_encode(mVPXContext, nullptr, mEncodedTimestamp,
- mEncodedFrameDuration, 0, VPX_DL_REALTIME)) {
+ 0, 0, VPX_DL_REALTIME)) {
return NS_ERROR_FAILURE;
}
} while(GetEncodedPartitions(aData));
}
return NS_OK ;
}
--- a/dom/media/encoder/VP8TrackEncoder.h
+++ b/dom/media/encoder/VP8TrackEncoder.h
@@ -11,20 +11,19 @@
namespace mozilla {
typedef struct vpx_codec_ctx vpx_codec_ctx_t;
typedef struct vpx_codec_enc_cfg vpx_codec_enc_cfg_t;
typedef struct vpx_image vpx_image_t;
/**
- * VP8TrackEncoder implements VideoTrackEncoder by using libvpx library.
- * We implement a realtime and fixed FPS encoder. In order to achieve that,
- * there is a pick target frame and drop frame encoding policy implemented in
- * GetEncodedTrack.
+ * VP8TrackEncoder implements VideoTrackEncoder by using the libvpx library.
+ * We implement a realtime and variable frame rate encoder. In order to achieve
+ * that, there is a frame-drop encoding policy implemented in GetEncodedTrack.
*/
class VP8TrackEncoder : public VideoTrackEncoder
{
enum EncodeOperation {
ENCODE_NORMAL_FRAME, // VP8 track encoder works normally.
ENCODE_I_FRAME, // The next frame will be encoded as I-Frame.
SKIP_FRAME, // Skip the next frame.
};
@@ -36,55 +35,40 @@ public:
nsresult GetEncodedTrack(EncodedFrameContainer& aData) final override;
protected:
nsresult Init(int32_t aWidth, int32_t aHeight,
int32_t aDisplayWidth, int32_t aDisplayHeight) final override;
private:
- // Calculate the target frame's encoded duration.
- StreamTime CalculateEncodedDuration(StreamTime aDurationCopied);
-
- // Calculate the mRemainingTicks for next target frame.
- StreamTime CalculateRemainingTicks(StreamTime aDurationCopied,
- StreamTime aEncodedDuration);
-
// Get the EncodeOperation for next target frame.
EncodeOperation GetNextEncodeOperation(TimeDuration aTimeElapsed,
StreamTime aProcessedDuration);
// Get the encoded data from encoder to aData.
// Return value: false if the vpx_codec_get_cx_data returns null
// for EOS detection.
bool GetEncodedPartitions(EncodedFrameContainer& aData);
// Prepare the input data to the mVPXImageWrapper for encoding.
nsresult PrepareRawFrame(VideoChunk &aChunk);
- // Output frame rate.
- uint32_t mEncodedFrameRate;
- // Duration for the output frame, reciprocal to mEncodedFrameRate.
- StreamTime mEncodedFrameDuration;
// Encoded timestamp.
StreamTime mEncodedTimestamp;
- // Duration to the next encode frame.
- StreamTime mRemainingTicks;
// Muted frame, we only create it once.
RefPtr<layers::Image> mMuteFrame;
// I420 frame, for converting to I420.
nsTArray<uint8_t> mI420Frame;
/**
* A local segment queue which takes the raw data out from mRawSegment in the
- * call of GetEncodedTrack(). Since we implement the fixed FPS encoding
- * policy, it needs to be global in order to store the leftover segments
- * taken from mRawSegment.
+ * call of GetEncodedTrack().
*/
VideoSegment mSourceSegment;
// VP8 relative members.
// Codec context structure.
nsAutoPtr<vpx_codec_ctx_t> mVPXContext;
// Image Descriptor.
nsAutoPtr<vpx_image_t> mVPXImageWrapper;