Bug 1354457 - Record time spent paused in media encoder and adjust time stamps when muxing. r?pehrsons
MozReview-Commit-ID: 7J6oQf7qn4u
--- a/dom/media/encoder/MediaEncoder.cpp
+++ b/dom/media/encoder/MediaEncoder.cpp
@@ -1,16 +1,17 @@
/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-*/
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this file,
* You can obtain one at http://mozilla.org/MPL/2.0/. */
#include "MediaEncoder.h"
#include "MediaDecoder.h"
#include "nsIPrincipal.h"
#include "nsMimeTypes.h"
+#include "TimeUnits.h"
#include "mozilla/Logging.h"
#include "mozilla/Preferences.h"
#include "mozilla/StaticPtr.h"
#include "mozilla/gfx/Point.h" // IntSize
#include"GeckoProfiler.h"
#include "OggWriter.h"
#include "OpusTrackEncoder.h"
@@ -35,16 +36,42 @@ MediaStreamVideoRecorderSink::SetCurrent
MOZ_ASSERT(mVideoEncoder);
// If we're suspended (paused) we don't forward frames
if (!mSuspended) {
mVideoEncoder->SetCurrentFrames(aSegment);
}
}
void
+MediaEncoder::Suspend()
+{
+ MOZ_ASSERT(NS_IsMainThread());
+ mLastPauseStartTime = TimeStamp::Now();
+ mSuspended = true;
+ mVideoSink->Suspend();
+}
+
+void
+MediaEncoder::Resume()
+{
+ MOZ_ASSERT(NS_IsMainThread());
+ if (!mSuspended) {
+ return;
+ }
+ media::TimeUnit timeSpentPaused =
+ media::TimeUnit::FromTimeDuration(
+ TimeStamp::Now() - mLastPauseStartTime);
+ MOZ_ASSERT(timeSpentPaused.ToMicroseconds() >= 0);
+ MOZ_RELEASE_ASSERT(timeSpentPaused.IsValid());
+ mMicrosecondsSpentPaused += timeSpentPaused.ToMicroseconds();;
+ mSuspended = false;
+ mVideoSink->Resume();
+}
+
+void
MediaEncoder::SetDirectConnect(bool aConnected)
{
mDirectConnected = aConnected;
}
void
MediaEncoder::NotifyRealtimeData(MediaStreamGraph* aGraph,
TrackID aID,
@@ -316,16 +343,39 @@ MediaEncoder::WriteEncodedDataToMuxer(Tr
EncodedFrameContainer encodedVideoData;
nsresult rv = aTrackEncoder->GetEncodedTrack(encodedVideoData);
if (NS_FAILED(rv)) {
// Encoding might be canceled.
LOG(LogLevel::Error, ("Error! Fail to get encoded data from video encoder."));
mState = ENCODE_ERROR;
return rv;
}
+
+ // Update timestamps to accommodate pauses
+ const nsTArray<RefPtr<EncodedFrame> >& encodedFrames =
+ encodedVideoData.GetEncodedFrames();
+ // Take a copy of the atomic so we don't continually access it
+ uint64_t microsecondsSpentPaused = mMicrosecondsSpentPaused;
+ for (size_t i = 0; i < encodedFrames.Length(); ++i) {
+ RefPtr<EncodedFrame> frame = encodedFrames[i];
+ if (frame->GetTimeStamp() > microsecondsSpentPaused &&
+ frame->GetTimeStamp() - microsecondsSpentPaused > mLastMuxedTimestamp) {
+ // Use the adjusted timestamp if it's after the last timestamp
+ frame->SetTimeStamp(frame->GetTimeStamp() - microsecondsSpentPaused);
+ } else {
+ // If not, we force the last time stamp. We do this so the frames are
+ // still around and in order in case the codec needs to reference them.
+ // Dropping them here may result in artifacts in playback.
+ frame->SetTimeStamp(mLastMuxedTimestamp);
+ }
+ MOZ_ASSERT(mLastMuxedTimestamp <= frame->GetTimeStamp(),
+ "Our frames should be ordered by this point!");
+ mLastMuxedTimestamp = frame->GetTimeStamp();
+ }
+
rv = mWriter->WriteEncodedTrack(encodedVideoData,
aTrackEncoder->IsEncodingComplete() ?
ContainerWriter::END_OF_STREAM : 0);
if (NS_FAILED(rv)) {
LOG(LogLevel::Error, ("Error! Fail to write encoded video track to the media container."));
mState = ENCODE_ERROR;
}
return rv;
--- a/dom/media/encoder/MediaEncoder.h
+++ b/dom/media/encoder/MediaEncoder.h
@@ -98,38 +98,29 @@ public :
, mVideoSink(new MediaStreamVideoRecorderSink(mVideoEncoder))
, mStartTime(TimeStamp::Now())
, mMIMEType(aMIMEType)
, mSizeOfBuffer(0)
, mState(MediaEncoder::ENCODE_METADDATA)
, mShutdown(false)
, mDirectConnected(false)
, mSuspended(false)
+ , mMicrosecondsSpentPaused(0)
+ , mLastMuxedTimestamp(0)
{}
~MediaEncoder() {};
/* Note - called from control code, not on MSG threads. */
- void Suspend()
- {
- MOZ_ASSERT(NS_IsMainThread());
- mSuspended = true;
- mVideoSink->Suspend();
- }
+ void Suspend();
/**
* Note - called from control code, not on MSG threads.
- * Arm to collect the Duration of the next video frame and give it
- * to the next frame, in order to avoid any possible loss of sync. */
- void Resume()
- {
- MOZ_ASSERT(NS_IsMainThread());
- mSuspended = false;
- mVideoSink->Resume();
- }
+ * Calculates time spent paused in order to offset frames. */
+ void Resume();
/**
* Tells us which Notify to pay attention to for media
*/
void SetDirectConnect(bool aConnected);
/**
* Notified by the AppendToTrack in MediaStreamGraph; aRealtimeMedia is the raw
@@ -239,17 +230,28 @@ private:
nsAutoPtr<VideoTrackEncoder> mVideoEncoder;
RefPtr<MediaStreamVideoRecorderSink> mVideoSink;
TimeStamp mStartTime;
nsString mMIMEType;
int64_t mSizeOfBuffer;
int mState;
bool mShutdown;
bool mDirectConnected;
+ // Tracks if the encoder is suspended (paused). Used on the main thread and
+ // MediaRecorder's read thread.
Atomic<bool> mSuspended;
+ // Timestamp of when the last pause happened. Should only be accessed on the
+ // main thread.
+ TimeStamp mLastPauseStartTime;
+ // Exposes the time spend paused in microseconds. Read by the main thread
+ // and MediaRecorder's read thread. Should only be written by main thread.
+ Atomic<uint64_t> mMicrosecondsSpentPaused;
+ // The timestamp of the last muxed sample. Should only be used on
+ // MediaRecorder's read thread.
+ uint64_t mLastMuxedTimestamp;
// Get duration from create encoder, for logging purpose
double GetEncodeTimeStamp()
{
TimeDuration decodeTime;
decodeTime = TimeStamp::Now() - mStartTime;
return decodeTime.ToMilliseconds();
}
};