--- a/dom/media/EncodedBufferCache.cpp
+++ b/dom/media/EncodedBufferCache.cpp
@@ -125,9 +125,16 @@ EncodedBufferCache::ExtractBlob(nsISuppo
mEncodedBuffers.Clear();
} else
return nullptr;
}
mDataSize = 0;
return blob.forget();
}
+size_t
+EncodedBufferCache::SizeOfExcludingThis(mozilla::MallocSizeOf aMallocSizeOf)
+{
+ MutexAutoLock lock(mMutex);
+ return mEncodedBuffers.ShallowSizeOfExcludingThis(aMallocSizeOf);
+}
+
} // namespace mozilla
--- a/dom/media/EncodedBufferCache.h
+++ b/dom/media/EncodedBufferCache.h
@@ -37,16 +37,19 @@ public:
~EncodedBufferCache()
{
}
// Append buffers in cache, check if the queue is too large then switch to write buffer to file system
// aBuf will append to mEncodedBuffers or temporary File, aBuf also be cleared
void AppendBuffer(nsTArray<uint8_t> & aBuf);
// Read all buffer from memory or file System, also Remove the temporary file or clean the buffers in memory.
already_AddRefed<dom::Blob> ExtractBlob(nsISupports* aParent, const nsAString &aContentType);
+ // Returns the heap size in bytes of our internal buffers.
+ // Note that this intentionally ignores the data in the temp file.
+ size_t SizeOfExcludingThis(mozilla::MallocSizeOf aMallocSizeOf);
private:
//array for storing the encoded data.
nsTArray<nsTArray<uint8_t> > mEncodedBuffers;
// File handle for the temporary file
PRFileDesc* mFD;
// Used to protect the mEncodedBuffer for avoiding AppendBuffer/Consume on different thread at the same time.
Mutex mMutex;
--- a/dom/media/MediaRecorder.cpp
+++ b/dom/media/MediaRecorder.cpp
@@ -1,61 +1,71 @@
/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* vim:set ts=2 sw=2 sts=2 et cindent: */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
#include "MediaRecorder.h"
+
#include "AudioNodeEngine.h"
#include "AudioNodeStream.h"
#include "DOMMediaStream.h"
#include "EncodedBufferCache.h"
+#include "GeckoProfiler.h"
#include "MediaDecoder.h"
#include "MediaEncoder.h"
-#include "mozilla/StaticPtr.h"
+#include "MediaStreamGraphImpl.h"
#include "mozilla/DOMEventTargetHelper.h"
-#include "mozilla/Preferences.h"
#include "mozilla/dom/AudioStreamTrack.h"
#include "mozilla/dom/BlobEvent.h"
#include "mozilla/dom/File.h"
#include "mozilla/dom/MediaRecorderErrorEvent.h"
#include "mozilla/dom/VideoStreamTrack.h"
+#include "mozilla/media/MediaUtils.h"
+#include "mozilla/MemoryReporting.h"
+#include "mozilla/Preferences.h"
+#include "mozilla/StaticPtr.h"
+#include "mozilla/TaskQueue.h"
#include "nsAutoPtr.h"
+#include "nsCharSeparatedTokenizer.h"
+#include "nsContentTypeParser.h"
#include "nsContentUtils.h"
#include "nsError.h"
#include "nsIDocument.h"
#include "nsIPermissionManager.h"
#include "nsIPrincipal.h"
#include "nsIScriptError.h"
#include "nsMimeTypes.h"
#include "nsProxyRelease.h"
#include "nsTArray.h"
-#include "GeckoProfiler.h"
-#include "nsContentTypeParser.h"
-#include "nsCharSeparatedTokenizer.h"
#ifdef LOG
#undef LOG
#endif
mozilla::LazyLogModule gMediaRecorderLog("MediaRecorder");
#define LOG(type, msg) MOZ_LOG(gMediaRecorderLog, type, msg)
namespace mozilla {
namespace dom {
+using namespace mozilla::media;
+
+/* static */ StaticRefPtr<nsIAsyncShutdownBlocker> gMediaRecorderShutdownBlocker;
+static nsTHashtable<nsRefPtrHashKey<MediaRecorder::Session>> gSessions;
+
/**
-+ * MediaRecorderReporter measures memory being used by the Media Recorder.
-+ *
-+ * It is a singleton reporter and the single class object lives as long as at
-+ * least one Recorder is registered. In MediaRecorder, the reporter is unregistered
-+ * when it is destroyed.
-+ */
+ * MediaRecorderReporter measures memory being used by the Media Recorder.
+ *
+ * It is a singleton reporter and the single class object lives as long as at
+ * least one Recorder is registered. In MediaRecorder, the reporter is unregistered
+ * when it is destroyed.
+ */
class MediaRecorderReporter final : public nsIMemoryReporter
{
public:
NS_DECL_THREADSAFE_ISUPPORTS
MediaRecorderReporter() {};
static MediaRecorderReporter* UniqueInstance();
void InitMemoryReporter();
@@ -72,25 +82,39 @@ public:
sUniqueInstance = nullptr;
}
}
NS_IMETHOD
CollectReports(nsIHandleReportCallback* aHandleReport,
nsISupports* aData, bool aAnonymize) override
{
- int64_t amount = 0;
RecordersArray& recorders = GetRecorders();
- for (size_t i = 0; i < recorders.Length(); ++i) {
- amount += recorders[i]->SizeOfExcludingThis(MallocSizeOf);
+ nsTArray<RefPtr<MediaRecorder::SizeOfPromise>> promises;
+ for (const RefPtr<MediaRecorder>& recorder: recorders) {
+ promises.AppendElement(recorder->SizeOfExcludingThis(MallocSizeOf));
}
- MOZ_COLLECT_REPORT(
- "explicit/media/recorder", KIND_HEAP, UNITS_BYTES, amount,
- "Memory used by media recorder.");
+ nsCOMPtr<nsIHandleReportCallback> handleReport = aHandleReport;
+ nsCOMPtr<nsISupports> data = aData;
+ MediaRecorder::SizeOfPromise::All(GetCurrentThreadSerialEventTarget(), promises)
+ ->Then(GetCurrentThreadSerialEventTarget(), __func__,
+ [handleReport, data](const nsTArray<size_t>& sizes) {
+ size_t sum = 0;
+ for (const size_t& size : sizes) {
+ sum += size;
+ }
+
+ handleReport->Callback(
+ EmptyCString(), NS_LITERAL_CSTRING("explicit/media/recorder"),
+ KIND_HEAP, UNITS_BYTES, sum,
+ NS_LITERAL_CSTRING("Memory used by media recorder."),
+ data);
+ },
+ [](size_t) { MOZ_CRASH("Unexpected reject"); });
return NS_OK;
}
private:
MOZ_DEFINE_MALLOC_SIZE_OF(MallocSizeOf)
virtual ~MediaRecorderReporter();
static StaticRefPtr<MediaRecorderReporter> sUniqueInstance;
@@ -154,31 +178,30 @@ NS_IMPL_RELEASE_INHERITED(MediaRecorder,
* Pull encoded A/V frames from MediaEncoder, dispatch to OnDataAvailable handler.
* Unless a client calls Session::Stop, Session object keeps stay in this stage.
* 3) Destroy Stage (in main thread)
* Switch from Extract stage to Destroy stage by calling Session::Stop.
* Release session resource and remove associated streams from MSG.
*
* Lifetime of MediaRecorder and Session objects.
* 1) MediaRecorder creates a Session in MediaRecorder::Start function and holds
- * a reference to Session. Then the Session registers itself to
- * ShutdownObserver and also holds a reference to MediaRecorder.
+ * a reference to Session. Then the Session registers itself to a
+ * ShutdownBlocker and also holds a reference to MediaRecorder.
* Therefore, the reference dependency in gecko is:
- * ShutdownObserver -> Session <-> MediaRecorder, note that there is a cycle
+ * ShutdownBlocker -> Session <-> MediaRecorder, note that there is a cycle
* reference between Session and MediaRecorder.
* 2) A Session is destroyed in DestroyRunnable after MediaRecorder::Stop being called
* _and_ all encoded media data been passed to OnDataAvailable handler.
* 3) MediaRecorder::Stop is called by user or the document is going to
* inactive or invisible.
*/
-class MediaRecorder::Session: public nsIObserver,
- public PrincipalChangeObserver<MediaStreamTrack>,
+class MediaRecorder::Session: public PrincipalChangeObserver<MediaStreamTrack>,
public DOMMediaStream::TrackListener
{
- NS_DECL_THREADSAFE_ISUPPORTS
+ NS_INLINE_DECL_THREADSAFE_REFCOUNTING(Session)
// Main thread task.
// Create a blob event and send back to client.
class PushBlobRunnable : public Runnable
{
public:
explicit PushBlobRunnable(Session* aSession)
: Runnable("dom::MediaRecorder::Session::PushBlobRunnable")
@@ -221,19 +244,17 @@ class MediaRecorder::Session: public nsI
LOG(LogLevel::Debug, ("Session.ErrorNotifyRunnable s=(%p)", mSession.get()));
MOZ_ASSERT(NS_IsMainThread());
RefPtr<MediaRecorder> recorder = mSession->mRecorder;
if (!recorder) {
return NS_OK;
}
- if (mSession->IsEncoderError()) {
- recorder->NotifyError(NS_ERROR_UNEXPECTED);
- }
+ recorder->NotifyError(NS_ERROR_UNEXPECTED);
return NS_OK;
}
private:
RefPtr<Session> mSession;
};
// Fire start event and set mimeType, run in main thread task.
@@ -260,85 +281,53 @@ class MediaRecorder::Session: public nsI
return NS_OK;
}
private:
RefPtr<Session> mSession;
nsString mEventName;
};
- // Record thread task and it run in Media Encoder thread.
- // Fetch encoded Audio/Video data from MediaEncoder.
- class ExtractRunnable : public Runnable
- {
- public:
- explicit ExtractRunnable(Session* aSession)
- : Runnable("dom::MediaRecorder::Session::ExtractRunnable")
- , mSession(aSession)
- {
- }
-
- ~ExtractRunnable()
- {}
-
- NS_IMETHOD Run() override
- {
- MOZ_ASSERT(mSession->mReadThread->EventTarget()->IsOnCurrentThread());
-
- LOG(LogLevel::Debug, ("Session.ExtractRunnable shutdown = %d", mSession->mEncoder->IsShutdown()));
- if (!mSession->mEncoder->IsShutdown()) {
- mSession->Extract(false);
- if (NS_FAILED(NS_DispatchToCurrentThread(this))) {
- NS_WARNING("Failed to dispatch ExtractRunnable to encoder thread");
- }
- } else {
- // Flush out remaining encoded data.
- mSession->Extract(true);
- if (NS_FAILED(NS_DispatchToMainThread(
- new DestroyRunnable(mSession.forget())))) {
- MOZ_ASSERT(false, "NS_DispatchToMainThread DestroyRunnable failed");
- }
- }
- return NS_OK;
- }
-
- private:
- RefPtr<Session> mSession;
- };
-
- // For Ensure recorder has tracks to record.
+ // To ensure that MediaRecorder has tracks to record.
class TracksAvailableCallback : public OnTracksAvailableCallback
{
public:
- explicit TracksAvailableCallback(Session *aSession, TrackRate aTrackRate)
- : mSession(aSession)
- , mTrackRate(aTrackRate) {}
+ explicit TracksAvailableCallback(Session *aSession)
+ : mSession(aSession) {}
virtual void NotifyTracksAvailable(DOMMediaStream* aStream)
{
if (mSession->mStopIssued) {
return;
}
MOZ_RELEASE_ASSERT(aStream);
mSession->MediaStreamReady(*aStream);
uint8_t trackTypes = 0;
nsTArray<RefPtr<mozilla::dom::AudioStreamTrack>> audioTracks;
aStream->GetAudioTracks(audioTracks);
if (!audioTracks.IsEmpty()) {
trackTypes |= ContainerWriter::CREATE_AUDIO_TRACK;
- mSession->ConnectMediaStreamTrack(*audioTracks[0]);
}
nsTArray<RefPtr<mozilla::dom::VideoStreamTrack>> videoTracks;
aStream->GetVideoTracks(videoTracks);
if (!videoTracks.IsEmpty()) {
trackTypes |= ContainerWriter::CREATE_VIDEO_TRACK;
- mSession->ConnectMediaStreamTrack(*videoTracks[0]);
+ }
+
+ nsTArray<RefPtr<mozilla::dom::MediaStreamTrack>> tracks;
+ aStream->GetTracks(tracks);
+ for (auto& track : tracks) {
+ if (track->Ended()) {
+ continue;
+ }
+
+ mSession->ConnectMediaStreamTrack(*track);
}
if (audioTracks.Length() > 1 ||
videoTracks.Length() > 1) {
// When MediaRecorder supports multiple tracks, we should set up a single
// MediaInputPort from the input stream, and let main thread check
// track principals async later.
nsPIDOMWindowInner* window = mSession->mRecorder->GetParentObject();
@@ -357,21 +346,20 @@ class MediaRecorder::Session: public nsI
// Check that we may access the tracks' content.
if (!mSession->MediaStreamTracksPrincipalSubsumes()) {
LOG(LogLevel::Warning, ("Session.NotifyTracksAvailable MediaStreamTracks principal check failed"));
mSession->DoSessionEndTask(NS_ERROR_DOM_SECURITY_ERR);
return;
}
LOG(LogLevel::Debug, ("Session.NotifyTracksAvailable track type = (%d)", trackTypes));
- mSession->InitEncoder(trackTypes, mTrackRate);
+ mSession->InitEncoder(trackTypes, aStream->GraphRate());
}
private:
RefPtr<Session> mSession;
- TrackRate mTrackRate;
};
// Main thread task.
// To delete RecordingSession object.
class DestroyRunnable : public Runnable
{
public:
explicit DestroyRunnable(Session* aSession)
: Runnable("dom::MediaRecorder::Session::DestroyRunnable")
@@ -407,39 +395,102 @@ class MediaRecorder::Session: public nsI
}
return NS_OK;
}
// Dispatch stop event and clear MIME type.
mSession->mMimeType = NS_LITERAL_STRING("");
recorder->SetMimeType(mSession->mMimeType);
recorder->DispatchSimpleEvent(NS_LITERAL_STRING("stop"));
- mSession->BreakCycle();
+
+ RefPtr<Session> session = mSession.forget();
+ session->Shutdown()->Then(
+ GetCurrentThreadSerialEventTarget(), __func__,
+ [session]() {
+ gSessions.RemoveEntry(session);
+ if (gSessions.Count() == 0 &&
+ gMediaRecorderShutdownBlocker) {
+ // All sessions finished before shutdown, no need to keep the blocker.
+ RefPtr<nsIAsyncShutdownClient> barrier = GetShutdownBarrier();
+ barrier->RemoveBlocker(gMediaRecorderShutdownBlocker);
+ gMediaRecorderShutdownBlocker = nullptr;
+ }
+ },
+ []() { MOZ_CRASH("Not reached"); });
return NS_OK;
}
private:
// Call mSession::Release automatically while DestroyRunnable be destroy.
RefPtr<Session> mSession;
};
+ class EncoderListener : public MediaEncoderListener
+ {
+ public:
+ EncoderListener(TaskQueue* aEncoderThread, Session* aSession)
+ : mEncoderThread(aEncoderThread)
+ , mSession(aSession)
+ {}
+
+ void Forget()
+ {
+ MOZ_ASSERT(mEncoderThread->IsCurrentThreadIn());
+ mSession = nullptr;
+ }
+
+ void Initialized() override
+ {
+ MOZ_ASSERT(mEncoderThread->IsCurrentThreadIn());
+ if (mSession) {
+ mSession->MediaEncoderInitialized();
+ }
+ }
+
+ void DataAvailable() override
+ {
+ MOZ_ASSERT(mEncoderThread->IsCurrentThreadIn());
+ if (mSession) {
+ mSession->MediaEncoderDataAvailable();
+ }
+ }
+
+ void Error() override
+ {
+ MOZ_ASSERT(mEncoderThread->IsCurrentThreadIn());
+ if (mSession) {
+ mSession->MediaEncoderError();
+ }
+ }
+
+ void Shutdown() override
+ {
+ MOZ_ASSERT(mEncoderThread->IsCurrentThreadIn());
+ if (mSession) {
+ mSession->MediaEncoderShutdown();
+ }
+ }
+
+ protected:
+ RefPtr<TaskQueue> mEncoderThread;
+ RefPtr<Session> mSession;
+ };
+
friend class EncoderErrorNotifierRunnable;
friend class PushBlobRunnable;
- friend class ExtractRunnable;
friend class DestroyRunnable;
friend class TracksAvailableCallback;
public:
Session(MediaRecorder* aRecorder, int32_t aTimeSlice)
: mRecorder(aRecorder)
, mTimeSlice(aTimeSlice)
, mStopIssued(false)
, mIsStartEventFired(false)
, mNeedSessionEndTask(true)
- , mSelectedVideoTrackID(TRACK_NONE)
{
MOZ_ASSERT(NS_IsMainThread());
uint32_t maxMem = Preferences::GetUint("media.recorder.max_memory",
MAX_ALLOW_MEMORY_BUFFER);
mEncodedBufferCache = new EncodedBufferCache(maxMem);
mLastBlobTimeStamp = TimeStamp::Now();
}
@@ -456,246 +507,213 @@ public:
void NotifyTrackAdded(const RefPtr<MediaStreamTrack>& aTrack) override
{
LOG(LogLevel::Warning, ("Session.NotifyTrackAdded %p Raising error due to track set change", this));
DoSessionEndTask(NS_ERROR_ABORT);
}
void NotifyTrackRemoved(const RefPtr<MediaStreamTrack>& aTrack) override
{
- // Handle possible early removal of direct video listener
- if (mEncoder) {
- RefPtr<VideoStreamTrack> videoTrack = aTrack->AsVideoStreamTrack();
- if (videoTrack) {
- videoTrack->RemoveDirectListener(mEncoder->GetVideoSink());
- }
+ if (aTrack->Ended()) {
+ // TrackEncoder will pickup tracks that end itself.
+ return;
}
- RefPtr<MediaInputPort> foundInputPort;
- for (RefPtr<MediaInputPort> inputPort : mInputPorts) {
- if (aTrack->IsForwardedThrough(inputPort)) {
- foundInputPort = inputPort;
- break;
- }
- }
-
- if (foundInputPort) {
- // A recorded track was removed or ended. End it in the recording.
- // Don't raise an error.
- foundInputPort->Destroy();
- DebugOnly<bool> removed = mInputPorts.RemoveElement(foundInputPort);
- MOZ_ASSERT(removed);
- return;
+ MOZ_ASSERT(mEncoder);
+ if (mEncoder) {
+ mEncoder->RemoveMediaStreamTrack(aTrack);
}
LOG(LogLevel::Warning, ("Session.NotifyTrackRemoved %p Raising error due to track set change", this));
DoSessionEndTask(NS_ERROR_ABORT);
}
void Start()
{
LOG(LogLevel::Debug, ("Session.Start %p", this));
MOZ_ASSERT(NS_IsMainThread());
- // Create a Track Union Stream
- MediaStreamGraph* gm = mRecorder->GetSourceMediaStream()->Graph();
- TrackRate trackRate = gm->GraphRate();
- mTrackUnionStream = gm->CreateTrackUnionStream();
- MOZ_ASSERT(mTrackUnionStream, "CreateTrackUnionStream failed");
-
- mTrackUnionStream->SetAutofinish(true);
-
DOMMediaStream* domStream = mRecorder->Stream();
if (domStream) {
- // Get the available tracks from the DOMMediaStream.
- // The callback will report back tracks that we have to connect to
- // mTrackUnionStream and listen to principal changes on.
- TracksAvailableCallback* tracksAvailableCallback = new TracksAvailableCallback(this, trackRate);
+ // The callback reports back when tracks are available and can be
+ // attached to MediaEncoder. This allows `recorder.start()` before any tracks are available.
+ // We have supported this historically and have mochitests assuming this.
+ TracksAvailableCallback* tracksAvailableCallback = new TracksAvailableCallback(this);
domStream->OnTracksAvailable(tracksAvailableCallback);
- } else {
+ } else if (mRecorder->mAudioNode) {
// Check that we may access the audio node's content.
if (!AudioNodePrincipalSubsumes()) {
LOG(LogLevel::Warning, ("Session.Start AudioNode principal check failed"));
DoSessionEndTask(NS_ERROR_DOM_SECURITY_ERR);
return;
}
- // Bind this Track Union Stream with Source Media.
- RefPtr<MediaInputPort> inputPort =
- mTrackUnionStream->AllocateInputPort(mRecorder->GetSourceMediaStream());
- mInputPorts.AppendElement(inputPort.forget());
- MOZ_ASSERT(mInputPorts[mInputPorts.Length()-1]);
+
+ TrackRate trackRate = mRecorder->mAudioNode->Context()->Graph()->GraphRate();
// Web Audio node has only audio.
InitEncoder(ContainerWriter::CREATE_AUDIO_TRACK, trackRate);
+ } else {
+ MOZ_ASSERT(false, "Unknown source");
}
}
void Stop()
{
LOG(LogLevel::Debug, ("Session.Stop %p", this));
MOZ_ASSERT(NS_IsMainThread());
mStopIssued = true;
- CleanupStreams();
+
+ if (mEncoder) {
+ mEncoder->Stop();
+ }
+
if (mNeedSessionEndTask) {
LOG(LogLevel::Debug, ("Session.Stop mNeedSessionEndTask %p", this));
// End the Session directly if there is no ExtractRunnable.
DoSessionEndTask(NS_OK);
}
- // If we don't do this, the Session will be purged only when the navigator exit
- // by the ShutdownObserver and the memory and number of threads will quickly
- // grows with each couple stop/start.
- nsContentUtils::UnregisterShutdownObserver(this);
}
nsresult Pause()
{
LOG(LogLevel::Debug, ("Session.Pause"));
MOZ_ASSERT(NS_IsMainThread());
- NS_ENSURE_TRUE(mTrackUnionStream, NS_ERROR_FAILURE);
- mTrackUnionStream->Suspend();
- if (mEncoder) {
- mEncoder->Suspend();
+ if (!mEncoder) {
+ return NS_ERROR_FAILURE;
}
+ mEncoder->Suspend(TimeStamp::Now());
return NS_OK;
}
nsresult Resume()
{
LOG(LogLevel::Debug, ("Session.Resume"));
MOZ_ASSERT(NS_IsMainThread());
- NS_ENSURE_TRUE(mTrackUnionStream, NS_ERROR_FAILURE);
- if (mEncoder) {
- mEncoder->Resume();
+ if (!mEncoder) {
+ return NS_ERROR_FAILURE;
}
- mTrackUnionStream->Resume();
+ mEncoder->Resume(TimeStamp::Now());
return NS_OK;
}
nsresult RequestData()
{
LOG(LogLevel::Debug, ("Session.RequestData"));
MOZ_ASSERT(NS_IsMainThread());
- if (NS_FAILED(NS_DispatchToMainThread(new EncoderErrorNotifierRunnable(this))) ||
- NS_FAILED(NS_DispatchToMainThread(new PushBlobRunnable(this)))) {
+ if (NS_FAILED(NS_DispatchToMainThread(new PushBlobRunnable(this)))) {
MOZ_ASSERT(false, "RequestData NS_DispatchToMainThread failed");
return NS_ERROR_FAILURE;
}
return NS_OK;
}
already_AddRefed<nsIDOMBlob> GetEncodedData()
{
MOZ_ASSERT(NS_IsMainThread());
return mEncodedBufferCache->ExtractBlob(mRecorder->GetParentObject(),
mMimeType);
}
- bool IsEncoderError()
+ RefPtr<SizeOfPromise>
+ SizeOfExcludingThis(mozilla::MallocSizeOf aMallocSizeOf)
{
- if (mEncoder && mEncoder->HasError()) {
- return true;
+ MOZ_ASSERT(NS_IsMainThread());
+ size_t encodedBufferSize =
+ mEncodedBufferCache->SizeOfExcludingThis(aMallocSizeOf);
+
+ if (!mEncoder) {
+ return SizeOfPromise::CreateAndResolve(encodedBufferSize, __func__);
}
- return false;
- }
- size_t
- SizeOfExcludingThis(mozilla::MallocSizeOf aMallocSizeOf) const
- {
- return (mEncoder ? mEncoder->SizeOfExcludingThis(aMallocSizeOf) : 0);
+ auto& encoder = mEncoder;
+ return InvokeAsync(mEncoderThread, __func__,
+ [encoder, encodedBufferSize, aMallocSizeOf]() {
+ return SizeOfPromise::CreateAndResolve(
+ encodedBufferSize + encoder->SizeOfExcludingThis(aMallocSizeOf), __func__);
+ });
}
-
private:
- // Only DestroyRunnable is allowed to delete Session object.
+ // Only DestroyRunnable is allowed to delete Session object on main thread.
virtual ~Session()
{
+ MOZ_ASSERT(NS_IsMainThread());
+ MOZ_ASSERT(mShutdownPromise);
LOG(LogLevel::Debug, ("Session.~Session (%p)", this));
- CleanupStreams();
- if (mReadThread) {
- mReadThread->Shutdown();
- mReadThread = nullptr;
- // Inside the if() so that if we delete after xpcom-shutdown's Observe(), we
- // won't try to remove it after the observer service is shut down.
- // Unregistering for safety in case Stop() was never called
- nsContentUtils::UnregisterShutdownObserver(this);
- }
}
// Pull encoded media data from MediaEncoder and put into EncodedBufferCache.
// Destroy this session object in the end of this function.
// If the bool aForceFlush is true, we will force to dispatch a
// PushBlobRunnable to main thread.
void Extract(bool aForceFlush)
{
- MOZ_ASSERT(mReadThread->EventTarget()->IsOnCurrentThread());
+ MOZ_ASSERT(mEncoderThread->IsCurrentThreadIn());
+
LOG(LogLevel::Debug, ("Session.Extract %p", this));
AUTO_PROFILER_LABEL("MediaRecorder::Session::Extract", OTHER);
// Pull encoded media data from MediaEncoder
nsTArray<nsTArray<uint8_t> > encodedBuf;
- mEncoder->GetEncodedData(&encodedBuf, mMimeType);
+ nsresult rv = mEncoder->GetEncodedData(&encodedBuf);
+ if (NS_FAILED(rv)) {
+ MOZ_RELEASE_ASSERT(encodedBuf.IsEmpty());
+ // Even if we failed to encode more data, it might be time to push a blob
+ // with already encoded data.
+ }
// Append pulled data into cache buffer.
for (uint32_t i = 0; i < encodedBuf.Length(); i++) {
if (!encodedBuf[i].IsEmpty()) {
mEncodedBufferCache->AppendBuffer(encodedBuf[i]);
- // Fire the start event when encoded data is available.
- if (!mIsStartEventFired) {
- NS_DispatchToMainThread(
- new DispatchStartEventRunnable(this, NS_LITERAL_STRING("start")));
- mIsStartEventFired = true;
- }
}
}
// Whether push encoded data back to onDataAvailable automatically or we
// need a flush.
- bool pushBlob = false;
- if ((mTimeSlice > 0) &&
- ((TimeStamp::Now()-mLastBlobTimeStamp).ToMilliseconds() > mTimeSlice)) {
+ bool pushBlob = aForceFlush;
+ if (!pushBlob &&
+ mTimeSlice > 0 &&
+ (TimeStamp::Now()-mLastBlobTimeStamp).ToMilliseconds() > mTimeSlice) {
pushBlob = true;
}
- if (pushBlob || aForceFlush) {
- // Fire the start event before the blob.
- if (!mIsStartEventFired) {
- NS_DispatchToMainThread(
- new DispatchStartEventRunnable(this, NS_LITERAL_STRING("start")));
- mIsStartEventFired = true;
- }
- if (NS_FAILED(NS_DispatchToMainThread(new EncoderErrorNotifierRunnable(this)))) {
- MOZ_ASSERT(false, "NS_DispatchToMainThread EncoderErrorNotifierRunnable failed");
- }
+ if (pushBlob) {
if (NS_FAILED(NS_DispatchToMainThread(new PushBlobRunnable(this)))) {
MOZ_ASSERT(false, "NS_DispatchToMainThread PushBlobRunnable failed");
} else {
mLastBlobTimeStamp = TimeStamp::Now();
}
}
}
void MediaStreamReady(DOMMediaStream& aStream) {
mMediaStream = &aStream;
aStream.RegisterTrackListener(this);
}
void ConnectMediaStreamTrack(MediaStreamTrack& aTrack)
{
+ for (auto& track : mMediaStreamTracks) {
+ if (track->AsAudioStreamTrack() && aTrack.AsAudioStreamTrack()) {
+ // We only allow one audio track. See bug 1276928.
+ return;
+ }
+ if (track->AsVideoStreamTrack() && aTrack.AsVideoStreamTrack()) {
+ // We only allow one video track. See bug 1276928.
+ return;
+ }
+ }
mMediaStreamTracks.AppendElement(&aTrack);
aTrack.AddPrincipalChangeObserver(this);
- RefPtr<MediaInputPort> inputPort =
- aTrack.ForwardTrackContentsTo(mTrackUnionStream);
- MOZ_ASSERT(inputPort);
- mInputPorts.AppendElement(inputPort.forget());
- MOZ_ASSERT(mInputPorts[mInputPorts.Length()-1]);
}
bool PrincipalSubsumes(nsIPrincipal* aPrincipal)
{
if (!mRecorder->GetOwner())
return false;
nsCOMPtr<nsIDocument> doc = mRecorder->GetOwner()->GetExtantDoc();
if (!doc) {
@@ -718,221 +736,335 @@ private:
for (RefPtr<MediaStreamTrack>& track : mMediaStreamTracks) {
nsContentUtils::CombineResourcePrincipals(&principal, track->GetPrincipal());
}
return PrincipalSubsumes(principal);
}
bool AudioNodePrincipalSubsumes()
{
- MOZ_ASSERT(mRecorder->mAudioNode != nullptr);
+ MOZ_ASSERT(mRecorder->mAudioNode);
nsIDocument* doc = mRecorder->mAudioNode->GetOwner()
? mRecorder->mAudioNode->GetOwner()->GetExtantDoc()
: nullptr;
nsCOMPtr<nsIPrincipal> principal = doc ? doc->NodePrincipal() : nullptr;
return PrincipalSubsumes(principal);
}
void InitEncoder(uint8_t aTrackTypes, TrackRate aTrackRate)
{
LOG(LogLevel::Debug, ("Session.InitEncoder %p", this));
MOZ_ASSERT(NS_IsMainThread());
- if (!mRecorder) {
- LOG(LogLevel::Debug, ("Session.InitEncoder failure, mRecorder is null %p", this));
+ // Create a TaskQueue to read encode media data from MediaEncoder.
+ MOZ_RELEASE_ASSERT(!mEncoderThread);
+ RefPtr<SharedThreadPool> pool =
+ SharedThreadPool::Get(NS_LITERAL_CSTRING("MediaRecorderReadThread"));
+ if (!pool) {
+ LOG(LogLevel::Debug, ("Session.InitEncoder %p Failed to create "
+ "MediaRecorderReadThread thread pool", this));
+ DoSessionEndTask(NS_ERROR_FAILURE);
return;
}
+
+ mEncoderThread = MakeAndAddRef<TaskQueue>(pool.forget());
+
+ if (!gMediaRecorderShutdownBlocker) {
+ // Add a shutdown blocker so mEncoderThread can be shutdown async.
+ class Blocker : public ShutdownBlocker
+ {
+ public:
+ Blocker()
+ : ShutdownBlocker(NS_LITERAL_STRING(
+ "MediaRecorder::Session: shutdown"))
+ {}
+
+ NS_IMETHOD BlockShutdown(nsIAsyncShutdownClient*) override
+ {
+ // Distribute the global async shutdown blocker in a ticket. If there
+ // are zero graphs then shutdown is unblocked when we go out of scope.
+ RefPtr<ShutdownTicket> ticket =
+ MakeAndAddRef<ShutdownTicket>(gMediaRecorderShutdownBlocker);
+ gMediaRecorderShutdownBlocker = nullptr;
+
+ nsTArray<RefPtr<ShutdownPromise>> promises(gSessions.Count());
+ for (auto iter = gSessions.Iter(); !iter.Done(); iter.Next()) {
+ promises.AppendElement(iter.Get()->GetKey()->Shutdown());
+ }
+ gSessions.Clear();
+ ShutdownPromise::All(GetCurrentThreadSerialEventTarget(), promises)->Then(
+ GetCurrentThreadSerialEventTarget(), __func__,
+ [ticket]() mutable {
+ MOZ_ASSERT(gSessions.Count() == 0);
+ // Unblock shutdown
+ ticket = nullptr;
+ },
+ []() { MOZ_CRASH("Not reached"); });
+ return NS_OK;
+ }
+ };
+
+ gMediaRecorderShutdownBlocker = MakeAndAddRef<Blocker>();
+ RefPtr<nsIAsyncShutdownClient> barrier = GetShutdownBarrier();
+ nsresult rv = barrier->AddBlocker(gMediaRecorderShutdownBlocker,
+ NS_LITERAL_STRING(__FILE__), __LINE__,
+ NS_LITERAL_STRING("MediaRecorder::Session: shutdown"));
+ MOZ_RELEASE_ASSERT(NS_SUCCEEDED(rv));
+ }
+
+ gSessions.PutEntry(this);
+
+ uint32_t audioBitrate = mRecorder->GetAudioBitrate();
+ uint32_t videoBitrate = mRecorder->GetVideoBitrate();
+ uint32_t bitrate = mRecorder->GetBitrate();
+ if (bitrate > 0) {
+ // There's a total cap set. We have to make sure the type-specific limits
+ // are within range.
+ if ((aTrackTypes & ContainerWriter::CREATE_AUDIO_TRACK) &&
+ (aTrackTypes & ContainerWriter::CREATE_VIDEO_TRACK) &&
+ audioBitrate + videoBitrate > bitrate) {
+ LOG(LogLevel::Info, ("Session.InitEncoder Bitrates higher than total cap. Recalculating."));
+ double factor = bitrate / static_cast<double>(audioBitrate + videoBitrate);
+ audioBitrate = static_cast<uint32_t>(audioBitrate * factor);
+ videoBitrate = static_cast<uint32_t>(videoBitrate * factor);
+ } else if ((aTrackTypes & ContainerWriter::CREATE_AUDIO_TRACK) &&
+ !(aTrackTypes & ContainerWriter::CREATE_VIDEO_TRACK)) {
+ audioBitrate = std::min(audioBitrate, bitrate);
+ videoBitrate = 0;
+ } else if (!(aTrackTypes & ContainerWriter::CREATE_AUDIO_TRACK) &&
+ (aTrackTypes & ContainerWriter::CREATE_VIDEO_TRACK)) {
+ audioBitrate = 0;
+ videoBitrate = std::min(videoBitrate, bitrate);
+ }
+ MOZ_ASSERT(audioBitrate + videoBitrate <= bitrate);
+ }
+
// Allocate encoder and bind with union stream.
// At this stage, the API doesn't allow UA to choose the output mimeType format.
- mEncoder = MediaEncoder::CreateEncoder(NS_LITERAL_STRING(""),
- mRecorder->GetAudioBitrate(),
- mRecorder->GetVideoBitrate(),
- mRecorder->GetBitrate(),
+ mEncoder = MediaEncoder::CreateEncoder(mEncoderThread,
+ NS_LITERAL_STRING(""),
+ audioBitrate, videoBitrate,
aTrackTypes, aTrackRate);
if (!mEncoder) {
- LOG(LogLevel::Debug, ("Session.InitEncoder !mEncoder %p", this));
+ LOG(LogLevel::Error, ("Session.InitEncoder !mEncoder %p", this));
DoSessionEndTask(NS_ERROR_ABORT);
return;
}
- // Media stream is ready but UA issues a stop method follow by start method.
- // The Session::stop would clean the mTrackUnionStream. If the AfterTracksAdded
- // comes after stop command, this function would crash.
- if (!mTrackUnionStream) {
- LOG(LogLevel::Debug, ("Session.InitEncoder !mTrackUnionStream %p", this));
- DoSessionEndTask(NS_OK);
- return;
- }
- mTrackUnionStream->AddListener(mEncoder.get());
+ mEncoderListener = MakeAndAddRef<EncoderListener>(mEncoderThread, this);
+ mEncoderThread->Dispatch(
+ NewRunnableMethod<RefPtr<EncoderListener>>(
+ "mozilla::MediaEncoder::RegisterListener",
+ mEncoder, &MediaEncoder::RegisterListener, mEncoderListener));
- nsTArray<RefPtr<mozilla::dom::VideoStreamTrack>> videoTracks;
- DOMMediaStream* domStream = mRecorder->Stream();
- if (domStream) {
- domStream->GetVideoTracks(videoTracks);
- if (!videoTracks.IsEmpty()) {
- // Right now, the MediaRecorder hasn't dealt with multiple video track
- // issues. So we just bind with the first video track. Bug 1276928 is
- // the following.
- videoTracks[0]->AddDirectListener(mEncoder->GetVideoSink());
- }
+ if (mRecorder->mAudioNode) {
+ mEncoder->ConnectAudioNode(mRecorder->mAudioNode,
+ mRecorder->mAudioNodeOutput);
}
- // Create a thread to read encode media data from MediaEncoder.
- if (!mReadThread) {
- nsresult rv = NS_NewNamedThread("Media_Encoder", getter_AddRefs(mReadThread));
- if (NS_FAILED(rv)) {
- LOG(LogLevel::Debug, ("Session.InitEncoder !mReadThread %p", this));
- DoSessionEndTask(rv);
- return;
- }
+ for (auto& track : mMediaStreamTracks) {
+ mEncoder->ConnectMediaStreamTrack(track);
}
- // In case source media stream does not notify track end, receive
- // shutdown notification and stop Read Thread.
- nsContentUtils::RegisterShutdownObserver(this);
-
- nsCOMPtr<nsIRunnable> event = new ExtractRunnable(this);
- if (NS_FAILED(mReadThread->EventTarget()->Dispatch(event.forget(), NS_DISPATCH_NORMAL))) {
- NS_WARNING("Failed to dispatch ExtractRunnable at beginning");
- LOG(LogLevel::Debug, ("Session.InitEncoder !ReadThread->Dispatch %p", this));
- DoSessionEndTask(NS_ERROR_ABORT);
- }
// Set mNeedSessionEndTask to false because the
// ExtractRunnable/DestroyRunnable will take the response to
// end the session.
mNeedSessionEndTask = false;
}
+
// application should get blob and onstop event
void DoSessionEndTask(nsresult rv)
{
MOZ_ASSERT(NS_IsMainThread());
- CleanupStreams();
if (!mIsStartEventFired) {
NS_DispatchToMainThread(
new DispatchStartEventRunnable(this, NS_LITERAL_STRING("start")));
}
if (NS_FAILED(rv)) {
mRecorder->ForceInactive();
NS_DispatchToMainThread(
NewRunnableMethod<nsresult>("dom::MediaRecorder::NotifyError",
mRecorder,
&MediaRecorder::NotifyError,
rv));
}
- if (NS_FAILED(NS_DispatchToMainThread(new EncoderErrorNotifierRunnable(this)))) {
- MOZ_ASSERT(false, "NS_DispatchToMainThread EncoderErrorNotifierRunnable failed");
- }
if (rv != NS_ERROR_DOM_SECURITY_ERR) {
// Don't push a blob if there was a security error.
if (NS_FAILED(NS_DispatchToMainThread(new PushBlobRunnable(this)))) {
MOZ_ASSERT(false, "NS_DispatchToMainThread PushBlobRunnable failed");
}
}
if (NS_FAILED(NS_DispatchToMainThread(new DestroyRunnable(this)))) {
MOZ_ASSERT(false, "NS_DispatchToMainThread DestroyRunnable failed");
}
mNeedSessionEndTask = false;
}
- void CleanupStreams()
+
+ void MediaEncoderInitialized()
{
- if (mTrackUnionStream) {
- if (mEncoder) {
- nsTArray<RefPtr<mozilla::dom::VideoStreamTrack>> videoTracks;
- DOMMediaStream* domStream = mRecorder->Stream();
- if (domStream) {
- domStream->GetVideoTracks(videoTracks);
- if (!videoTracks.IsEmpty()) {
- videoTracks[0]->RemoveDirectListener(mEncoder->GetVideoSink());
- }
- }
+ MOZ_ASSERT(mEncoderThread->IsCurrentThreadIn());
+
+ // Pull encoded metadata from MediaEncoder
+ nsTArray<nsTArray<uint8_t> > encodedBuf;
+ nsresult rv = mEncoder->GetEncodedMetadata(&encodedBuf, mMimeType);
+ if (NS_FAILED(rv)) {
+ MOZ_ASSERT(false);
+ return;
+ }
+
+ // Append pulled data into cache buffer.
+ for (uint32_t i = 0; i < encodedBuf.Length(); i++) {
+ if (!encodedBuf[i].IsEmpty()) {
+ mEncodedBufferCache->AppendBuffer(encodedBuf[i]);
}
+ }
+ }
- // Sometimes the MediaEncoder might be initialized fail and go to
- // |CleanupStreams|. So the mEncoder might be a nullptr in this case.
- if (mEncoder && mSelectedVideoTrackID != TRACK_NONE) {
- mTrackUnionStream->RemoveVideoOutput(mEncoder->GetVideoSink(), mSelectedVideoTrackID);
- }
- if (mEncoder) {
- mTrackUnionStream->RemoveListener(mEncoder.get());
- }
- mTrackUnionStream->Destroy();
- mTrackUnionStream = nullptr;
+ void MediaEncoderDataAvailable()
+ {
+ MOZ_ASSERT(mEncoderThread->IsCurrentThreadIn());
+
+ if (!mIsStartEventFired) {
+ NS_DispatchToMainThread(
+ new DispatchStartEventRunnable(this, NS_LITERAL_STRING("start")));
+ mIsStartEventFired = true;
}
- for (RefPtr<MediaInputPort>& inputPort : mInputPorts) {
- MOZ_ASSERT(inputPort);
- inputPort->Destroy();
+ Extract(false);
+ }
+
+ void MediaEncoderError()
+ {
+ MOZ_ASSERT(mEncoderThread->IsCurrentThreadIn());
+ NS_DispatchToMainThread(
+ NewRunnableMethod<nsresult>(
+ "dom::MediaRecorder::Session::DoSessionEndTask",
+ this, &Session::DoSessionEndTask, NS_ERROR_FAILURE));
+ }
+
+ void MediaEncoderShutdown()
+ {
+ MOZ_ASSERT(mEncoderThread->IsCurrentThreadIn());
+ MOZ_ASSERT(mEncoder->IsShutdown());
+
+ // Forces the last blob even if it's not time for it yet.
+ Extract(true);
+
+ // For the stop event.
+ if (NS_FAILED(NS_DispatchToMainThread(
+ new DestroyRunnable(this)))) {
+ MOZ_ASSERT(false, "NS_DispatchToMainThread DestroyRunnable failed");
}
- mInputPorts.Clear();
+
+ // Clean up.
+ mEncoderListener->Forget();
+ DebugOnly<bool> unregistered =
+ mEncoder->UnregisterListener(mEncoderListener);
+ MOZ_ASSERT(unregistered);
+ }
+
+ RefPtr<ShutdownPromise> Shutdown()
+ {
+ MOZ_ASSERT(NS_IsMainThread());
+ LOG(LogLevel::Debug, ("Session Shutdown %p", this));
+
+ if (mShutdownPromise) {
+ return mShutdownPromise;
+ }
+
+ mShutdownPromise = ShutdownPromise::CreateAndResolve(true, __func__);
+ RefPtr<Session> self = this;
+ if (mEncoder) {
+ auto& encoder = mEncoder;
+ encoder->Cancel();
+
+ MOZ_RELEASE_ASSERT(mEncoderListener);
+ auto& encoderListener = mEncoderListener;
+ mShutdownPromise = mShutdownPromise->Then(
+ mEncoderThread, __func__,
+ [encoder, encoderListener]() {
+ encoder->UnregisterListener(encoderListener);
+ encoderListener->Forget();
+ return ShutdownPromise::CreateAndResolve(true, __func__);
+ },
+ []() {
+ MOZ_ASSERT_UNREACHABLE("Unexpected reject");
+ return ShutdownPromise::CreateAndReject(false, __func__);
+ });
+ }
+
+ // Remove main thread state.
if (mMediaStream) {
mMediaStream->UnregisterTrackListener(this);
mMediaStream = nullptr;
}
- for (RefPtr<MediaStreamTrack>& track : mMediaStreamTracks) {
- track->RemovePrincipalChangeObserver(this);
- }
- mMediaStreamTracks.Clear();
- }
-
- NS_IMETHOD Observe(nsISupports *aSubject, const char *aTopic, const char16_t *aData) override
- {
- MOZ_ASSERT(NS_IsMainThread());
- LOG(LogLevel::Debug, ("Session.Observe XPCOM_SHUTDOWN %p", this));
- if (strcmp(aTopic, NS_XPCOM_SHUTDOWN_OBSERVER_ID) == 0) {
- // Force stop Session to terminate Read Thread.
- mEncoder->Cancel();
- if (mReadThread) {
- mReadThread->Shutdown();
- mReadThread = nullptr;
+ {
+ auto tracks(Move(mMediaStreamTracks));
+ for (RefPtr<MediaStreamTrack>& track : tracks) {
+ track->RemovePrincipalChangeObserver(this);
}
- nsContentUtils::UnregisterShutdownObserver(this);
- BreakCycle();
- Stop();
}
- return NS_OK;
- }
+ // Break the cycle reference between Session and MediaRecorder.
+ if (mRecorder) {
+ mShutdownPromise = mShutdownPromise->Then(
+ GetCurrentThreadSerialEventTarget(), __func__,
+ [self]() {
+ self->mRecorder->RemoveSession(self);
+ self->mRecorder = nullptr;
+ return ShutdownPromise::CreateAndResolve(true, __func__);
+ },
+ []() {
+ MOZ_ASSERT_UNREACHABLE("Unexpected reject");
+ return ShutdownPromise::CreateAndReject(false, __func__);
+ });
+ }
- // Break the cycle reference between Session and MediaRecorder.
- void BreakCycle()
- {
- MOZ_ASSERT(NS_IsMainThread());
- if (mRecorder) {
- mRecorder->RemoveSession(this);
- mRecorder = nullptr;
+ if (mEncoderThread) {
+ RefPtr<TaskQueue>& encoderThread = mEncoderThread;
+ mShutdownPromise = mShutdownPromise->Then(
+ GetCurrentThreadSerialEventTarget(), __func__,
+ [encoderThread]() {
+ return encoderThread->BeginShutdown();
+ },
+ []() {
+ MOZ_ASSERT_UNREACHABLE("Unexpected reject");
+ return ShutdownPromise::CreateAndReject(false, __func__);
+ });
}
+
+ return mShutdownPromise;
}
private:
// Hold reference to MediaRecoder that ensure MediaRecorder is alive
// if there is an active session. Access ONLY on main thread.
RefPtr<MediaRecorder> mRecorder;
- // Receive track data from source and dispatch to Encoder.
- // Pause/ Resume controller.
- RefPtr<ProcessedMediaStream> mTrackUnionStream;
- nsTArray<RefPtr<MediaInputPort>> mInputPorts;
-
// Stream currently recorded.
RefPtr<DOMMediaStream> mMediaStream;
// Tracks currently recorded. This should be a subset of mMediaStream's track
// set.
nsTArray<RefPtr<MediaStreamTrack>> mMediaStreamTracks;
- // Runnable thread for read data from MediaEncode.
- nsCOMPtr<nsIThread> mReadThread;
+ // Runnable thread for reading data from MediaEncoder.
+ RefPtr<TaskQueue> mEncoderThread;
// MediaEncoder pipeline.
RefPtr<MediaEncoder> mEncoder;
- // A buffer to cache encoded meda data.
+ // Listener through which MediaEncoder signals us.
+ RefPtr<EncoderListener> mEncoderListener;
+ // Set in Shutdown() and resolved when shutdown is complete.
+ RefPtr<ShutdownPromise> mShutdownPromise;
+ // A buffer to cache encoded media data.
nsAutoPtr<EncodedBufferCache> mEncodedBufferCache;
// Current session mimeType
nsString mMimeType;
// Timestamp of the last fired dataavailable event.
TimeStamp mLastBlobTimeStamp;
// The interval of passing encoded data from EncodedBufferCache to onDataAvailable
// handler. "mTimeSlice < 0" means Session object does not push encoded data to
// onDataAvailable, instead, it passive wait the client side pull encoded data
@@ -941,71 +1073,47 @@ private:
// Indicate this session's stop has been called.
bool mStopIssued;
// Indicate the session had fire start event. Encoding thread only.
bool mIsStartEventFired;
// False if the InitEncoder called successfully, ensure the
// ExtractRunnable/DestroyRunnable will end the session.
// Main thread only.
bool mNeedSessionEndTask;
- TrackID mSelectedVideoTrackID;
};
-NS_IMPL_ISUPPORTS(MediaRecorder::Session, nsIObserver)
-
MediaRecorder::~MediaRecorder()
{
- if (mPipeStream != nullptr) {
- mInputPort->Destroy();
- mPipeStream->Destroy();
- }
LOG(LogLevel::Debug, ("~MediaRecorder (%p)", this));
UnRegisterActivityObserver();
}
MediaRecorder::MediaRecorder(DOMMediaStream& aSourceMediaStream,
nsPIDOMWindowInner* aOwnerWindow)
: DOMEventTargetHelper(aOwnerWindow)
+ , mAudioNodeOutput(0)
, mState(RecordingState::Inactive)
{
MOZ_ASSERT(aOwnerWindow);
MOZ_ASSERT(aOwnerWindow->IsInnerWindow());
mDOMStream = &aSourceMediaStream;
RegisterActivityObserver();
}
MediaRecorder::MediaRecorder(AudioNode& aSrcAudioNode,
uint32_t aSrcOutput,
nsPIDOMWindowInner* aOwnerWindow)
: DOMEventTargetHelper(aOwnerWindow)
+ , mAudioNodeOutput(aSrcOutput)
, mState(RecordingState::Inactive)
{
MOZ_ASSERT(aOwnerWindow);
MOZ_ASSERT(aOwnerWindow->IsInnerWindow());
- // Only AudioNodeStream of kind EXTERNAL_STREAM stores output audio data in
- // the track (see AudioNodeStream::AdvanceOutputSegment()). That means track
- // union stream in recorder session won't be able to copy data from the
- // stream of non-destination node. Create a pipe stream in this case.
- if (aSrcAudioNode.NumberOfOutputs() > 0) {
- AudioContext* ctx = aSrcAudioNode.Context();
- AudioNodeEngine* engine = new AudioNodeEngine(nullptr);
- AudioNodeStream::Flags flags =
- AudioNodeStream::EXTERNAL_OUTPUT |
- AudioNodeStream::NEED_MAIN_THREAD_FINISHED;
- mPipeStream = AudioNodeStream::Create(ctx, engine, flags, ctx->Graph());
- AudioNodeStream* ns = aSrcAudioNode.GetStream();
- if (ns) {
- mInputPort =
- mPipeStream->AllocateInputPort(aSrcAudioNode.GetStream(),
- TRACK_ANY, TRACK_ANY,
- 0, aSrcOutput);
- }
- }
mAudioNode = &aSrcAudioNode;
RegisterActivityObserver();
}
void
MediaRecorder::RegisterActivityObserver()
{
@@ -1046,21 +1154,16 @@ MediaRecorder::Start(const Optional<int3
InitializeDomExceptions();
if (mState != RecordingState::Inactive) {
aResult.Throw(NS_ERROR_DOM_INVALID_STATE_ERR);
return;
}
- if (GetSourceMediaStream()->IsFinished() || GetSourceMediaStream()->IsDestroyed()) {
- aResult.Throw(NS_ERROR_DOM_INVALID_STATE_ERR);
- return;
- }
-
nsTArray<RefPtr<MediaStreamTrack>> tracks;
if (mDOMStream) {
mDOMStream->GetTracks(tracks);
}
if (!tracks.IsEmpty()) {
// If there are tracks already available that we're not allowed
// to record, we should throw a security error.
bool subsumes = false;
@@ -1215,18 +1318,18 @@ MediaRecorder::Constructor(const GlobalO
}
if (!IsTypeSupported(aInitDict.mMimeType)) {
aRv.Throw(NS_ERROR_DOM_NOT_SUPPORTED_ERR);
return nullptr;
}
RefPtr<MediaRecorder> object = new MediaRecorder(aSrcAudioNode,
- aSrcOutput,
- ownerWindow);
+ aSrcOutput,
+ ownerWindow);
object->SetOptions(aInitDict);
return object.forget();
}
void
MediaRecorder::SetOptions(const MediaRecorderOptions& aInitDict)
{
SetMimeType(aInitDict.mMimeType);
@@ -1444,26 +1547,16 @@ MediaRecorder::NotifyOwnerDocumentActivi
if (!doc->IsActive() || !doc->IsVisible()) {
// Stop the session.
ErrorResult result;
Stop(result);
result.SuppressException();
}
}
-MediaStream*
-MediaRecorder::GetSourceMediaStream()
-{
- if (mDOMStream != nullptr) {
- return mDOMStream->GetPlaybackStream();
- }
- MOZ_ASSERT(mAudioNode != nullptr);
- return mPipeStream ? mPipeStream.get() : mAudioNode->GetStream();
-}
-
void
MediaRecorder::ForceInactive()
{
LOG(LogLevel::Debug, ("MediaRecorder.ForceInactive %p", this));
mState = RecordingState::Inactive;
}
void
@@ -1480,40 +1573,61 @@ MediaRecorder::StopForSessionDestruction
void
MediaRecorder::InitializeDomExceptions()
{
mSecurityDomException = DOMException::Create(NS_ERROR_DOM_SECURITY_ERR);
mUnknownDomException = DOMException::Create(NS_ERROR_DOM_UNKNOWN_ERR);
}
-size_t
-MediaRecorder::SizeOfExcludingThis(mozilla::MallocSizeOf aMallocSizeOf) const
+RefPtr<MediaRecorder::SizeOfPromise>
+MediaRecorder::SizeOfExcludingThis(mozilla::MallocSizeOf aMallocSizeOf)
{
- size_t amount = 42;
- for (size_t i = 0; i < mSessions.Length(); ++i) {
- amount += mSessions[i]->SizeOfExcludingThis(aMallocSizeOf);
+ MOZ_ASSERT(NS_IsMainThread());
+
+ // The return type of a chained MozPromise cannot be changed, so we create a
+ // holder for our desired return type and resolve that from All()->Then().
+ auto holder = MakeRefPtr<Refcountable<MozPromiseHolder<SizeOfPromise>>>();
+ RefPtr<SizeOfPromise> promise = holder->Ensure(__func__);
+
+ nsTArray<RefPtr<SizeOfPromise>> promises(mSessions.Length());
+ for (const RefPtr<Session>& session : mSessions) {
+ promises.AppendElement(session->SizeOfExcludingThis(aMallocSizeOf));
}
- return amount;
+
+ SizeOfPromise::All(GetCurrentThreadSerialEventTarget(), promises)->Then(
+ GetCurrentThreadSerialEventTarget(), __func__,
+ [holder](const nsTArray<size_t>& sizes) {
+ size_t total = 0;
+ for (const size_t& size : sizes) {
+ total += size;
+ }
+ holder->Resolve(total, __func__);
+ },
+ []() {
+ MOZ_CRASH("Unexpected reject");
+ });
+
+ return promise;
}
StaticRefPtr<MediaRecorderReporter> MediaRecorderReporter::sUniqueInstance;
MediaRecorderReporter* MediaRecorderReporter::UniqueInstance()
{
if (!sUniqueInstance) {
sUniqueInstance = new MediaRecorderReporter();
sUniqueInstance->InitMemoryReporter();
}
return sUniqueInstance;
}
void MediaRecorderReporter::InitMemoryReporter()
{
- RegisterWeakMemoryReporter(this);
+ RegisterWeakAsyncMemoryReporter(this);
}
MediaRecorderReporter::~MediaRecorderReporter()
{
UnregisterWeakMemoryReporter(this);
}
} // namespace dom
--- a/dom/media/MediaRecorder.h
+++ b/dom/media/MediaRecorder.h
@@ -4,28 +4,25 @@
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
#ifndef MediaRecorder_h
#define MediaRecorder_h
#include "mozilla/dom/MediaRecorderBinding.h"
#include "mozilla/DOMEventTargetHelper.h"
-#include "mozilla/MemoryReporting.h"
#include "nsIDocumentActivity.h"
// Max size for allowing queue encoded data in memory
#define MAX_ALLOW_MEMORY_BUFFER 1024000
namespace mozilla {
-class AbstractThread;
class AudioNodeStream;
class DOMMediaStream;
class ErrorResult;
-class MediaInputPort;
struct MediaRecorderOptions;
class MediaStream;
class GlobalObject;
namespace dom {
class AudioNode;
class DOMException;
@@ -39,43 +36,45 @@ class DOMException;
* Thread model:
* When the recorder starts, it creates a "Media Encoder" thread to read data from MediaEncoder object and store buffer in EncodedBufferCache object.
* Also extract the encoded data and create blobs on every timeslice passed from start function or RequestData function called by UA.
*/
class MediaRecorder final : public DOMEventTargetHelper,
public nsIDocumentActivity
{
+public:
class Session;
-public:
MediaRecorder(DOMMediaStream& aSourceMediaStream,
nsPIDOMWindowInner* aOwnerWindow);
MediaRecorder(AudioNode& aSrcAudioNode, uint32_t aSrcOutput,
nsPIDOMWindowInner* aOwnerWindow);
+ static nsTArray<RefPtr<Session>> GetSessions();
+
// nsWrapperCache
JSObject* WrapObject(JSContext* aCx, JS::Handle<JSObject*> aGivenProto) override;
nsPIDOMWindowInner* GetParentObject() { return GetOwner(); }
NS_DECL_ISUPPORTS_INHERITED
NS_DECL_CYCLE_COLLECTION_CLASS_INHERITED(MediaRecorder,
DOMEventTargetHelper)
// WebIDL
// Start recording. If timeSlice has been provided, mediaRecorder will
// raise a dataavailable event containing the Blob of collected data on every timeSlice milliseconds.
// If timeSlice isn't provided, UA should call the RequestData to obtain the Blob data, also set the mTimeSlice to zero.
void Start(const Optional<int32_t>& timeSlice, ErrorResult & aResult);
// Stop the recording activiy. Including stop the Media Encoder thread, un-hook the mediaStreamListener to encoder.
void Stop(ErrorResult& aResult);
- // Pause the mTrackUnionStream
+ // Pause a recording.
void Pause(ErrorResult& aResult);
-
+ // Resume a paused recording.
void Resume(ErrorResult& aResult);
// Extract encoded data Blob from EncodedBufferCache.
void RequestData(ErrorResult& aResult);
// Return the The DOMMediaStream passed from UA.
DOMMediaStream* Stream() const { return mDOMStream; }
// The current state of the MediaRecorder object.
RecordingState State() const { return mState; }
// Return the current encoding MIME type selected by the MediaEncoder.
@@ -94,20 +93,21 @@ public:
static already_AddRefed<MediaRecorder>
Constructor(const GlobalObject& aGlobal,
AudioNode& aSrcAudioNode,
uint32_t aSrcOutput,
const MediaRecorderOptions& aInitDict,
ErrorResult& aRv);
/*
- * Measure the size of the buffer, and memory occupied by mAudioEncoder
- * and mVideoEncoder
+ * Measure the size of the buffer, and heap memory in bytes occupied by
+ * mAudioEncoder and mVideoEncoder.
*/
- size_t SizeOfExcludingThis(mozilla::MallocSizeOf aMallocSizeOf) const;
+ typedef MozPromise<size_t, size_t, true> SizeOfPromise;
+ RefPtr<SizeOfPromise> SizeOfExcludingThis(mozilla::MallocSizeOf aMallocSizeOf);
// EventHandler
IMPL_EVENT_HANDLER(dataavailable)
IMPL_EVENT_HANDLER(error)
IMPL_EVENT_HANDLER(start)
IMPL_EVENT_HANDLER(stop)
IMPL_EVENT_HANDLER(warning)
NS_DECL_NSIDOCUMENTACTIVITY
@@ -127,18 +127,16 @@ protected:
void NotifyError(nsresult aRv);
// Set encoded MIME type.
void SetMimeType(const nsString &aMimeType);
void SetOptions(const MediaRecorderOptions& aInitDict);
MediaRecorder(const MediaRecorder& x) = delete; // prevent bad usage
// Remove session pointer.
void RemoveSession(Session* aSession);
- // Functions for Session to query input source info.
- MediaStream* GetSourceMediaStream();
// Create DOMExceptions capturing the JS stack for async errors. These are
// created ahead of time rather than on demand when firing an error as the JS
// stack of the operation that started the async behavior will not be
// available at the time the error event is fired. Note, depending on when
// this is called there may not be a JS stack to capture.
void InitializeDomExceptions();
// Set the recorder state to inactive. This is needed to handle error states
// in the recorder where state must transition to inactive before full
@@ -146,22 +144,18 @@ protected:
void ForceInactive();
// Stop the recorder and its internal session. This should be used by
// sessions that are in the process of being destroyed.
void StopForSessionDestruction();
// DOM wrapper for source media stream. Will be null when input is audio node.
RefPtr<DOMMediaStream> mDOMStream;
// Source audio node. Will be null when input is a media stream.
RefPtr<AudioNode> mAudioNode;
- // Pipe stream connecting non-destination source node and session track union
- // stream of recorder. Will be null when input is media stream or destination
- // node.
- RefPtr<AudioNodeStream> mPipeStream;
- // Connect source node to the pipe stream.
- RefPtr<MediaInputPort> mInputPort;
+ // Source audio node's output index. Will be zero when input is a media stream.
+ const uint32_t mAudioNodeOutput;
// The current state of the MediaRecorder object.
RecordingState mState;
// Hold the sessions reference and clean it when the DestroyRunnable for a
// session is running.
nsTArray<RefPtr<Session> > mSessions;
nsCOMPtr<nsIDocument> mDocument;
--- a/dom/media/encoder/MediaEncoder.cpp
+++ b/dom/media/encoder/MediaEncoder.cpp
@@ -1,430 +1,1059 @@
/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-*/
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this file,
* You can obtain one at http://mozilla.org/MPL/2.0/. */
+
#include "MediaEncoder.h"
+
+#include <algorithm>
+#include "AudioNodeEngine.h"
+#include "AudioNodeStream.h"
+#include "GeckoProfiler.h"
#include "MediaDecoder.h"
+#include "MediaStreamVideoSink.h"
+#include "mozilla/dom/AudioNode.h"
+#include "mozilla/dom/AudioStreamTrack.h"
+#include "mozilla/dom/MediaStreamTrack.h"
+#include "mozilla/dom/VideoStreamTrack.h"
+#include "mozilla/gfx/Point.h" // IntSize
+#include "mozilla/Logging.h"
+#include "mozilla/media/MediaUtils.h"
+#include "mozilla/Preferences.h"
+#include "mozilla/StaticPtr.h"
+#include "mozilla/TaskQueue.h"
#include "nsIPrincipal.h"
#include "nsMimeTypes.h"
-#include "TimeUnits.h"
-#include "mozilla/Logging.h"
-#include "mozilla/Preferences.h"
-#include "mozilla/StaticPtr.h"
-#include "mozilla/gfx/Point.h" // IntSize
-
-#include"GeckoProfiler.h"
#include "OggWriter.h"
#include "OpusTrackEncoder.h"
+#include "TimeUnits.h"
#ifdef MOZ_WEBM_ENCODER
#include "VP8TrackEncoder.h"
#include "WebMWriter.h"
#endif
#ifdef LOG
#undef LOG
#endif
mozilla::LazyLogModule gMediaEncoderLog("MediaEncoder");
#define LOG(type, msg) MOZ_LOG(gMediaEncoderLog, type, msg)
namespace mozilla {
-void
-MediaStreamVideoRecorderSink::SetCurrentFrames(const VideoSegment& aSegment)
+using namespace dom;
+using namespace media;
+
+class MediaEncoder::AudioTrackListener : public DirectMediaStreamTrackListener
+{
+public:
+ AudioTrackListener(AudioTrackEncoder* aEncoder,
+ TaskQueue* aEncoderThread)
+ : mDirectConnected(false)
+ , mInitialized(false)
+ , mRemoved(false)
+ , mEncoder(aEncoder)
+ , mEncoderThread(aEncoderThread)
+ {
+ MOZ_ASSERT(mEncoder);
+ MOZ_ASSERT(mEncoderThread);
+ }
+
+ void NotifyShutdown() {
+ mShutdown = true;
+ }
+
+ void NotifyDirectListenerInstalled(InstallationResult aResult) override
+ {
+ if (aResult == InstallationResult::SUCCESS) {
+ LOG(LogLevel::Info, ("Audio track direct listener installed"));
+ mDirectConnected = true;
+ } else {
+ LOG(LogLevel::Info, ("Audio track failed to install direct listener"));
+ MOZ_ASSERT(!mDirectConnected);
+ }
+ }
+
+ void NotifyDirectListenerUninstalled() override
+ {
+ mDirectConnected = false;
+
+ if (mRemoved) {
+ mEncoder = nullptr;
+ mEncoderThread = nullptr;
+ }
+ }
+
+ void NotifyQueuedChanges(MediaStreamGraph* aGraph,
+ StreamTime aTrackOffset,
+ const MediaSegment& aQueuedMedia) override
+ {
+ MOZ_ASSERT(mEncoder);
+ MOZ_ASSERT(mEncoderThread);
+
+ if (mShutdown) {
+ return;
+ }
+
+ if (!mInitialized) {
+ mEncoderThread->Dispatch(
+ NewRunnableMethod<StreamTime>(
+ "mozilla::AudioTrackEncoder::SetStartOffset",
+ mEncoder, &AudioTrackEncoder::SetStartOffset, aTrackOffset));
+ mInitialized = true;
+ }
+
+ if (mDirectConnected) {
+ if (aQueuedMedia.IsNull()) {
+ mEncoderThread->Dispatch(
+ NewRunnableMethod<StreamTime>(
+ "mozilla::AudioTrackEncoder::AdvanceBlockedInput",
+ mEncoder, &AudioTrackEncoder::AdvanceBlockedInput,
+ aQueuedMedia.GetDuration()));
+ return;
+ }
+ } else {
+ NotifyRealtimeTrackData(aGraph, aTrackOffset, aQueuedMedia);
+ }
+
+ mEncoderThread->Dispatch(
+ NewRunnableMethod<StreamTime>(
+ "mozilla::AudioTrackEncoder::AdvanceCurrentTime",
+ mEncoder, &AudioTrackEncoder::AdvanceCurrentTime,
+ aQueuedMedia.GetDuration()));
+ }
+
+ void NotifyRealtimeTrackData(MediaStreamGraph* aGraph,
+ StreamTime aTrackOffset,
+ const MediaSegment& aMedia) override
+ {
+ MOZ_ASSERT(mEncoder);
+ MOZ_ASSERT(mEncoderThread);
+
+ if (mShutdown) {
+ return;
+ }
+
+ const AudioSegment& audio = static_cast<const AudioSegment&>(aMedia);
+
+ AudioSegment copy;
+ copy.AppendSlice(audio, 0, audio.GetDuration());
+
+ mEncoderThread->Dispatch(
+ NewRunnableMethod<StoreCopyPassByRRef<AudioSegment>>(
+ "mozilla::AudioTrackEncoder::AppendAudioSegment",
+ mEncoder, &AudioTrackEncoder::AppendAudioSegment, Move(copy)));
+ }
+
+ void NotifyEnded() override
+ {
+ MOZ_ASSERT(mEncoder);
+ MOZ_ASSERT(mEncoderThread);
+
+ if (mShutdown) {
+ return;
+ }
+
+ mEncoderThread->Dispatch(
+ NewRunnableMethod("mozilla::AudioTrackEncoder::NotifyEndOfStream",
+ mEncoder, &AudioTrackEncoder::NotifyEndOfStream));
+ }
+
+ void NotifyRemoved() override
+ {
+ if (!mShutdown) {
+ mEncoderThread->Dispatch(
+ NewRunnableMethod("mozilla::AudioTrackEncoder::NotifyEndOfStream",
+ mEncoder, &AudioTrackEncoder::NotifyEndOfStream));
+ }
+
+ mRemoved = true;
+
+ if (!mDirectConnected) {
+ mEncoder = nullptr;
+ mEncoderThread = nullptr;
+ }
+ }
+
+private:
+ // True when MediaEncoder has shutdown and destroyed the TaskQueue.
+ Atomic<bool> mShutdown;
+ bool mDirectConnected;
+ bool mInitialized;
+ bool mRemoved;
+ RefPtr<AudioTrackEncoder> mEncoder;
+ RefPtr<TaskQueue> mEncoderThread;
+};
+
+class MediaEncoder::VideoTrackListener : public MediaStreamVideoSink
{
- MOZ_ASSERT(mVideoEncoder);
- // If we're suspended (paused) we don't forward frames
- if (!mSuspended) {
- mVideoEncoder->SetCurrentFrames(aSegment);
+public:
+ VideoTrackListener(VideoTrackEncoder* aEncoder,
+ TaskQueue* aEncoderThread)
+ : mDirectConnected(false)
+ , mInitialized(false)
+ , mRemoved(false)
+ , mEncoder(aEncoder)
+ , mEncoderThread(aEncoderThread)
+ {
+ MOZ_ASSERT(mEncoder);
+ MOZ_ASSERT(mEncoderThread);
+ }
+
+ void NotifyShutdown() {
+ mShutdown = true;
+ }
+
+ void NotifyDirectListenerInstalled(InstallationResult aResult) override
+ {
+ if (aResult == InstallationResult::SUCCESS) {
+ LOG(LogLevel::Info, ("Video track direct listener installed"));
+ mDirectConnected = true;
+ } else {
+ LOG(LogLevel::Info, ("Video track failed to install direct listener"));
+ MOZ_ASSERT(!mDirectConnected);
+ return;
+ }
+ }
+
+ void NotifyDirectListenerUninstalled() override
+ {
+ mDirectConnected = false;
+
+ if (mRemoved) {
+ mEncoder = nullptr;
+ mEncoderThread = nullptr;
+ }
+ }
+
+ void NotifyQueuedChanges(MediaStreamGraph* aGraph,
+ StreamTime aTrackOffset,
+ const MediaSegment& aQueuedMedia) override
+ {
+ MOZ_ASSERT(mEncoder);
+ MOZ_ASSERT(mEncoderThread);
+
+ if (mShutdown) {
+ return;
+ }
+
+ if (!mInitialized) {
+ mEncoderThread->Dispatch(
+ NewRunnableMethod<StreamTime>(
+ "mozilla::VideoTrackEncoder::SetStartOffset",
+ mEncoder, &VideoTrackEncoder::SetStartOffset, aTrackOffset));
+ mInitialized = true;
+ }
+
+ if (aQueuedMedia.IsNull()) {
+ mEncoderThread->Dispatch(
+ NewRunnableMethod<StreamTime>(
+ "mozilla::VideoTrackEncoder::AdvanceBlockedInput",
+ mEncoder, &VideoTrackEncoder::AdvanceBlockedInput,
+ aQueuedMedia.GetDuration()));
+ return;
+ }
+
+ mEncoderThread->Dispatch(
+ NewRunnableMethod<StreamTime>(
+ "mozilla::VideoTrackEncoder::AdvanceCurrentTime",
+ mEncoder, &VideoTrackEncoder::AdvanceCurrentTime,
+ aQueuedMedia.GetDuration()));
+ }
+
+ void SetCurrentFrames(const VideoSegment& aMedia) override
+ {
+ MOZ_ASSERT(mEncoder);
+ MOZ_ASSERT(mEncoderThread);
+
+ if (mShutdown) {
+ return;
+ }
+
+ VideoSegment copy;
+ copy.AppendSlice(aMedia, 0, aMedia.GetDuration());
+
+ mEncoderThread->Dispatch(
+ NewRunnableMethod<StoreCopyPassByRRef<VideoSegment>>(
+ "mozilla::VideoTrackEncoder::AppendVideoSegment",
+ mEncoder, &VideoTrackEncoder::AppendVideoSegment, Move(copy)));
+ }
+
+ void ClearFrames() override {}
+
+ void NotifyEnded() override
+ {
+ MOZ_ASSERT(mEncoder);
+ MOZ_ASSERT(mEncoderThread);
+
+ if (mShutdown) {
+ return;
+ }
+
+ mEncoderThread->Dispatch(
+ NewRunnableMethod("mozilla::VideoTrackEncoder::NotifyEndOfStream",
+ mEncoder, &VideoTrackEncoder::NotifyEndOfStream));
+ }
+
+ void NotifyRemoved() override
+ {
+ if (!mShutdown) {
+ mEncoderThread->Dispatch(
+ NewRunnableMethod("mozilla::VideoTrackEncoder::NotifyEndOfStream",
+ mEncoder, &VideoTrackEncoder::NotifyEndOfStream));
+ }
+
+ mRemoved = true;
+
+ if (!mDirectConnected) {
+ mEncoder = nullptr;
+ mEncoderThread = nullptr;
+ }
+ }
+
+private:
+ // True when MediaEncoder has shutdown and destroyed the TaskQueue.
+ Atomic<bool> mShutdown;
+ bool mDirectConnected;
+ bool mInitialized;
+ bool mRemoved;
+ RefPtr<VideoTrackEncoder> mEncoder;
+ RefPtr<TaskQueue> mEncoderThread;
+};
+
+class MediaEncoder::EncoderListener : public TrackEncoderListener
+{
+public:
+ EncoderListener(TaskQueue* aEncoderThread, MediaEncoder* aEncoder)
+ : mEncoderThread(aEncoderThread)
+ , mEncoder(aEncoder)
+ , mPendingDataAvailable(false)
+ {}
+
+ void Forget()
+ {
+ MOZ_ASSERT(mEncoderThread->IsCurrentThreadIn());
+ mEncoder = nullptr;
+ }
+
+ void Initialized(TrackEncoder* aTrackEncoder) override
+ {
+ MOZ_ASSERT(mEncoderThread->IsCurrentThreadIn());
+ MOZ_ASSERT(aTrackEncoder->IsInitialized());
+
+ if (!mEncoder) {
+ return;
+ }
+
+ mEncoderThread->Dispatch(
+ NewRunnableMethod("mozilla::MediaEncoder::NotifyInitialized",
+ mEncoder, &MediaEncoder::NotifyInitialized));
+ }
+
+ void DataAvailable(TrackEncoder* aTrackEncoder) override
+ {
+ MOZ_ASSERT(mEncoderThread->IsCurrentThreadIn());
+ MOZ_ASSERT(aTrackEncoder->IsInitialized());
+
+ if (!mEncoder) {
+ return;
+ }
+
+ if (mPendingDataAvailable) {
+ return;
+ }
+
+ mEncoderThread->Dispatch(
+ NewRunnableMethod("mozilla::MediaEncoder::EncoderListener::DataAvailableImpl",
+ this, &EncoderListener::DataAvailableImpl));
+
+ mPendingDataAvailable = true;
+ }
+
+ void DataAvailableImpl()
+ {
+ MOZ_ASSERT(mEncoderThread->IsCurrentThreadIn());
+
+ if (!mEncoder) {
+ return;
+ }
+
+ mEncoder->NotifyDataAvailable();
+ mPendingDataAvailable = false;
+ }
+
+ void Error(TrackEncoder* aTrackEncoder) override
+ {
+ MOZ_ASSERT(mEncoderThread->IsCurrentThreadIn());
+
+ if (!mEncoder) {
+ return;
+ }
+
+ mEncoderThread->Dispatch(
+ NewRunnableMethod("mozilla::MediaEncoder::SetError",
+ mEncoder, &MediaEncoder::SetError));
+ }
+
+protected:
+ RefPtr<TaskQueue> mEncoderThread;
+ RefPtr<MediaEncoder> mEncoder;
+ bool mPendingDataAvailable;
+};
+
+MediaEncoder::MediaEncoder(TaskQueue* aEncoderThread,
+ UniquePtr<ContainerWriter> aWriter,
+ AudioTrackEncoder* aAudioEncoder,
+ VideoTrackEncoder* aVideoEncoder,
+ const nsAString& aMIMEType)
+ : mEncoderThread(aEncoderThread)
+ , mWriter(Move(aWriter))
+ , mAudioEncoder(aAudioEncoder)
+ , mVideoEncoder(aVideoEncoder)
+ , mEncoderListener(MakeAndAddRef<EncoderListener>(mEncoderThread, this))
+ , mStartTime(TimeStamp::Now())
+ , mMIMEType(aMIMEType)
+ , mInitialized(false)
+ , mMetadataEncoded(false)
+ , mCompleted(false)
+ , mError(false)
+ , mCanceled(false)
+ , mShutdown(false)
+{
+ if (mAudioEncoder) {
+ mAudioListener =
+ MakeAndAddRef<AudioTrackListener>(mAudioEncoder, mEncoderThread);
+ mEncoderThread->Dispatch(
+ NewRunnableMethod<RefPtr<EncoderListener>>(
+ "mozilla::AudioTrackEncoder::RegisterListener",
+ mAudioEncoder, &AudioTrackEncoder::RegisterListener, mEncoderListener));
+ }
+ if (mVideoEncoder) {
+ mVideoListener =
+ MakeAndAddRef<VideoTrackListener>(mVideoEncoder, mEncoderThread);
+ mEncoderThread->Dispatch(
+ NewRunnableMethod<RefPtr<EncoderListener>>(
+ "mozilla::VideoTrackEncoder::RegisterListener",
+ mVideoEncoder, &VideoTrackEncoder::RegisterListener, mEncoderListener));
+ }
+}
+
+MediaEncoder::~MediaEncoder()
+{
+ MOZ_ASSERT(mListeners.IsEmpty());
+}
+
+void
+MediaEncoder::Suspend(TimeStamp aTime)
+{
+ auto& ae = mAudioEncoder;
+ auto& ve = mVideoEncoder;
+ mEncoderThread->Dispatch(NewRunnableFrom([ae, ve, aTime]() {
+ if (ae) {
+ ae->Suspend(aTime);
+ }
+ if (ve) {
+ ve->Suspend(aTime);
+ }
+ return NS_OK;
+ }));
+}
+
+void
+MediaEncoder::Resume(TimeStamp aTime)
+{
+ auto& ae = mAudioEncoder;
+ auto& ve = mVideoEncoder;
+ mEncoderThread->Dispatch(NewRunnableFrom([ae, ve, aTime]() {
+ if (ae) {
+ ae->Resume(aTime);
+ }
+ if (ve) {
+ ve->Resume(aTime);
+ }
+ return NS_OK;
+ }));
+}
+
+void
+MediaEncoder::ConnectAudioNode(AudioNode* aNode, uint32_t aOutput)
+{
+ MOZ_ASSERT(NS_IsMainThread());
+
+ if (mAudioNode) {
+ MOZ_ASSERT(false, "Only one audio node supported");
+ return;
+ }
+
+ // Only AudioNodeStream of kind EXTERNAL_OUTPUT stores output audio data in
+ // the track (see AudioNodeStream::AdvanceOutputSegment()). That means track
+ // union stream in recorder session won't be able to copy data from the
+ // stream of non-destination node. Create a pipe stream in this case.
+ if (aNode->NumberOfOutputs() > 0) {
+ AudioContext* ctx = aNode->Context();
+ AudioNodeEngine* engine = new AudioNodeEngine(nullptr);
+ AudioNodeStream::Flags flags =
+ AudioNodeStream::EXTERNAL_OUTPUT |
+ AudioNodeStream::NEED_MAIN_THREAD_FINISHED;
+ mPipeStream = AudioNodeStream::Create(ctx, engine, flags, ctx->Graph());
+ AudioNodeStream* ns = aNode->GetStream();
+ if (ns) {
+ mInputPort =
+ mPipeStream->AllocateInputPort(aNode->GetStream(),
+ TRACK_ANY, TRACK_ANY,
+ 0, aOutput);
+ }
+ }
+
+ mAudioNode = aNode;
+
+ if (mPipeStream) {
+ mPipeStream->AddTrackListener(mAudioListener, AudioNodeStream::AUDIO_TRACK);
+ } else {
+ mAudioNode->GetStream()->AddTrackListener(mAudioListener, AudioNodeStream::AUDIO_TRACK);
}
}
void
-MediaEncoder::Suspend()
+MediaEncoder::ConnectMediaStreamTrack(MediaStreamTrack* aTrack)
{
MOZ_ASSERT(NS_IsMainThread());
- mLastPauseStartTime = TimeStamp::Now();
- mSuspended = true;
- mVideoSink->Suspend();
-}
-void
-MediaEncoder::Resume()
-{
- MOZ_ASSERT(NS_IsMainThread());
- if (!mSuspended) {
+ if (aTrack->Ended()) {
+ NS_ASSERTION(false, "Cannot connect ended track");
return;
}
- media::TimeUnit timeSpentPaused =
- media::TimeUnit::FromTimeDuration(
- TimeStamp::Now() - mLastPauseStartTime);
- MOZ_ASSERT(timeSpentPaused.ToMicroseconds() >= 0);
- MOZ_RELEASE_ASSERT(timeSpentPaused.IsValid());
- mMicrosecondsSpentPaused += timeSpentPaused.ToMicroseconds();;
- mSuspended = false;
- mVideoSink->Resume();
-}
-void
-MediaEncoder::SetDirectConnect(bool aConnected)
-{
- mDirectConnected = aConnected;
-}
+ if (AudioStreamTrack* audio = aTrack->AsAudioStreamTrack()) {
+ if (!mAudioEncoder) {
+ MOZ_ASSERT(false, "No audio encoder for this audio track");
+ return;
+ }
+ if (mAudioTrack) {
+ MOZ_ASSERT(false, "Only one audio track supported.");
+ return;
+ }
+ if (!mAudioListener) {
+ MOZ_ASSERT(false, "No audio listener for this audio track");
+ return;
+ }
-void
-MediaEncoder::NotifyRealtimeData(MediaStreamGraph* aGraph,
- TrackID aID,
- StreamTime aTrackOffset,
- uint32_t aTrackEvents,
- const MediaSegment& aRealtimeMedia)
-{
- if (mSuspended) {
- return;
- }
- // Process the incoming raw track data from MediaStreamGraph, called on the
- // thread of MediaStreamGraph.
- if (mAudioEncoder && aRealtimeMedia.GetType() == MediaSegment::AUDIO) {
- mAudioEncoder->NotifyQueuedTrackChanges(aGraph, aID,
- aTrackOffset, aTrackEvents,
- aRealtimeMedia);
- } else if (mVideoEncoder &&
- aRealtimeMedia.GetType() == MediaSegment::VIDEO &&
- aTrackEvents != TrackEventCommand::TRACK_EVENT_NONE) {
- mVideoEncoder->NotifyQueuedTrackChanges(aGraph, aID,
- aTrackOffset, aTrackEvents,
- aRealtimeMedia);
+ mAudioTrack = audio;
+ audio->AddDirectListener(mAudioListener);
+ audio->AddListener(mAudioListener);
+ } else if (VideoStreamTrack* video = aTrack->AsVideoStreamTrack()) {
+ if(!mVideoEncoder) {
+ MOZ_ASSERT(false, "No video encoder for this video track");
+ return;
+ }
+ if (mVideoTrack) {
+ MOZ_ASSERT(false, "Only one video track supported.");
+ return;
+ }
+ if (!mVideoListener) {
+ MOZ_ASSERT(false, "No video listener for this audio track");
+ return;
+ }
+
+ mVideoTrack = video;
+ video->AddVideoOutput(mVideoListener);
+ video->AddListener(mVideoListener);
+ } else {
+ MOZ_ASSERT(false, "Unknown track type");
}
}
void
-MediaEncoder::NotifyQueuedTrackChanges(MediaStreamGraph* aGraph,
- TrackID aID,
- StreamTime aTrackOffset,
- TrackEventCommand aTrackEvents,
- const MediaSegment& aQueuedMedia,
- MediaStream* aInputStream,
- TrackID aInputTrackID)
+MediaEncoder::RemoveMediaStreamTrack(MediaStreamTrack* aTrack)
{
- if (!mDirectConnected) {
- NotifyRealtimeData(aGraph, aID, aTrackOffset, aTrackEvents, aQueuedMedia);
- } else {
- if (aTrackEvents != TrackEventCommand::TRACK_EVENT_NONE) {
- // forward events (TRACK_EVENT_ENDED) but not the media
- if (aQueuedMedia.GetType() == MediaSegment::VIDEO) {
- VideoSegment segment;
- NotifyRealtimeData(aGraph, aID, aTrackOffset, aTrackEvents, segment);
- } else {
- AudioSegment segment;
- NotifyRealtimeData(aGraph, aID, aTrackOffset, aTrackEvents, segment);
- }
+ if (!aTrack) {
+ MOZ_ASSERT(false);
+ return;
+ }
+
+ if (AudioStreamTrack* audio = aTrack->AsAudioStreamTrack()) {
+ if (audio != mAudioTrack) {
+ MOZ_ASSERT(false, "Not connected to this audio track");
+ return;
}
- }
-}
-void
-MediaEncoder::NotifyQueuedAudioData(MediaStreamGraph* aGraph, TrackID aID,
- StreamTime aTrackOffset,
- const AudioSegment& aQueuedMedia,
- MediaStream* aInputStream,
- TrackID aInputTrackID)
-{
- if (!mDirectConnected) {
- NotifyRealtimeData(aGraph, aID, aTrackOffset, 0, aQueuedMedia);
- }
-}
+ if (mAudioListener) {
+ audio->RemoveDirectListener(mAudioListener);
+ audio->RemoveListener(mAudioListener);
+ }
+ mAudioTrack = nullptr;
+ } else if (VideoStreamTrack* video = aTrack->AsVideoStreamTrack()) {
+ if (video != mVideoTrack) {
+ MOZ_ASSERT(false, "Not connected to this video track");
+ return;
+ }
-void
-MediaEncoder::NotifyEvent(MediaStreamGraph* aGraph,
- MediaStreamGraphEvent event)
-{
- // In case that MediaEncoder does not receive a TRACK_EVENT_ENDED event.
- LOG(LogLevel::Debug, ("NotifyRemoved in [MediaEncoder]."));
- if (mAudioEncoder) {
- mAudioEncoder->NotifyEvent(aGraph, event);
- }
- if (mVideoEncoder) {
- mVideoEncoder->NotifyEvent(aGraph, event);
+ if (mVideoListener) {
+ video->RemoveVideoOutput(mVideoListener);
+ video->RemoveListener(mVideoListener);
+ }
+ mVideoTrack = nullptr;
}
}
/* static */
already_AddRefed<MediaEncoder>
-MediaEncoder::CreateEncoder(const nsAString& aMIMEType, uint32_t aAudioBitrate,
- uint32_t aVideoBitrate, uint32_t aBitrate,
+MediaEncoder::CreateEncoder(TaskQueue* aEncoderThread,
+ const nsAString& aMIMEType,
+ uint32_t aAudioBitrate,
+ uint32_t aVideoBitrate,
uint8_t aTrackTypes,
TrackRate aTrackRate)
{
AUTO_PROFILER_LABEL("MediaEncoder::CreateEncoder", OTHER);
- nsAutoPtr<ContainerWriter> writer;
- nsAutoPtr<AudioTrackEncoder> audioEncoder;
- nsAutoPtr<VideoTrackEncoder> videoEncoder;
- RefPtr<MediaEncoder> encoder;
+ UniquePtr<ContainerWriter> writer;
+ RefPtr<AudioTrackEncoder> audioEncoder;
+ RefPtr<VideoTrackEncoder> videoEncoder;
nsString mimeType;
+
if (!aTrackTypes) {
- LOG(LogLevel::Error, ("NO TrackTypes!!!"));
+ MOZ_ASSERT(false);
+ LOG(LogLevel::Error, ("No TrackTypes"));
return nullptr;
}
#ifdef MOZ_WEBM_ENCODER
else if (MediaEncoder::IsWebMEncoderEnabled() &&
- (aMIMEType.EqualsLiteral(VIDEO_WEBM) ||
- (aTrackTypes & ContainerWriter::CREATE_VIDEO_TRACK))) {
+ (aMIMEType.EqualsLiteral(VIDEO_WEBM) ||
+ (aTrackTypes & ContainerWriter::CREATE_VIDEO_TRACK))) {
if (aTrackTypes & ContainerWriter::CREATE_AUDIO_TRACK &&
MediaDecoder::IsOpusEnabled()) {
- audioEncoder = new OpusTrackEncoder();
+ audioEncoder = MakeAndAddRef<OpusTrackEncoder>(aTrackRate);
NS_ENSURE_TRUE(audioEncoder, nullptr);
}
- videoEncoder = new VP8TrackEncoder(aTrackRate);
- writer = new WebMWriter(aTrackTypes);
+ videoEncoder = MakeAndAddRef<VP8TrackEncoder>(aTrackRate);
+ writer = MakeUnique<WebMWriter>(aTrackTypes);
NS_ENSURE_TRUE(writer, nullptr);
NS_ENSURE_TRUE(videoEncoder, nullptr);
mimeType = NS_LITERAL_STRING(VIDEO_WEBM);
}
#endif //MOZ_WEBM_ENCODER
else if (MediaDecoder::IsOggEnabled() && MediaDecoder::IsOpusEnabled() &&
(aMIMEType.EqualsLiteral(AUDIO_OGG) ||
- (aTrackTypes & ContainerWriter::CREATE_AUDIO_TRACK))) {
- writer = new OggWriter();
- audioEncoder = new OpusTrackEncoder();
+ (aTrackTypes & ContainerWriter::CREATE_AUDIO_TRACK))) {
+ writer = MakeUnique<OggWriter>();
+ audioEncoder = MakeAndAddRef<OpusTrackEncoder>(aTrackRate);
NS_ENSURE_TRUE(writer, nullptr);
NS_ENSURE_TRUE(audioEncoder, nullptr);
mimeType = NS_LITERAL_STRING(AUDIO_OGG);
}
else {
LOG(LogLevel::Error, ("Can not find any encoder to record this media stream"));
return nullptr;
}
- LOG(LogLevel::Debug, ("Create encoder result:a[%d] v[%d] w[%d] mimeType = %s.",
- audioEncoder != nullptr, videoEncoder != nullptr,
- writer != nullptr, NS_ConvertUTF16toUTF8(mimeType).get()));
- if (videoEncoder && aVideoBitrate != 0) {
- videoEncoder->SetBitrate(aVideoBitrate);
+
+ LOG(LogLevel::Info, ("Create encoder result:a[%p](%u bps) v[%p](%u bps) w[%p] mimeType = %s.",
+ audioEncoder.get(), aAudioBitrate,
+ videoEncoder.get(), aVideoBitrate,
+ writer.get(), NS_ConvertUTF16toUTF8(mimeType).get()));
+
+ if (audioEncoder) {
+ audioEncoder->SetWorkerThread(aEncoderThread);
+ if (aAudioBitrate != 0) {
+ audioEncoder->SetBitrate(aAudioBitrate);
+ }
}
- if (audioEncoder && aAudioBitrate != 0) {
- audioEncoder->SetBitrate(aAudioBitrate);
+ if (videoEncoder) {
+ videoEncoder->SetWorkerThread(aEncoderThread);
+ if (aVideoBitrate != 0) {
+ videoEncoder->SetBitrate(aVideoBitrate);
+ }
}
- encoder = new MediaEncoder(writer.forget(), audioEncoder.forget(),
- videoEncoder.forget(), mimeType, aAudioBitrate,
- aVideoBitrate, aBitrate);
- return encoder.forget();
+ return MakeAndAddRef<MediaEncoder>(aEncoderThread,
+ Move(writer),
+ audioEncoder,
+ videoEncoder,
+ mimeType);
}
-/**
- * GetEncodedData() runs as a state machine, starting with mState set to
- * GET_METADDATA, the procedure should be as follow:
- *
- * While non-stop
- * If mState is GET_METADDATA
- * Get the meta data from audio/video encoder
- * If a meta data is generated
- * Get meta data from audio/video encoder
- * Set mState to ENCODE_TRACK
- * Return the final container data
- *
- * If mState is ENCODE_TRACK
- * Get encoded track data from audio/video encoder
- * If a packet of track data is generated
- * Insert encoded track data into the container stream of writer
- * If the final container data is copied to aOutput
- * Return the copy of final container data
- * If this is the last packet of input stream
- * Set mState to ENCODE_DONE
- *
- * If mState is ENCODE_DONE or ENCODE_ERROR
- * Stop the loop
- */
-void
-MediaEncoder::GetEncodedData(nsTArray<nsTArray<uint8_t> >* aOutputBufs,
- nsAString& aMIMEType)
+nsresult
+MediaEncoder::GetEncodedMetadata(nsTArray<nsTArray<uint8_t>>* aOutputBufs,
+ nsAString& aMIMEType)
{
- MOZ_ASSERT(!NS_IsMainThread());
+ AUTO_PROFILER_LABEL("MediaEncoder::GetEncodedMetadata", OTHER);
+
+ MOZ_ASSERT(mEncoderThread->IsCurrentThreadIn());
+
+ if (mShutdown) {
+ MOZ_ASSERT(false);
+ return NS_ERROR_FAILURE;
+ }
+
+ if (!mInitialized) {
+ MOZ_ASSERT(false);
+ return NS_ERROR_FAILURE;
+ }
+
+ if (mMetadataEncoded) {
+ MOZ_ASSERT(false);
+ return NS_ERROR_FAILURE;
+ }
aMIMEType = mMIMEType;
+
+ LOG(LogLevel::Verbose, ("GetEncodedMetadata TimeStamp = %f", GetEncodeTimeStamp()));
+
+ nsresult rv;
+
+ if (mAudioEncoder) {
+ if (!mAudioEncoder->IsInitialized()) {
+ LOG(LogLevel::Error, ("GetEncodedMetadata Audio encoder not initialized"));
+ MOZ_ASSERT(false);
+ return NS_ERROR_FAILURE;
+ }
+ rv = CopyMetadataToMuxer(mAudioEncoder);
+ if (NS_FAILED(rv)) {
+ LOG(LogLevel::Error, ("Failed to Set Audio Metadata"));
+ SetError();
+ return rv;
+ }
+ }
+ if (mVideoEncoder) {
+ if (!mVideoEncoder->IsInitialized()) {
+ LOG(LogLevel::Error, ("GetEncodedMetadata Video encoder not initialized"));
+ MOZ_ASSERT(false);
+ return NS_ERROR_FAILURE;
+ }
+ rv = CopyMetadataToMuxer(mVideoEncoder.get());
+ if (NS_FAILED(rv)) {
+ LOG(LogLevel::Error, ("Failed to Set Video Metadata"));
+ SetError();
+ return rv;
+ }
+ }
+
+ rv = mWriter->GetContainerData(aOutputBufs,
+ ContainerWriter::GET_HEADER);
+ if (NS_FAILED(rv)) {
+ LOG(LogLevel::Error,("Writer fail to generate header!"));
+ SetError();
+ return rv;
+ }
+ LOG(LogLevel::Verbose, ("Finish GetEncodedMetadata TimeStamp = %f", GetEncodeTimeStamp()));
+ mMetadataEncoded = true;
+
+ return NS_OK;
+}
+
+nsresult
+MediaEncoder::GetEncodedData(nsTArray<nsTArray<uint8_t> >* aOutputBufs)
+{
AUTO_PROFILER_LABEL("MediaEncoder::GetEncodedData", OTHER);
- bool reloop = true;
- while (reloop) {
- switch (mState) {
- case ENCODE_METADDATA: {
- LOG(LogLevel::Debug, ("ENCODE_METADDATA TimeStamp = %f", GetEncodeTimeStamp()));
- nsresult rv = CopyMetadataToMuxer(mAudioEncoder.get());
- if (NS_FAILED(rv)) {
- LOG(LogLevel::Error, ("Error! Fail to Set Audio Metadata"));
- break;
- }
- rv = CopyMetadataToMuxer(mVideoEncoder.get());
- if (NS_FAILED(rv)) {
- LOG(LogLevel::Error, ("Error! Fail to Set Video Metadata"));
- break;
- }
+ MOZ_ASSERT(mEncoderThread->IsCurrentThreadIn());
+
+ if (!mMetadataEncoded) {
+ MOZ_ASSERT(false);
+ return NS_ERROR_FAILURE;
+ }
+
+ nsresult rv;
+ LOG(LogLevel::Verbose, ("GetEncodedData TimeStamp = %f", GetEncodeTimeStamp()));
+ EncodedFrameContainer encodedData;
- rv = mWriter->GetContainerData(aOutputBufs,
- ContainerWriter::GET_HEADER);
- if (aOutputBufs != nullptr) {
- mSizeOfBuffer = aOutputBufs->ShallowSizeOfExcludingThis(MallocSizeOf);
- }
- if (NS_FAILED(rv)) {
- LOG(LogLevel::Error,("Error! writer fail to generate header!"));
- mState = ENCODE_ERROR;
- break;
- }
- LOG(LogLevel::Debug, ("Finish ENCODE_METADDATA TimeStamp = %f", GetEncodeTimeStamp()));
- mState = ENCODE_TRACK;
- break;
+ if (mVideoEncoder) {
+ // We're most likely to actually wait for a video frame, so do that first
+ // to minimize capture offset/lipsync issues.
+ rv = WriteEncodedDataToMuxer(mVideoEncoder);
+ LOG(LogLevel::Verbose, ("Video encoded TimeStamp = %f", GetEncodeTimeStamp()));
+ if (NS_FAILED(rv)) {
+ LOG(LogLevel::Warning, ("Failed to write encoded video data to muxer"));
+ return rv;
}
+ }
+
+ if (mAudioEncoder) {
+ rv = WriteEncodedDataToMuxer(mAudioEncoder);
+ LOG(LogLevel::Verbose, ("Audio encoded TimeStamp = %f", GetEncodeTimeStamp()));
+ if (NS_FAILED(rv)) {
+ LOG(LogLevel::Warning, ("Failed to write encoded audio data to muxer"));
+ return rv;
+ }
+ }
+
+ // In audio only or video only case, let unavailable track's flag to be true.
+ bool isAudioCompleted = !mAudioEncoder || mAudioEncoder->IsEncodingComplete();
+ bool isVideoCompleted = !mVideoEncoder || mVideoEncoder->IsEncodingComplete();
+ rv = mWriter->GetContainerData(aOutputBufs,
+ isAudioCompleted && isVideoCompleted ?
+ ContainerWriter::FLUSH_NEEDED : 0);
+ if (mWriter->IsWritingComplete()) {
+ mCompleted = true;
+ Shutdown();
+ }
- case ENCODE_TRACK: {
- LOG(LogLevel::Debug, ("ENCODE_TRACK TimeStamp = %f", GetEncodeTimeStamp()));
- EncodedFrameContainer encodedData;
- nsresult rv = NS_OK;
- // We're most likely to actually wait for a video frame, so do that first to minimize
- // capture offset/lipsync issues
- rv = WriteEncodedDataToMuxer(mVideoEncoder.get());
- if (NS_FAILED(rv)) {
- LOG(LogLevel::Error, ("Fail to write video encoder data to muxer"));
- break;
- }
- rv = WriteEncodedDataToMuxer(mAudioEncoder.get());
- if (NS_FAILED(rv)) {
- LOG(LogLevel::Error, ("Error! Fail to write audio encoder data to muxer"));
- break;
- }
- LOG(LogLevel::Debug, ("Audio encoded TimeStamp = %f", GetEncodeTimeStamp()));
- LOG(LogLevel::Debug, ("Video encoded TimeStamp = %f", GetEncodeTimeStamp()));
- // In audio only or video only case, let unavailable track's flag to be true.
- bool isAudioCompleted = (mAudioEncoder && mAudioEncoder->IsEncodingComplete()) || !mAudioEncoder;
- bool isVideoCompleted = (mVideoEncoder && mVideoEncoder->IsEncodingComplete()) || !mVideoEncoder;
- rv = mWriter->GetContainerData(aOutputBufs,
- isAudioCompleted && isVideoCompleted ?
- ContainerWriter::FLUSH_NEEDED : 0);
- if (aOutputBufs != nullptr) {
- mSizeOfBuffer = aOutputBufs->ShallowSizeOfExcludingThis(MallocSizeOf);
- }
- if (NS_SUCCEEDED(rv)) {
- // Successfully get the copy of final container data from writer.
- reloop = false;
- }
- mState = (mWriter->IsWritingComplete()) ? ENCODE_DONE : ENCODE_TRACK;
- LOG(LogLevel::Debug, ("END ENCODE_TRACK TimeStamp = %f "
- "mState = %d aComplete %d vComplete %d",
- GetEncodeTimeStamp(), mState, isAudioCompleted, isVideoCompleted));
- break;
- }
+ LOG(LogLevel::Verbose, ("END GetEncodedData TimeStamp=%f "
+ "mCompleted=%d, aComplete=%d, vComplete=%d",
+ GetEncodeTimeStamp(), mCompleted, isAudioCompleted, isVideoCompleted));
+
+ return rv;
+}
+
+void
+MediaEncoder::Shutdown()
+{
+ MOZ_ASSERT(mEncoderThread->IsCurrentThreadIn());
+ if (mShutdown) {
+ return;
+ }
+ mShutdown = true;
- case ENCODE_DONE:
- case ENCODE_ERROR:
- LOG(LogLevel::Debug, ("MediaEncoder has been shutdown."));
- mSizeOfBuffer = 0;
- mShutdown = true;
- reloop = false;
- break;
- default:
- MOZ_CRASH("Invalid encode state");
- }
+ LOG(LogLevel::Info, ("MediaEncoder has been shut down."));
+ if (mAudioEncoder) {
+ mAudioEncoder->UnregisterListener(mEncoderListener);
+ }
+ if (mAudioListener) {
+ mAudioListener->NotifyShutdown();
+ }
+ if (mVideoEncoder) {
+ mVideoEncoder->UnregisterListener(mEncoderListener);
+ }
+ if (mVideoListener) {
+ mVideoListener->NotifyShutdown();
+ }
+ mEncoderListener->Forget();
+
+ if (mCanceled) {
+ // Shutting down after being canceled. We cannot use the encoder thread.
+ return;
+ }
+
+ auto listeners(mListeners);
+ for (auto& l : listeners) {
+ // We dispatch here since this method is typically called from
+ // a DataAvailable() handler.
+ mEncoderThread->Dispatch(
+ NewRunnableMethod("mozilla::MediaEncoderListener::Shutdown",
+ l, &MediaEncoderListener::Shutdown));
}
}
nsresult
MediaEncoder::WriteEncodedDataToMuxer(TrackEncoder *aTrackEncoder)
{
- if (aTrackEncoder == nullptr) {
- return NS_OK;
+ AUTO_PROFILER_LABEL("MediaEncoder::WriteEncodedDataToMuxer", OTHER);
+
+ MOZ_ASSERT(mEncoderThread->IsCurrentThreadIn());
+
+ if (!aTrackEncoder) {
+ NS_ERROR("No track encoder to get data from");
+ return NS_ERROR_FAILURE;
}
+
if (aTrackEncoder->IsEncodingComplete()) {
return NS_OK;
}
- AUTO_PROFILER_LABEL("MediaEncoder::WriteEncodedDataToMuxer", OTHER);
-
- EncodedFrameContainer encodedVideoData;
- nsresult rv = aTrackEncoder->GetEncodedTrack(encodedVideoData);
+ EncodedFrameContainer encodedData;
+ nsresult rv = aTrackEncoder->GetEncodedTrack(encodedData);
if (NS_FAILED(rv)) {
// Encoding might be canceled.
- LOG(LogLevel::Error, ("Error! Fail to get encoded data from video encoder."));
- mState = ENCODE_ERROR;
+ LOG(LogLevel::Error, ("Failed to get encoded data from encoder."));
+ SetError();
return rv;
}
-
- // Update timestamps to accommodate pauses
- const nsTArray<RefPtr<EncodedFrame> >& encodedFrames =
- encodedVideoData.GetEncodedFrames();
- // Take a copy of the atomic so we don't continually access it
- uint64_t microsecondsSpentPaused = mMicrosecondsSpentPaused;
- for (size_t i = 0; i < encodedFrames.Length(); ++i) {
- RefPtr<EncodedFrame> frame = encodedFrames[i];
- if (frame->GetTimeStamp() > microsecondsSpentPaused &&
- frame->GetTimeStamp() - microsecondsSpentPaused > mLastMuxedTimestamp) {
- // Use the adjusted timestamp if it's after the last timestamp
- frame->SetTimeStamp(frame->GetTimeStamp() - microsecondsSpentPaused);
- } else {
- // If not, we force the last time stamp. We do this so the frames are
- // still around and in order in case the codec needs to reference them.
- // Dropping them here may result in artifacts in playback.
- frame->SetTimeStamp(mLastMuxedTimestamp);
- }
- MOZ_ASSERT(mLastMuxedTimestamp <= frame->GetTimeStamp(),
- "Our frames should be ordered by this point!");
- mLastMuxedTimestamp = frame->GetTimeStamp();
- }
-
- rv = mWriter->WriteEncodedTrack(encodedVideoData,
+ rv = mWriter->WriteEncodedTrack(encodedData,
aTrackEncoder->IsEncodingComplete() ?
ContainerWriter::END_OF_STREAM : 0);
if (NS_FAILED(rv)) {
- LOG(LogLevel::Error, ("Error! Fail to write encoded video track to the media container."));
- mState = ENCODE_ERROR;
+ LOG(LogLevel::Error, ("Failed to write encoded track to the media container."));
+ SetError();
}
return rv;
}
nsresult
MediaEncoder::CopyMetadataToMuxer(TrackEncoder *aTrackEncoder)
{
- if (aTrackEncoder == nullptr) {
- return NS_OK;
+ AUTO_PROFILER_LABEL("MediaEncoder::CopyMetadataToMuxer", OTHER);
+
+ MOZ_ASSERT(mEncoderThread->IsCurrentThreadIn());
+
+ if (!aTrackEncoder) {
+ NS_ERROR("No track encoder to get metadata from");
+ return NS_ERROR_FAILURE;
}
- AUTO_PROFILER_LABEL("MediaEncoder::CopyMetadataToMuxer", OTHER);
-
RefPtr<TrackMetadataBase> meta = aTrackEncoder->GetMetadata();
if (meta == nullptr) {
- LOG(LogLevel::Error, ("Error! metadata = null"));
- mState = ENCODE_ERROR;
+ LOG(LogLevel::Error, ("metadata == null"));
+ SetError();
return NS_ERROR_ABORT;
}
nsresult rv = mWriter->SetMetadata(meta);
if (NS_FAILED(rv)) {
- LOG(LogLevel::Error, ("Error! SetMetadata fail"));
- mState = ENCODE_ERROR;
+ LOG(LogLevel::Error, ("SetMetadata failed"));
+ SetError();
}
return rv;
}
+bool
+MediaEncoder::IsShutdown()
+{
+ MOZ_ASSERT(mEncoderThread->IsCurrentThreadIn());
+ return mShutdown;
+}
+
+void
+MediaEncoder::Cancel()
+{
+ MOZ_ASSERT(NS_IsMainThread());
+
+ RefPtr<MediaEncoder> self = this;
+ mEncoderThread->Dispatch(NewRunnableFrom([self]() mutable {
+ self->mCanceled = true;
+
+ if (self->mAudioEncoder) {
+ self->mAudioEncoder->Cancel();
+ }
+ if (self->mVideoEncoder) {
+ self->mVideoEncoder->Cancel();
+ }
+ self->Shutdown();
+ return NS_OK;
+ }));
+}
+
+bool
+MediaEncoder::HasError()
+{
+ MOZ_ASSERT(mEncoderThread->IsCurrentThreadIn());
+ return mError;
+}
+
+void
+MediaEncoder::SetError()
+{
+ MOZ_ASSERT(mEncoderThread->IsCurrentThreadIn());
+
+ if (mError) {
+ return;
+ }
+
+ mError = true;
+ auto listeners(mListeners);
+ for (auto& l : listeners) {
+ l->Error();
+ }
+}
+
+void
+MediaEncoder::Stop()
+{
+ MOZ_ASSERT(NS_IsMainThread());
+
+ if (mAudioNode) {
+ mAudioNode->GetStream()->RemoveTrackListener(mAudioListener, AudioNodeStream::AUDIO_TRACK);
+ if (mInputPort) {
+ mInputPort->Destroy();
+ mInputPort = nullptr;
+ }
+ if (mPipeStream) {
+ mPipeStream->RemoveTrackListener(mAudioListener, AudioNodeStream::AUDIO_TRACK);
+ mPipeStream->Destroy();
+ mPipeStream = nullptr;
+ }
+ mAudioNode = nullptr;
+ }
+
+ if (mAudioTrack) {
+ RemoveMediaStreamTrack(mAudioTrack);
+ }
+
+ if (mVideoTrack) {
+ RemoveMediaStreamTrack(mVideoTrack);
+ }
+}
+
#ifdef MOZ_WEBM_ENCODER
bool
MediaEncoder::IsWebMEncoderEnabled()
{
return Preferences::GetBool("media.encoder.webm.enabled");
}
#endif
+void
+MediaEncoder::NotifyInitialized()
+{
+ MOZ_ASSERT(mEncoderThread->IsCurrentThreadIn());
+
+ if (mInitialized) {
+ // This could happen if an encoder re-inits due to a resolution change.
+ return;
+ }
+
+ if (mAudioEncoder && !mAudioEncoder->IsInitialized()) {
+ return;
+ }
+
+ if (mVideoEncoder && !mVideoEncoder->IsInitialized()) {
+ return;
+ }
+
+ mInitialized = true;
+
+ auto listeners(mListeners);
+ for (auto& l : listeners) {
+ l->Initialized();
+ }
+}
+
+void
+MediaEncoder::NotifyDataAvailable()
+{
+ MOZ_ASSERT(mEncoderThread->IsCurrentThreadIn());
+
+ if (!mInitialized) {
+ return;
+ }
+
+ auto listeners(mListeners);
+ for (auto& l : listeners) {
+ l->DataAvailable();
+ }
+}
+
+void
+MediaEncoder::RegisterListener(MediaEncoderListener* aListener)
+{
+ MOZ_ASSERT(mEncoderThread->IsCurrentThreadIn());
+ MOZ_ASSERT(!mListeners.Contains(aListener));
+ mListeners.AppendElement(aListener);
+}
+
+bool
+MediaEncoder::UnregisterListener(MediaEncoderListener* aListener)
+{
+ MOZ_ASSERT(mEncoderThread->IsCurrentThreadIn());
+ return mListeners.RemoveElement(aListener);
+}
+
/*
* SizeOfExcludingThis measures memory being used by the Media Encoder.
* Currently it measures the size of the Encoder buffer and memory occupied
* by mAudioEncoder and mVideoEncoder.
*/
size_t
-MediaEncoder::SizeOfExcludingThis(mozilla::MallocSizeOf aMallocSizeOf) const
+MediaEncoder::SizeOfExcludingThis(mozilla::MallocSizeOf aMallocSizeOf)
{
- size_t amount = 0;
- if (mState == ENCODE_TRACK) {
- amount = mSizeOfBuffer +
- (mAudioEncoder != nullptr ? mAudioEncoder->SizeOfExcludingThis(aMallocSizeOf) : 0) +
- (mVideoEncoder != nullptr ? mVideoEncoder->SizeOfExcludingThis(aMallocSizeOf) : 0);
+ MOZ_ASSERT(mEncoderThread->IsCurrentThreadIn());
+
+ size_t size = 0;
+ if (mAudioEncoder) {
+ size += mAudioEncoder->SizeOfExcludingThis(aMallocSizeOf);
}
- return amount;
+ if (mVideoEncoder) {
+ size += mVideoEncoder->SizeOfExcludingThis(aMallocSizeOf);
+ }
+ return size;
}
} // namespace mozilla
--- a/dom/media/encoder/MediaEncoder.h
+++ b/dom/media/encoder/MediaEncoder.h
@@ -1,257 +1,288 @@
/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-*/
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this file,
* You can obtain one at http://mozilla.org/MPL/2.0/. */
#ifndef MediaEncoder_h_
#define MediaEncoder_h_
-#include "mozilla/DebugOnly.h"
-#include "TrackEncoder.h"
#include "ContainerWriter.h"
#include "CubebUtils.h"
#include "MediaStreamGraph.h"
#include "MediaStreamListener.h"
-#include "nsAutoPtr.h"
-#include "MediaStreamVideoSink.h"
+#include "mozilla/DebugOnly.h"
+#include "mozilla/MemoryReporting.h"
+#include "mozilla/UniquePtr.h"
#include "nsIMemoryReporter.h"
-#include "mozilla/MemoryReporting.h"
-#include "mozilla/Atomics.h"
+#include "TrackEncoder.h"
namespace mozilla {
-class MediaStreamVideoRecorderSink : public MediaStreamVideoSink
+class TaskQueue;
+
+namespace dom {
+class AudioNode;
+class AudioStreamTrack;
+class MediaStreamTrack;
+class VideoStreamTrack;
+}
+
+class MediaEncoder;
+
+class MediaEncoderListener
{
public:
- explicit MediaStreamVideoRecorderSink(VideoTrackEncoder* aEncoder)
- : mVideoEncoder(aEncoder)
- , mSuspended(false) {}
-
- // MediaStreamVideoSink methods
- virtual void SetCurrentFrames(const VideoSegment& aSegment) override;
- virtual void ClearFrames() override {}
-
- void Resume() { mSuspended = false; }
- void Suspend() { mSuspended = true; }
-
-private:
- virtual ~MediaStreamVideoRecorderSink() {}
- VideoTrackEncoder* mVideoEncoder;
- Atomic<bool> mSuspended;
+ NS_INLINE_DECL_THREADSAFE_REFCOUNTING(MediaEncoderListener)
+ virtual void Initialized() = 0;
+ virtual void DataAvailable() = 0;
+ virtual void Error() = 0;
+ virtual void Shutdown() = 0;
+protected:
+ virtual ~MediaEncoderListener() {}
};
/**
* MediaEncoder is the framework of encoding module, it controls and manages
* procedures between ContainerWriter and TrackEncoder. ContainerWriter packs
- * the encoded track data with a specific container (e.g. ogg, mp4).
+ * the encoded track data with a specific container (e.g. ogg, webm).
* AudioTrackEncoder and VideoTrackEncoder are subclasses of TrackEncoder, and
* are responsible for encoding raw data coming from MediaStreamGraph.
*
- * Also, MediaEncoder is a type of MediaStreamListener, it starts to receive raw
- * segments after itself is added to the source stream. In the mean time,
- * encoded track data is pulled by its owner periodically on a worker thread. A
- * reentrant monitor is used to protect the push and pull of resource.
+ * MediaEncoder solves threading issues by doing message passing to a TaskQueue
+ * (the "encoder thread") as passed in to the constructor. Each
+ * MediaStreamTrack to be recorded is set up with a MediaStreamTrackListener.
+ * Typically there are a non-direct track listeners for audio, direct listeners
+ * for video, and there is always a non-direct listener on each track for
+ * time-keeping. The listeners forward data to their corresponding TrackEncoders
+ * on the encoder thread.
*
- * MediaEncoder is designed to be a passive component, neither it owns nor in
- * charge of managing threads. However, a monitor is used in function
- * TrackEncoder::GetEncodedTrack() for the purpose of thread safety (e.g.
- * between callbacks of MediaStreamListener and others), a call to this function
- * might block. Therefore, MediaEncoder should not run on threads that forbid
- * blocking, such as main thread or I/O thread.
+ * The MediaEncoder listens to events from all TrackEncoders, and in turn
+ * signals events to interested parties. Typically a MediaRecorder::Session.
+ * The event that there's data available in the TrackEncoders is what typically
+ * drives the extraction and muxing of data.
*
- * For example, an usage from MediaRecorder of this component would be:
+ * MediaEncoder is designed to be a passive component, neither does it own or is
+ * in charge of managing threads. Instead this is done by its owner.
+ *
+ * For example, usage from MediaRecorder of this component would be:
* 1) Create an encoder with a valid MIME type.
* => encoder = MediaEncoder::CreateEncoder(aMIMEType);
- * It then generate a ContainerWriter according to the MIME type, and an
- * AudioTrackEncoder (or a VideoTrackEncoder too) associated with the media
- * type.
+ * It then creates a ContainerWriter according to the MIME type
+ *
+ * 2) Connect a MediaEncoderListener to be notified when the MediaEncoder has
+ * been initialized and when there's data available.
+ * => encoder->RegisterListener(listener);
*
- * 2) Dispatch the task GetEncodedData() to a worker thread.
+ * 3) Connect the MediaStreamTracks to be recorded.
+ * => encoder->ConnectMediaStreamTrack(track);
+ * This creates the corresponding TrackEncoder and connects the track and
+ * the TrackEncoder through a track listener. This also starts encoding.
+ *
+ * 4) When the MediaEncoderListener is notified that the MediaEncoder is
+ * initialized, we can encode metadata.
+ * => encoder->GetEncodedMetadata(...);
*
- * 3) To start encoding, add this component to its source stream.
- * => sourceStream->AddListener(encoder);
+ * 5) When the MediaEncoderListener is notified that the MediaEncoder has
+ * data available, we can encode data.
+ * => encoder->GetEncodedData(...);
+ *
+ * 6) To stop encoding, there are multiple options:
*
- * 4) To stop encoding, remove this component from its source stream.
- * => sourceStream->RemoveListener(encoder);
+ * 6.1) Stop() for a graceful stop.
+ * => encoder->Stop();
+ *
+ * 6.2) Cancel() for an immediate stop, if you don't need the data currently
+ * buffered.
+ * => encoder->Cancel();
+ *
+ * 6.3) When all input tracks end, the MediaEncoder will automatically stop
+ * and shut down.
*/
-class MediaEncoder : public DirectMediaStreamListener
+class MediaEncoder
{
- friend class MediaStreamVideoRecorderSink;
+private:
+ class AudioTrackListener;
+ class VideoTrackListener;
+ class EncoderListener;
+
public :
- enum {
- ENCODE_METADDATA,
- ENCODE_TRACK,
- ENCODE_DONE,
- ENCODE_ERROR,
- };
+ NS_INLINE_DECL_THREADSAFE_REFCOUNTING(MediaEncoder)
- MediaEncoder(ContainerWriter* aWriter,
+ MediaEncoder(TaskQueue* aEncoderThread,
+ UniquePtr<ContainerWriter> aWriter,
AudioTrackEncoder* aAudioEncoder,
VideoTrackEncoder* aVideoEncoder,
- const nsAString& aMIMEType,
- uint32_t aAudioBitrate,
- uint32_t aVideoBitrate,
- uint32_t aBitrate)
- : mWriter(aWriter)
- , mAudioEncoder(aAudioEncoder)
- , mVideoEncoder(aVideoEncoder)
- , mVideoSink(new MediaStreamVideoRecorderSink(mVideoEncoder))
- , mStartTime(TimeStamp::Now())
- , mMIMEType(aMIMEType)
- , mSizeOfBuffer(0)
- , mState(MediaEncoder::ENCODE_METADDATA)
- , mShutdown(false)
- , mDirectConnected(false)
- , mSuspended(false)
- , mMicrosecondsSpentPaused(0)
- , mLastMuxedTimestamp(0)
-{}
-
- ~MediaEncoder() {};
+ const nsAString& aMIMEType);
/* Note - called from control code, not on MSG threads. */
- void Suspend();
+ void Suspend(TimeStamp aTime);
/**
* Note - called from control code, not on MSG threads.
* Calculates time spent paused in order to offset frames. */
- void Resume();
-
- /**
- * Tells us which Notify to pay attention to for media
- */
- void SetDirectConnect(bool aConnected);
+ void Resume(TimeStamp aTime);
/**
- * Notified by the AppendToTrack in MediaStreamGraph; aRealtimeMedia is the raw
- * track data in form of MediaSegment.
+ * Stops the current encoding, and disconnects the input tracks.
*/
- void NotifyRealtimeData(MediaStreamGraph* aGraph, TrackID aID,
- StreamTime aTrackOffset,
- uint32_t aTrackEvents,
- const MediaSegment& aRealtimeMedia) override;
+ void Stop();
/**
- * Notified by the control loop of MediaStreamGraph; aQueueMedia is the raw
- * track data in form of MediaSegment.
+ * Connects an AudioNode with the appropriate encoder.
*/
- void NotifyQueuedTrackChanges(MediaStreamGraph* aGraph, TrackID aID,
- StreamTime aTrackOffset,
- TrackEventCommand aTrackEvents,
- const MediaSegment& aQueuedMedia,
- MediaStream* aInputStream,
- TrackID aInputTrackID) override;
+ void ConnectAudioNode(dom::AudioNode* aNode, uint32_t aOutput);
/**
- * Notifed by the control loop of MediaStreamGraph; aQueueMedia is the audio
- * data in the form of an AudioSegment.
+ * Connects a MediaStreamTrack with the appropriate encoder.
*/
- void NotifyQueuedAudioData(MediaStreamGraph* aGraph, TrackID aID,
- StreamTime aTrackOffset,
- const AudioSegment& aQueuedMedia,
- MediaStream* aInputStream,
- TrackID aInputTrackID) override;
+ void ConnectMediaStreamTrack(dom::MediaStreamTrack* aTrack);
/**
- * * Notified the stream is being removed.
+ * Removes a connected MediaStreamTrack.
*/
- void NotifyEvent(MediaStreamGraph* aGraph,
- MediaStreamGraphEvent event) override;
+ void RemoveMediaStreamTrack(dom::MediaStreamTrack* aTrack);
/**
* Creates an encoder with a given MIME type. Returns null if we are unable
* to create the encoder. For now, default aMIMEType to "audio/ogg" and use
* Ogg+Opus if it is empty.
*/
- static already_AddRefed<MediaEncoder> CreateEncoder(const nsAString& aMIMEType,
- uint32_t aAudioBitrate, uint32_t aVideoBitrate,
- uint32_t aBitrate,
- uint8_t aTrackTypes = ContainerWriter::CREATE_AUDIO_TRACK,
- TrackRate aTrackRate = CubebUtils::PreferredSampleRate());
+ static already_AddRefed<MediaEncoder>
+ CreateEncoder(TaskQueue* aEncoderThread,
+ const nsAString& aMIMEType,
+ uint32_t aAudioBitrate,
+ uint32_t aVideoBitrate,
+ uint8_t aTrackTypes,
+ TrackRate aTrackRate);
+
/**
- * Encodes the raw track data and returns the final container data. Assuming
- * it is called on a single worker thread. The buffer of container data is
- * allocated in ContainerWriter::GetContainerData(), and is appended to
- * aOutputBufs. aMIMEType is the valid mime-type of this returned container
- * data.
+ * Encodes raw metadata for all tracks to aOutputBufs. aMIMEType is the valid
+ * mime-type for the returned container data. The buffer of container data is
+ * allocated in ContainerWriter::GetContainerData().
+ *
+ * Should there be insufficient input data for either track encoder to infer
+ * the metadata, or if metadata has already been encoded, we return an error
+ * and the output arguments are undefined. Otherwise we return NS_OK.
*/
- void GetEncodedData(nsTArray<nsTArray<uint8_t> >* aOutputBufs,
- nsAString& aMIMEType);
+ nsresult GetEncodedMetadata(nsTArray<nsTArray<uint8_t> >* aOutputBufs,
+ nsAString& aMIMEType);
+ /**
+ * Encodes raw data for all tracks to aOutputBufs. The buffer of container
+ * data is allocated in ContainerWriter::GetContainerData().
+ *
+ * This implies that metadata has already been encoded and that all track
+ * encoders are still active. Should either implication break, we return an
+ * error and the output argument is undefined. Otherwise we return NS_OK.
+ */
+ nsresult GetEncodedData(nsTArray<nsTArray<uint8_t> >* aOutputBufs);
/**
* Return true if MediaEncoder has been shutdown. Reasons are encoding
* complete, encounter an error, or being canceled by its caller.
*/
- bool IsShutdown()
- {
- return mShutdown;
- }
+ bool IsShutdown();
/**
- * Cancel the encoding, and wakes up the lock of reentrant monitor in encoder.
+ * Cancels the encoding and shuts down the encoder using Shutdown().
+ * Listeners are not notified of the shutdown.
*/
- void Cancel()
- {
- if (mAudioEncoder) {
- mAudioEncoder->NotifyCancel();
- }
- if (mVideoEncoder) {
- mVideoEncoder->NotifyCancel();
- }
- }
+ void Cancel();
- bool HasError()
- {
- return mState == ENCODE_ERROR;
- }
+ bool HasError();
#ifdef MOZ_WEBM_ENCODER
static bool IsWebMEncoderEnabled();
#endif
+ /**
+ * Notifies listeners that this MediaEncoder has been initialized.
+ */
+ void NotifyInitialized();
+
+ /**
+ * Notifies listeners that this MediaEncoder has data available in some
+ * TrackEncoders.
+ */
+ void NotifyDataAvailable();
+
+ /**
+ * Registers a listener to events from this MediaEncoder.
+ * We hold a strong reference to the listener.
+ */
+ void RegisterListener(MediaEncoderListener* aListener);
+
+ /**
+ * Unregisters a listener from events from this MediaEncoder.
+ * The listener will stop receiving events synchronously.
+ */
+ bool UnregisterListener(MediaEncoderListener* aListener);
+
MOZ_DEFINE_MALLOC_SIZE_OF(MallocSizeOf)
/*
- * Measure the size of the buffer, and memory occupied by mAudioEncoder
- * and mVideoEncoder
+ * Measure the size of the buffer, and heap memory in bytes occupied by
+ * mAudioEncoder and mVideoEncoder.
*/
- size_t SizeOfExcludingThis(mozilla::MallocSizeOf aMallocSizeOf) const;
+ size_t SizeOfExcludingThis(mozilla::MallocSizeOf aMallocSizeOf);
- MediaStreamVideoRecorderSink* GetVideoSink() {
- return mVideoSink.get();
- }
+protected:
+ ~MediaEncoder();
private:
+ /**
+ * Shuts down the MediaEncoder and cleans up track encoders.
+ * Listeners will be notified of the shutdown unless we were Cancel()ed first.
+ */
+ void Shutdown();
+
+ /**
+ * Sets mError to true, notifies listeners of the error if mError changed,
+ * and stops encoding.
+ */
+ void SetError();
+
// Get encoded data from trackEncoder and write to muxer
nsresult WriteEncodedDataToMuxer(TrackEncoder *aTrackEncoder);
// Get metadata from trackEncoder and copy to muxer
nsresult CopyMetadataToMuxer(TrackEncoder* aTrackEncoder);
- nsAutoPtr<ContainerWriter> mWriter;
- nsAutoPtr<AudioTrackEncoder> mAudioEncoder;
- nsAutoPtr<VideoTrackEncoder> mVideoEncoder;
- RefPtr<MediaStreamVideoRecorderSink> mVideoSink;
+
+ const RefPtr<TaskQueue> mEncoderThread;
+
+ UniquePtr<ContainerWriter> mWriter;
+ RefPtr<AudioTrackEncoder> mAudioEncoder;
+ RefPtr<AudioTrackListener> mAudioListener;
+ RefPtr<VideoTrackEncoder> mVideoEncoder;
+ RefPtr<VideoTrackListener> mVideoListener;
+ RefPtr<EncoderListener> mEncoderListener;
+ nsTArray<RefPtr<MediaEncoderListener>> mListeners;
+
+ // The AudioNode we are encoding.
+ // Will be null when input is media stream or destination node.
+ RefPtr<dom::AudioNode> mAudioNode;
+ // Pipe-stream for allowing a track listener on a non-destination AudioNode.
+ // Will be null when input is media stream or destination node.
+ RefPtr<AudioNodeStream> mPipeStream;
+ // Input port that connect mAudioNode to mPipeStream.
+ // Will be null when input is media stream or destination node.
+ RefPtr<MediaInputPort> mInputPort;
+ // An audio track that we are encoding. Will be null if the input stream
+ // doesn't contain audio on start() or if the input is an AudioNode.
+ RefPtr<dom::AudioStreamTrack> mAudioTrack;
+ // A video track that we are encoding. Will be null if the input stream
+ // doesn't contain video on start() or if the input is an AudioNode.
+ RefPtr<dom::VideoStreamTrack> mVideoTrack;
TimeStamp mStartTime;
nsString mMIMEType;
- int64_t mSizeOfBuffer;
- int mState;
+ bool mInitialized;
+ bool mMetadataEncoded;
+ bool mCompleted;
+ bool mError;
+ bool mCanceled;
bool mShutdown;
- bool mDirectConnected;
- // Tracks if the encoder is suspended (paused). Used on the main thread and
- // MediaRecorder's read thread.
- Atomic<bool> mSuspended;
- // Timestamp of when the last pause happened. Should only be accessed on the
- // main thread.
- TimeStamp mLastPauseStartTime;
- // Exposes the time spend paused in microseconds. Read by the main thread
- // and MediaRecorder's read thread. Should only be written by main thread.
- Atomic<uint64_t> mMicrosecondsSpentPaused;
- // The timestamp of the last muxed sample. Should only be used on
- // MediaRecorder's read thread.
- uint64_t mLastMuxedTimestamp;
// Get duration from create encoder, for logging purpose
double GetEncodeTimeStamp()
{
TimeDuration decodeTime;
decodeTime = TimeStamp::Now() - mStartTime;
return decodeTime.ToMilliseconds();
}
};
--- a/dom/media/encoder/OpusTrackEncoder.cpp
+++ b/dom/media/encoder/OpusTrackEncoder.cpp
@@ -116,18 +116,18 @@ SerializeOpusCommentHeader(const nsCStri
SerializeToBuffer((uint32_t)aComments.Length(), aOutput);
for (uint32_t i = 0; i < aComments.Length(); ++i) {
SerializeToBuffer(aComments[i], aOutput);
}
}
} // Anonymous namespace.
-OpusTrackEncoder::OpusTrackEncoder()
- : AudioTrackEncoder()
+OpusTrackEncoder::OpusTrackEncoder(TrackRate aTrackRate)
+ : AudioTrackEncoder(aTrackRate)
, mEncoder(nullptr)
, mLookahead(0)
, mResampler(nullptr)
, mOutputTimeStamp(0)
{
}
OpusTrackEncoder::~OpusTrackEncoder()
@@ -139,20 +139,16 @@ OpusTrackEncoder::~OpusTrackEncoder()
speex_resampler_destroy(mResampler);
mResampler = nullptr;
}
}
nsresult
OpusTrackEncoder::Init(int aChannels, int aSamplingRate)
{
- // This monitor is used to wake up other methods that are waiting for encoder
- // to be completely initialized.
- ReentrantMonitorAutoEnter mon(mReentrantMonitor);
-
NS_ENSURE_TRUE((aChannels <= MAX_SUPPORTED_AUDIO_CHANNELS) && (aChannels > 0),
NS_ERROR_FAILURE);
// This version of encoder API only support 1 or 2 channels,
// So set the mChannels less or equal 2 and
// let InterleaveTrackData downmix pcm data.
mChannels = aChannels > MAX_CHANNELS ? MAX_CHANNELS : aChannels;
@@ -181,24 +177,24 @@ OpusTrackEncoder::Init(int aChannels, in
mSamplingRate = aSamplingRate;
NS_ENSURE_TRUE(mSamplingRate > 0, NS_ERROR_FAILURE);
int error = 0;
mEncoder = opus_encoder_create(GetOutputSampleRate(), mChannels,
OPUS_APPLICATION_AUDIO, &error);
- mInitialized = (error == OPUS_OK);
+ if (error == OPUS_OK) {
+ SetInitialized();
+ }
if (mAudioBitrate) {
opus_encoder_ctl(mEncoder, OPUS_SET_BITRATE(static_cast<int>(mAudioBitrate)));
}
- mReentrantMonitor.NotifyAll();
-
return error == OPUS_OK ? NS_OK : NS_ERROR_FAILURE;
}
int
OpusTrackEncoder::GetOutputSampleRate()
{
return mResampler ? kOpusSamplingRate : mSamplingRate;
}
@@ -208,25 +204,24 @@ OpusTrackEncoder::GetPacketDuration()
{
return GetOutputSampleRate() * kFrameDurationMs / 1000;
}
already_AddRefed<TrackMetadataBase>
OpusTrackEncoder::GetMetadata()
{
AUTO_PROFILER_LABEL("OpusTrackEncoder::GetMetadata", OTHER);
- {
- // Wait if mEncoder is not initialized.
- ReentrantMonitorAutoEnter mon(mReentrantMonitor);
- while (!mCanceled && !mInitialized) {
- mReentrantMonitor.Wait();
- }
+
+ MOZ_ASSERT(mInitialized || mCanceled);
+
+ if (mCanceled || mEncodingComplete) {
+ return nullptr;
}
- if (mCanceled || mEncodingComplete) {
+ if (!mInitialized) {
return nullptr;
}
RefPtr<OpusMetadata> meta = new OpusMetadata();
meta->mChannels = mChannels;
meta->mSamplingFrequency = mSamplingRate;
mLookahead = 0;
@@ -251,86 +246,69 @@ OpusTrackEncoder::GetMetadata()
return meta.forget();
}
nsresult
OpusTrackEncoder::GetEncodedTrack(EncodedFrameContainer& aData)
{
AUTO_PROFILER_LABEL("OpusTrackEncoder::GetEncodedTrack", OTHER);
- {
- ReentrantMonitorAutoEnter mon(mReentrantMonitor);
- // Wait until initialized or cancelled.
- while (!mCanceled && !mInitialized) {
- mReentrantMonitor.Wait();
- }
- if (mCanceled || mEncodingComplete) {
- return NS_ERROR_FAILURE;
- }
+
+ MOZ_ASSERT(mInitialized || mCanceled);
+
+ if (mCanceled || mEncodingComplete) {
+ return NS_ERROR_FAILURE;
}
- // calculation below depends on the truth that mInitialized is true.
- MOZ_ASSERT(mInitialized);
+ if (!mInitialized) {
+ // calculation below depends on the truth that mInitialized is true.
+ return NS_ERROR_FAILURE;
+ }
- bool wait = true;
+ TakeTrackData(mSourceSegment);
+
int result = 0;
- // Only wait once, then loop until we run out of packets of input data
+ // Loop until we run out of packets of input data
while (result >= 0 && !mEncodingComplete) {
// re-sampled frames left last time which didn't fit into an Opus packet duration.
const int framesLeft = mResampledLeftover.Length() / mChannels;
// When framesLeft is 0, (GetPacketDuration() - framesLeft) is a multiple
// of kOpusSamplingRate. There is not precision loss in the integer division
// in computing framesToFetch. If frameLeft > 0, we need to add 1 to
// framesToFetch to ensure there will be at least n frames after re-sampling.
const int frameRoundUp = framesLeft ? 1 : 0;
MOZ_ASSERT(GetPacketDuration() >= framesLeft);
// Try to fetch m frames such that there will be n frames
// where (n + frameLeft) >= GetPacketDuration() after re-sampling.
const int framesToFetch = !mResampler ? GetPacketDuration()
: (GetPacketDuration() - framesLeft) * mSamplingRate / kOpusSamplingRate
+ frameRoundUp;
- {
- // Move all the samples from mRawSegment to mSourceSegment. We only hold
- // the monitor in this block.
- ReentrantMonitorAutoEnter mon(mReentrantMonitor);
+
+ if (!mEndOfStream && mSourceSegment.GetDuration() < framesToFetch) {
+ // Not enough raw data
+ return NS_OK;
+ }
- // Wait until enough raw data, end of stream or cancelled.
- while (!mCanceled && mRawSegment.GetDuration() +
- mSourceSegment.GetDuration() < framesToFetch &&
- !mEndOfStream) {
- if (wait) {
- mReentrantMonitor.Wait();
- wait = false;
- } else {
- goto done; // nested while's...
- }
- }
-
- if (mCanceled) {
- return NS_ERROR_FAILURE;
- }
-
- mSourceSegment.AppendFrom(&mRawSegment);
-
- // Pad |mLookahead| samples to the end of source stream to prevent lost of
- // original data, the pcm duration will be calculated at rate 48K later.
- if (mEndOfStream && !mEosSetInEncoder) {
- mEosSetInEncoder = true;
- mSourceSegment.AppendNullData(mLookahead);
- }
+ // Pad |mLookahead| samples to the end of source stream to prevent lost of
+ // original data, the pcm duration will be calculated at rate 48K later.
+ if (mEndOfStream && !mEosSetInEncoder) {
+ mEosSetInEncoder = true;
+ mSourceSegment.AppendNullData(mLookahead);
}
// Start encoding data.
AutoTArray<AudioDataValue, 9600> pcm;
pcm.SetLength(GetPacketDuration() * mChannels);
- AudioSegment::ChunkIterator iter(mSourceSegment);
+
int frameCopied = 0;
- while (!iter.IsEnded() && frameCopied < framesToFetch) {
+ for (AudioSegment::ChunkIterator iter(mSourceSegment);
+ !iter.IsEnded() && frameCopied < framesToFetch;
+ iter.Next()) {
AudioChunk chunk = *iter;
// Chunk to the required frame size.
StreamTime frameToCopy = chunk.GetDuration();
if (frameToCopy > framesToFetch - frameCopied) {
frameToCopy = framesToFetch - frameCopied;
}
// Possible greatest value of framesToFetch = 3844: see
@@ -352,17 +330,16 @@ OpusTrackEncoder::GetEncodedTrack(Encode
MOZ_ASSERT_UNREACHABLE("memsetLength invalid!");
return NS_ERROR_FAILURE;
}
memset(pcm.Elements() + frameCopied * mChannels, 0,
memsetLength.value());
}
frameCopied += frameToCopy;
- iter.Next();
}
// Possible greatest value of framesToFetch = 3844: see
// https://bugzilla.mozilla.org/show_bug.cgi?id=1349421#c8. frameCopied
// should not be able to exceed this value.
MOZ_ASSERT(frameCopied <= 3844, "frameCopied exceeded expected range");
RefPtr<EncodedFrame> audiodata = new EncodedFrame();
@@ -463,13 +440,13 @@ OpusTrackEncoder::GetEncodedTrack(Encode
audiodata->SwapInFrameData(frameData);
// timestamp should be the time of the first sample
audiodata->SetTimeStamp(mOutputTimeStamp);
mOutputTimeStamp += FramesToUsecs(GetPacketDuration(), kOpusSamplingRate).value();
LOG("[Opus] mOutputTimeStamp %lld.",mOutputTimeStamp);
aData.AppendEncodedFrame(audiodata);
}
-done:
+
return result >= 0 ? NS_OK : NS_ERROR_FAILURE;
}
} // namespace mozilla
--- a/dom/media/encoder/OpusTrackEncoder.h
+++ b/dom/media/encoder/OpusTrackEncoder.h
@@ -25,17 +25,17 @@ public:
int32_t mChannels;
float mSamplingFrequency;
MetadataKind GetKind() const override { return METADATA_OPUS; }
};
class OpusTrackEncoder : public AudioTrackEncoder
{
public:
- OpusTrackEncoder();
+ explicit OpusTrackEncoder(TrackRate aTrackRate);
virtual ~OpusTrackEncoder();
already_AddRefed<TrackMetadataBase> GetMetadata() override;
nsresult GetEncodedTrack(EncodedFrameContainer& aData) override;
protected:
int GetPacketDuration() override;
--- a/dom/media/encoder/TrackEncoder.cpp
+++ b/dom/media/encoder/TrackEncoder.cpp
@@ -1,16 +1,20 @@
/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-*/
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this file,
* You can obtain one at http://mozilla.org/MPL/2.0/. */
+
#include "TrackEncoder.h"
+
#include "AudioChannelFormat.h"
+#include "GeckoProfiler.h"
#include "MediaStreamGraph.h"
#include "MediaStreamListener.h"
+#include "mozilla/AbstractThread.h"
#include "mozilla/Logging.h"
#include "VideoUtils.h"
#include "mozilla/Logging.h"
namespace mozilla {
LazyLogModule gTrackEncoderLog("TrackEncoder");
#define TRACK_LOG(type, msg) MOZ_LOG(gTrackEncoderLog, type, msg)
@@ -20,155 +24,332 @@ static const int DEFAULT_SAMPLING_RATE =
static const int DEFAULT_FRAME_WIDTH = 640;
static const int DEFAULT_FRAME_HEIGHT = 480;
static const int DEFAULT_TRACK_RATE = USECS_PER_S;
// 1 second threshold if the audio encoder cannot be initialized.
static const int AUDIO_INIT_FAILED_DURATION = 1;
// 30 second threshold if the video encoder cannot be initialized.
static const int VIDEO_INIT_FAILED_DURATION = 30;
-TrackEncoder::TrackEncoder()
- : mReentrantMonitor("media.TrackEncoder")
- , mEncodingComplete(false)
+TrackEncoder::TrackEncoder(TrackRate aTrackRate)
+ : mEncodingComplete(false)
, mEosSetInEncoder(false)
, mInitialized(false)
, mEndOfStream(false)
, mCanceled(false)
+ , mCurrentTime(0)
, mInitCounter(0)
, mNotInitDuration(0)
+ , mSuspended(false)
+ , mTrackRate(aTrackRate)
{
}
-void TrackEncoder::NotifyEvent(MediaStreamGraph* aGraph,
- MediaStreamGraphEvent event)
+bool
+TrackEncoder::IsInitialized()
+{
+ MOZ_ASSERT(!mWorkerThread || mWorkerThread->IsCurrentThreadIn());
+ return mInitialized;
+}
+
+bool
+TrackEncoder::IsEncodingComplete()
+{
+ MOZ_ASSERT(!mWorkerThread || mWorkerThread->IsCurrentThreadIn());
+ return mEncodingComplete;
+}
+
+void
+TrackEncoder::SetInitialized()
{
- if (event == MediaStreamGraphEvent::EVENT_REMOVED) {
- NotifyEndOfStream();
+ MOZ_ASSERT(!mWorkerThread || mWorkerThread->IsCurrentThreadIn());
+
+ if (mInitialized) {
+ return;
+ }
+
+ mInitialized = true;
+
+ auto listeners(mListeners);
+ for (auto& l : listeners) {
+ l->Initialized(this);
+ }
+}
+
+void
+TrackEncoder::OnDataAvailable()
+{
+ MOZ_ASSERT(!mWorkerThread || mWorkerThread->IsCurrentThreadIn());
+
+ auto listeners(mListeners);
+ for (auto& l : listeners) {
+ l->DataAvailable(this);
+ }
+}
+
+void
+TrackEncoder::OnError()
+{
+ MOZ_ASSERT(!mWorkerThread || mWorkerThread->IsCurrentThreadIn());
+
+ Cancel();
+
+ auto listeners(mListeners);
+ for (auto& l : listeners) {
+ l->Error(this);
}
}
-nsresult
-AudioTrackEncoder::TryInit(const AudioSegment& aSegment, int aSamplingRate)
+void
+TrackEncoder::RegisterListener(TrackEncoderListener* aListener)
+{
+ MOZ_ASSERT(!mWorkerThread || mWorkerThread->IsCurrentThreadIn());
+ MOZ_ASSERT(!mListeners.Contains(aListener));
+ mListeners.AppendElement(aListener);
+}
+
+bool
+TrackEncoder::UnregisterListener(TrackEncoderListener* aListener)
+{
+ MOZ_ASSERT(!mWorkerThread || mWorkerThread->IsCurrentThreadIn());
+ return mListeners.RemoveElement(aListener);
+}
+
+void
+TrackEncoder::SetWorkerThread(AbstractThread* aWorkerThread)
+{
+ mWorkerThread = aWorkerThread;
+}
+
+void
+AudioTrackEncoder::Suspend(TimeStamp)
+{
+ MOZ_ASSERT(!mWorkerThread || mWorkerThread->IsCurrentThreadIn());
+ TRACK_LOG(LogLevel::Info,
+ ("[AudioTrackEncoder %p]: Suspend(), was %s",
+ this, mSuspended ? "suspended" : "live"));
+
+ if (mSuspended) {
+ return;
+ }
+
+ mSuspended = true;
+}
+
+void
+AudioTrackEncoder::Resume(TimeStamp)
{
- ReentrantMonitorAutoEnter mon(mReentrantMonitor);
+ MOZ_ASSERT(!mWorkerThread || mWorkerThread->IsCurrentThreadIn());
+ TRACK_LOG(LogLevel::Info,
+ ("[AudioTrackEncoder %p]: Resume(), was %s",
+ this, mSuspended ? "suspended" : "live"));
+
+ if (!mSuspended) {
+ return;
+ }
+
+ mSuspended = false;
+}
+
+void
+AudioTrackEncoder::AppendAudioSegment(AudioSegment&& aSegment)
+{
+ MOZ_ASSERT(!mWorkerThread || mWorkerThread->IsCurrentThreadIn());
+ TRACK_LOG(LogLevel::Verbose,
+ ("[AudioTrackEncoder %p]: AppendAudioSegment() duration=%" PRIu64,
+ this, aSegment.GetDuration()));
+
+ if (mCanceled) {
+ return;
+ }
+
+ if (mEndOfStream) {
+ return;
+ }
+
+ mIncomingBuffer.AppendFrom(&aSegment);
+}
+
+void
+AudioTrackEncoder::TakeTrackData(AudioSegment& aSegment)
+{
+ MOZ_ASSERT(!mWorkerThread || mWorkerThread->IsCurrentThreadIn());
+
+ if (mCanceled) {
+ return;
+ }
+
+ aSegment.AppendFrom(&mOutgoingBuffer);
+}
+
+void
+AudioTrackEncoder::TryInit(const AudioSegment& aSegment, StreamTime aDuration)
+{
+ MOZ_ASSERT(!mWorkerThread || mWorkerThread->IsCurrentThreadIn());
if (mInitialized) {
- return NS_OK;
+ return;
}
mInitCounter++;
- TRACK_LOG(LogLevel::Debug, ("Init the audio encoder %d times", mInitCounter));
- AudioSegment::ConstChunkIterator iter(aSegment);
- while (!iter.IsEnded()) {
- AudioChunk chunk = *iter;
+ TRACK_LOG(LogLevel::Debug,
+ ("[AudioTrackEncoder %p]: Inited the audio encoder %d times",
+ this, mInitCounter));
+ for (AudioSegment::ConstChunkIterator iter(aSegment); !iter.IsEnded(); iter.Next()) {
// The number of channels is determined by the first non-null chunk, and
// thus the audio encoder is initialized at this time.
- if (!chunk.IsNull()) {
- nsresult rv = Init(chunk.mChannelData.Length(), aSamplingRate);
- if (NS_FAILED(rv)) {
- TRACK_LOG(LogLevel::Error,
- ("[AudioTrackEncoder]: Fail to initialize the encoder!"));
- NotifyCancel();
- return rv;
- }
- break;
+ if (iter->IsNull()) {
+ continue;
}
- iter.Next();
+ nsresult rv = Init(iter->mChannelData.Length(), mTrackRate);
+
+ if (NS_SUCCEEDED(rv)) {
+ TRACK_LOG(LogLevel::Info,
+ ("[AudioTrackEncoder %p]: Successfully initialized!", this));
+ return;
+ } else {
+ TRACK_LOG(LogLevel::Error,
+ ("[AudioTrackEncoder %p]: Failed to initialize the encoder!", this));
+ OnError();
+ return;
+ }
+ break;
}
- mNotInitDuration += aSegment.GetDuration();
+ mNotInitDuration += aDuration;
if (!mInitialized &&
- (mNotInitDuration / aSamplingRate >= AUDIO_INIT_FAILED_DURATION) &&
+ (mNotInitDuration / mTrackRate > AUDIO_INIT_FAILED_DURATION) &&
mInitCounter > 1) {
// Perform a best effort initialization since we haven't gotten any
// data yet. Motivated by issues like Bug 1336367
TRACK_LOG(LogLevel::Warning,
("[AudioTrackEncoder]: Initialize failed "
"for %ds. Attempting to init with %d "
"(default) channels!",
AUDIO_INIT_FAILED_DURATION,
DEFAULT_CHANNELS));
- nsresult rv = Init(DEFAULT_CHANNELS, aSamplingRate);
+ nsresult rv = Init(DEFAULT_CHANNELS, mTrackRate);
if (NS_FAILED(rv)) {
TRACK_LOG(LogLevel::Error,
- ("[AudioTrackEncoder]: Fail to initialize the encoder!"));
- NotifyCancel();
- return rv;
+ ("[AudioTrackEncoder %p]: Default-channel-init failed.", this));
+ OnError();
+ return;
}
}
+}
- return NS_OK;
+void
+AudioTrackEncoder::Cancel()
+{
+ MOZ_ASSERT(!mWorkerThread || mWorkerThread->IsCurrentThreadIn());
+ TRACK_LOG(LogLevel::Info,
+ ("[AudioTrackEncoder %p]: Cancel(), currentTime=%" PRIu64,
+ this, mCurrentTime));
+ mCanceled = true;
+ mIncomingBuffer.Clear();
+ mOutgoingBuffer.Clear();
}
void
-AudioTrackEncoder::NotifyQueuedTrackChanges(MediaStreamGraph* aGraph,
- TrackID aID,
- StreamTime aTrackOffset,
- uint32_t aTrackEvents,
- const MediaSegment& aQueuedMedia)
+AudioTrackEncoder::NotifyEndOfStream()
+{
+ MOZ_ASSERT(!mWorkerThread || mWorkerThread->IsCurrentThreadIn());
+ TRACK_LOG(LogLevel::Info,
+ ("[AudioTrackEncoder %p]: NotifyEndOfStream(), currentTime=%" PRIu64,
+ this, mCurrentTime));
+
+ if (!mCanceled && !mInitialized) {
+ // If source audio track is completely silent till the end of encoding,
+ // initialize the encoder with default channel counts and sampling rate.
+ Init(DEFAULT_CHANNELS, DEFAULT_SAMPLING_RATE);
+ }
+
+ mEndOfStream = true;
+
+ mIncomingBuffer.Clear();
+
+ if (mInitialized && !mCanceled) {
+ OnDataAvailable();
+ }
+}
+
+void
+AudioTrackEncoder::SetStartOffset(StreamTime aStartOffset)
{
- ReentrantMonitorAutoEnter mon(mReentrantMonitor);
+ MOZ_ASSERT(!mWorkerThread || mWorkerThread->IsCurrentThreadIn());
+ MOZ_ASSERT(mCurrentTime == 0);
+ TRACK_LOG(LogLevel::Info,
+ ("[AudioTrackEncoder %p]: SetStartOffset(), aStartOffset=%" PRIu64,
+ this, aStartOffset));
+ mIncomingBuffer.InsertNullDataAtStart(aStartOffset);
+ mCurrentTime = aStartOffset;
+}
+
+void
+AudioTrackEncoder::AdvanceBlockedInput(StreamTime aDuration)
+{
+ MOZ_ASSERT(!mWorkerThread || mWorkerThread->IsCurrentThreadIn());
+ TRACK_LOG(LogLevel::Verbose,
+ ("[AudioTrackEncoder %p]: AdvanceBlockedInput(), aDuration=%" PRIu64,
+ this, aDuration));
+
+ // We call Init here so it can account for aDuration towards the Init timeout
+ TryInit(mOutgoingBuffer, aDuration);
+
+ mIncomingBuffer.InsertNullDataAtStart(aDuration);
+ mCurrentTime += aDuration;
+}
+
+void
+AudioTrackEncoder::AdvanceCurrentTime(StreamTime aDuration)
+{
+ AUTO_PROFILER_LABEL("AudioTrackEncoder::AdvanceCurrentTime", OTHER);
+
+ MOZ_ASSERT(!mWorkerThread || mWorkerThread->IsCurrentThreadIn());
if (mCanceled) {
return;
}
- const AudioSegment& audio = static_cast<const AudioSegment&>(aQueuedMedia);
+ if (mEndOfStream) {
+ return;
+ }
- nsresult rv = TryInit(audio, aGraph->GraphRate());
- if (NS_FAILED(rv)) {
+ TRACK_LOG(LogLevel::Verbose,
+ ("[AudioTrackEncoder %p]: AdvanceCurrentTime() %" PRIu64,
+ this, aDuration));
+
+ StreamTime currentTime = mCurrentTime + aDuration;
+
+ if (mSuspended) {
+ mCurrentTime = currentTime;
+ mIncomingBuffer.ForgetUpTo(mCurrentTime);
return;
}
- // Append and consume this raw segment.
- AppendAudioSegment(audio);
-
+ if (currentTime <= mIncomingBuffer.GetDuration()) {
+ mOutgoingBuffer.AppendSlice(mIncomingBuffer, mCurrentTime, currentTime);
- // The stream has stopped and reached the end of track.
- if (aTrackEvents == TrackEventCommand::TRACK_EVENT_ENDED) {
- TRACK_LOG(LogLevel::Info, ("[AudioTrackEncoder]: Receive TRACK_EVENT_ENDED ."));
- NotifyEndOfStream();
- }
-}
-
-void
-AudioTrackEncoder::NotifyEndOfStream()
-{
- ReentrantMonitorAutoEnter mon(mReentrantMonitor);
-
- // If source audio track is completely silent till the end of encoding,
- // initialize the encoder with default channel counts and sampling rate.
- if (!mCanceled && !mInitialized) {
- Init(DEFAULT_CHANNELS, DEFAULT_SAMPLING_RATE);
+ TryInit(mOutgoingBuffer, aDuration);
+ if (mInitialized && mOutgoingBuffer.GetDuration() >= GetPacketDuration()) {
+ OnDataAvailable();
+ }
+ } else {
+ NS_ASSERTION(false, "AudioTrackEncoder::AdvanceCurrentTime Not enough data");
+ TRACK_LOG(LogLevel::Error,
+ ("[AudioTrackEncoder %p]: AdvanceCurrentTime() Not enough data. "
+ "In incoming=%" PRIu64 ", aDuration=%" PRIu64 ", currentTime=%" PRIu64,
+ this, mIncomingBuffer.GetDuration(), aDuration, currentTime));
+ OnError();
}
- mEndOfStream = true;
- mReentrantMonitor.NotifyAll();
-}
-
-nsresult
-AudioTrackEncoder::AppendAudioSegment(const AudioSegment& aSegment)
-{
- ReentrantMonitorAutoEnter mon(mReentrantMonitor);
-
- AudioSegment::ChunkIterator iter(const_cast<AudioSegment&>(aSegment));
- while (!iter.IsEnded()) {
- AudioChunk chunk = *iter;
- // Append and consume both non-null and null chunks.
- mRawSegment.AppendAndConsumeChunk(&chunk);
- iter.Next();
- }
-
- if (mRawSegment.GetDuration() >= GetPacketDuration()) {
- mReentrantMonitor.NotifyAll();
- }
-
- return NS_OK;
+ mCurrentTime = currentTime;
+ mIncomingBuffer.ForgetUpTo(mCurrentTime);
}
/*static*/
void
AudioTrackEncoder::InterleaveTrackData(AudioChunk& aChunk,
int32_t aDuration,
uint32_t aOutputChannels,
AudioDataValue* aOutput)
@@ -210,121 +391,277 @@ AudioTrackEncoder::DeInterleaveTrackData
for (int32_t i = 0; i < aChannels; ++i) {
for(int32_t j = 0; j < aDuration; ++j) {
aOutput[i * aDuration + j] = aInput[i + j * aChannels];
}
}
}
size_t
-AudioTrackEncoder::SizeOfExcludingThis(mozilla::MallocSizeOf aMallocSizeOf) const
+AudioTrackEncoder::SizeOfExcludingThis(mozilla::MallocSizeOf aMallocSizeOf)
+{
+ MOZ_ASSERT(!mWorkerThread || mWorkerThread->IsCurrentThreadIn());
+ return mIncomingBuffer.SizeOfExcludingThis(aMallocSizeOf) +
+ mOutgoingBuffer.SizeOfExcludingThis(aMallocSizeOf);
+}
+
+void
+VideoTrackEncoder::Suspend(TimeStamp aTime)
{
- return mRawSegment.SizeOfExcludingThis(aMallocSizeOf);
+ MOZ_ASSERT(!mWorkerThread || mWorkerThread->IsCurrentThreadIn());
+ TRACK_LOG(LogLevel::Info,
+ ("[VideoTrackEncoder %p]: Suspend(), was %s",
+ this, mSuspended ? "suspended" : "live"));
+
+ if (mSuspended) {
+ return;
+ }
+
+ mSuspended = true;
+ mSuspendTime = aTime;
}
void
-VideoTrackEncoder::Init(const VideoSegment& aSegment)
+VideoTrackEncoder::Resume(TimeStamp aTime)
+{
+ MOZ_ASSERT(!mWorkerThread || mWorkerThread->IsCurrentThreadIn());
+ TRACK_LOG(LogLevel::Info,
+ ("[VideoTrackEncoder %p]: Resume(), was %s",
+ this, mSuspended ? "suspended" : "live"));
+
+ if (!mSuspended) {
+ return;
+ }
+
+ mSuspended = false;
+
+ TimeDuration suspendDuration = aTime - mSuspendTime;
+ if (!mLastChunk.mTimeStamp.IsNull()) {
+ VideoChunk* nextChunk = mIncomingBuffer.FindChunkContaining(mCurrentTime);
+ if (nextChunk && nextChunk->mTimeStamp < aTime) {
+ nextChunk->mTimeStamp = aTime;
+ }
+ mLastChunk.mTimeStamp += suspendDuration;
+ }
+ if (!mStartTime.IsNull()) {
+ mStartTime += suspendDuration;
+ }
+
+ mSuspendTime = TimeStamp();
+}
+
+void
+VideoTrackEncoder::AppendVideoSegment(VideoSegment&& aSegment)
{
- ReentrantMonitorAutoEnter mon(mReentrantMonitor);
+ MOZ_ASSERT(!mWorkerThread || mWorkerThread->IsCurrentThreadIn());
+ TRACK_LOG(LogLevel::Verbose,
+ ("[VideoTrackEncoder %p]: AppendVideoSegment() duration=%" PRIu64,
+ this, aSegment.GetDuration()));
+
+ if (mCanceled) {
+ return;
+ }
+
+ if (mEndOfStream) {
+ return;
+ }
+
+ mIncomingBuffer.AppendFrom(&aSegment);
+}
+
+void
+VideoTrackEncoder::TakeTrackData(VideoSegment& aSegment)
+{
+ MOZ_ASSERT(!mWorkerThread || mWorkerThread->IsCurrentThreadIn());
+
+ if (mCanceled) {
+ return;
+ }
+
+ aSegment.AppendFrom(&mOutgoingBuffer);
+ mOutgoingBuffer.Clear();
+}
+
+void
+VideoTrackEncoder::Init(const VideoSegment& aSegment, StreamTime aDuration)
+{
+ MOZ_ASSERT(!mWorkerThread || mWorkerThread->IsCurrentThreadIn());
if (mInitialized) {
return;
}
mInitCounter++;
- TRACK_LOG(LogLevel::Debug, ("Init the video encoder %d times", mInitCounter));
- VideoSegment::ConstChunkIterator iter(aSegment);
- while (!iter.IsEnded()) {
- VideoChunk chunk = *iter;
- if (!chunk.IsNull()) {
- gfx::IntSize imgsize = chunk.mFrame.GetImage()->GetSize();
- gfx::IntSize intrinsicSize = chunk.mFrame.GetIntrinsicSize();
- nsresult rv = Init(imgsize.width, imgsize.height,
- intrinsicSize.width, intrinsicSize.height);
+ TRACK_LOG(LogLevel::Debug,
+ ("[VideoTrackEncoder %p]: Init the video encoder %d times",
+ this, mInitCounter));
+
+ for (VideoSegment::ConstChunkIterator iter(aSegment); !iter.IsEnded(); iter.Next()) {
+ if (iter->IsNull()) {
+ continue;
+ }
- if (NS_FAILED(rv)) {
- TRACK_LOG(LogLevel::Error, ("[VideoTrackEncoder]: Fail to initialize the encoder!"));
- NotifyCancel();
- }
- break;
- }
+ gfx::IntSize imgsize = iter->mFrame.GetImage()->GetSize();
+ gfx::IntSize intrinsicSize = iter->mFrame.GetIntrinsicSize();
+ nsresult rv = Init(imgsize.width, imgsize.height,
+ intrinsicSize.width, intrinsicSize.height);
- iter.Next();
+ if (NS_SUCCEEDED(rv)) {
+ TRACK_LOG(LogLevel::Info,
+ ("[VideoTrackEncoder %p]: Successfully initialized!", this));
+ return;
+ } else {
+ TRACK_LOG(LogLevel::Error,
+ ("[VideoTrackEncoder %p]: Failed to initialize the encoder!", this));
+ OnError();
+ }
+ break;
}
- mNotInitDuration += aSegment.GetDuration();
- if ((mNotInitDuration / mTrackRate >= VIDEO_INIT_FAILED_DURATION) &&
+ mNotInitDuration += aDuration;
+ if ((mNotInitDuration / mTrackRate > VIDEO_INIT_FAILED_DURATION) &&
mInitCounter > 1) {
- TRACK_LOG(LogLevel::Debug,
- ("[VideoTrackEncoder]: Initialize failed for %ds.",
- VIDEO_INIT_FAILED_DURATION));
- NotifyEndOfStream();
+ TRACK_LOG(LogLevel::Warning,
+ ("[VideoTrackEncoder %p]: No successful init for %ds.",
+ this, VIDEO_INIT_FAILED_DURATION));
+ OnError();
return;
}
}
void
-VideoTrackEncoder::SetCurrentFrames(const VideoSegment& aSegment)
+VideoTrackEncoder::Cancel()
+{
+ MOZ_ASSERT(!mWorkerThread || mWorkerThread->IsCurrentThreadIn());
+ TRACK_LOG(LogLevel::Info,
+ ("[VideoTrackEncoder %p]: Cancel(), currentTime=%" PRIu64,
+ this, mCurrentTime));
+ mCanceled = true;
+ mIncomingBuffer.Clear();
+ mOutgoingBuffer.Clear();
+ mLastChunk.SetNull(0);
+}
+
+void
+VideoTrackEncoder::NotifyEndOfStream()
{
- ReentrantMonitorAutoEnter mon(mReentrantMonitor);
+ MOZ_ASSERT(!mWorkerThread || mWorkerThread->IsCurrentThreadIn());
+
+ if (!mCanceled && !mInitialized) {
+ // If source video track is muted till the end of encoding, initialize the
+ // encoder with default frame width, frame height, and track rate.
+ Init(DEFAULT_FRAME_WIDTH, DEFAULT_FRAME_HEIGHT,
+ DEFAULT_FRAME_WIDTH, DEFAULT_FRAME_HEIGHT);
+ }
+
+ if (mEndOfStream) {
+ // We have already been notified.
+ return;
+ }
+
+ mEndOfStream = true;
+ TRACK_LOG(LogLevel::Info,
+ ("[VideoTrackEncoder %p]: NotifyEndOfStream(), currentTime=%" PRIu64,
+ this, mCurrentTime));
+
+ if (!mLastChunk.IsNull() && mLastChunk.mDuration > 0) {
+ RefPtr<layers::Image> lastImage = mLastChunk.mFrame.GetImage();
+ TRACK_LOG(LogLevel::Debug,
+ ("[VideoTrackEncoder]: Appending last video frame %p, "
+ "duration=%.5f", lastImage.get(),
+ FramesToTimeUnit(mLastChunk.mDuration, mTrackRate).ToSeconds()));
+ mOutgoingBuffer.AppendFrame(lastImage.forget(),
+ mLastChunk.mDuration,
+ mLastChunk.mFrame.GetIntrinsicSize(),
+ PRINCIPAL_HANDLE_NONE,
+ mLastChunk.mFrame.GetForceBlack(),
+ mLastChunk.mTimeStamp);
+ }
+
+ mIncomingBuffer.Clear();
+ mLastChunk.SetNull(0);
+
+ if (mInitialized && !mCanceled) {
+ OnDataAvailable();
+ }
+}
+
+void
+VideoTrackEncoder::SetStartOffset(StreamTime aStartOffset)
+{
+ MOZ_ASSERT(!mWorkerThread || mWorkerThread->IsCurrentThreadIn());
+ MOZ_ASSERT(mCurrentTime == 0);
+ TRACK_LOG(LogLevel::Info,
+ ("[VideoTrackEncoder %p]: SetStartOffset(), aStartOffset=%" PRIu64,
+ this, aStartOffset));
+ mIncomingBuffer.InsertNullDataAtStart(aStartOffset);
+ mCurrentTime = aStartOffset;
+}
+
+void
+VideoTrackEncoder::AdvanceBlockedInput(StreamTime aDuration)
+{
+ MOZ_ASSERT(!mWorkerThread || mWorkerThread->IsCurrentThreadIn());
+ TRACK_LOG(LogLevel::Verbose,
+ ("[VideoTrackEncoder %p]: AdvanceBlockedInput(), aDuration=%" PRIu64,
+ this, aDuration));
+
+ // We call Init here so it can account for aDuration towards the Init timeout
+ Init(mOutgoingBuffer, aDuration);
+
+ mIncomingBuffer.InsertNullDataAtStart(aDuration);
+ mCurrentTime += aDuration;
+}
+
+void
+VideoTrackEncoder::AdvanceCurrentTime(StreamTime aDuration)
+{
+ AUTO_PROFILER_LABEL("VideoTrackEncoder::AdvanceCurrentTime", OTHER);
+
+ MOZ_ASSERT(!mWorkerThread || mWorkerThread->IsCurrentThreadIn());
if (mCanceled) {
return;
}
- Init(aSegment);
- AppendVideoSegment(aSegment);
-}
-
-void
-VideoTrackEncoder::NotifyQueuedTrackChanges(MediaStreamGraph* aGraph,
- TrackID aID,
- StreamTime aTrackOffset,
- uint32_t aTrackEvents,
- const MediaSegment& aQueuedMedia)
-{
- ReentrantMonitorAutoEnter mon(mReentrantMonitor);
-
- if (mCanceled) {
+ if (mEndOfStream) {
return;
}
- if (!(aTrackEvents == TRACK_EVENT_CREATED ||
- aTrackEvents == TRACK_EVENT_ENDED)) {
+ TRACK_LOG(LogLevel::Verbose,
+ ("[VideoTrackEncoder %p]: AdvanceCurrentTime() %" PRIu64,
+ this, aDuration));
+
+ StreamTime currentTime = mCurrentTime + aDuration;
+
+ if (mSuspended) {
+ mCurrentTime = currentTime;
+ mIncomingBuffer.ForgetUpTo(mCurrentTime);
return;
}
- const VideoSegment& video = static_cast<const VideoSegment&>(aQueuedMedia);
-
- // Check and initialize parameters for codec encoder.
- Init(video);
-
- AppendVideoSegment(video);
-
- // The stream has stopped and reached the end of track.
- if (aTrackEvents == TrackEventCommand::TRACK_EVENT_ENDED) {
- TRACK_LOG(LogLevel::Info, ("[VideoTrackEncoder]: Receive TRACK_EVENT_ENDED ."));
- NotifyEndOfStream();
+ VideoSegment tempSegment;
+ if (currentTime <= mIncomingBuffer.GetDuration()) {
+ tempSegment.AppendSlice(mIncomingBuffer, mCurrentTime, currentTime);
+ } else {
+ NS_ASSERTION(false, "VideoTrackEncoder::AdvanceCurrentTime Not enough data");
+ TRACK_LOG(LogLevel::Error,
+ ("[VideoTrackEncoder %p]: AdvanceCurrentTime() Not enough data. "
+ "In incoming=%" PRIu64 ", aDuration=%" PRIu64 ", currentTime=%" PRIu64,
+ this, mIncomingBuffer.GetDuration(), aDuration, currentTime));
+ OnError();
}
-}
-
-nsresult
-VideoTrackEncoder::AppendVideoSegment(const VideoSegment& aSegment)
-{
- ReentrantMonitorAutoEnter mon(mReentrantMonitor);
+ mCurrentTime = currentTime;
+ mIncomingBuffer.ForgetUpTo(mCurrentTime);
- if (mEndOfStream) {
- MOZ_ASSERT(false);
- return NS_OK;
- }
+ bool chunkAppended = false;
- // Append all video segments from MediaStreamGraph, including null an
- // non-null frames.
- VideoSegment::ConstChunkIterator iter(aSegment);
+ // Convert tempSegment timestamps to durations and add it to mOutgoingBuffer.
+ VideoSegment::ConstChunkIterator iter(tempSegment);
for (; !iter.IsEnded(); iter.Next()) {
VideoChunk chunk = *iter;
if (mLastChunk.mTimeStamp.IsNull()) {
if (chunk.IsNull()) {
// The start of this track is frameless. We need to track the time
// it takes to get the first frame.
mLastChunk.mDuration += chunk.mDuration;
@@ -341,17 +678,17 @@ VideoTrackEncoder::AppendVideoSegment(co
TRACK_LOG(LogLevel::Verbose,
("[VideoTrackEncoder]: Got first video chunk after %" PRId64 " ticks.",
nullDuration));
// Adapt to the time before the first frame. This extends the first frame
// from [start, end] to [0, end], but it'll do for now.
auto diff = FramesToTimeUnit(nullDuration, mTrackRate);
if (!diff.IsValid()) {
NS_ERROR("null duration overflow");
- return NS_ERROR_DOM_MEDIA_OVERFLOW_ERR;
+ return;
}
mLastChunk.mTimeStamp -= diff.ToTimeDuration();
mLastChunk.mDuration += nullDuration;
}
MOZ_ASSERT(!mLastChunk.IsNull());
if (mLastChunk.CanCombineWithFollowing(chunk) || chunk.IsNull()) {
@@ -370,116 +707,76 @@ VideoTrackEncoder::AppendVideoSegment(co
TRACK_LOG(LogLevel::Verbose,
("[VideoTrackEncoder]: Chunk >1 second. duration=%" PRId64 ", "
"trackRate=%" PRId32, mLastChunk.mDuration, mTrackRate));
// If we have gotten dupes for over a second, we force send one
// to the encoder to make sure there is some output.
chunk.mTimeStamp = mLastChunk.mTimeStamp + TimeDuration::FromSeconds(1);
-
- // chunk's duration has already been accounted for.
- chunk.mDuration = 0;
+ chunk.mDuration = mLastChunk.mDuration - mTrackRate;
+ mLastChunk.mDuration = mTrackRate;
if (chunk.IsNull()) {
// Ensure that we don't pass null to the encoder by making mLastChunk
// null later on.
chunk.mFrame = mLastChunk.mFrame;
}
}
- if (mStartOffset.IsNull()) {
- mStartOffset = mLastChunk.mTimeStamp;
+ if (mStartTime.IsNull()) {
+ mStartTime = mLastChunk.mTimeStamp;
}
- TimeDuration relativeTime = chunk.mTimeStamp - mStartOffset;
+ TimeDuration relativeTime = chunk.mTimeStamp - mStartTime;
RefPtr<layers::Image> lastImage = mLastChunk.mFrame.GetImage();
TRACK_LOG(LogLevel::Verbose,
("[VideoTrackEncoder]: Appending video frame %p, at pos %.5fs",
lastImage.get(), relativeTime.ToSeconds()));
- CheckedInt64 totalDuration =
- UsecsToFrames(relativeTime.ToMicroseconds(), mTrackRate);
- if (!totalDuration.isValid()) {
- NS_ERROR("Duration overflow");
- return NS_ERROR_DOM_MEDIA_OVERFLOW_ERR;
- }
-
- CheckedInt64 duration = totalDuration - mEncodedTicks;
+ CheckedInt64 duration = UsecsToFrames(relativeTime.ToMicroseconds(),
+ mTrackRate)
+ - mEncodedTicks;
if (!duration.isValid()) {
NS_ERROR("Duration overflow");
- return NS_ERROR_DOM_MEDIA_OVERFLOW_ERR;
+ return;
}
- if (duration.isValid()) {
- if (duration.value() <= 0) {
- // The timestamp for mLastChunk is newer than for chunk.
- // This means the durations reported from MediaStreamGraph for
- // mLastChunk were larger than the timestamp diff - and durations were
- // used to trigger the 1-second frame above. This could happen due to
- // drift or underruns in the graph.
- TRACK_LOG(LogLevel::Warning,
- ("[VideoTrackEncoder]: Underrun detected. Diff=%" PRId64,
- duration.value()));
- chunk.mTimeStamp = mLastChunk.mTimeStamp;
- } else {
- mEncodedTicks += duration.value();
- mRawSegment.AppendFrame(lastImage.forget(),
- duration.value(),
- mLastChunk.mFrame.GetIntrinsicSize(),
- PRINCIPAL_HANDLE_NONE,
- mLastChunk.mFrame.GetForceBlack(),
- mLastChunk.mTimeStamp);
- }
+ if (duration.value() <= 0) {
+ // The timestamp for mLastChunk is newer than for chunk.
+ // This means the durations reported from MediaStreamGraph for
+ // mLastChunk were larger than the timestamp diff - and durations were
+ // used to trigger the 1-second frame above. This could happen due to
+ // drift or underruns in the graph.
+ TRACK_LOG(LogLevel::Warning,
+ ("[VideoTrackEncoder]: Underrun detected. Diff=%" PRId64,
+ duration.value()));
+ chunk.mTimeStamp = mLastChunk.mTimeStamp;
+ } else {
+ mEncodedTicks += duration.value();
+ mOutgoingBuffer.AppendFrame(lastImage.forget(),
+ duration.value(),
+ mLastChunk.mFrame.GetIntrinsicSize(),
+ PRINCIPAL_HANDLE_NONE,
+ mLastChunk.mFrame.GetForceBlack(),
+ mLastChunk.mTimeStamp);
+ chunkAppended = true;
}
mLastChunk = chunk;
}
- if (mRawSegment.GetDuration() > 0) {
- mReentrantMonitor.NotifyAll();
- }
-
- return NS_OK;
-}
-
-void
-VideoTrackEncoder::NotifyEndOfStream()
-{
- ReentrantMonitorAutoEnter mon(mReentrantMonitor);
-
- // If source video track is muted till the end of encoding, initialize the
- // encoder with default frame width, frame height, and track rate.
- if (!mCanceled && !mInitialized) {
- Init(DEFAULT_FRAME_WIDTH, DEFAULT_FRAME_HEIGHT,
- DEFAULT_FRAME_WIDTH, DEFAULT_FRAME_HEIGHT);
+ if (chunkAppended) {
+ Init(mOutgoingBuffer, aDuration);
+ if (mInitialized) {
+ OnDataAvailable();
+ }
}
-
- if (mEndOfStream) {
- // We have already been notified.
- return;
- }
-
- mEndOfStream = true;
- TRACK_LOG(LogLevel::Info, ("[VideoTrackEncoder]: Reached end of stream"));
-
- if (!mLastChunk.IsNull() && mLastChunk.mDuration > 0) {
- RefPtr<layers::Image> lastImage = mLastChunk.mFrame.GetImage();
- TRACK_LOG(LogLevel::Debug,
- ("[VideoTrackEncoder]: Appending last video frame %p, "
- "duration=%.5f", lastImage.get(),
- FramesToTimeUnit(mLastChunk.mDuration, mTrackRate).ToSeconds()));
- mRawSegment.AppendFrame(lastImage.forget(),
- mLastChunk.mDuration,
- mLastChunk.mFrame.GetIntrinsicSize(),
- PRINCIPAL_HANDLE_NONE,
- mLastChunk.mFrame.GetForceBlack(),
- mLastChunk.mTimeStamp);
- }
- mReentrantMonitor.NotifyAll();
}
size_t
-VideoTrackEncoder::SizeOfExcludingThis(mozilla::MallocSizeOf aMallocSizeOf) const
+VideoTrackEncoder::SizeOfExcludingThis(mozilla::MallocSizeOf aMallocSizeOf)
{
- return mRawSegment.SizeOfExcludingThis(aMallocSizeOf);
+ MOZ_ASSERT(!mWorkerThread || mWorkerThread->IsCurrentThreadIn());
+ return mIncomingBuffer.SizeOfExcludingThis(aMallocSizeOf) +
+ mOutgoingBuffer.SizeOfExcludingThis(aMallocSizeOf);
}
} // namespace mozilla
--- a/dom/media/encoder/TrackEncoder.h
+++ b/dom/media/encoder/TrackEncoder.h
@@ -1,155 +1,256 @@
/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-*/
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this file,
* You can obtain one at http://mozilla.org/MPL/2.0/. */
#ifndef TrackEncoder_h_
#define TrackEncoder_h_
-#include "mozilla/ReentrantMonitor.h"
-
#include "AudioSegment.h"
#include "EncodedFrameContainer.h"
+#include "MediaStreamGraph.h"
#include "StreamTracks.h"
#include "TrackMetadataBase.h"
#include "VideoSegment.h"
-#include "MediaStreamGraph.h"
namespace mozilla {
+class AbstractThread;
+class TrackEncoder;
+
+class TrackEncoderListener
+{
+public:
+ NS_INLINE_DECL_THREADSAFE_REFCOUNTING(TrackEncoderListener)
+
+ /**
+ * Called when the TrackEncoder's underlying encoder has been successfully
+ * initialized and there's non-null data ready to be encoded.
+ */
+ virtual void Initialized(TrackEncoder* aEncoder) = 0;
+
+ /**
+ * Called when there's new data ready to be encoded.
+ * Always called after Initialized().
+ */
+ virtual void DataAvailable(TrackEncoder* aEncoder) = 0;
+
+ /**
+ * Called after the TrackEncoder hit an unexpected error, causing it to
+ * abort operation.
+ */
+ virtual void Error(TrackEncoder* aEncoder) = 0;
+protected:
+ virtual ~TrackEncoderListener() {}
+};
+
/**
- * Base class of AudioTrackEncoder and VideoTrackEncoder. Lifetimes managed by
- * MediaEncoder. Most methods can only be called on the MediaEncoder's thread,
- * but some subclass methods can be called on other threads when noted.
+ * Base class of AudioTrackEncoder and VideoTrackEncoder. Lifetime managed by
+ * MediaEncoder. All methods are to be called only on the worker thread.
*
- * NotifyQueuedTrackChanges is called on subclasses of this class from the
- * MediaStreamGraph thread, and AppendAudioSegment/AppendVideoSegment is then
- * called to store media data in the TrackEncoder. Later on, GetEncodedTrack is
- * called on MediaEncoder's thread to encode and retrieve the encoded data.
+ * MediaStreamTrackListeners will get store raw data in mIncomingBuffer, so
+ * mIncomingBuffer is protected by a lock. The control APIs are all called by
+ * MediaEncoder on its dedicated thread, where GetEncodedTrack is called
+ * periodically to swap out mIncomingBuffer, feed it to the encoder, and return
+ * the encoded data.
*/
class TrackEncoder
{
+ NS_INLINE_DECL_THREADSAFE_REFCOUNTING(TrackEncoder);
+
public:
- TrackEncoder();
+ explicit TrackEncoder(TrackRate aTrackRate);
+
+ virtual void Suspend(TimeStamp aTime) = 0;
- virtual ~TrackEncoder() {}
+ virtual void Resume(TimeStamp aTime) = 0;
+
+ /**
+ * Called by MediaEncoder to cancel the encoding.
+ */
+ virtual void Cancel() = 0;
/**
- * Notified by the same callbcak of MediaEncoder when it has received a track
- * change from MediaStreamGraph. Called on the MediaStreamGraph thread.
+ * Notifies us that we have reached the end of the stream and no more data
+ * will be appended.
*/
- virtual void NotifyQueuedTrackChanges(MediaStreamGraph* aGraph, TrackID aID,
- StreamTime aTrackOffset,
- uint32_t aTrackEvents,
- const MediaSegment& aQueuedMedia) = 0;
+ virtual void NotifyEndOfStream() = 0;
+
+ /**
+ * MediaStreamGraph notifies us about the time of the track's start.
+ * This gets called on the MediaEncoder thread after a dispatch.
+ */
+ virtual void SetStartOffset(StreamTime aStartOffset) = 0;
/**
- * Notified by the same callback of MediaEncoder when it has been removed from
- * MediaStreamGraph. Called on the MediaStreamGraph thread.
+ * Dispatched from MediaStreamGraph when it has run an iteration where the
+ * input track of the track this TrackEncoder is associated with didn't have
+ * any data.
*/
- void NotifyEvent(MediaStreamGraph* aGraph,
- MediaStreamGraphEvent event);
+ virtual void AdvanceBlockedInput(StreamTime aDuration) = 0;
+
+ /**
+ * MediaStreamGraph notifies us about the duration of data that has just been
+ * processed. This gets called on the MediaEncoder thread after a dispatch.
+ */
+ virtual void AdvanceCurrentTime(StreamTime aDuration) = 0;
/**
* Creates and sets up meta data for a specific codec, called on the worker
* thread.
*/
virtual already_AddRefed<TrackMetadataBase> GetMetadata() = 0;
/**
* Encodes raw segments. Result data is returned in aData, and called on the
* worker thread.
*/
virtual nsresult GetEncodedTrack(EncodedFrameContainer& aData) = 0;
/**
+ * Returns true once this TrackEncoder is initialized.
+ */
+ bool IsInitialized();
+
+ /**
* True if the track encoder has encoded all source segments coming from
* MediaStreamGraph. Call on the worker thread.
*/
- bool IsEncodingComplete() { return mEncodingComplete; }
+ bool IsEncodingComplete();
+
+ /**
+ * If this TrackEncoder was not already initialized, it is set to initialized
+ * and listeners are notified.
+ */
+ void SetInitialized();
+
+ /**
+ * Notifies listeners that there is data available for encoding.
+ */
+ void OnDataAvailable();
+
+ /**
+ * Called after an error. Cancels the encoding and notifies listeners.
+ */
+ void OnError();
/**
- * Notifies from MediaEncoder to cancel the encoding, and wakes up
- * mReentrantMonitor if encoder is waiting on it.
+ * Registers a listener to events from this TrackEncoder.
+ * We hold a strong reference to the listener.
+ */
+ void RegisterListener(TrackEncoderListener* aListener);
+
+ /**
+ * Unregisters a listener from events from this TrackEncoder.
+ * The listener will stop receiving events synchronously.
*/
- void NotifyCancel()
- {
- ReentrantMonitorAutoEnter mon(mReentrantMonitor);
- mCanceled = true;
- NotifyEndOfStream();
- }
+ bool UnregisterListener(TrackEncoderListener* aListener);
+
+ virtual void SetBitrate(const uint32_t aBitrate) = 0;
- virtual void SetBitrate(const uint32_t aBitrate) {}
+ /**
+ * It's optional to set the worker thread, but if you do we'll assert that
+ * we are in the worker thread in every method that gets called.
+ */
+ void SetWorkerThread(AbstractThread* aWorkerThread);
+
+ /**
+ * Measure size of internal buffers.
+ */
+ virtual size_t SizeOfExcludingThis(mozilla::MallocSizeOf aMallocSizeOf) = 0;
protected:
- /**
- * Notifies track encoder that we have reached the end of source stream, and
- * wakes up mReentrantMonitor if encoder is waiting for any source data.
- */
- virtual void NotifyEndOfStream() = 0;
-
- /**
- * A ReentrantMonitor to protect the pushing and pulling of mRawSegment which
- * is declared in its subclasses, and the following flags: mInitialized,
- * EndOfStream and mCanceled. The control of protection is managed by its
- * subclasses.
- */
- ReentrantMonitor mReentrantMonitor;
+ virtual ~TrackEncoder()
+ {
+ MOZ_ASSERT(mListeners.IsEmpty());
+ }
/**
* True if the track encoder has encoded all source data.
*/
bool mEncodingComplete;
/**
* True if flag of EOS or any form of indicating EOS has set in the codec-
* encoder.
*/
bool mEosSetInEncoder;
/**
- * True if the track encoder has initialized successfully, protected by
- * mReentrantMonitor.
+ * True if the track encoder has been initialized successfully.
*/
bool mInitialized;
/**
- * True if the TrackEncoder has received an event of TRACK_EVENT_ENDED from
- * MediaStreamGraph, or the MediaEncoder is removed from its source stream,
- * protected by mReentrantMonitor.
+ * True once all data until the end of the input track has been received.
*/
bool mEndOfStream;
/**
- * True if a cancellation of encoding is sent from MediaEncoder, protected by
- * mReentrantMonitor.
+ * True once this encoding has been cancelled.
*/
bool mCanceled;
+ /**
+ * The latest current time reported to us from the MSG.
+ */
+ StreamTime mCurrentTime;
+
// How many times we have tried to initialize the encoder.
uint32_t mInitCounter;
StreamTime mNotInitDuration;
+
+ bool mSuspended;
+
+ /**
+ * The track rate of source media.
+ */
+ TrackRate mTrackRate;
+
+ /**
+ * If set we assert that all methods are called on this thread.
+ */
+ RefPtr<AbstractThread> mWorkerThread;
+
+ nsTArray<RefPtr<TrackEncoderListener>> mListeners;
};
class AudioTrackEncoder : public TrackEncoder
{
public:
- AudioTrackEncoder()
- : TrackEncoder()
+ explicit AudioTrackEncoder(TrackRate aTrackRate)
+ : TrackEncoder(aTrackRate)
, mChannels(0)
, mSamplingRate(0)
, mAudioBitrate(0)
{}
- void NotifyQueuedTrackChanges(MediaStreamGraph* aGraph, TrackID aID,
- StreamTime aTrackOffset,
- uint32_t aTrackEvents,
- const MediaSegment& aQueuedMedia) override;
+ /**
+ * Suspends encoding from mCurrentTime, i.e., all audio data until the next
+ * Resume() will be dropped.
+ */
+ void Suspend(TimeStamp aTime) override;
+
+ /**
+ * Resumes encoding starting at mCurrentTime.
+ */
+ void Resume(TimeStamp aTime) override;
+
+ /**
+ * Appends and consumes track data from aSegment.
+ */
+ void AppendAudioSegment(AudioSegment&& aSegment);
+
+ /**
+ * Takes track data from the last time TakeTrackData ran until mCurrentTime
+ * and moves it to aSegment.
+ */
+ void TakeTrackData(AudioSegment& aSegment);
template<typename T>
static
void InterleaveTrackData(nsTArray<const T*>& aInput,
int32_t aDuration,
uint32_t aOutputChannels,
AudioDataValue* aOutput,
float aVolume)
@@ -179,162 +280,221 @@ public:
AudioDataValue* aOutput);
/**
* De-interleaves the aInput data and stores the result into aOutput.
* No up-mix or down-mix operations inside.
*/
static void DeInterleaveTrackData(AudioDataValue* aInput, int32_t aDuration,
int32_t aChannels, AudioDataValue* aOutput);
+
/**
- * Measure size of mRawSegment
- */
- size_t SizeOfExcludingThis(mozilla::MallocSizeOf aMallocSizeOf) const;
+ * Measure size of internal buffers.
+ */
+ size_t SizeOfExcludingThis(mozilla::MallocSizeOf aMallocSizeOf) override;
void SetBitrate(const uint32_t aBitrate) override
{
mAudioBitrate = aBitrate;
}
+
+ /**
+ * Tries to initiate the AudioEncoder based on data in aSegment.
+ * This can be re-called often, as it will exit early should we already be
+ * initiated. mInitiated will only be set if there was enough data in
+ * aSegment to infer metadata. If mInitiated gets set, listeners are notified.
+ *
+ * Not having enough data in aSegment to initiate the encoder for an accumulated aDuration of one second will make us initiate with a default number of channels.
+ *
+ * If we attempt to initiate the underlying encoder but fail, we Cancel() and
+ * notify listeners.
+ */
+ void TryInit(const AudioSegment& aSegment, StreamTime aDuration);
+
+ void Cancel() override;
+
+ /**
+ * Dispatched from MediaStreamGraph when we have finished feeding data to
+ * mIncomingBuffer.
+ */
+ void NotifyEndOfStream() override;
+
+ void SetStartOffset(StreamTime aStartOffset) override;
+
+ /**
+ * Dispatched from MediaStreamGraph when it has run an iteration where the
+ * input track of the track this TrackEncoder is associated with didn't have
+ * any data.
+ *
+ * Since we sometimes use a direct listener for AudioSegments we miss periods
+ * of time for which the source didn't have any data. This ensures that the
+ * latest frame gets displayed while we wait for more data to be pushed.
+ */
+ void AdvanceBlockedInput(StreamTime aDuration) override;
+
+ /**
+ * Dispatched from MediaStreamGraph when it has run an iteration so we can
+ * hand more data to the encoder.
+ */
+ void AdvanceCurrentTime(StreamTime aDuration) override;
protected:
/**
* Number of samples per channel in a pcm buffer. This is also the value of
- * frame size required by audio encoder, and mReentrantMonitor will be
- * notified when at least this much data has been added to mRawSegment.
+ * frame size required by audio encoder, and listeners will be notified when
+ * at least this much data has been added to mOutgoingBuffer.
*/
virtual int GetPacketDuration() { return 0; }
/**
- * Attempt to initialize the audio encoder. The call of this method is
- * delayed until we have received the first valid track from
- * MediaStreamGraph, and the mReentrantMonitor will be notified if other
- * methods is waiting for encoder to be completely initialized. This method
- * is called on the MediaStreamGraph thread. This method will attempt to
- * initialize with best effort if all the following are met:
- * - it has been called multiple times
- * - reached a threshold duration of audio data
- * - the encoder has not yet initialized.
- * Returns NS_OK on init, as well as when deferring for more data, so check
- * mInitialized after calling as necessary.
- */
- virtual nsresult TryInit(const AudioSegment& aSegment, int aSamplingRate);
-
- /**
* Initializes the audio encoder. The call of this method is delayed until we
- * have received the first valid track from MediaStreamGraph, and the
- * mReentrantMonitor will be notified if other methods is waiting for encoder
- * to be completely initialized. This method is called on the MediaStreamGraph
- * thread.
+ * have received the first valid track from MediaStreamGraph.
*/
virtual nsresult Init(int aChannels, int aSamplingRate) = 0;
/**
- * Appends and consumes track data from aSegment, this method is called on
- * the MediaStreamGraph thread. mReentrantMonitor will be notified when at
- * least GetPacketDuration() data has been added to mRawSegment, wake up other
- * method which is waiting for more data from mRawSegment.
- */
- nsresult AppendAudioSegment(const AudioSegment& aSegment);
-
- /**
- * Notifies the audio encoder that we have reached the end of source stream,
- * and wakes up mReentrantMonitor if encoder is waiting for more track data.
- */
- void NotifyEndOfStream() override;
-
- /**
* The number of channels are used for processing PCM data in the audio encoder.
* This value comes from the first valid audio chunk. If encoder can't support
* the channels in the chunk, downmix PCM stream can be performed.
* This value also be used to initialize the audio encoder.
*/
int mChannels;
/**
* The sampling rate of source audio data.
*/
int mSamplingRate;
/**
- * A segment queue of audio track data, protected by mReentrantMonitor.
+ * A segment queue of incoming audio track data, from listeners.
+ * The duration of mIncomingBuffer is strictly increasing as it gets fed more
+ * data. Consumed data is replaced by null data.
*/
- AudioSegment mRawSegment;
+ AudioSegment mIncomingBuffer;
+
+ /**
+ * A segment queue of outgoing audio track data to the encoder.
+ * The contents of mOutgoingBuffer will always be what has been consumed from
+ * mIncomingBuffer (up to mCurrentTime) but not yet consumed by the encoder
+ * sub class.
+ */
+ AudioSegment mOutgoingBuffer;
uint32_t mAudioBitrate;
+
+ // This may only be accessed on the MSG thread.
+ // I.e., in the regular NotifyQueuedChanges for audio to avoid adding data
+ // from that callback when the direct one is active.
+ bool mDirectConnected;
};
class VideoTrackEncoder : public TrackEncoder
{
public:
explicit VideoTrackEncoder(TrackRate aTrackRate)
- : TrackEncoder()
+ : TrackEncoder(aTrackRate)
, mFrameWidth(0)
, mFrameHeight(0)
, mDisplayWidth(0)
, mDisplayHeight(0)
- , mTrackRate(aTrackRate)
, mEncodedTicks(0)
, mVideoBitrate(0)
{
mLastChunk.mDuration = 0;
}
/**
- * Notified by the same callback of MediaEncoder when it has received a track
- * change from MediaStreamGraph. Called on the MediaStreamGraph thread.
+ * Suspends encoding from aTime, i.e., all video frame with a timestamp
+ * between aTime and the timestamp of the next Resume() will be dropped.
+ */
+ void Suspend(TimeStamp aTime) override;
+
+ /**
+ * Resumes encoding starting at aTime.
*/
- void NotifyQueuedTrackChanges(MediaStreamGraph* aGraph, TrackID aID,
- StreamTime aTrackOffset,
- uint32_t aTrackEvents,
- const MediaSegment& aQueuedMedia) override;
+ void Resume(TimeStamp aTime) override;
+
+ /**
+ * Appends source video frames to mIncomingBuffer. We only append the source
+ * chunk if the image is different from mLastChunk's image. Called on the
+ * MediaStreamGraph thread.
+ */
+ void AppendVideoSegment(VideoSegment&& aSegment);
+
/**
- * Measure size of mRawSegment
- */
- size_t SizeOfExcludingThis(mozilla::MallocSizeOf aMallocSizeOf) const;
+ * Takes track data from the last time TakeTrackData ran until mCurrentTime
+ * and moves it to aSegment.
+ */
+ void TakeTrackData(VideoSegment& aSegment);
+
+ /**
+ * Measure size of internal buffers.
+ */
+ size_t SizeOfExcludingThis(mozilla::MallocSizeOf aMallocSizeOf) override;
void SetBitrate(const uint32_t aBitrate) override
{
mVideoBitrate = aBitrate;
}
- void Init(const VideoSegment& aSegment);
-
- void SetCurrentFrames(const VideoSegment& aSegment);
+ /**
+ * Tries to initiate the VideoEncoder based on data in aSegment.
+ * This can be re-called often, as it will exit early should we already be
+ * initiated. mInitiated will only be set if there was enough data in
+ * aSegment to infer metadata. If mInitiated gets set, listeners are notified.
+ *
+ * Failing to initiate the encoder for an accumulated aDuration of 30 seconds
+ * is seen as an error and will cancel the current encoding.
+ */
+ void Init(const VideoSegment& aSegment, StreamTime aDuration);
StreamTime SecondsToMediaTime(double aS) const
{
NS_ASSERTION(0 <= aS && aS <= TRACK_TICKS_MAX/TRACK_RATE_MAX,
"Bad seconds");
return mTrackRate * aS;
}
+ void Cancel() override;
+
+ /**
+ * Notifies us that we have reached the end of the stream and no more data
+ * will be appended to mIncomingBuffer.
+ */
+ void NotifyEndOfStream() override;
+
+ void SetStartOffset(StreamTime aStartOffset) override;
+
+ /**
+ * Dispatched from MediaStreamGraph when it has run an iteration where the
+ * input track of the track this TrackEncoder is associated with didn't have
+ * any data.
+ *
+ * Since we use a direct listener for VideoSegments we miss periods of time
+ * for which the source didn't have any data. This ensures that the latest
+ * frame gets displayed while we wait for more data to be pushed.
+ */
+ void AdvanceBlockedInput(StreamTime aDuration) override;
+
+ /**
+ * Dispatched from MediaStreamGraph when it has run an iteration so we can
+ * hand more data to the encoder.
+ */
+ void AdvanceCurrentTime(StreamTime aDuration) override;
+
protected:
/**
- * Initialized the video encoder. In order to collect the value of width and
+ * Initialize the video encoder. In order to collect the value of width and
* height of source frames, this initialization is delayed until we have
- * received the first valid video frame from MediaStreamGraph;
- * mReentrantMonitor will be notified after it has successfully initialized,
- * and this method is called on the MediaStramGraph thread.
+ * received the first valid video frame from MediaStreamGraph.
+ * Listeners will be notified after it has been successfully initialized.
*/
virtual nsresult Init(int aWidth, int aHeight, int aDisplayWidth,
int aDisplayHeight) = 0;
/**
- * Appends source video frames to mRawSegment. We only append the source chunk
- * if it is unique to mLastChunk. Called on the MediaStreamGraph thread.
- */
- nsresult AppendVideoSegment(const VideoSegment& aSegment);
-
- /**
- * Tells the video track encoder that we've reached the end of source stream,
- * and wakes up mReentrantMonitor if encoder is waiting for more track data.
- * Called on the MediaStreamGraph thread.
- */
- void NotifyEndOfStream() override;
-
- /**
* The width of source video frame, ceiled if the source width is odd.
*/
int mFrameWidth;
/**
* The height of source video frame, ceiled if the source height is odd.
*/
int mFrameHeight;
@@ -345,41 +505,54 @@ protected:
int mDisplayWidth;
/**
* The display height of source video frame.
*/
int mDisplayHeight;
/**
- * The track rate of source video.
- */
- TrackRate mTrackRate;
-
- /**
- * The last unique frame and duration we've sent to track encoder,
- * kept track of in subclasses.
+ * The last unique frame and duration so far handled by NotifyAdvanceCurrentTime.
+ * When a new frame is detected, mLastChunk is added to mOutgoingBuffer.
*/
VideoChunk mLastChunk;
/**
- * A segment queue of audio track data, protected by mReentrantMonitor.
+ * A segment queue of incoming video track data, from listeners.
+ * The duration of mIncomingBuffer is strictly increasing as it gets fed more
+ * data. Consumed data is replaced by null data.
*/
- VideoSegment mRawSegment;
+ VideoSegment mIncomingBuffer;
/**
- * The number of mTrackRate ticks we have passed to the encoder.
- * Only accessed in AppendVideoSegment().
+ * A segment queue of outgoing video track data to the encoder.
+ * The contents of mOutgoingBuffer will always be what has been consumed from
+ * mIncomingBuffer (up to mCurrentTime) but not yet consumed by the encoder
+ * sub class. There won't be any null data at the beginning of mOutgoingBuffer
+ * unless explicitly pushed by the producer.
+ */
+ VideoSegment mOutgoingBuffer;
+
+ /**
+ * The number of mTrackRate ticks we have passed to mOutgoingBuffer.
*/
StreamTime mEncodedTicks;
/**
- * The time of the first real video frame passed to the encoder.
- * Only accessed in AppendVideoSegment().
+ * The time of the first real video frame passed to mOutgoingBuffer (at t=0).
+ *
+ * Note that this time will progress during suspension, to make sure the
+ * incoming frames stay in sync with the output.
*/
- TimeStamp mStartOffset;
+ TimeStamp mStartTime;
+
+ /**
+ * The time Suspend was called on the MediaRecorder, so we can calculate the
+ * duration on the next Resume().
+ */
+ TimeStamp mSuspendTime;
uint32_t mVideoBitrate;
};
} // namespace mozilla
#endif
--- a/dom/media/encoder/VP8TrackEncoder.cpp
+++ b/dom/media/encoder/VP8TrackEncoder.cpp
@@ -69,17 +69,16 @@ VP8TrackEncoder::~VP8TrackEncoder()
{
Destroy();
MOZ_COUNT_DTOR(VP8TrackEncoder);
}
void
VP8TrackEncoder::Destroy()
{
- ReentrantMonitorAutoEnter mon(mReentrantMonitor);
if (mInitialized) {
vpx_codec_destroy(mVPXContext);
}
if (mVPXImageWrapper) {
vpx_img_free(mVPXImageWrapper);
}
mInitialized = false;
@@ -88,17 +87,16 @@ VP8TrackEncoder::Destroy()
nsresult
VP8TrackEncoder::Init(int32_t aWidth, int32_t aHeight, int32_t aDisplayWidth,
int32_t aDisplayHeight)
{
if (aWidth < 1 || aHeight < 1 || aDisplayWidth < 1 || aDisplayHeight < 1) {
return NS_ERROR_FAILURE;
}
- ReentrantMonitorAutoEnter mon(mReentrantMonitor);
if (mInitialized) {
MOZ_ASSERT(false);
return NS_ERROR_FAILURE;
}
// Encoder configuration structure.
vpx_codec_enc_cfg_t config;
nsresult rv = SetConfigurationValues(aWidth, aHeight, aDisplayWidth, aDisplayHeight, config);
@@ -116,51 +114,47 @@ VP8TrackEncoder::Init(int32_t aWidth, in
return NS_ERROR_FAILURE;
}
vpx_codec_control(mVPXContext, VP8E_SET_STATIC_THRESHOLD, 1);
vpx_codec_control(mVPXContext, VP8E_SET_CPUUSED, -6);
vpx_codec_control(mVPXContext, VP8E_SET_TOKEN_PARTITIONS,
VP8_ONE_TOKENPARTITION);
- mInitialized = true;
- mon.NotifyAll();
+ SetInitialized();
return NS_OK;
}
nsresult
VP8TrackEncoder::Reconfigure(int32_t aWidth, int32_t aHeight,
int32_t aDisplayWidth, int32_t aDisplayHeight)
{
if(aWidth <= 0 || aHeight <= 0 || aDisplayWidth <= 0 || aDisplayHeight <= 0) {
MOZ_ASSERT(false);
return NS_ERROR_FAILURE;
}
- ReentrantMonitorAutoEnter mon(mReentrantMonitor);
if (!mInitialized) {
MOZ_ASSERT(false);
return NS_ERROR_FAILURE;
}
- mInitialized = false;
// Recreate image wrapper
vpx_img_free(mVPXImageWrapper);
vpx_img_wrap(mVPXImageWrapper, VPX_IMG_FMT_I420, aWidth, aHeight, 1, nullptr);
// Encoder configuration structure.
vpx_codec_enc_cfg_t config;
nsresult rv = SetConfigurationValues(aWidth, aHeight, aDisplayWidth, aDisplayHeight, config);
NS_ENSURE_SUCCESS(rv, NS_ERROR_FAILURE);
// Set new configuration
if (vpx_codec_enc_config_set(mVPXContext.get(), &config) != VPX_CODEC_OK) {
VP8LOG(LogLevel::Error, "Failed to set new configuration");
return NS_ERROR_FAILURE;
}
- mInitialized = true;
return NS_OK;
}
nsresult
VP8TrackEncoder::SetConfigurationValues(int32_t aWidth, int32_t aHeight, int32_t aDisplayWidth,
int32_t aDisplayHeight, vpx_codec_enc_cfg_t& config)
{
mFrameWidth = aWidth;
@@ -219,34 +213,37 @@ VP8TrackEncoder::SetConfigurationValues(
return NS_OK;
}
already_AddRefed<TrackMetadataBase>
VP8TrackEncoder::GetMetadata()
{
AUTO_PROFILER_LABEL("VP8TrackEncoder::GetMetadata", OTHER);
- {
- // Wait if mEncoder is not initialized.
- ReentrantMonitorAutoEnter mon(mReentrantMonitor);
- while (!mCanceled && !mInitialized) {
- mon.Wait();
- }
+
+ MOZ_ASSERT(mInitialized || mCanceled);
+
+ if (mCanceled || mEncodingComplete) {
+ return nullptr;
}
- if (mCanceled || mEncodingComplete) {
+ if (!mInitialized) {
return nullptr;
}
RefPtr<VP8Metadata> meta = new VP8Metadata();
meta->mWidth = mFrameWidth;
meta->mHeight = mFrameHeight;
meta->mDisplayWidth = mDisplayWidth;
meta->mDisplayHeight = mDisplayHeight;
+ VP8LOG(LogLevel::Info, "GetMetadata() width=%d, height=%d, "
+ "displayWidht=%d, displayHeight=%d",
+ meta->mWidth, meta->mHeight, meta->mDisplayWidth, meta->mDisplayHeight);
+
return meta.forget();
}
nsresult
VP8TrackEncoder::GetEncodedPartitions(EncodedFrameContainer& aData)
{
vpx_codec_iter_t iter = nullptr;
EncodedFrame::FrameType frameType = EncodedFrame::VP8_P_FRAME;
@@ -608,35 +605,29 @@ VP8TrackEncoder::GetNextEncodeOperation(
* mSourceSegment is 100ms, means that we can't spend more than 100ms to
* encode it.
* 4. Remove the encoded chunks in mSourceSegment after for-loop.
*/
nsresult
VP8TrackEncoder::GetEncodedTrack(EncodedFrameContainer& aData)
{
AUTO_PROFILER_LABEL("VP8TrackEncoder::GetEncodedTrack", OTHER);
- bool EOS;
- {
- // Move all the samples from mRawSegment to mSourceSegment. We only hold
- // the monitor in this block.
- ReentrantMonitorAutoEnter mon(mReentrantMonitor);
- // Wait if mEncoder is not initialized, or when not enough raw data, but is
- // not the end of stream nor is being canceled.
- while (!mCanceled && (!mInitialized ||
- (mRawSegment.GetDuration() + mSourceSegment.GetDuration() == 0 &&
- !mEndOfStream))) {
- mon.Wait();
- }
- if (mCanceled || mEncodingComplete) {
- return NS_ERROR_FAILURE;
- }
- mSourceSegment.AppendFrom(&mRawSegment);
- EOS = mEndOfStream;
+
+ MOZ_ASSERT(mInitialized || mCanceled);
+
+ if (mCanceled || mEncodingComplete) {
+ return NS_ERROR_FAILURE;
}
+ if (!mInitialized) {
+ return NS_ERROR_FAILURE;
+ }
+
+ TakeTrackData(mSourceSegment);
+
StreamTime totalProcessedDuration = 0;
TimeStamp timebase = TimeStamp::Now();
EncodeOperation nextEncodeOperation = ENCODE_NORMAL_FRAME;
for (VideoSegment::ChunkIterator iter(mSourceSegment);
!iter.IsEnded(); iter.Next()) {
VideoChunk &chunk = *iter;
VP8LOG(LogLevel::Verbose, "nextEncodeOperation is %d for frame of duration %" PRId64,
@@ -696,22 +687,21 @@ VP8TrackEncoder::GetEncodedTrack(Encoded
nextEncodeOperation = GetNextEncodeOperation(elapsedTime,
totalProcessedDuration);
}
// Remove the chunks we have processed.
mSourceSegment.Clear();
// End of stream, pull the rest frames in encoder.
- if (EOS) {
+ if (mEndOfStream) {
VP8LOG(LogLevel::Debug, "mEndOfStream is true");
mEncodingComplete = true;
// Bug 1243611, keep calling vpx_codec_encode and vpx_codec_get_cx_data
// until vpx_codec_get_cx_data return null.
-
do {
if (vpx_codec_encode(mVPXContext, nullptr, mEncodedTimestamp,
0, 0, VPX_DL_REALTIME)) {
return NS_ERROR_FAILURE;
}
} while(NS_SUCCEEDED(GetEncodedPartitions(aData)));
}
--- a/dom/media/gtest/TestAudioTrackEncoder.cpp
+++ b/dom/media/gtest/TestAudioTrackEncoder.cpp
@@ -32,68 +32,46 @@ public:
private:
SineWaveGenerator mGenerator;
const int32_t mChannels;
};
class TestOpusTrackEncoder : public OpusTrackEncoder
{
public:
+ TestOpusTrackEncoder() : OpusTrackEncoder(90000) {}
+
// Return true if it has successfully initialized the Opus encoder.
- bool TestOpusCreation(int aChannels, int aSamplingRate)
+ bool TestOpusRawCreation(int aChannels, int aSamplingRate)
{
if (Init(aChannels, aSamplingRate) == NS_OK) {
- if (GetPacketDuration()) {
- return true;
- }
- }
- return false;
- }
-
- bool TestOpusTryCreation(const AudioSegment& aSegment, int aSamplingRate)
- {
- if (TryInit(aSegment, aSamplingRate) == NS_OK) {
- if (GetPacketDuration()) {
+ if (IsInitialized()) {
return true;
}
}
return false;
}
// Return the sample rate of data to be fed to the Opus encoder, could be
// re-sampled if it was not one of the Opus supported sampling rates.
// Init() is expected to be called first.
int TestGetOutputSampleRate()
{
return mInitialized ? GetOutputSampleRate() : 0;
}
};
-static AudioSegment
-CreateTestSegment()
-{
- RefPtr<SharedBuffer> dummyBuffer = SharedBuffer::Create(2);
- AutoTArray<const int16_t*, 1> channels;
- const int16_t* channelData = static_cast<const int16_t*>(dummyBuffer->Data());
- channels.AppendElement(channelData);
-
- AudioSegment testSegment;
- testSegment.AppendFrames(
- dummyBuffer.forget(), channels, 1 /* #samples */, PRINCIPAL_HANDLE_NONE);
- return testSegment;
-}
-
static bool
TestOpusInit(int aChannels, int aSamplingRate)
{
TestOpusTrackEncoder encoder;
- return encoder.TestOpusCreation(aChannels, aSamplingRate);
+ return encoder.TestOpusRawCreation(aChannels, aSamplingRate);
}
-TEST(OpusAudioTrackEncoder, Init)
+TEST(OpusAudioTrackEncoder, InitRaw)
{
// Expect false with 0 or negative channels of input signal.
EXPECT_FALSE(TestOpusInit(0, 16000));
EXPECT_FALSE(TestOpusInit(-1, 16000));
// The Opus format supports up to 8 channels, and supports multitrack audio up
// to 255 channels, but the current implementation supports only mono and
// stereo, and downmixes any more than that.
@@ -114,84 +92,102 @@ TEST(OpusAudioTrackEncoder, Init)
EXPECT_FALSE(TestOpusInit(2, 4000));
EXPECT_FALSE(TestOpusInit(2, 7999));
EXPECT_TRUE(TestOpusInit(2, 8000));
EXPECT_TRUE(TestOpusInit(2, 192000));
EXPECT_FALSE(TestOpusInit(2, 192001));
EXPECT_FALSE(TestOpusInit(2, 200000));
}
-TEST(OpusAudioTrackEncoder, TryInit)
+TEST(OpusAudioTrackEncoder, Init)
{
{
// The encoder does not normally recieve enough info from null data to
// init. However, multiple attempts to do so, with sufficiently long
// duration segments, should result in a best effort attempt. The first
// attempt should never do this though, even if the duration is long:
- TestOpusTrackEncoder encoder;
- AudioSegment testSegment;
- testSegment.AppendNullData(48000 * 100);
- EXPECT_FALSE(encoder.TestOpusTryCreation(testSegment, 48000));
+ OpusTrackEncoder encoder(48000);
+ AudioSegment segment;
+ segment.AppendNullData(48000 * 100);
+ encoder.TryInit(segment, segment.GetDuration());
+ EXPECT_FALSE(encoder.IsInitialized());
// Multiple init attempts should result in best effort init:
- EXPECT_TRUE(encoder.TestOpusTryCreation(testSegment, 48000));
+ encoder.TryInit(segment, segment.GetDuration());
+ EXPECT_TRUE(encoder.IsInitialized());
}
{
// If the duration of the segments given to the encoder is not long then
// we shouldn't try a best effort init:
- TestOpusTrackEncoder encoder;
- AudioSegment testSegment;
- testSegment.AppendNullData(1);
- EXPECT_FALSE(encoder.TestOpusTryCreation(testSegment, 48000));
- EXPECT_FALSE(encoder.TestOpusTryCreation(testSegment, 48000));
+ OpusTrackEncoder encoder(48000);
+ AudioSegment segment;
+ segment.AppendNullData(1);
+ encoder.TryInit(segment, segment.GetDuration());
+ EXPECT_FALSE(encoder.IsInitialized());
+ encoder.TryInit(segment, segment.GetDuration());
+ EXPECT_FALSE(encoder.IsInitialized());
}
{
// For non-null segments we should init immediately
- TestOpusTrackEncoder encoder;
- AudioSegment testSegment = CreateTestSegment();
- EXPECT_TRUE(encoder.TestOpusTryCreation(testSegment, 48000));
+ OpusTrackEncoder encoder(48000);
+ AudioSegment segment;
+ AudioGenerator generator(2, 48000);
+ generator.Generate(segment, 1);
+ encoder.TryInit(segment, segment.GetDuration());
+ EXPECT_TRUE(encoder.IsInitialized());
}
{
// Test low sample rate bound
- TestOpusTrackEncoder encoder;
- AudioSegment testSegment = CreateTestSegment();
- EXPECT_FALSE(encoder.TestOpusTryCreation(testSegment, 7999));
+ OpusTrackEncoder encoder(7999);
+ AudioSegment segment;
+ AudioGenerator generator(2, 7999);
+ generator.Generate(segment, 1);
+ encoder.TryInit(segment, segment.GetDuration());
+ EXPECT_FALSE(encoder.IsInitialized());
}
{
// Test low sample rate bound
- TestOpusTrackEncoder encoder;
- AudioSegment testSegment = CreateTestSegment();
- EXPECT_FALSE(encoder.TestOpusTryCreation(testSegment, 7999));
- EXPECT_TRUE(encoder.TestOpusTryCreation(testSegment, 8000));
+ OpusTrackEncoder encoder(8000);
+ AudioSegment segment;
+ AudioGenerator generator(2, 8000);
+ generator.Generate(segment, 1);
+ encoder.TryInit(segment, segment.GetDuration());
+ EXPECT_TRUE(encoder.IsInitialized());
}
{
// Test high sample rate bound
- TestOpusTrackEncoder encoder;
- AudioSegment testSegment = CreateTestSegment();
- EXPECT_FALSE(encoder.TestOpusTryCreation(testSegment, 192001));
+ OpusTrackEncoder encoder(192001);
+ AudioSegment segment;
+ AudioGenerator generator(2, 192001);
+ generator.Generate(segment, 1);
+ encoder.TryInit(segment, segment.GetDuration());
+ EXPECT_FALSE(encoder.IsInitialized());
}
{
// Test high sample rate bound
- TestOpusTrackEncoder encoder;
- AudioSegment testSegment = CreateTestSegment();
- EXPECT_TRUE(encoder.TestOpusTryCreation(testSegment, 192000));
+ OpusTrackEncoder encoder(192000);
+ AudioSegment segment;
+ AudioGenerator generator(2, 192000);
+ generator.Generate(segment, 1);
+ encoder.TryInit(segment, segment.GetDuration());
+ EXPECT_TRUE(encoder.IsInitialized());
}
}
static int
TestOpusResampler(int aChannels, int aSamplingRate)
{
TestOpusTrackEncoder encoder;
- EXPECT_TRUE(encoder.TestOpusCreation(aChannels, aSamplingRate));
+ EXPECT_TRUE(encoder.TestOpusRawCreation(aChannels, aSamplingRate));
return encoder.TestGetOutputSampleRate();
}
TEST(OpusAudioTrackEncoder, Resample)
{
// Sampling rates of data to be fed to Opus encoder, should remain unchanged
// if it is one of Opus supported rates (8000, 12000, 16000, 24000 and 48000
// (kHz)) at initialization.
@@ -206,42 +202,42 @@ TEST(OpusAudioTrackEncoder, Resample)
EXPECT_TRUE(TestOpusResampler(1, 44100) == 48000);
}
TEST(OpusAudioTrackEncoder, FetchMetadata)
{
const int32_t channels = 1;
const int32_t sampleRate = 44100;
TestOpusTrackEncoder encoder;
- EXPECT_TRUE(encoder.TestOpusCreation(channels, sampleRate));
+ EXPECT_TRUE(encoder.TestOpusRawCreation(channels, sampleRate));
RefPtr<TrackMetadataBase> metadata = encoder.GetMetadata();
ASSERT_EQ(TrackMetadataBase::METADATA_OPUS, metadata->GetKind());
RefPtr<OpusMetadata> opusMeta =
static_cast<OpusMetadata*>(metadata.get());
EXPECT_EQ(channels, opusMeta->mChannels);
EXPECT_EQ(sampleRate, opusMeta->mSamplingFrequency);
}
TEST(OpusAudioTrackEncoder, FrameEncode)
{
const int32_t channels = 1;
const int32_t sampleRate = 44100;
TestOpusTrackEncoder encoder;
- EXPECT_TRUE(encoder.TestOpusCreation(channels, sampleRate));
+ EXPECT_TRUE(encoder.TestOpusRawCreation(channels, sampleRate));
// Generate five seconds of raw audio data.
AudioGenerator generator(channels, sampleRate);
AudioSegment segment;
const int32_t samples = sampleRate * 5;
generator.Generate(segment, samples);
encoder.AppendAudioSegment(Move(segment));
- encoder.NotifyCurrentTime(samples);
+ encoder.AdvanceCurrentTime(samples);
EncodedFrameContainer container;
EXPECT_TRUE(NS_SUCCEEDED(encoder.GetEncodedTrack(container)));
// Verify that encoded data is 5 seconds long.
uint64_t totalDuration = 0;
for (auto& frame : container.GetEncodedFrames()) {
totalDuration += frame->GetDuration();
--- a/dom/media/gtest/TestVideoTrackEncoder.cpp
+++ b/dom/media/gtest/TestVideoTrackEncoder.cpp
@@ -283,18 +283,18 @@ TEST(VP8VideoTrackEncoder, FrameEncode)
segment.AppendFrame(image.forget(),
mozilla::StreamTime(90000),
generator.GetSize(),
PRINCIPAL_HANDLE_NONE,
false,
now + TimeDuration::FromSeconds(i));
}
- // track change notification.
- encoder.SetCurrentFrames(segment);
+ encoder.AppendVideoSegment(Move(segment));
+ encoder.AdvanceCurrentTime(images.Length() * 90000);
// Pull Encoded Data back from encoder.
EncodedFrameContainer container;
EXPECT_TRUE(NS_SUCCEEDED(encoder.GetEncodedTrack(container)));
}
// Test that encoding a single frame gives useful output.
TEST(VP8VideoTrackEncoder, SingleFrameEncode)
@@ -308,21 +308,19 @@ TEST(VP8VideoTrackEncoder, SingleFrameEn
YUVBufferGenerator generator;
generator.Init(mozilla::gfx::IntSize(640, 480));
VideoSegment segment;
segment.AppendFrame(generator.GenerateI420Image(),
mozilla::StreamTime(45000), // 1/2 second
generator.GetSize(),
PRINCIPAL_HANDLE_NONE);
- encoder.SetCurrentFrames(segment);
-
- // End the track.
- segment.Clear();
- encoder.NotifyQueuedTrackChanges(nullptr, 0, 0, TrackEventCommand::TRACK_EVENT_ENDED, segment);
+ encoder.AppendVideoSegment(Move(segment));
+ encoder.AdvanceCurrentTime(45000);
+ encoder.NotifyEndOfStream();
EncodedFrameContainer container;
ASSERT_TRUE(NS_SUCCEEDED(encoder.GetEncodedTrack(container)));
EXPECT_TRUE(encoder.IsEncodingComplete());
// Read out encoded data, and verify.
const nsTArray<RefPtr<EncodedFrame>>& frames = container.GetEncodedFrames();
@@ -354,21 +352,19 @@ TEST(VP8VideoTrackEncoder, SameFrameEnco
segment.AppendFrame(do_AddRef(image),
mozilla::StreamTime(9000), // 100ms
generator.GetSize(),
PRINCIPAL_HANDLE_NONE,
false,
now + TimeDuration::FromSeconds(i * 0.1));
}
- encoder.SetCurrentFrames(segment);
-
- // End the track.
- segment.Clear();
- encoder.NotifyQueuedTrackChanges(nullptr, 0, 0, TrackEventCommand::TRACK_EVENT_ENDED, segment);
+ encoder.AppendVideoSegment(Move(segment));
+ encoder.AdvanceCurrentTime(15 * 9000);
+ encoder.NotifyEndOfStream();
EncodedFrameContainer container;
ASSERT_TRUE(NS_SUCCEEDED(encoder.GetEncodedTrack(container)));
EXPECT_TRUE(encoder.IsEncodingComplete());
// Verify total duration being 1.5s.
uint64_t totalDuration = 0;
@@ -405,21 +401,19 @@ TEST(VP8VideoTrackEncoder, NullFrameFirs
// Pass a real 100ms frame to the encoder.
segment.AppendFrame(image.forget(),
mozilla::StreamTime(9000), // 100ms
generator.GetSize(),
PRINCIPAL_HANDLE_NONE,
false,
now + TimeDuration::FromSeconds(0.3));
- encoder.SetCurrentFrames(segment);
-
- // End the track.
- segment.Clear();
- encoder.NotifyQueuedTrackChanges(nullptr, 0, 0, TrackEventCommand::TRACK_EVENT_ENDED, segment);
+ encoder.AppendVideoSegment(Move(segment));
+ encoder.AdvanceCurrentTime(3 * 9000);
+ encoder.NotifyEndOfStream();
EncodedFrameContainer container;
ASSERT_TRUE(NS_SUCCEEDED(encoder.GetEncodedTrack(container)));
EXPECT_TRUE(encoder.IsEncodingComplete());
// Verify total duration being 0.3s.
uint64_t totalDuration = 0;
@@ -448,21 +442,19 @@ TEST(VP8VideoTrackEncoder, SkippedFrames
segment.AppendFrame(generator.GenerateI420Image(),
mozilla::StreamTime(90), // 1ms
generator.GetSize(),
PRINCIPAL_HANDLE_NONE,
false,
now + TimeDuration::FromMilliseconds(i));
}
- encoder.SetCurrentFrames(segment);
-
- // End the track.
- segment.Clear();
- encoder.NotifyQueuedTrackChanges(nullptr, 0, 0, TrackEventCommand::TRACK_EVENT_ENDED, segment);
+ encoder.AppendVideoSegment(Move(segment));
+ encoder.AdvanceCurrentTime(100 * 90);
+ encoder.NotifyEndOfStream();
EncodedFrameContainer container;
ASSERT_TRUE(NS_SUCCEEDED(encoder.GetEncodedTrack(container)));
EXPECT_TRUE(encoder.IsEncodingComplete());
// Verify total duration being 100 * 1ms = 100ms.
uint64_t totalDuration = 0;
@@ -500,21 +492,19 @@ TEST(VP8VideoTrackEncoder, RoundingError
// This last frame has timestamp start + 0.9s and duration 0.1s.
segment.AppendFrame(generator.GenerateI420Image(),
mozilla::StreamTime(9000), // 100ms
generator.GetSize(),
PRINCIPAL_HANDLE_NONE,
false,
now + TimeDuration::FromSeconds(0.9));
- encoder.SetCurrentFrames(segment);
-
- // End the track.
- segment.Clear();
- encoder.NotifyQueuedTrackChanges(nullptr, 0, 0, TrackEventCommand::TRACK_EVENT_ENDED, segment);
+ encoder.AppendVideoSegment(Move(segment));
+ encoder.AdvanceCurrentTime(10 * 9000);
+ encoder.NotifyEndOfStream();
EncodedFrameContainer container;
ASSERT_TRUE(NS_SUCCEEDED(encoder.GetEncodedTrack(container)));
EXPECT_TRUE(encoder.IsEncodingComplete());
// Verify total duration being 1s.
uint64_t totalDuration = 0;
@@ -554,17 +544,17 @@ TEST(VP8VideoTrackEncoder, TimestampFram
segment.AppendFrame(generator.GenerateI420Image(),
mozilla::StreamTime(9000), // 0.1s
generator.GetSize(),
PRINCIPAL_HANDLE_NONE,
false,
now + TimeDuration::FromSeconds(0.2));
encoder.AppendVideoSegment(Move(segment));
- encoder.NotifyCurrentTime(3 * 9000);
+ encoder.AdvanceCurrentTime(3 * 9000);
encoder.NotifyEndOfStream();
EncodedFrameContainer container;
ASSERT_TRUE(NS_SUCCEEDED(encoder.GetEncodedTrack(container)));
EXPECT_TRUE(encoder.IsEncodingComplete());
// Verify total duration being 4s and individual frames being [0.5s, 1.5s, 1s, 1s]
@@ -598,39 +588,39 @@ TEST(VP8VideoTrackEncoder, Suspended)
segment.AppendFrame(generator.GenerateI420Image(),
mozilla::StreamTime(9000), // 0.1s
generator.GetSize(),
PRINCIPAL_HANDLE_NONE,
false,
now);
encoder.AppendVideoSegment(Move(segment));
- encoder.NotifyCurrentTime(9000);
+ encoder.AdvanceCurrentTime(9000);
- encoder.Suspend();
+ encoder.Suspend(now + TimeDuration::FromSeconds(0.1));
segment.AppendFrame(generator.GenerateI420Image(),
mozilla::StreamTime(9000), // 0.1s
generator.GetSize(),
PRINCIPAL_HANDLE_NONE,
false,
now + TimeDuration::FromSeconds(0.1));
encoder.AppendVideoSegment(Move(segment));
- encoder.NotifyCurrentTime(2 * 9000);
+ encoder.AdvanceCurrentTime(9000);
- encoder.Resume();
+ encoder.Resume(now + TimeDuration::FromSeconds(0.2));
segment.AppendFrame(generator.GenerateI420Image(),
mozilla::StreamTime(9000), // 0.1s
generator.GetSize(),
PRINCIPAL_HANDLE_NONE,
false,
now + TimeDuration::FromSeconds(0.2));
encoder.AppendVideoSegment(Move(segment));
- encoder.NotifyCurrentTime(3 * 9000);
+ encoder.AdvanceCurrentTime(9000);
encoder.NotifyEndOfStream();
EncodedFrameContainer container;
ASSERT_TRUE(NS_SUCCEEDED(encoder.GetEncodedTrack(container)));
EXPECT_TRUE(encoder.IsEncodingComplete());
@@ -662,28 +652,28 @@ TEST(VP8VideoTrackEncoder, SuspendedUnti
segment.AppendFrame(generator.GenerateI420Image(),
mozilla::StreamTime(9000), // 0.1s
generator.GetSize(),
PRINCIPAL_HANDLE_NONE,
false,
now);
encoder.AppendVideoSegment(Move(segment));
- encoder.NotifyCurrentTime(9000);
+ encoder.AdvanceCurrentTime(9000);
- encoder.Suspend();
+ encoder.Suspend(now + TimeDuration::FromSeconds(0.1));
segment.AppendFrame(generator.GenerateI420Image(),
mozilla::StreamTime(9000), // 0.1s
generator.GetSize(),
PRINCIPAL_HANDLE_NONE,
false,
now + TimeDuration::FromSeconds(0.1));
encoder.AppendVideoSegment(Move(segment));
- encoder.NotifyCurrentTime(2 * 9000);
+ encoder.AdvanceCurrentTime(9000);
encoder.NotifyEndOfStream();
EncodedFrameContainer container;
ASSERT_TRUE(NS_SUCCEEDED(encoder.GetEncodedTrack(container)));
EXPECT_TRUE(encoder.IsEncodingComplete());
@@ -706,28 +696,30 @@ TEST(VP8VideoTrackEncoder, AlwaysSuspend
TestVP8TrackEncoder encoder;
InitParam param = {true, 640, 480};
encoder.TestInit(param);
// Suspend and then pass a frame with duration 2s.
YUVBufferGenerator generator;
generator.Init(mozilla::gfx::IntSize(640, 480));
- encoder.Suspend();
+ TimeStamp now = TimeStamp::Now();
+
+ encoder.Suspend(now);
VideoSegment segment;
segment.AppendFrame(generator.GenerateI420Image(),
mozilla::StreamTime(180000), // 2s
generator.GetSize(),
PRINCIPAL_HANDLE_NONE,
false,
- TimeStamp::Now());
+ now);
encoder.AppendVideoSegment(Move(segment));
- encoder.NotifyCurrentTime(180000);
+ encoder.AdvanceCurrentTime(180000);
encoder.NotifyEndOfStream();
EncodedFrameContainer container;
ASSERT_TRUE(NS_SUCCEEDED(encoder.GetEncodedTrack(container)));
EXPECT_TRUE(encoder.IsEncodingComplete());
@@ -738,62 +730,62 @@ TEST(VP8VideoTrackEncoder, AlwaysSuspend
// Test that encoding a track that is suspended in the beginning works.
TEST(VP8VideoTrackEncoder, SuspendedBeginning)
{
// Initiate VP8 encoder
TestVP8TrackEncoder encoder;
InitParam param = {true, 640, 480};
encoder.TestInit(param);
+ TimeStamp now = TimeStamp::Now();
- // Suspend and pass a frame with duration 1.5s. Then resume and pass one more.
- encoder.Suspend();
+ // Suspend and pass a frame with duration 0.5s. Then resume and pass one more.
+ encoder.Suspend(now);
YUVBufferGenerator generator;
generator.Init(mozilla::gfx::IntSize(640, 480));
- TimeStamp now = TimeStamp::Now();
VideoSegment segment;
segment.AppendFrame(generator.GenerateI420Image(),
- mozilla::StreamTime(135000), // 1.5s
+ mozilla::StreamTime(45000), // 0.5s
generator.GetSize(),
PRINCIPAL_HANDLE_NONE,
false,
now);
encoder.AppendVideoSegment(Move(segment));
- encoder.NotifyCurrentTime(135000);
+ encoder.AdvanceCurrentTime(45000);
- encoder.Resume();
+ encoder.Resume(now + TimeDuration::FromSeconds(0.5));
segment.AppendFrame(generator.GenerateI420Image(),
- mozilla::StreamTime(135000), // 1.5s
+ mozilla::StreamTime(45000), // 0.5s
generator.GetSize(),
PRINCIPAL_HANDLE_NONE,
false,
- now + TimeDuration::FromSeconds(1.5));
+ now + TimeDuration::FromSeconds(0.5));
encoder.AppendVideoSegment(Move(segment));
- encoder.NotifyCurrentTime(270000);
+ encoder.AdvanceCurrentTime(45000);
encoder.NotifyEndOfStream();
EncodedFrameContainer container;
ASSERT_TRUE(NS_SUCCEEDED(encoder.GetEncodedTrack(container)));
EXPECT_TRUE(encoder.IsEncodingComplete());
// Verify that we have one encoded frames and a total duration of 0.1s.
const uint64_t one = 1;
EXPECT_EQ(one, container.GetEncodedFrames().Length());
uint64_t totalDuration = 0;
for (auto& frame : container.GetEncodedFrames()) {
totalDuration += frame->GetDuration();
}
- const uint64_t oneAndAHalf = PR_USEC_PER_SEC / 2 * 3;
- EXPECT_EQ(oneAndAHalf, totalDuration);
+ const uint64_t half = PR_USEC_PER_SEC / 2;
+ EXPECT_EQ(half, totalDuration);
}
// Test that suspending and resuming in the middle of already pushed data
// works.
TEST(VP8VideoTrackEncoder, SuspendedOverlap)
{
// Initiate VP8 encoder
TestVP8TrackEncoder encoder;
@@ -809,30 +801,30 @@ TEST(VP8VideoTrackEncoder, SuspendedOver
mozilla::StreamTime(90000), // 1s
generator.GetSize(),
PRINCIPAL_HANDLE_NONE,
false,
now);
encoder.AppendVideoSegment(Move(segment));
- encoder.NotifyCurrentTime(45000);
- encoder.Suspend();
+ encoder.AdvanceCurrentTime(45000);
+ encoder.Suspend(now + TimeDuration::FromSeconds(0.5));
// Pass another 1s frame and resume after 0.3 of this new frame.
segment.AppendFrame(generator.GenerateI420Image(),
mozilla::StreamTime(90000), // 1s
generator.GetSize(),
PRINCIPAL_HANDLE_NONE,
false,
now + TimeDuration::FromSeconds(1));
encoder.AppendVideoSegment(Move(segment));
- encoder.NotifyCurrentTime(117000);
- encoder.Resume();
- encoder.NotifyCurrentTime(180000);
+ encoder.AdvanceCurrentTime(72000);
+ encoder.Resume(now + TimeDuration::FromSeconds(1.3));
+ encoder.AdvanceCurrentTime(63000);
encoder.NotifyEndOfStream();
EncodedFrameContainer container;
ASSERT_TRUE(NS_SUCCEEDED(encoder.GetEncodedTrack(container)));
EXPECT_TRUE(encoder.IsEncodingComplete());
@@ -864,17 +856,17 @@ TEST(VP8VideoTrackEncoder, PrematureEndi
segment.AppendFrame(generator.GenerateI420Image(),
mozilla::StreamTime(90000), // 1s
generator.GetSize(),
PRINCIPAL_HANDLE_NONE,
false,
now);
encoder.AppendVideoSegment(Move(segment));
- encoder.NotifyCurrentTime(45000);
+ encoder.AdvanceCurrentTime(45000);
encoder.NotifyEndOfStream();
EncodedFrameContainer container;
ASSERT_TRUE(NS_SUCCEEDED(encoder.GetEncodedTrack(container)));
EXPECT_TRUE(encoder.IsEncodingComplete());
uint64_t totalDuration = 0;
@@ -889,18 +881,17 @@ TEST(VP8VideoTrackEncoder, PrematureEndi
TEST(VP8VideoTrackEncoder, EncodeComplete)
{
// Initiate VP8 encoder
TestVP8TrackEncoder encoder;
InitParam param = {true, 640, 480};
encoder.TestInit(param);
// track end notification.
- VideoSegment segment;
- encoder.NotifyQueuedTrackChanges(nullptr, 0, 0, TrackEventCommand::TRACK_EVENT_ENDED, segment);
+ encoder.NotifyEndOfStream();
// Pull Encoded Data back from encoder. Since we have sent
// EOS to encoder, encoder.GetEncodedTrack should return
// NS_OK immidiately.
EncodedFrameContainer container;
EXPECT_TRUE(NS_SUCCEEDED(encoder.GetEncodedTrack(container)));
EXPECT_TRUE(encoder.IsEncodingComplete());
--- a/dom/media/gtest/TestWebMWriter.cpp
+++ b/dom/media/gtest/TestWebMWriter.cpp
@@ -11,16 +11,18 @@
#include "VP8TrackEncoder.h"
#include "WebMWriter.h"
using namespace mozilla;
class WebMOpusTrackEncoder : public OpusTrackEncoder
{
public:
+ explicit WebMOpusTrackEncoder(TrackRate aTrackRate)
+ : OpusTrackEncoder(aTrackRate) {}
bool TestOpusCreation(int aChannels, int aSamplingRate)
{
if (NS_SUCCEEDED(Init(aChannels, aSamplingRate))) {
return true;
}
return false;
}
};
@@ -47,18 +49,18 @@ const uint32_t FIXED_FRAMESIZE = 500;
class TestWebMWriter: public WebMWriter
{
public:
explicit TestWebMWriter(int aTrackTypes)
: WebMWriter(aTrackTypes),
mTimestamp(0)
{}
- void SetOpusMetadata(int aChannels, int aSampleRate) {
- WebMOpusTrackEncoder opusEncoder;
+ void SetOpusMetadata(int aChannels, int aSampleRate, TrackRate aTrackRate) {
+ WebMOpusTrackEncoder opusEncoder(aTrackRate);
EXPECT_TRUE(opusEncoder.TestOpusCreation(aChannels, aSampleRate));
RefPtr<TrackMetadataBase> opusMeta = opusEncoder.GetMetadata();
SetMetadata(opusMeta);
}
void SetVP8Metadata(int32_t aWidth, int32_t aHeight, int32_t aDisplayWidth,
int32_t aDisplayHeight,TrackRate aTrackRate) {
WebMVP8TrackEncoder vp8Encoder;
EXPECT_TRUE(vp8Encoder.TestVP8Creation(aWidth, aHeight, aDisplayWidth,
@@ -108,52 +110,52 @@ TEST(WebMWriter, Metadata)
writer.GetContainerData(&encodedBuf, ContainerWriter::GET_HEADER);
EXPECT_TRUE(encodedBuf.Length() == 0);
writer.GetContainerData(&encodedBuf, ContainerWriter::FLUSH_NEEDED);
EXPECT_TRUE(encodedBuf.Length() == 0);
// Set opus metadata.
int channel = 1;
int sampleRate = 44100;
- writer.SetOpusMetadata(channel, sampleRate);
+ TrackRate aTrackRate = 90000;
+ writer.SetOpusMetadata(channel, sampleRate, aTrackRate);
// No output data since we didn't set both audio/video
// metadata in writer.
writer.GetContainerData(&encodedBuf, ContainerWriter::GET_HEADER);
EXPECT_TRUE(encodedBuf.Length() == 0);
writer.GetContainerData(&encodedBuf, ContainerWriter::FLUSH_NEEDED);
EXPECT_TRUE(encodedBuf.Length() == 0);
// Set vp8 metadata
int32_t width = 640;
int32_t height = 480;
int32_t displayWidth = 640;
int32_t displayHeight = 480;
- TrackRate aTrackRate = 90000;
writer.SetVP8Metadata(width, height, displayWidth,
displayHeight, aTrackRate);
writer.GetContainerData(&encodedBuf, ContainerWriter::GET_HEADER);
EXPECT_TRUE(encodedBuf.Length() > 0);
}
TEST(WebMWriter, Cluster)
{
TestWebMWriter writer(ContainerWriter::CREATE_AUDIO_TRACK |
ContainerWriter::CREATE_VIDEO_TRACK);
// Set opus metadata.
int channel = 1;
int sampleRate = 48000;
- writer.SetOpusMetadata(channel, sampleRate);
+ TrackRate aTrackRate = 90000;
+ writer.SetOpusMetadata(channel, sampleRate, aTrackRate);
// Set vp8 metadata
int32_t width = 320;
int32_t height = 240;
int32_t displayWidth = 320;
int32_t displayHeight = 240;
- TrackRate aTrackRate = 90000;
writer.SetVP8Metadata(width, height, displayWidth,
displayHeight, aTrackRate);
nsTArray<nsTArray<uint8_t> > encodedBuf;
writer.GetContainerData(&encodedBuf, ContainerWriter::GET_HEADER);
EXPECT_TRUE(encodedBuf.Length() > 0);
encodedBuf.Clear();
@@ -180,23 +182,23 @@ TEST(WebMWriter, Cluster)
TEST(WebMWriter, FLUSH_NEEDED)
{
TestWebMWriter writer(ContainerWriter::CREATE_AUDIO_TRACK |
ContainerWriter::CREATE_VIDEO_TRACK);
// Set opus metadata.
int channel = 2;
int sampleRate = 44100;
- writer.SetOpusMetadata(channel, sampleRate);
+ TrackRate aTrackRate = 100000;
+ writer.SetOpusMetadata(channel, sampleRate, aTrackRate);
// Set vp8 metadata
int32_t width = 176;
int32_t height = 352;
int32_t displayWidth = 176;
int32_t displayHeight = 352;
- TrackRate aTrackRate = 100000;
writer.SetVP8Metadata(width, height, displayWidth,
displayHeight, aTrackRate);
// write the first I-Frame.
writer.AppendDummyFrame(EncodedFrame::VP8_I_FRAME, FIXED_DURATION);
// P-Frame
writer.AppendDummyFrame(EncodedFrame::VP8_P_FRAME, FIXED_DURATION);
@@ -305,23 +307,23 @@ static int64_t webm_tell(void* aUserData
TEST(WebMWriter, bug970774_aspect_ratio)
{
TestWebMWriter writer(ContainerWriter::CREATE_AUDIO_TRACK |
ContainerWriter::CREATE_VIDEO_TRACK);
// Set opus metadata.
int channel = 1;
int sampleRate = 44100;
- writer.SetOpusMetadata(channel, sampleRate);
+ TrackRate aTrackRate = 90000;
+ writer.SetOpusMetadata(channel, sampleRate, aTrackRate);
// Set vp8 metadata
int32_t width = 640;
int32_t height = 480;
int32_t displayWidth = 1280;
int32_t displayHeight = 960;
- TrackRate aTrackRate = 90000;
writer.SetVP8Metadata(width, height, displayWidth,
displayHeight, aTrackRate);
// write the first I-Frame.
writer.AppendDummyFrame(EncodedFrame::VP8_I_FRAME, FIXED_DURATION);
// write the second I-Frame.
writer.AppendDummyFrame(EncodedFrame::VP8_I_FRAME, FIXED_DURATION);
--- a/dom/media/gtest/moz.build
+++ b/dom/media/gtest/moz.build
@@ -26,17 +26,16 @@ UNIFIED_SOURCES += [
'TestVideoSegment.cpp',
'TestVideoUtils.cpp',
'TestVPXDecoding.cpp',
'TestWebMBuffered.cpp',
]
if CONFIG['MOZ_WEBM_ENCODER']:
UNIFIED_SOURCES += [
- 'TestAudioTrackEncoder.cpp',
'TestVideoTrackEncoder.cpp',
'TestWebMWriter.cpp',
]
TEST_HARNESS_FILES.gtest += [
'../test/gizmo-frag.mp4',
'../test/gizmo.mp4',
'../test/vp9cake.webm',