Bug 1201363 - Let MediaStreamVideoSink bind with particular video track. r?jesup,r?pehrsons
In some cases, MSG need to deal with the same MediaStreamVideoSink will be exposed to different SourceMediaStream. For example, if MediaStream.addTrack() is called, the MSG should take care this case. Or you will see flicker due to multiple SourceMediaStream trying to call |SetCurrentFrames| in the same time.
MozReview-Commit-ID: DezAIg1Cg8h
--- a/dom/camera/CameraPreviewMediaStream.cpp
+++ b/dom/camera/CameraPreviewMediaStream.cpp
@@ -54,28 +54,28 @@ CameraPreviewMediaStream::SetAudioOutput
}
void
CameraPreviewMediaStream::RemoveAudioOutput(void* aKey)
{
}
void
-CameraPreviewMediaStream::AddVideoOutput(MediaStreamVideoSink* aSink)
+CameraPreviewMediaStream::AddVideoOutput(MediaStreamVideoSink* aSink, TrackID aID)
{
MutexAutoLock lock(mMutex);
RefPtr<MediaStreamVideoSink> sink = aSink;
- AddVideoOutputImpl(sink.forget());
+ AddVideoOutputImpl(sink.forget(), aID);
}
void
-CameraPreviewMediaStream::RemoveVideoOutput(MediaStreamVideoSink* aSink)
+CameraPreviewMediaStream::RemoveVideoOutput(MediaStreamVideoSink* aSink, TrackID aID)
{
MutexAutoLock lock(mMutex);
- RemoveVideoOutputImpl(aSink);
+ RemoveVideoOutputImpl(aSink, aID);
}
void
CameraPreviewMediaStream::AddListener(MediaStreamListener* aListener)
{
MutexAutoLock lock(mMutex);
MediaStreamListener* listener = *mListeners.AppendElement() = aListener;
@@ -120,18 +120,18 @@ CameraPreviewMediaStream::Destroy()
DestroyImpl();
}
void
CameraPreviewMediaStream::Invalidate()
{
MutexAutoLock lock(mMutex);
--mInvalidatePending;
- for (MediaStreamVideoSink* sink : mVideoOutputs) {
- VideoFrameContainer* output = sink->AsVideoFrameContainer();
+ for (auto sinkPair : mVideoOutputs) {
+ VideoFrameContainer* output = sinkPair.first()->AsVideoFrameContainer();
if (!output) {
continue;
}
output->Invalidate();
}
}
void
@@ -155,18 +155,18 @@ CameraPreviewMediaStream::SetCurrentFram
}
DOM_CAMERA_LOGI("Update preview frame, %d invalidation(s) pending",
mInvalidatePending);
}
mDiscardedFrames = 0;
TimeStamp now = TimeStamp::Now();
- for (MediaStreamVideoSink* sink : mVideoOutputs) {
- VideoFrameContainer* output = sink->AsVideoFrameContainer();
+ for (auto sinkPair : mVideoOutputs) {
+ VideoFrameContainer* output = sinkPair.first()->AsVideoFrameContainer();
if (!output) {
continue;
}
output->SetCurrentFrame(aIntrinsicSize, aImage, now);
}
++mInvalidatePending;
}
@@ -176,18 +176,18 @@ CameraPreviewMediaStream::SetCurrentFram
NS_DispatchToMainThread(event);
}
void
CameraPreviewMediaStream::ClearCurrentFrame()
{
MutexAutoLock lock(mMutex);
- for (MediaStreamVideoSink* sink : mVideoOutputs) {
- VideoFrameContainer* output = sink->AsVideoFrameContainer();
+ for (auto sinkPair : mVideoOutputs) {
+ VideoFrameContainer* output = sinkPair.first()->AsVideoFrameContainer();
if (!output) {
continue;
}
output->ClearCurrentFrame();
nsCOMPtr<nsIRunnable> event =
NS_NewRunnableMethod(output, &VideoFrameContainer::Invalidate);
NS_DispatchToMainThread(event);
}
--- a/dom/camera/CameraPreviewMediaStream.h
+++ b/dom/camera/CameraPreviewMediaStream.h
@@ -41,18 +41,18 @@ class CameraPreviewMediaStream : public
typedef mozilla::layers::Image Image;
public:
explicit CameraPreviewMediaStream(DOMMediaStream* aWrapper);
virtual void AddAudioOutput(void* aKey) override;
virtual void SetAudioOutputVolume(void* aKey, float aVolume) override;
virtual void RemoveAudioOutput(void* aKey) override;
- virtual void AddVideoOutput(MediaStreamVideoSink* aSink) override;
- virtual void RemoveVideoOutput(MediaStreamVideoSink* aSink) override;
+ virtual void AddVideoOutput(MediaStreamVideoSink* aSink, TrackID aID) override;
+ virtual void RemoveVideoOutput(MediaStreamVideoSink* aSink, TrackID aID) override;
virtual void Suspend() override {}
virtual void Resume() override {}
virtual void AddListener(MediaStreamListener* aListener) override;
virtual void RemoveListener(MediaStreamListener* aListener) override;
virtual void Destroy() override;
void OnPreviewStateChange(bool aActive);
void Invalidate();
--- a/dom/html/HTMLMediaElement.cpp
+++ b/dom/html/HTMLMediaElement.cpp
@@ -2117,17 +2117,19 @@ HTMLMediaElement::HTMLMediaElement(alrea
mAudioChannelVolume(1.0),
mPlayingThroughTheAudioChannel(false),
mDisableVideo(false),
mPlayBlockedBecauseHidden(false),
mMediaStreamTrackListener(nullptr),
mElementInTreeState(ELEMENT_NOT_INTREE),
mHasUserInteraction(false),
mFirstFrameLoaded(false),
- mDefaultPlaybackStartPosition(0.0)
+ mDefaultPlaybackStartPosition(0.0),
+ mSelectedVideoTrackID(TRACK_NONE),
+ bDeferAddVideoOutput(false)
{
mAudioChannel = AudioChannelService::GetDefaultAudioChannel();
mPaused.SetOuter(this);
RegisterActivityObserver();
NotifyOwnerDocumentActivityChangedInternal();
@@ -2991,16 +2993,24 @@ public:
if (mElement) {
RefPtr<HTMLMediaElement> deathGrip = mElement;
mElement->FirstFrameLoaded();
}
NotifyWatchers();
DoNotifyOutput();
}
+ void DoNotifyQueuedTrackChanges()
+ {
+ if (mElement) {
+ RefPtr<HTMLMediaElement> deathGrip = mElement;
+ mElement->AddVideoOutput();
+ }
+ }
+
// These notifications run on the media graph thread so we need to
// dispatch events to the main thread.
virtual void NotifyBlockingChanged(MediaStreamGraph* aGraph, Blocking aBlocked) override
{
nsCOMPtr<nsIRunnable> event;
if (aBlocked == BLOCKED) {
event = NS_NewRunnableMethod(this, &StreamListener::DoNotifyBlocked);
} else {
@@ -3031,16 +3041,31 @@ public:
if (mPendingNotifyOutput)
return;
mPendingNotifyOutput = true;
nsCOMPtr<nsIRunnable> event =
NS_NewRunnableMethod(this, &StreamListener::DoNotifyOutput);
aGraph->DispatchToMainThreadAfterStreamStateUpdate(event.forget());
}
+ virtual void NotifyQueuedTrackChanges(MediaStreamGraph* aGraph, TrackID aID,
+ StreamTime aTrackOffset,
+ uint32_t aTrackEvents,
+ const MediaSegment& aQueuedMedia,
+ MediaStream* aInputStream = nullptr,
+ TrackID aInputTrackID = TRACK_INVALID) override
+ {
+ if (aQueuedMedia.GetType() == MediaSegment::AUDIO ||
+ !(aTrackEvents & TRACK_EVENT_CREATED)) {
+ return;
+ }
+ nsCOMPtr<nsIRunnable> event =
+ NS_NewRunnableMethod(this, &StreamListener::DoNotifyQueuedTrackChanges);
+ aGraph->DispatchToMainThreadAfterStreamStateUpdate(event.forget());
+ }
private:
// These fields may only be accessed on the main thread
HTMLMediaElement* mElement;
bool mHaveCurrentData;
bool mBlocked;
bool mFinished;
// mMutex protects the fields below; they can be accessed on any thread
@@ -3136,16 +3161,42 @@ public:
}
protected:
~MediaStreamTrackListener() {}
HTMLMediaElement* const mElement;
};
+void HTMLMediaElement::AddVideoOutput()
+{
+ if (mSelectedVideoTrackID != TRACK_NONE) {
+ return;
+ }
+
+ MOZ_ASSERT(mSrcStream);
+ nsTArray<RefPtr<mozilla::dom::VideoStreamTrack>> videoTracks;
+ mSrcStream->GetVideoTracks(videoTracks);
+ // The VideoStreamTrack might be removed after VideoFrameContainer is created.
+ // In such situation, no need to add video output into MSG.
+ if (videoTracks.IsEmpty()) {
+ return;
+ }
+
+ // Assuming the MediaStream only have one video track. Will deal with
+ // multiple video track case in DOMMediaStream::AddTrack.
+ DOMMediaStream::TrackPort* trackPort = mSrcStream->FindPlaybackTrackPort(*videoTracks[0]);
+ mSelectedVideoTrackID = trackPort->GetPlaybackTrackID();
+ if (mSelectedVideoTrackID == TRACK_NONE) {
+ bDeferAddVideoOutput = true;
+ return;
+ }
+ GetSrcMediaStream()->AddVideoOutput(GetVideoFrameContainer(), mSelectedVideoTrackID);
+}
+
void HTMLMediaElement::UpdateSrcMediaStreamPlaying(uint32_t aFlags)
{
if (!mSrcStream) {
return;
}
// We might be in cycle collection with mSrcStream->GetPlaybackStream() already
// returning null due to unlinking.
@@ -3173,29 +3224,44 @@ void HTMLMediaElement::UpdateSrcMediaStr
&HTMLMediaElement::UpdateReadyStateInternal);
stream->AddAudioOutput(this);
SetVolumeInternal();
VideoFrameContainer* container = GetVideoFrameContainer();
if (container) {
container->AddImageSizeChangedListener(mMediaStreamSizeListener);
- stream->AddVideoOutput(container);
+ AddVideoOutput();
}
} else {
if (stream) {
mSrcStreamPausedCurrentTime = CurrentTime();
stream->RemoveListener(mMediaStreamListener);
stream->RemoveAudioOutput(this);
VideoFrameContainer* container = GetVideoFrameContainer();
if (container) {
container->RemoveImageSizeChangedListener(mMediaStreamSizeListener);
- stream->RemoveVideoOutput(container);
+ MOZ_ASSERT(mSrcStream);
+ if (mSrcStream) {
+ nsTArray<RefPtr<mozilla::dom::VideoStreamTrack>> videoTracks;
+ mSrcStream->GetVideoTracks(videoTracks);
+ if (videoTracks.IsEmpty() || bDeferAddVideoOutput) {
+ bDeferAddVideoOutput = false;
+ } else {
+ // The mSelectedVideoTrackID might be set to TRACK_NONE in the
+ // function |NotifyMediaStreamTrackRemoved|.
+ if (!bDeferAddVideoOutput && mSelectedVideoTrackID != TRACK_NONE) {
+ stream->RemoveVideoOutput(container, mSelectedVideoTrackID);
+ mSelectedVideoTrackID = TRACK_NONE;
+ }
+ bDeferAddVideoOutput = false;
+ }
+ }
}
}
// If stream is null, then DOMMediaStream::Destroy must have been
// called and that will remove all listeners/outputs.
mWatchManager.Unwatch(*mMediaStreamListener,
&HTMLMediaElement::UpdateReadyStateInternal);
@@ -3349,16 +3415,46 @@ HTMLMediaElement::NotifyMediaStreamTrack
VideoTracks()->RemoveTrack(t);
} else {
// XXX (bug 1208328) Uncomment this when DOMMediaStream doesn't call
// NotifyTrackRemoved multiple times for the same track, i.e., when it
// implements the "addtrack" and "removetrack" events.
// NS_ASSERTION(false, "MediaStreamTrack ended but did not exist in track lists");
return;
}
+ DOMMediaStream::TrackPort* trackPort = mSrcStream->FindPlaybackTrackPort(*aTrack);
+ TrackID playbackID = trackPort->GetPlaybackTrackID();
+ if (playbackID == mSelectedVideoTrackID) {
+ MediaStream* stream = GetSrcMediaStream();
+ if (stream) {
+ stream->RemoveVideoOutput(GetVideoFrameContainer(), mSelectedVideoTrackID);
+ }
+ mSelectedVideoTrackID = TRACK_NONE;
+
+ nsTArray<RefPtr<mozilla::dom::VideoStreamTrack>> videoTracks;
+ mSrcStream->GetVideoTracks(videoTracks);
+ if (videoTracks.IsEmpty()) {
+ return;
+ }
+
+ // Find the first non-removed video track.
+ for (auto track : videoTracks) {
+ if (track == aTrack) {
+ continue;
+ }
+ DOMMediaStream::TrackPort* trackPort = mSrcStream->FindPlaybackTrackPort(*track);
+ mSelectedVideoTrackID = trackPort->GetPlaybackTrackID();
+ if (mSelectedVideoTrackID == TRACK_NONE) {
+ bDeferAddVideoOutput = true;
+ return;
+ }
+ GetSrcMediaStream()->AddVideoOutput(GetVideoFrameContainer(), mSelectedVideoTrackID);
+ return;
+ }
+ }
}
void HTMLMediaElement::ProcessMediaFragmentURI()
{
nsMediaFragmentURIParser parser(mLoadingSrc);
if (mDecoder && parser.HasEndTime()) {
--- a/dom/html/HTMLMediaElement.h
+++ b/dom/html/HTMLMediaElement.h
@@ -745,16 +745,18 @@ protected:
/**
* Changes mHasPlayedOrSeeked to aValue. If mHasPlayedOrSeeked changes
* we'll force a reflow so that the video frame gets reflowed to reflect
* the poster hiding or showing immediately.
*/
void SetPlayedOrSeeked(bool aValue);
+ void AddVideoOutput();
+
/**
* Initialize the media element for playback of aStream
*/
void SetupSrcMediaStreamPlayback(DOMMediaStream* aStream);
/**
* Stop playback on mSrcStream.
*/
void EndSrcMediaStreamPlayback();
@@ -1520,14 +1522,20 @@ private:
// True if the first frame has been successfully loaded.
bool mFirstFrameLoaded;
// Media elements also have a default playback start position, which must
// initially be set to zero seconds. This time is used to allow the element to
// be seeked even before the media is loaded.
double mDefaultPlaybackStartPosition;
+
+ // The selected video track id of playback stream in the mSrcStream.
+ TrackID mSelectedVideoTrackID;
+
+ // Defer the AddVideoOutput until the track is added into StreamTracks.
+ bool bDeferAddVideoOutput;
};
} // namespace dom
} // namespace mozilla
#endif // mozilla_dom_HTMLMediaElement_h
--- a/dom/media/DOMMediaStream.cpp
+++ b/dom/media/DOMMediaStream.cpp
@@ -38,16 +38,17 @@ static LazyLogModule gMediaStreamLog("Me
const TrackID TRACK_VIDEO_PRIMARY = 1;
DOMMediaStream::TrackPort::TrackPort(MediaInputPort* aInputPort,
MediaStreamTrack* aTrack,
const InputPortOwnership aOwnership)
: mInputPort(aInputPort)
, mTrack(aTrack)
+ , mPlaybackTrackID(TRACK_NONE)
, mOwnership(aOwnership)
{
// XXX Bug 1124630. nsDOMCameraControl requires adding a track without and
// input port.
// MOZ_ASSERT(mInputPort);
MOZ_ASSERT(mTrack);
MOZ_COUNT_CTOR(TrackPort);
@@ -207,17 +208,17 @@ public:
mStream->FindPlaybackDOMTrack(aInputStream, aInputTrackID);
if (!track) {
LOG(LogLevel::Debug, ("DOMMediaStream %p Not a playback track.", mStream));
return;
}
LOG(LogLevel::Debug, ("DOMMediaStream %p Playback track; notifying stream listeners.",
mStream));
- mStream->NotifyTrackRemoved(track);
+ mStream->NotifyTrackRemoved(track, aInputTrackID);
RefPtr<TrackPort> endedPort = mStream->FindPlaybackTrackPort(*track);
NS_ASSERTION(endedPort, "Playback track should have a TrackPort");
if (endedPort &&
endedPort->GetSourceTrackId() != TRACK_ANY &&
endedPort->GetSourceTrackId() != TRACK_INVALID &&
endedPort->GetSourceTrackId() != TRACK_NONE) {
// If a track connected to a locked-track input port ends, we destroy the
@@ -234,24 +235,43 @@ public:
if (!mStream) {
return;
}
mStream->NotifyTracksCreated();
}
+ void DoNotifyVideoTrackCreated(TrackID aOutputTrackID,
+ MediaStream* aInputStream,
+ TrackID aInputTrackID)
+ {
+ MOZ_ASSERT(NS_IsMainThread());
+
+ if (!mStream) {
+ return;
+ }
+
+ mStream->AddPlaybackIDToAssociatedTrackPort(aOutputTrackID, aInputStream, aInputTrackID);
+ }
+
// The methods below are called on the MediaStreamGraph thread.
void NotifyQueuedTrackChanges(MediaStreamGraph* aGraph, TrackID aID,
StreamTime aTrackOffset, uint32_t aTrackEvents,
const MediaSegment& aQueuedMedia,
MediaStream* aInputStream,
TrackID aInputTrackID) override
{
+ if (aTrackEvents & TRACK_EVENT_CREATED) {
+ nsCOMPtr<nsIRunnable> runnable =
+ NS_NewRunnableMethodWithArgs<TrackID, StorensRefPtrPassByPtr<MediaStream>, TrackID>(
+ this, &PlaybackStreamListener::DoNotifyVideoTrackCreated, aID, aInputStream, aInputTrackID);
+ aGraph->DispatchToMainThreadAfterStreamStateUpdate(runnable.forget());
+ }
if (aTrackEvents & TRACK_EVENT_ENDED) {
nsCOMPtr<nsIRunnable> runnable =
NS_NewRunnableMethodWithArgs<StorensRefPtrPassByPtr<MediaStream>, TrackID>(
this, &PlaybackStreamListener::DoNotifyTrackEnded, aInputStream, aInputTrackID);
aGraph->DispatchToMainThreadAfterStreamStateUpdate(runnable.forget());
}
}
@@ -861,16 +881,30 @@ DOMMediaStream::FindPlaybackTrackPort(co
for (const RefPtr<TrackPort>& info : mTracks) {
if (info->GetTrack() == &aTrack) {
return info;
}
}
return nullptr;
}
+
+void
+DOMMediaStream::AddPlaybackIDToAssociatedTrackPort(TrackID aOutputTrackID,
+ MediaStream* aInputStream,
+ TrackID aInputTrackID)
+{
+ for (auto trackPort : mTracks) {
+ if (trackPort->GetSource() == aInputStream &&
+ trackPort->GetTrack()->GetTrackID() == aInputTrackID) {
+ trackPort->SetPlaybackTrackID(aOutputTrackID);
+ }
+ }
+}
+
void
DOMMediaStream::NotifyMediaStreamGraphShutdown()
{
// No more tracks will ever be added, so just clear these callbacks now
// to prevent leaks.
mNotifiedOfMediaStreamGraphShutdown = true;
mRunOnTracksAvailable.Clear();
mTrackListeners.Clear();
@@ -945,17 +979,18 @@ DOMMediaStream::NotifyTrackAdded(
for (int32_t i = mTrackListeners.Length() - 1; i >= 0; --i) {
const RefPtr<TrackListener>& listener = mTrackListeners[i];
listener->NotifyTrackAdded(aTrack);
}
}
void
DOMMediaStream::NotifyTrackRemoved(
- const RefPtr<MediaStreamTrack>& aTrack)
+ const RefPtr<MediaStreamTrack>& aTrack,
+ TrackID aID)
{
MOZ_ASSERT(NS_IsMainThread());
for (int32_t i = mTrackListeners.Length() - 1; i >= 0; --i) {
const RefPtr<TrackListener>& listener = mTrackListeners[i];
listener->NotifyTrackRemoved(aTrack);
}
}
--- a/dom/media/DOMMediaStream.h
+++ b/dom/media/DOMMediaStream.h
@@ -269,20 +269,26 @@ public:
MediaStreamTrack* GetTrack() const { return mTrack; }
/**
* Blocks aTrackId from going into mInputPort unless the port has been
* destroyed.
*/
void BlockTrackId(TrackID aTrackId);
+ TrackID GetPlaybackTrackID() const { return mPlaybackTrackID; }
+
+ void SetPlaybackTrackID(TrackID aTrackID) { mPlaybackTrackID = aTrackID;}
+
private:
RefPtr<MediaInputPort> mInputPort;
RefPtr<MediaStreamTrack> mTrack;
+ TrackID mPlaybackTrackID;
+
// Defines if we've been given ownership of the input port or if it's owned
// externally. The owner is responsible for destroying the port.
const InputPortOwnership mOwnership;
};
DOMMediaStream();
NS_DECL_ISUPPORTS_INHERITED
@@ -345,16 +351,20 @@ public:
*/
MediaStreamTrack* FindPlaybackDOMTrack(MediaStream* aOwningStream, TrackID aTrackID) const;
/**
* Returns the TrackPort connecting mOwnedStream to mPlaybackStream for aTrack.
*/
TrackPort* FindPlaybackTrackPort(const MediaStreamTrack& aTrack) const;
+ void AddPlaybackIDToAssociatedTrackPort(TrackID aOutputTrackID,
+ MediaStream* aInputStream,
+ TrackID aInputTrackID);
+
MediaStream* GetInputStream() const { return mInputStream; }
ProcessedMediaStream* GetOwnedStream() const { return mOwnedStream; }
ProcessedMediaStream* GetPlaybackStream() const { return mPlaybackStream; }
/**
* Allows a video element to identify this stream as a camera stream, which
* needs special treatment.
*/
@@ -537,17 +547,17 @@ protected:
// Called when MediaStreamGraph has finished an iteration where tracks were
// created.
void NotifyTracksCreated();
// Dispatches NotifyTrackAdded() to all registered track listeners.
void NotifyTrackAdded(const RefPtr<MediaStreamTrack>& aTrack);
// Dispatches NotifyTrackRemoved() to all registered track listeners.
- void NotifyTrackRemoved(const RefPtr<MediaStreamTrack>& aTrack);
+ void NotifyTrackRemoved(const RefPtr<MediaStreamTrack>& aTrack, TrackID aID);
class OwnedStreamListener;
friend class OwnedStreamListener;
class PlaybackStreamListener;
friend class PlaybackStreamListener;
// XXX Bug 1124630. Remove with CameraPreviewMediaStream.
--- a/dom/media/MediaStreamGraph.cpp
+++ b/dom/media/MediaStreamGraph.cpp
@@ -862,20 +862,20 @@ MediaStreamGraphImpl::PlayVideo(MediaStr
TimeDuration::FromSeconds(MediaTimeToSeconds(frameTime - IterationEnd()));
if (frame->GetForceBlack()) {
if (!blackImage) {
// Fixme: PlayVideo will be replaced in latter changeset
// "Call MediaStreamVideoSink::setCurrentFrames in SourceMediaStream::AppendToTrack."
// of this bug.
// This is a temp workaround to pass the build and test.
- if (!aStream->mVideoOutputs[0]->AsVideoFrameContainer()) {
+ if (!aStream->mVideoOutputs[0].first()->AsVideoFrameContainer()) {
return;
}
- blackImage = aStream->mVideoOutputs[0]->AsVideoFrameContainer()->
+ blackImage = aStream->mVideoOutputs[0].first()->AsVideoFrameContainer()->
GetImageContainer()->CreatePlanarYCbCrImage();
if (blackImage) {
// Sets the image to a single black pixel, which will be scaled to
// fill the rendered size.
SetImageToBlackPixel(blackImage->AsPlanarYCbCrImage());
}
}
if (blackImage) {
@@ -888,18 +888,18 @@ MediaStreamGraphImpl::PlayVideo(MediaStr
}
if (!aStream->mLastPlayedVideoFrame.GetImage())
return;
nsAutoTArray<ImageContainer::NonOwningImage,4> images;
bool haveMultipleImages = false;
- for (MediaStreamVideoSink* sink : aStream->mVideoOutputs) {
- VideoFrameContainer* output = sink->AsVideoFrameContainer();
+ for (auto sinkPair : aStream->mVideoOutputs) {
+ VideoFrameContainer* output = sinkPair.first()->AsVideoFrameContainer();
if (!output) {
continue;
}
// Find previous frames that may still be valid.
nsAutoTArray<ImageContainer::OwningImage,4> previousImages;
output->GetImageContainer()->GetCurrentImages(&previousImages);
uint32_t j = previousImages.Length();
@@ -1926,65 +1926,80 @@ MediaStream::RemoveAudioOutput(void* aKe
mStream->RemoveAudioOutputImpl(mKey);
}
void* mKey;
};
GraphImpl()->AppendMessage(new Message(this, aKey));
}
void
-MediaStream::AddVideoOutputImpl(already_AddRefed<MediaStreamVideoSink> aSink)
+MediaStream::AddVideoOutputImpl(already_AddRefed<MediaStreamVideoSink> aSink, TrackID aID)
{
RefPtr<MediaStreamVideoSink> sink = aSink;
STREAM_LOG(LogLevel::Info, ("MediaStream %p Adding MediaStreamVideoSink %p as output",
this, sink.get()));
- *mVideoOutputs.AppendElement() = sink.forget();
+ MOZ_ASSERT(aID != TRACK_NONE);
+ for (auto entry : mVideoOutputs) {
+ if (entry.first() == sink &&
+ (entry.second() == TRACK_ANY || entry.second() == aID)) {
+ return;
+ }
+ }
+ mVideoOutputs.AppendElement(MakePair(sink, aID));
}
void
-MediaStream::RemoveVideoOutputImpl(MediaStreamVideoSink* aSink)
+MediaStream::RemoveVideoOutputImpl(MediaStreamVideoSink* aSink, TrackID aID)
{
STREAM_LOG(LogLevel::Info, ("MediaStream %p Removing MediaStreamVideoSink %p as output",
this, aSink));
+ MOZ_ASSERT(aID != TRACK_NONE);
// Ensure that any frames currently queued for playback by the compositor
// are removed.
aSink->ClearFrames();
- mVideoOutputs.RemoveElement(aSink);
+ for (int32_t i = 0; i < mVideoOutputs.Length(); ++i) {
+ if (aSink == mVideoOutputs[i].first() &&
+ (aID == TRACK_ANY || aID == mVideoOutputs[i].second())) {
+ mVideoOutputs.RemoveElementAt(i);
+ }
+ }
}
void
-MediaStream::AddVideoOutput(MediaStreamVideoSink* aSink)
+MediaStream::AddVideoOutput(MediaStreamVideoSink* aSink, TrackID aID)
{
class Message : public ControlMessage {
public:
- Message(MediaStream* aStream, MediaStreamVideoSink* aSink) :
- ControlMessage(aStream), mSink(aSink) {}
+ Message(MediaStream* aStream, MediaStreamVideoSink* aSink, TrackID aID) :
+ ControlMessage(aStream), mSink(aSink), mID(aID) {}
void Run() override
{
- mStream->AddVideoOutputImpl(mSink.forget());
+ mStream->AddVideoOutputImpl(mSink.forget(), mID);
}
RefPtr<MediaStreamVideoSink> mSink;
+ TrackID mID;
};
- GraphImpl()->AppendMessage(new Message(this, aSink));
+ GraphImpl()->AppendMessage(new Message(this, aSink, aID));
}
void
-MediaStream::RemoveVideoOutput(MediaStreamVideoSink* aSink)
+MediaStream::RemoveVideoOutput(MediaStreamVideoSink* aSink, TrackID aID)
{
class Message : public ControlMessage {
public:
- Message(MediaStream* aStream, MediaStreamVideoSink* aSink) :
- ControlMessage(aStream), mSink(aSink) {}
+ Message(MediaStream* aStream, MediaStreamVideoSink* aSink, TrackID aID) :
+ ControlMessage(aStream), mSink(aSink), mID(aID) {}
void Run() override
{
- mStream->RemoveVideoOutputImpl(mSink);
+ mStream->RemoveVideoOutputImpl(mSink, mID);
}
RefPtr<MediaStreamVideoSink> mSink;
+ TrackID mID;
};
- GraphImpl()->AppendMessage(new Message(this, aSink));
+ GraphImpl()->AppendMessage(new Message(this, aSink, aID));
}
void
MediaStream::Suspend()
{
class Message : public ControlMessage {
public:
explicit Message(MediaStream* aStream) :
--- a/dom/media/MediaStreamGraph.h
+++ b/dom/media/MediaStreamGraph.h
@@ -3,16 +3,17 @@
* License, v. 2.0. If a copy of the MPL was not distributed with this file,
* You can obtain one at http://mozilla.org/MPL/2.0/. */
#ifndef MOZILLA_MEDIASTREAMGRAPH_H_
#define MOZILLA_MEDIASTREAMGRAPH_H_
#include "mozilla/LinkedList.h"
#include "mozilla/Mutex.h"
+#include "Mozilla/Pair.h"
#include "mozilla/TaskQueue.h"
#include "mozilla/dom/AudioChannelBinding.h"
#include "AudioSegment.h"
#include "AudioStream.h"
#include "nsTArray.h"
#include "nsIRunnable.h"
@@ -243,16 +244,17 @@ struct AudioNodeSizes
size_t mStream;
size_t mEngine;
nsCString mNodeType;
};
class MediaStreamGraphImpl;
class SourceMediaStream;
class ProcessedMediaStream;
+class TrackUnionStream;
class MediaInputPort;
class AudioNodeEngine;
class AudioNodeExternalInputStream;
class AudioNodeStream;
class CameraPreviewMediaStream;
/**
* A stream of synchronized audio and video data. All (not blocked) streams
@@ -356,21 +358,21 @@ public:
// volume settings. The aKey parameter is used to keep volume settings
// separate. Since the stream is always playing the same contents, only
// a single audio output stream is used; the volumes are combined.
// Currently only the first enabled audio track is played.
// XXX change this so all enabled audio tracks are mixed and played.
virtual void AddAudioOutput(void* aKey);
virtual void SetAudioOutputVolume(void* aKey, float aVolume);
virtual void RemoveAudioOutput(void* aKey);
- // Since a stream can be played multiple ways, we need to be able to
- // play to multiple MediaStreamVideoSinks.
- // Only the first enabled video track is played.
- virtual void AddVideoOutput(MediaStreamVideoSink* aSink);
- virtual void RemoveVideoOutput(MediaStreamVideoSink* aSink);
+ // The MediaStreamVideoSink should be bound with particular track id in
+ // the StreamTrack of corresponding MediaStream to support multiple video
+ // track use cases.
+ virtual void AddVideoOutput(MediaStreamVideoSink* aSink, TrackID aID = TRACK_ANY);
+ virtual void RemoveVideoOutput(MediaStreamVideoSink* aSink, TrackID aID = TRACK_ANY);
// Explicitly suspend. Useful for example if a media element is pausing
// and we need to stop its stream emitting its buffered data. As soon as the
// Suspend message reaches the graph, the stream stops processing. It
// ignores its inputs and produces silence/no video until Resumed. Its
// current time does not advance.
virtual void Suspend();
virtual void Resume();
// Events will be dispatched by calling methods of aListener.
@@ -431,16 +433,17 @@ public:
friend class MediaStreamGraphImpl;
friend class MediaInputPort;
friend class AudioNodeExternalInputStream;
virtual SourceMediaStream* AsSourceStream() { return nullptr; }
virtual ProcessedMediaStream* AsProcessedStream() { return nullptr; }
virtual AudioNodeStream* AsAudioNodeStream() { return nullptr; }
+ virtual TrackUnionStream* AsTrackUnionStream() { return nullptr; }
// These Impl methods perform the core functionality of the control methods
// above, on the media graph thread.
/**
* Stop all stream activity and disconnect it from all inputs and outputs.
* This must be idempotent.
*/
virtual void DestroyImpl();
@@ -451,18 +454,18 @@ public:
void SetAudioOutputVolumeImpl(void* aKey, float aVolume);
void AddAudioOutputImpl(void* aKey);
// Returns true if this stream has an audio output.
bool HasAudioOutput()
{
return !mAudioOutputs.IsEmpty();
}
void RemoveAudioOutputImpl(void* aKey);
- void AddVideoOutputImpl(already_AddRefed<MediaStreamVideoSink> aSink);
- void RemoveVideoOutputImpl(MediaStreamVideoSink* aSink);
+ void AddVideoOutputImpl(already_AddRefed<MediaStreamVideoSink> aSink, TrackID aID);
+ void RemoveVideoOutputImpl(MediaStreamVideoSink* aSink, TrackID aID);
void AddListenerImpl(already_AddRefed<MediaStreamListener> aListener);
void RemoveListenerImpl(MediaStreamListener* aListener);
void RemoveAllListenersImpl();
virtual void SetTrackEnabledImpl(TrackID aTrackID, bool aEnabled);
void AddConsumer(MediaInputPort* aPort)
{
mConsumers.AppendElement(aPort);
@@ -557,16 +560,18 @@ public:
bool IsSuspended() { return mSuspendedCount > 0; }
void IncrementSuspendCount() { ++mSuspendedCount; }
void DecrementSuspendCount()
{
NS_ASSERTION(mSuspendedCount > 0, "Suspend count underrun");
--mSuspendedCount;
}
+ typedef Pair<RefPtr<MediaStreamVideoSink>, TrackID> VideoOutputElement;
+
protected:
void AdvanceTimeVaryingValuesToCurrentTime(GraphTime aCurrentTime, GraphTime aBlockedTime)
{
mTracksStartTime += aBlockedTime;
mTracks.ForgetUpTo(aCurrentTime - mTracksStartTime);
}
void NotifyMainThreadListeners()
@@ -604,17 +609,17 @@ protected:
// Client-set volume of this stream
struct AudioOutput {
explicit AudioOutput(void* aKey) : mKey(aKey), mVolume(1.0f) {}
void* mKey;
float mVolume;
};
nsTArray<AudioOutput> mAudioOutputs;
- nsTArray<RefPtr<MediaStreamVideoSink>> mVideoOutputs;
+ nsTArray<VideoOutputElement> mVideoOutputs;
// We record the last played video frame to avoid playing the frame again
// with a different frame id.
VideoFrame mLastPlayedVideoFrame;
nsTArray<RefPtr<MediaStreamListener> > mListeners;
nsTArray<MainThreadMediaStreamListener*> mMainThreadListeners;
nsTArray<TrackID> mDisabledTrackIDs;
// GraphTime at which this stream starts blocking.
--- a/dom/media/TrackUnionStream.cpp
+++ b/dom/media/TrackUnionStream.cpp
@@ -181,18 +181,18 @@ TrackUnionStream::TrackUnionStream(DOMMe
MediaStreamListener::TRACK_EVENT_CREATED,
*segment,
aPort->GetSource(), aTrack->GetID());
}
segment->AppendNullData(outputStart);
StreamTracks::Track* track =
&mTracks.AddTrack(id, outputStart, segment.forget());
STREAM_LOG(LogLevel::Debug, ("TrackUnionStream %p adding track %d for input stream %p track %d, start ticks %lld",
- this, id, aPort->GetSource(), aTrack->GetID(),
- (long long)outputStart));
+ this, id, aPort->GetSource(), aTrack->GetID(),
+ (long long)outputStart));
TrackMapEntry* map = mTrackMap.AppendElement();
map->mEndOfConsumedInputTicks = 0;
map->mEndOfLastInputIntervalInInputStream = -1;
map->mEndOfLastInputIntervalInOutputStream = -1;
map->mInputPort = aPort;
map->mInputTrackID = aTrack->GetID();
map->mOutputTrackID = track->GetID();
--- a/dom/media/TrackUnionStream.h
+++ b/dom/media/TrackUnionStream.h
@@ -13,16 +13,19 @@ namespace mozilla {
/**
* See MediaStreamGraph::CreateTrackUnionStream.
*/
class TrackUnionStream : public ProcessedMediaStream {
public:
explicit TrackUnionStream(DOMMediaStream* aWrapper);
+ virtual TrackUnionStream* AsTrackUnionStream() override { return this; }
+ friend class DOMMediaStream;
+
void RemoveInput(MediaInputPort* aPort) override;
void ProcessInput(GraphTime aFrom, GraphTime aTo, uint32_t aFlags) override;
protected:
// Only non-ended tracks are allowed to persist in this map.
struct TrackMapEntry {
// mEndOfConsumedInputTicks is the end of the input ticks that we've consumed.
// 0 if we haven't consumed any yet.