Bug 1201363 - Let MediaStreamVideoSink bind with particular video track. r?jesup
MozReview-Commit-ID: FcjnmDKuRQI
--- a/dom/camera/CameraPreviewMediaStream.cpp
+++ b/dom/camera/CameraPreviewMediaStream.cpp
@@ -55,28 +55,28 @@ CameraPreviewMediaStream::SetAudioOutput
}
void
CameraPreviewMediaStream::RemoveAudioOutput(void* aKey)
{
}
void
-CameraPreviewMediaStream::AddVideoOutput(MediaStreamVideoSink* aSink)
+CameraPreviewMediaStream::AddVideoOutput(MediaStreamVideoSink* aSink, TrackID aID)
{
MutexAutoLock lock(mMutex);
RefPtr<MediaStreamVideoSink> sink = aSink;
- AddVideoOutputImpl(sink.forget());
+ AddVideoOutputImpl(sink.forget(), aID);
}
void
-CameraPreviewMediaStream::RemoveVideoOutput(MediaStreamVideoSink* aSink)
+CameraPreviewMediaStream::RemoveVideoOutput(MediaStreamVideoSink* aSink, TrackID aID)
{
MutexAutoLock lock(mMutex);
- RemoveVideoOutputImpl(aSink);
+ RemoveVideoOutputImpl(aSink, aID);
}
void
CameraPreviewMediaStream::AddListener(MediaStreamListener* aListener)
{
MutexAutoLock lock(mMutex);
MediaStreamListener* listener = *mListeners.AppendElement() = aListener;
@@ -121,18 +121,18 @@ CameraPreviewMediaStream::Destroy()
DestroyImpl();
}
void
CameraPreviewMediaStream::Invalidate()
{
MutexAutoLock lock(mMutex);
--mInvalidatePending;
- for (MediaStreamVideoSink* sink : mVideoOutputs) {
- VideoFrameContainer* output = sink->AsVideoFrameContainer();
+ for (const TrackBound<MediaStreamVideoSink>& sink : mVideoOutputs) {
+ VideoFrameContainer* output = sink.mListener->AsVideoFrameContainer();
if (!output) {
continue;
}
output->Invalidate();
}
}
void
@@ -163,18 +163,18 @@ CameraPreviewMediaStream::SetCurrentFram
}
DOM_CAMERA_LOGI("Update preview frame, %d invalidation(s) pending",
mInvalidatePending);
}
mDiscardedFrames = 0;
TimeStamp now = TimeStamp::Now();
- for (MediaStreamVideoSink* sink : mVideoOutputs) {
- VideoFrameContainer* output = sink->AsVideoFrameContainer();
+ for (const TrackBound<MediaStreamVideoSink>& sink : mVideoOutputs) {
+ VideoFrameContainer* output = sink.mListener->AsVideoFrameContainer();
if (!output) {
continue;
}
output->SetCurrentFrame(aIntrinsicSize, aImage, now);
}
++mInvalidatePending;
}
@@ -182,18 +182,18 @@ CameraPreviewMediaStream::SetCurrentFram
NS_DispatchToMainThread(NewRunnableMethod(this, &CameraPreviewMediaStream::Invalidate));
}
void
CameraPreviewMediaStream::ClearCurrentFrame()
{
MutexAutoLock lock(mMutex);
- for (MediaStreamVideoSink* sink : mVideoOutputs) {
- VideoFrameContainer* output = sink->AsVideoFrameContainer();
+ for (const TrackBound<MediaStreamVideoSink>& sink : mVideoOutputs) {
+ VideoFrameContainer* output = sink.mListener->AsVideoFrameContainer();
if (!output) {
continue;
}
output->ClearCurrentFrame();
NS_DispatchToMainThread(NewRunnableMethod(output, &VideoFrameContainer::Invalidate));
}
}
--- a/dom/camera/CameraPreviewMediaStream.h
+++ b/dom/camera/CameraPreviewMediaStream.h
@@ -38,26 +38,26 @@ protected:
*/
class CameraPreviewMediaStream : public ProcessedMediaStream
{
typedef mozilla::layers::Image Image;
public:
CameraPreviewMediaStream();
- virtual void AddAudioOutput(void* aKey) override;
- virtual void SetAudioOutputVolume(void* aKey, float aVolume) override;
- virtual void RemoveAudioOutput(void* aKey) override;
- virtual void AddVideoOutput(MediaStreamVideoSink* aSink) override;
- virtual void RemoveVideoOutput(MediaStreamVideoSink* aSink) override;
- virtual void Suspend() override {}
- virtual void Resume() override {}
- virtual void AddListener(MediaStreamListener* aListener) override;
- virtual void RemoveListener(MediaStreamListener* aListener) override;
- virtual void Destroy() override;
+ void AddAudioOutput(void* aKey) override;
+ void SetAudioOutputVolume(void* aKey, float aVolume) override;
+ void RemoveAudioOutput(void* aKey) override;
+ void AddVideoOutput(MediaStreamVideoSink* aSink, TrackID aID) override;
+ void RemoveVideoOutput(MediaStreamVideoSink* aSink, TrackID aID) override;
+ void Suspend() override {}
+ void Resume() override {}
+ void AddListener(MediaStreamListener* aListener) override;
+ void RemoveListener(MediaStreamListener* aListener) override;
+ void Destroy() override;
void OnPreviewStateChange(bool aActive);
void Invalidate();
void ProcessInput(GraphTime aFrom, GraphTime aTo, uint32_t aFlags) override;
// Call these on any thread.
void SetCurrentFrame(const gfx::IntSize& aIntrinsicSize, Image* aImage);
--- a/dom/media/MediaStreamGraph.cpp
+++ b/dom/media/MediaStreamGraph.cpp
@@ -1020,20 +1020,20 @@ MediaStreamGraphImpl::PlayVideo(MediaStr
TimeDuration::FromSeconds(MediaTimeToSeconds(frameTime - IterationEnd()));
if (frame->GetForceBlack()) {
if (!blackImage) {
// Fixme: PlayVideo will be replaced in latter changeset
// "Call MediaStreamVideoSink::setCurrentFrames in SourceMediaStream::AppendToTrack."
// of this bug.
// This is a temp workaround to pass the build and test.
- if (!aStream->mVideoOutputs[0]->AsVideoFrameContainer()) {
+ if (!aStream->mVideoOutputs[0].mListener->AsVideoFrameContainer()) {
return;
}
- blackImage = aStream->mVideoOutputs[0]->AsVideoFrameContainer()->
+ blackImage = aStream->mVideoOutputs[0].mListener->AsVideoFrameContainer()->
GetImageContainer()->CreatePlanarYCbCrImage();
if (blackImage) {
// Sets the image to a single black pixel, which will be scaled to
// fill the rendered size.
SetImageToBlackPixel(blackImage->AsPlanarYCbCrImage());
}
}
if (blackImage) {
@@ -1048,18 +1048,18 @@ MediaStreamGraphImpl::PlayVideo(MediaStr
}
if (!aStream->mLastPlayedVideoFrame.GetImage())
return;
AutoTArray<ImageContainer::NonOwningImage,4> images;
bool haveMultipleImages = false;
- for (MediaStreamVideoSink* sink : aStream->mVideoOutputs) {
- VideoFrameContainer* output = sink->AsVideoFrameContainer();
+ for (const TrackBound<MediaStreamVideoSink>& sink : aStream->mVideoOutputs) {
+ VideoFrameContainer* output = sink.mListener->AsVideoFrameContainer();
if (!output) {
continue;
}
bool principalHandleChanged =
lastPrincipalHandle != PRINCIPAL_HANDLE_NONE &&
lastPrincipalHandle != output->GetLastPrincipalHandle();
@@ -2266,65 +2266,86 @@ MediaStream::RemoveAudioOutput(void* aKe
mStream->RemoveAudioOutputImpl(mKey);
}
void* mKey;
};
GraphImpl()->AppendMessage(MakeUnique<Message>(this, aKey));
}
void
-MediaStream::AddVideoOutputImpl(already_AddRefed<MediaStreamVideoSink> aSink)
+MediaStream::AddVideoOutputImpl(already_AddRefed<MediaStreamVideoSink> aSink,
+ TrackID aID)
{
RefPtr<MediaStreamVideoSink> sink = aSink;
STREAM_LOG(LogLevel::Info, ("MediaStream %p Adding MediaStreamVideoSink %p as output",
this, sink.get()));
- *mVideoOutputs.AppendElement() = sink.forget();
+ MOZ_ASSERT(aID != TRACK_NONE);
+ for (auto entry : mVideoOutputs) {
+ if (entry.mListener == sink &&
+ (entry.mTrackID == TRACK_ANY || entry.mTrackID == aID)) {
+ return;
+ }
+ }
+ TrackBound<MediaStreamVideoSink>* l = mVideoOutputs.AppendElement();
+ l->mListener = sink;
+ l->mTrackID = aID;
}
void
-MediaStream::RemoveVideoOutputImpl(MediaStreamVideoSink* aSink)
+MediaStream::RemoveVideoOutputImpl(MediaStreamVideoSink* aSink,
+ TrackID aID)
{
STREAM_LOG(LogLevel::Info, ("MediaStream %p Removing MediaStreamVideoSink %p as output",
this, aSink));
+ MOZ_ASSERT(aID != TRACK_NONE);
+
// Ensure that any frames currently queued for playback by the compositor
// are removed.
aSink->ClearFrames();
- mVideoOutputs.RemoveElement(aSink);
+ for (size_t i = 0; i < mVideoOutputs.Length(); ++i) {
+ if (mVideoOutputs[i].mListener == aSink &&
+ (mVideoOutputs[i].mTrackID == TRACK_ANY ||
+ mVideoOutputs[i].mTrackID == aID)) {
+ mVideoOutputs.RemoveElementAt(i);
+ }
+ }
}
void
-MediaStream::AddVideoOutput(MediaStreamVideoSink* aSink)
+MediaStream::AddVideoOutput(MediaStreamVideoSink* aSink, TrackID aID)
{
class Message : public ControlMessage {
public:
- Message(MediaStream* aStream, MediaStreamVideoSink* aSink) :
- ControlMessage(aStream), mSink(aSink) {}
+ Message(MediaStream* aStream, MediaStreamVideoSink* aSink, TrackID aID) :
+ ControlMessage(aStream), mSink(aSink), mID(aID) {}
void Run() override
{
- mStream->AddVideoOutputImpl(mSink.forget());
+ mStream->AddVideoOutputImpl(mSink.forget(), mID);
}
RefPtr<MediaStreamVideoSink> mSink;
+ TrackID mID;
};
- GraphImpl()->AppendMessage(MakeUnique<Message>(this, aSink));
+ GraphImpl()->AppendMessage(MakeUnique<Message>(this, aSink, aID));
}
void
-MediaStream::RemoveVideoOutput(MediaStreamVideoSink* aSink)
+MediaStream::RemoveVideoOutput(MediaStreamVideoSink* aSink, TrackID aID)
{
class Message : public ControlMessage {
public:
- Message(MediaStream* aStream, MediaStreamVideoSink* aSink) :
- ControlMessage(aStream), mSink(aSink) {}
+ Message(MediaStream* aStream, MediaStreamVideoSink* aSink, TrackID aID) :
+ ControlMessage(aStream), mSink(aSink), mID(aID) {}
void Run() override
{
- mStream->RemoveVideoOutputImpl(mSink);
+ mStream->RemoveVideoOutputImpl(mSink, mID);
}
RefPtr<MediaStreamVideoSink> mSink;
+ TrackID mID;
};
- GraphImpl()->AppendMessage(MakeUnique<Message>(this, aSink));
+ GraphImpl()->AppendMessage(MakeUnique<Message>(this, aSink, aID));
}
void
MediaStream::Suspend()
{
class Message : public ControlMessage {
public:
explicit Message(MediaStream* aStream) :
--- a/dom/media/MediaStreamGraph.h
+++ b/dom/media/MediaStreamGraph.h
@@ -12,17 +12,16 @@
#include "mozilla/dom/AudioChannelBinding.h"
#include "AudioStream.h"
#include "nsTArray.h"
#include "nsIRunnable.h"
#include "VideoSegment.h"
#include "StreamTracks.h"
-#include "MediaStreamVideoSink.h"
#include "MainThreadUtils.h"
#include "StreamTracks.h"
#include "nsAutoPtr.h"
#include "nsAutoRef.h"
#include <speex/speex_resampler.h>
class nsIRunnable;
@@ -162,18 +161,20 @@ class AudioNodeStream;
class AudioSegment;
class CameraPreviewMediaStream;
class DirectMediaStreamListener;
class DirectMediaStreamTrackListener;
class MediaInputPort;
class MediaStreamGraphImpl;
class MediaStreamListener;
class MediaStreamTrackListener;
+class MediaStreamVideoSink;
class ProcessedMediaStream;
class SourceMediaStream;
+class TrackUnionStream;
enum MediaStreamGraphEvent : uint32_t;
enum TrackEventCommand : uint32_t;
/**
* Helper struct for binding a track listener to a specific TrackID.
*/
template<typename Listener>
@@ -289,18 +290,20 @@ public:
// Currently only the first enabled audio track is played.
// XXX change this so all enabled audio tracks are mixed and played.
virtual void AddAudioOutput(void* aKey);
virtual void SetAudioOutputVolume(void* aKey, float aVolume);
virtual void RemoveAudioOutput(void* aKey);
// Since a stream can be played multiple ways, we need to be able to
// play to multiple MediaStreamVideoSinks.
// Only the first enabled video track is played.
- virtual void AddVideoOutput(MediaStreamVideoSink* aSink);
- virtual void RemoveVideoOutput(MediaStreamVideoSink* aSink);
+ virtual void AddVideoOutput(MediaStreamVideoSink* aSink,
+ TrackID aID = TRACK_ANY);
+ virtual void RemoveVideoOutput(MediaStreamVideoSink* aSink,
+ TrackID aID = TRACK_ANY);
// Explicitly suspend. Useful for example if a media element is pausing
// and we need to stop its stream emitting its buffered data. As soon as the
// Suspend message reaches the graph, the stream stops processing. It
// ignores its inputs and produces silence/no video until Resumed. Its
// current time does not advance.
virtual void Suspend();
virtual void Resume();
// Events will be dispatched by calling methods of aListener.
@@ -399,16 +402,17 @@ public:
friend class MediaStreamGraphImpl;
friend class MediaInputPort;
friend class AudioNodeExternalInputStream;
virtual SourceMediaStream* AsSourceStream() { return nullptr; }
virtual ProcessedMediaStream* AsProcessedStream() { return nullptr; }
virtual AudioNodeStream* AsAudioNodeStream() { return nullptr; }
+ virtual TrackUnionStream* AsTrackUnionStream() { return nullptr; }
// These Impl methods perform the core functionality of the control methods
// above, on the media graph thread.
/**
* Stop all stream activity and disconnect it from all inputs and outputs.
* This must be idempotent.
*/
virtual void DestroyImpl();
@@ -419,18 +423,19 @@ public:
void SetAudioOutputVolumeImpl(void* aKey, float aVolume);
void AddAudioOutputImpl(void* aKey);
// Returns true if this stream has an audio output.
bool HasAudioOutput()
{
return !mAudioOutputs.IsEmpty();
}
void RemoveAudioOutputImpl(void* aKey);
- void AddVideoOutputImpl(already_AddRefed<MediaStreamVideoSink> aSink);
- void RemoveVideoOutputImpl(MediaStreamVideoSink* aSink);
+ void AddVideoOutputImpl(already_AddRefed<MediaStreamVideoSink> aSink,
+ TrackID aID);
+ void RemoveVideoOutputImpl(MediaStreamVideoSink* aSink, TrackID aID);
void AddListenerImpl(already_AddRefed<MediaStreamListener> aListener);
void RemoveListenerImpl(MediaStreamListener* aListener);
void RemoveAllListenersImpl();
virtual void AddTrackListenerImpl(already_AddRefed<MediaStreamTrackListener> aListener,
TrackID aTrackID);
virtual void RemoveTrackListenerImpl(MediaStreamTrackListener* aListener,
TrackID aTrackID);
virtual void AddDirectTrackListenerImpl(already_AddRefed<DirectMediaStreamTrackListener> aListener,
@@ -579,17 +584,17 @@ protected:
// Client-set volume of this stream
struct AudioOutput {
explicit AudioOutput(void* aKey) : mKey(aKey), mVolume(1.0f) {}
void* mKey;
float mVolume;
};
nsTArray<AudioOutput> mAudioOutputs;
- nsTArray<RefPtr<MediaStreamVideoSink>> mVideoOutputs;
+ nsTArray<TrackBound<MediaStreamVideoSink>> mVideoOutputs;
// We record the last played video frame to avoid playing the frame again
// with a different frame id.
VideoFrame mLastPlayedVideoFrame;
nsTArray<RefPtr<MediaStreamListener> > mListeners;
nsTArray<TrackBound<MediaStreamTrackListener>> mTrackListeners;
nsTArray<MainThreadMediaStreamListener*> mMainThreadListeners;
nsTArray<TrackID> mDisabledTrackIDs;
--- a/dom/media/TrackUnionStream.h
+++ b/dom/media/TrackUnionStream.h
@@ -14,16 +14,19 @@ namespace mozilla {
/**
* See MediaStreamGraph::CreateTrackUnionStream.
*/
class TrackUnionStream : public ProcessedMediaStream {
public:
explicit TrackUnionStream();
+ virtual TrackUnionStream* AsTrackUnionStream() override { return this; }
+ friend class DOMMediaStream;
+
void RemoveInput(MediaInputPort* aPort) override;
void ProcessInput(GraphTime aFrom, GraphTime aTo, uint32_t aFlags) override;
void SetTrackEnabledImpl(TrackID aTrackID, bool aEnabled) override;
MediaStream* GetInputStreamFor(TrackID aTrackID) override;
TrackID GetInputTrackIDFor(TrackID aTrackID) override;