Bug 1201363 - Let MediaStreamVideoSink bind with particular video track. r?jesup draft
authorctai <ctai@mozilla.com>
Fri, 27 May 2016 14:33:50 +0800
changeset 396579 c3d308e5c7ca9e7cf94cc077c26f111687370421
parent 396578 00387ad857ee878ee92f2c5426eb766fae0939a6
child 396580 982605739b9605d31a717e3837f4a5a2dec05377
push id25058
push userbmo:ctai@mozilla.com
push dateThu, 04 Aug 2016 05:45:45 +0000
reviewersjesup
bugs1201363
milestone51.0a1
Bug 1201363 - Let MediaStreamVideoSink bind with particular video track. r?jesup MozReview-Commit-ID: FcjnmDKuRQI
dom/camera/CameraPreviewMediaStream.cpp
dom/camera/CameraPreviewMediaStream.h
dom/media/MediaStreamGraph.cpp
dom/media/MediaStreamGraph.h
dom/media/TrackUnionStream.h
--- a/dom/camera/CameraPreviewMediaStream.cpp
+++ b/dom/camera/CameraPreviewMediaStream.cpp
@@ -55,28 +55,28 @@ CameraPreviewMediaStream::SetAudioOutput
 }
 
 void
 CameraPreviewMediaStream::RemoveAudioOutput(void* aKey)
 {
 }
 
 void
-CameraPreviewMediaStream::AddVideoOutput(MediaStreamVideoSink* aSink)
+CameraPreviewMediaStream::AddVideoOutput(MediaStreamVideoSink* aSink, TrackID aID)
 {
   MutexAutoLock lock(mMutex);
   RefPtr<MediaStreamVideoSink> sink = aSink;
-  AddVideoOutputImpl(sink.forget());
+  AddVideoOutputImpl(sink.forget(), aID);
 }
 
 void
-CameraPreviewMediaStream::RemoveVideoOutput(MediaStreamVideoSink* aSink)
+CameraPreviewMediaStream::RemoveVideoOutput(MediaStreamVideoSink* aSink, TrackID aID)
 {
   MutexAutoLock lock(mMutex);
-  RemoveVideoOutputImpl(aSink);
+  RemoveVideoOutputImpl(aSink, aID);
 }
 
 void
 CameraPreviewMediaStream::AddListener(MediaStreamListener* aListener)
 {
   MutexAutoLock lock(mMutex);
 
   MediaStreamListener* listener = *mListeners.AppendElement() = aListener;
@@ -121,18 +121,18 @@ CameraPreviewMediaStream::Destroy()
   DestroyImpl();
 }
 
 void
 CameraPreviewMediaStream::Invalidate()
 {
   MutexAutoLock lock(mMutex);
   --mInvalidatePending;
-  for (MediaStreamVideoSink* sink : mVideoOutputs) {
-    VideoFrameContainer* output = sink->AsVideoFrameContainer();
+  for (const TrackBound<MediaStreamVideoSink>& sink : mVideoOutputs) {
+    VideoFrameContainer* output = sink.mListener->AsVideoFrameContainer();
     if (!output) {
       continue;
     }
     output->Invalidate();
   }
 }
 
 void
@@ -163,18 +163,18 @@ CameraPreviewMediaStream::SetCurrentFram
       }
 
       DOM_CAMERA_LOGI("Update preview frame, %d invalidation(s) pending",
         mInvalidatePending);
     }
     mDiscardedFrames = 0;
 
     TimeStamp now = TimeStamp::Now();
-    for (MediaStreamVideoSink* sink : mVideoOutputs) {
-      VideoFrameContainer* output = sink->AsVideoFrameContainer();
+    for (const TrackBound<MediaStreamVideoSink>& sink : mVideoOutputs) {
+      VideoFrameContainer* output = sink.mListener->AsVideoFrameContainer();
       if (!output) {
         continue;
       }
       output->SetCurrentFrame(aIntrinsicSize, aImage, now);
     }
 
     ++mInvalidatePending;
   }
@@ -182,18 +182,18 @@ CameraPreviewMediaStream::SetCurrentFram
   NS_DispatchToMainThread(NewRunnableMethod(this, &CameraPreviewMediaStream::Invalidate));
 }
 
 void
 CameraPreviewMediaStream::ClearCurrentFrame()
 {
   MutexAutoLock lock(mMutex);
 
-  for (MediaStreamVideoSink* sink : mVideoOutputs) {
-    VideoFrameContainer* output = sink->AsVideoFrameContainer();
+  for (const TrackBound<MediaStreamVideoSink>& sink : mVideoOutputs) {
+    VideoFrameContainer* output = sink.mListener->AsVideoFrameContainer();
     if (!output) {
       continue;
     }
     output->ClearCurrentFrame();
     NS_DispatchToMainThread(NewRunnableMethod(output, &VideoFrameContainer::Invalidate));
   }
 }
 
--- a/dom/camera/CameraPreviewMediaStream.h
+++ b/dom/camera/CameraPreviewMediaStream.h
@@ -38,26 +38,26 @@ protected:
  */
 class CameraPreviewMediaStream : public ProcessedMediaStream
 {
   typedef mozilla::layers::Image Image;
 
 public:
   CameraPreviewMediaStream();
 
-  virtual void AddAudioOutput(void* aKey) override;
-  virtual void SetAudioOutputVolume(void* aKey, float aVolume) override;
-  virtual void RemoveAudioOutput(void* aKey) override;
-  virtual void AddVideoOutput(MediaStreamVideoSink* aSink) override;
-  virtual void RemoveVideoOutput(MediaStreamVideoSink* aSink) override;
-  virtual void Suspend() override {}
-  virtual void Resume() override {}
-  virtual void AddListener(MediaStreamListener* aListener) override;
-  virtual void RemoveListener(MediaStreamListener* aListener) override;
-  virtual void Destroy() override;
+  void AddAudioOutput(void* aKey) override;
+  void SetAudioOutputVolume(void* aKey, float aVolume) override;
+  void RemoveAudioOutput(void* aKey) override;
+  void AddVideoOutput(MediaStreamVideoSink* aSink, TrackID aID) override;
+  void RemoveVideoOutput(MediaStreamVideoSink* aSink, TrackID aID) override;
+  void Suspend() override {}
+  void Resume() override {}
+  void AddListener(MediaStreamListener* aListener) override;
+  void RemoveListener(MediaStreamListener* aListener) override;
+  void Destroy() override;
   void OnPreviewStateChange(bool aActive);
 
   void Invalidate();
 
   void ProcessInput(GraphTime aFrom, GraphTime aTo, uint32_t aFlags) override;
 
   // Call these on any thread.
   void SetCurrentFrame(const gfx::IntSize& aIntrinsicSize, Image* aImage);
--- a/dom/media/MediaStreamGraph.cpp
+++ b/dom/media/MediaStreamGraph.cpp
@@ -1020,20 +1020,20 @@ MediaStreamGraphImpl::PlayVideo(MediaStr
       TimeDuration::FromSeconds(MediaTimeToSeconds(frameTime - IterationEnd()));
 
     if (frame->GetForceBlack()) {
       if (!blackImage) {
         // Fixme: PlayVideo will be replaced in latter changeset
         // "Call MediaStreamVideoSink::setCurrentFrames in SourceMediaStream::AppendToTrack."
         // of this bug.
         // This is a temp workaround to pass the build and test.
-        if (!aStream->mVideoOutputs[0]->AsVideoFrameContainer()) {
+        if (!aStream->mVideoOutputs[0].mListener->AsVideoFrameContainer()) {
           return;
         }
-        blackImage = aStream->mVideoOutputs[0]->AsVideoFrameContainer()->
+        blackImage = aStream->mVideoOutputs[0].mListener->AsVideoFrameContainer()->
           GetImageContainer()->CreatePlanarYCbCrImage();
         if (blackImage) {
           // Sets the image to a single black pixel, which will be scaled to
           // fill the rendered size.
           SetImageToBlackPixel(blackImage->AsPlanarYCbCrImage());
         }
       }
       if (blackImage) {
@@ -1048,18 +1048,18 @@ MediaStreamGraphImpl::PlayVideo(MediaStr
   }
 
   if (!aStream->mLastPlayedVideoFrame.GetImage())
     return;
 
   AutoTArray<ImageContainer::NonOwningImage,4> images;
   bool haveMultipleImages = false;
 
-  for (MediaStreamVideoSink* sink : aStream->mVideoOutputs) {
-    VideoFrameContainer* output = sink->AsVideoFrameContainer();
+  for (const TrackBound<MediaStreamVideoSink>& sink : aStream->mVideoOutputs) {
+    VideoFrameContainer* output = sink.mListener->AsVideoFrameContainer();
     if (!output) {
       continue;
     }
 
     bool principalHandleChanged =
       lastPrincipalHandle != PRINCIPAL_HANDLE_NONE &&
       lastPrincipalHandle != output->GetLastPrincipalHandle();
 
@@ -2266,65 +2266,86 @@ MediaStream::RemoveAudioOutput(void* aKe
       mStream->RemoveAudioOutputImpl(mKey);
     }
     void* mKey;
   };
   GraphImpl()->AppendMessage(MakeUnique<Message>(this, aKey));
 }
 
 void
-MediaStream::AddVideoOutputImpl(already_AddRefed<MediaStreamVideoSink> aSink)
+MediaStream::AddVideoOutputImpl(already_AddRefed<MediaStreamVideoSink> aSink,
+                                TrackID aID)
 {
   RefPtr<MediaStreamVideoSink> sink = aSink;
   STREAM_LOG(LogLevel::Info, ("MediaStream %p Adding MediaStreamVideoSink %p as output",
                               this, sink.get()));
-  *mVideoOutputs.AppendElement() = sink.forget();
+  MOZ_ASSERT(aID != TRACK_NONE);
+   for (auto entry : mVideoOutputs) {
+     if (entry.mListener == sink &&
+         (entry.mTrackID == TRACK_ANY || entry.mTrackID == aID)) {
+       return;
+     }
+   }
+   TrackBound<MediaStreamVideoSink>* l = mVideoOutputs.AppendElement();
+   l->mListener = sink;
+   l->mTrackID = aID;
 }
 
 void
-MediaStream::RemoveVideoOutputImpl(MediaStreamVideoSink* aSink)
+MediaStream::RemoveVideoOutputImpl(MediaStreamVideoSink* aSink,
+                                   TrackID aID)
 {
   STREAM_LOG(LogLevel::Info, ("MediaStream %p Removing MediaStreamVideoSink %p as output",
                               this, aSink));
+  MOZ_ASSERT(aID != TRACK_NONE);
+
   // Ensure that any frames currently queued for playback by the compositor
   // are removed.
   aSink->ClearFrames();
-  mVideoOutputs.RemoveElement(aSink);
+  for (size_t i = 0; i < mVideoOutputs.Length(); ++i) {
+    if (mVideoOutputs[i].mListener == aSink &&
+        (mVideoOutputs[i].mTrackID == TRACK_ANY ||
+         mVideoOutputs[i].mTrackID == aID)) {
+      mVideoOutputs.RemoveElementAt(i);
+    }
+  }
 }
 
 void
-MediaStream::AddVideoOutput(MediaStreamVideoSink* aSink)
+MediaStream::AddVideoOutput(MediaStreamVideoSink* aSink, TrackID aID)
 {
   class Message : public ControlMessage {
   public:
-    Message(MediaStream* aStream, MediaStreamVideoSink* aSink) :
-      ControlMessage(aStream), mSink(aSink) {}
+    Message(MediaStream* aStream, MediaStreamVideoSink* aSink, TrackID aID) :
+      ControlMessage(aStream), mSink(aSink), mID(aID) {}
     void Run() override
     {
-      mStream->AddVideoOutputImpl(mSink.forget());
+      mStream->AddVideoOutputImpl(mSink.forget(), mID);
     }
     RefPtr<MediaStreamVideoSink> mSink;
+    TrackID mID;
   };
-  GraphImpl()->AppendMessage(MakeUnique<Message>(this, aSink));
+  GraphImpl()->AppendMessage(MakeUnique<Message>(this, aSink, aID));
 }
 
 void
-MediaStream::RemoveVideoOutput(MediaStreamVideoSink* aSink)
+MediaStream::RemoveVideoOutput(MediaStreamVideoSink* aSink, TrackID aID)
 {
   class Message : public ControlMessage {
   public:
-    Message(MediaStream* aStream, MediaStreamVideoSink* aSink) :
-      ControlMessage(aStream), mSink(aSink) {}
+    Message(MediaStream* aStream, MediaStreamVideoSink* aSink, TrackID aID) :
+      ControlMessage(aStream), mSink(aSink), mID(aID) {}
     void Run() override
     {
-      mStream->RemoveVideoOutputImpl(mSink);
+      mStream->RemoveVideoOutputImpl(mSink, mID);
     }
     RefPtr<MediaStreamVideoSink> mSink;
+    TrackID mID;
   };
-  GraphImpl()->AppendMessage(MakeUnique<Message>(this, aSink));
+  GraphImpl()->AppendMessage(MakeUnique<Message>(this, aSink, aID));
 }
 
 void
 MediaStream::Suspend()
 {
   class Message : public ControlMessage {
   public:
     explicit Message(MediaStream* aStream) :
--- a/dom/media/MediaStreamGraph.h
+++ b/dom/media/MediaStreamGraph.h
@@ -12,17 +12,16 @@
 
 #include "mozilla/dom/AudioChannelBinding.h"
 
 #include "AudioStream.h"
 #include "nsTArray.h"
 #include "nsIRunnable.h"
 #include "VideoSegment.h"
 #include "StreamTracks.h"
-#include "MediaStreamVideoSink.h"
 #include "MainThreadUtils.h"
 #include "StreamTracks.h"
 #include "nsAutoPtr.h"
 #include "nsAutoRef.h"
 #include <speex/speex_resampler.h>
 
 class nsIRunnable;
 
@@ -162,18 +161,20 @@ class AudioNodeStream;
 class AudioSegment;
 class CameraPreviewMediaStream;
 class DirectMediaStreamListener;
 class DirectMediaStreamTrackListener;
 class MediaInputPort;
 class MediaStreamGraphImpl;
 class MediaStreamListener;
 class MediaStreamTrackListener;
+class MediaStreamVideoSink;
 class ProcessedMediaStream;
 class SourceMediaStream;
+class TrackUnionStream;
 
 enum MediaStreamGraphEvent : uint32_t;
 enum TrackEventCommand : uint32_t;
 
 /**
  * Helper struct for binding a track listener to a specific TrackID.
  */
 template<typename Listener>
@@ -289,18 +290,20 @@ public:
   // Currently only the first enabled audio track is played.
   // XXX change this so all enabled audio tracks are mixed and played.
   virtual void AddAudioOutput(void* aKey);
   virtual void SetAudioOutputVolume(void* aKey, float aVolume);
   virtual void RemoveAudioOutput(void* aKey);
   // Since a stream can be played multiple ways, we need to be able to
   // play to multiple MediaStreamVideoSinks.
   // Only the first enabled video track is played.
-  virtual void AddVideoOutput(MediaStreamVideoSink* aSink);
-  virtual void RemoveVideoOutput(MediaStreamVideoSink* aSink);
+  virtual void AddVideoOutput(MediaStreamVideoSink* aSink,
+                              TrackID aID = TRACK_ANY);
+  virtual void RemoveVideoOutput(MediaStreamVideoSink* aSink,
+                                 TrackID aID = TRACK_ANY);
   // Explicitly suspend. Useful for example if a media element is pausing
   // and we need to stop its stream emitting its buffered data. As soon as the
   // Suspend message reaches the graph, the stream stops processing. It
   // ignores its inputs and produces silence/no video until Resumed. Its
   // current time does not advance.
   virtual void Suspend();
   virtual void Resume();
   // Events will be dispatched by calling methods of aListener.
@@ -399,16 +402,17 @@ public:
 
   friend class MediaStreamGraphImpl;
   friend class MediaInputPort;
   friend class AudioNodeExternalInputStream;
 
   virtual SourceMediaStream* AsSourceStream() { return nullptr; }
   virtual ProcessedMediaStream* AsProcessedStream() { return nullptr; }
   virtual AudioNodeStream* AsAudioNodeStream() { return nullptr; }
+  virtual TrackUnionStream* AsTrackUnionStream() { return nullptr; }
 
   // These Impl methods perform the core functionality of the control methods
   // above, on the media graph thread.
   /**
    * Stop all stream activity and disconnect it from all inputs and outputs.
    * This must be idempotent.
    */
   virtual void DestroyImpl();
@@ -419,18 +423,19 @@ public:
   void SetAudioOutputVolumeImpl(void* aKey, float aVolume);
   void AddAudioOutputImpl(void* aKey);
   // Returns true if this stream has an audio output.
   bool HasAudioOutput()
   {
     return !mAudioOutputs.IsEmpty();
   }
   void RemoveAudioOutputImpl(void* aKey);
-  void AddVideoOutputImpl(already_AddRefed<MediaStreamVideoSink> aSink);
-  void RemoveVideoOutputImpl(MediaStreamVideoSink* aSink);
+  void AddVideoOutputImpl(already_AddRefed<MediaStreamVideoSink> aSink,
+                          TrackID aID);
+  void RemoveVideoOutputImpl(MediaStreamVideoSink* aSink, TrackID aID);
   void AddListenerImpl(already_AddRefed<MediaStreamListener> aListener);
   void RemoveListenerImpl(MediaStreamListener* aListener);
   void RemoveAllListenersImpl();
   virtual void AddTrackListenerImpl(already_AddRefed<MediaStreamTrackListener> aListener,
                                     TrackID aTrackID);
   virtual void RemoveTrackListenerImpl(MediaStreamTrackListener* aListener,
                                        TrackID aTrackID);
   virtual void AddDirectTrackListenerImpl(already_AddRefed<DirectMediaStreamTrackListener> aListener,
@@ -579,17 +584,17 @@ protected:
 
   // Client-set volume of this stream
   struct AudioOutput {
     explicit AudioOutput(void* aKey) : mKey(aKey), mVolume(1.0f) {}
     void* mKey;
     float mVolume;
   };
   nsTArray<AudioOutput> mAudioOutputs;
-  nsTArray<RefPtr<MediaStreamVideoSink>> mVideoOutputs;
+  nsTArray<TrackBound<MediaStreamVideoSink>> mVideoOutputs;
   // We record the last played video frame to avoid playing the frame again
   // with a different frame id.
   VideoFrame mLastPlayedVideoFrame;
   nsTArray<RefPtr<MediaStreamListener> > mListeners;
   nsTArray<TrackBound<MediaStreamTrackListener>> mTrackListeners;
   nsTArray<MainThreadMediaStreamListener*> mMainThreadListeners;
   nsTArray<TrackID> mDisabledTrackIDs;
 
--- a/dom/media/TrackUnionStream.h
+++ b/dom/media/TrackUnionStream.h
@@ -14,16 +14,19 @@ namespace mozilla {
 
 /**
  * See MediaStreamGraph::CreateTrackUnionStream.
  */
 class TrackUnionStream : public ProcessedMediaStream {
 public:
   explicit TrackUnionStream();
 
+  virtual TrackUnionStream* AsTrackUnionStream() override { return this; }
+  friend class DOMMediaStream;
+
   void RemoveInput(MediaInputPort* aPort) override;
   void ProcessInput(GraphTime aFrom, GraphTime aTo, uint32_t aFlags) override;
 
   void SetTrackEnabledImpl(TrackID aTrackID, bool aEnabled) override;
 
   MediaStream* GetInputStreamFor(TrackID aTrackID) override;
   TrackID GetInputTrackIDFor(TrackID aTrackID) override;