Bug 1201363 - MediaStreamVideoSink for MediaPipelineTransmit case. r?jesup draft
authorctai <ctai@mozilla.com>
Tue, 31 May 2016 14:59:13 +0800
changeset 396714 0b79b59146725bbaf7d4eefb73370cf284ec5c35
parent 396713 e9224290d6cf081258d1e2179099b2912442ea1d
child 396715 214dc507b9cb1696d6674ec0670ae87da0ab9826
push id25079
push userbmo:ctai@mozilla.com
push dateThu, 04 Aug 2016 08:39:16 +0000
reviewersjesup
bugs1201363
milestone51.0a1
Bug 1201363 - MediaStreamVideoSink for MediaPipelineTransmit case. r?jesup Replace |MediaPipelineTransmit::PipelineListener::NotifyQueuedTrackChanges| with |MediaPipelineTransmit::PipelineVideoSink::SetCurrentFrames|. We only need to deal with the video case since audio will be routed to |NotifyQueuedAudioData|. MozReview-Commit-ID: EVpMVgJynGT
media/webrtc/signaling/src/mediapipeline/MediaPipeline.cpp
media/webrtc/signaling/src/mediapipeline/MediaPipeline.h
media/webrtc/signaling/test/FakeMediaStreams.h
--- a/media/webrtc/signaling/src/mediapipeline/MediaPipeline.cpp
+++ b/media/webrtc/signaling/src/mediapipeline/MediaPipeline.cpp
@@ -20,17 +20,19 @@
 #include "VideoSegment.h"
 #include "Layers.h"
 #include "LayersLogging.h"
 #include "ImageTypes.h"
 #include "ImageContainer.h"
 #include "DOMMediaStream.h"
 #include "MediaStreamTrack.h"
 #include "MediaStreamListener.h"
+#include "MediaStreamVideoSink.h"
 #include "VideoUtils.h"
+#include "VideoStreamTrack.h"
 #ifdef WEBRTC_GONK
 #include "GrallocImages.h"
 #include "mozilla/layers/GrallocTextureClient.h"
 #endif
 #endif
 
 #include "nsError.h"
 #include "AudioSegment.h"
@@ -1253,30 +1255,59 @@ protected:
     MOZ_COUNT_DTOR(VideoFrameFeeder);
   }
 
   RefPtr<PipelineListener> listener_;
   Mutex mutex_;
 };
 #endif
 
+class MediaPipelineTransmit::PipelineVideoSink :
+  public MediaStreamVideoSink
+{
+public:
+  explicit PipelineVideoSink(const RefPtr<MediaSessionConduit>& conduit,
+                             MediaPipelineTransmit::PipelineListener* listener)
+    : conduit_(conduit)
+    , pipelineListener_(listener)
+  {
+  }
+
+  virtual void SetCurrentFrames(const VideoSegment& aSegment) override;
+  virtual void ClearFrames() override {}
+
+private:
+  ~PipelineVideoSink() {
+    // release conduit on mainthread.  Must use forget()!
+    nsresult rv = NS_DispatchToMainThread(new
+      ConduitDeleteEvent(conduit_.forget()));
+    MOZ_ASSERT(!NS_FAILED(rv),"Could not dispatch conduit shutdown to main");
+    if (NS_FAILED(rv)) {
+      MOZ_CRASH();
+    }
+  }
+  RefPtr<MediaSessionConduit> conduit_;
+  MediaPipelineTransmit::PipelineListener* pipelineListener_;
+};
+
 MediaPipelineTransmit::MediaPipelineTransmit(
     const std::string& pc,
     nsCOMPtr<nsIEventTarget> main_thread,
     nsCOMPtr<nsIEventTarget> sts_thread,
     dom::MediaStreamTrack* domtrack,
     const std::string& track_id,
     int level,
     RefPtr<MediaSessionConduit> conduit,
     RefPtr<TransportFlow> rtp_transport,
     RefPtr<TransportFlow> rtcp_transport,
     nsAutoPtr<MediaPipelineFilter> filter) :
   MediaPipeline(pc, TRANSMIT, main_thread, sts_thread, track_id, level,
                 conduit, rtp_transport, rtcp_transport, filter),
   listener_(new PipelineListener(conduit)),
+  video_sink_(new PipelineVideoSink(conduit, listener_)),
   domtrack_(domtrack)
 {
 #if !defined(MOZILLA_EXTERNAL_LINKAGE)
   if (IsVideo()) {
     // For video we send frames to an async VideoFrameConverter that calls
     // back to a VideoFrameFeeder that feeds I420 frames to VideoConduit.
 
     feeder_ = MakeAndAddRef<VideoFrameFeeder>(listener_);
@@ -1320,16 +1351,20 @@ void MediaPipelineTransmit::AttachToTrac
 
   // Register the Listener directly with the source if we can.
   // We also register it as a non-direct listener so we fall back to that
   // if installing the direct listener fails. As a direct listener we get access
   // to direct unqueued (and not resampled) data.
   domtrack_->AddDirectListener(listener_);
   domtrack_->AddListener(listener_);
 
+#if !defined(MOZILLA_EXTERNAL_LINKAGE)
+  domtrack_->AddDirectListener(video_sink_);
+#endif
+
 #ifndef MOZILLA_INTERNAL_API
   // this enables the unit tests that can't fiddle with principals and the like
   listener_->SetEnabled(true);
 #endif
 }
 
 bool
 MediaPipelineTransmit::IsVideo() const
@@ -1368,16 +1403,17 @@ void MediaPipelineTransmit::UpdateSinkId
 
 void
 MediaPipelineTransmit::DetachMedia()
 {
   ASSERT_ON_THREAD(main_thread_);
   if (domtrack_) {
     domtrack_->RemoveDirectListener(listener_);
     domtrack_->RemoveListener(listener_);
+    domtrack_->RemoveDirectListener(video_sink_);
     domtrack_ = nullptr;
   }
   // Let the listener be destroyed with the pipeline (or later).
 }
 
 nsresult MediaPipelineTransmit::TransportReady_s(TransportInfo &info) {
   ASSERT_ON_THREAD(sts_thread_);
   // Call base ready function.
@@ -1638,27 +1674,16 @@ NewData(MediaStreamGraph* graph,
       rate = Fake_MediaStream::GraphRate();
 #else
       rate = graph->GraphRate();
 #endif
       ProcessAudioChunk(static_cast<AudioSessionConduit*>(conduit_.get()),
                         rate, *iter);
       iter.Next();
     }
-  } else if (media.GetType() == MediaSegment::VIDEO) {
-#if !defined(MOZILLA_EXTERNAL_LINKAGE)
-    VideoSegment* video = const_cast<VideoSegment *>(
-        static_cast<const VideoSegment *>(&media));
-
-    VideoSegment::ChunkIterator iter(*video);
-    while(!iter.IsEnded()) {
-      converter_->QueueVideoChunk(*iter, !enabled_);
-      iter.Next();
-    }
-#endif
   } else {
     // Ignore
   }
 }
 
 void MediaPipelineTransmit::PipelineListener::ProcessAudioChunk(
     AudioSessionConduit *conduit,
     TrackRate rate,
@@ -1722,16 +1747,42 @@ void MediaPipelineTransmit::PipelineList
 
     packetizer_->Output(packet);
     conduit->SendAudioFrame(packet,
                             samplesPerPacket,
                             rate, 0);
   }
 }
 
+void MediaPipelineTransmit::PipelineVideoSink::
+SetCurrentFrames(const VideoSegment& aSegment)
+{
+  MOZ_ASSERT(pipelineListener_);
+
+  if (!pipelineListener_->active_) {
+    MOZ_MTLOG(ML_DEBUG, "Discarding packets because transport not ready");
+    return;
+  }
+
+  if (conduit_->type() != MediaSessionConduit::VIDEO) {
+    // Ignore data of wrong kind in case we have a muxed stream
+    return;
+  }
+
+#if !defined(MOZILLA_EXTERNAL_LINKAGE)
+    VideoSegment* video = const_cast<VideoSegment *>(&aSegment);
+
+    VideoSegment::ChunkIterator iter(*video);
+    while(!iter.IsEnded()) {
+      pipelineListener_->converter_->QueueVideoChunk(*iter, !pipelineListener_->enabled_);
+      iter.Next();
+    }
+#endif
+}
+
 class TrackAddedCallback {
  public:
   virtual void TrackAdded(TrackTicks current_ticks) = 0;
 
   NS_INLINE_DECL_THREADSAFE_REFCOUNTING(TrackAddedCallback);
 
  protected:
   virtual ~TrackAddedCallback() {}
--- a/media/webrtc/signaling/src/mediapipeline/MediaPipeline.h
+++ b/media/webrtc/signaling/src/mediapipeline/MediaPipeline.h
@@ -337,26 +337,28 @@ public:
   // In non-compliance with the likely final spec, allow the new
   // track to be part of a different stream (since we don't support
   // multiple tracks of a type in a stream yet).  bug 1056650
   virtual nsresult ReplaceTrack(dom::MediaStreamTrack& domtrack);
 
   // Separate classes to allow ref counting
   class PipelineListener;
   class VideoFrameFeeder;
+  class PipelineVideoSink;
 
  protected:
   ~MediaPipelineTransmit();
 
  private:
   RefPtr<PipelineListener> listener_;
 #if !defined(MOZILLA_EXTERNAL_LINKAGE)
   RefPtr<VideoFrameFeeder> feeder_;
   RefPtr<VideoFrameConverter> converter_;
 #endif
+  RefPtr<PipelineVideoSink> video_sink_;
   dom::MediaStreamTrack* domtrack_;
 };
 
 
 // A specialization of pipeline for reading from the network and
 // rendering video.
 class MediaPipelineReceive : public MediaPipeline {
  public:
--- a/media/webrtc/signaling/test/FakeMediaStreams.h
+++ b/media/webrtc/signaling/test/FakeMediaStreams.h
@@ -18,16 +18,17 @@
 #include "nsISupportsImpl.h"
 #include "nsServiceManagerUtils.h"
 
 // #includes from MediaStream.h
 #include "mozilla/Mutex.h"
 #include "AudioSegment.h"
 #include "MediaSegment.h"
 #include "StreamTracks.h"
+#include "VideoSegment.h"
 #include "nsTArray.h"
 #include "nsIRunnable.h"
 #include "nsISupportsImpl.h"
 
 class nsPIDOMWindowInner;
 
 namespace mozilla {
    class MediaStreamGraphImpl;
@@ -142,16 +143,37 @@ public:
   enum class InstallationResult {
     STREAM_NOT_SUPPORTED,
     SUCCESS
   };
   virtual void NotifyDirectListenerInstalled(InstallationResult aResult) = 0;
   virtual void NotifyDirectListenerUninstalled() = 0;
 };
 
+class Fake_MediaStreamVideoSink : public Fake_DirectMediaStreamTrackListener{
+public:
+  Fake_MediaStreamVideoSink() {}
+
+  void NotifyQueuedChanges(mozilla::MediaStreamGraph* aGraph,
+                           mozilla::StreamTime aTrackOffset,
+                           const mozilla::MediaSegment& aQueuedMedia) override {}
+
+  void NotifyRealtimeTrackData(mozilla::MediaStreamGraph* aGraph,
+                               mozilla::StreamTime aTrackOffset,
+                               const mozilla::MediaSegment& aMedia) override {}
+  void NotifyDirectListenerInstalled(InstallationResult aResult) override {}
+  void NotifyDirectListenerUninstalled() override {}
+
+  virtual void SetCurrentFrames(const mozilla::VideoSegment& aSegment) {};
+  virtual void ClearFrames() {};
+
+protected:
+  virtual ~Fake_MediaStreamVideoSink() {}
+};
+
 // Note: only one listener supported
 class Fake_MediaStream {
  protected:
   virtual ~Fake_MediaStream() { Stop(); }
 
   struct BoundTrackListener
   {
     BoundTrackListener(Fake_MediaStreamTrackListener* aListener,
@@ -614,16 +636,17 @@ namespace mozilla {
 typedef Fake_MediaStream MediaStream;
 typedef Fake_SourceMediaStream SourceMediaStream;
 typedef Fake_MediaStreamListener MediaStreamListener;
 typedef Fake_DirectMediaStreamListener DirectMediaStreamListener;
 typedef Fake_MediaStreamTrackListener MediaStreamTrackListener;
 typedef Fake_DirectMediaStreamTrackListener DirectMediaStreamTrackListener;
 typedef Fake_DOMMediaStream DOMMediaStream;
 typedef Fake_DOMMediaStream DOMLocalMediaStream;
+typedef Fake_MediaStreamVideoSink MediaStreamVideoSink;
 
 namespace dom {
 typedef Fake_MediaStreamTrack MediaStreamTrack;
 typedef Fake_MediaStreamTrackSource MediaStreamTrackSource;
 }
 }
 
 #endif