Bug 1201363 - MediaStreamVideoSink for MediaPipelineTransmit case. r?roc draft
authorctai <ctai@mozilla.com>
Wed, 06 Jan 2016 21:34:08 +0800
changeset 324273 b072bca5c2641d278defe1dba11c218ed52a8905
parent 324272 f05599202f920e10160a536d90630c6ee144e650
child 324274 2f3419720ad9a023a250902f3ba3d0f8eea29daf
push id9869
push userbmo:ctai@mozilla.com
push dateFri, 22 Jan 2016 07:09:26 +0000
reviewersroc
bugs1201363
milestone46.0a1
Bug 1201363 - MediaStreamVideoSink for MediaPipelineTransmit case. r?roc Replace |MediaPipelineTransmit::PipelineListener::NotifyQueuedTrackChanges| with |MediaPipelineTransmit::PipelineVideoSink::SetCurrentFrames|. We only need to deal with the video case since audio will be routed to |NotifyQueuedAudioData|.
media/webrtc/signaling/src/mediapipeline/MediaPipeline.cpp
media/webrtc/signaling/src/mediapipeline/MediaPipeline.h
media/webrtc/signaling/test/FakeMediaStreams.h
--- a/media/webrtc/signaling/src/mediapipeline/MediaPipeline.cpp
+++ b/media/webrtc/signaling/src/mediapipeline/MediaPipeline.cpp
@@ -636,16 +636,17 @@ void MediaPipelineTransmit::AttachToTrac
   description_ += "]";
 
   // TODO(ekr@rtfm.com): Check for errors
   MOZ_MTLOG(ML_DEBUG, "Attaching pipeline to stream "
             << static_cast<void *>(stream_) << " conduit type=" <<
             (conduit_->type() == MediaSessionConduit::AUDIO ?"audio":"video"));
 
   stream_->AddListener(listener_);
+  stream_->AddVideoOutput(video_sink_);
 
   // Is this a gUM mediastream?  If so, also register the Listener directly with
   // the SourceMediaStream that's attached to the TrackUnion so we can get direct
   // unqueued (and not resampled) data
   listener_->direct_connect_ = domstream_->AddDirectListener(listener_);
 
 #ifndef MOZILLA_INTERNAL_API
   // this enables the unit tests that can't fiddle with principals and the like
@@ -836,16 +837,43 @@ nsresult MediaPipeline::PipelineTranspor
                       RefPtr<MediaPipeline::PipelineTransport>(this),
                       &MediaPipeline::PipelineTransport::SendRtpRtcpPacket_s,
                       buf, false),
                   NS_DISPATCH_NORMAL);
 
     return NS_OK;
 }
 
+void MediaPipelineTransmit::PipelineVideoSink::
+SetCurrentFrames(const VideoSegment& aSegment)
+{
+  MOZ_ASSERT(pipelineListener_);
+
+  if (!pipelineListener_->active_) {
+    MOZ_MTLOG(ML_DEBUG, "Discarding packets because transport not ready");
+    return;
+  }
+
+  if (conduit_->type() != MediaSessionConduit::VIDEO) {
+    // Ignore data of wrong kind in case we have a muxed stream
+    return;
+  }
+
+#if !defined(MOZILLA_EXTERNAL_LINKAGE)
+    VideoSegment* video = const_cast<VideoSegment *>(&aSegment);
+
+    VideoSegment::ChunkIterator iter(*video);
+    while(!iter.IsEnded()) {
+      pipelineListener_->ProcessVideoChunk(static_cast<VideoSessionConduit*>(conduit_.get()),
+                                           *iter);
+      iter.Next();
+    }
+#endif
+}
+
 void MediaPipelineTransmit::PipelineListener::
 UnsetTrackId(MediaStreamGraphImpl* graph) {
 #ifndef USE_FAKE_MEDIA_STREAMS
   class Message : public ControlMessage {
   public:
     explicit Message(PipelineListener* listener) :
       ControlMessage(nullptr), listener_(listener) {}
     virtual void Run() override
@@ -866,31 +894,16 @@ NotifyRealtimeData(MediaStreamGraph* gra
                    uint32_t events,
                    const MediaSegment& media) {
   MOZ_MTLOG(ML_DEBUG, "MediaPipeline::NotifyRealtimeData()");
 
   NewData(graph, tid, offset, events, media);
 }
 
 void MediaPipelineTransmit::PipelineListener::
-NotifyQueuedTrackChanges(MediaStreamGraph* graph, TrackID tid,
-                         StreamTime offset,
-                         uint32_t events,
-                         const MediaSegment& queued_media,
-                         MediaStream* input_stream,
-                         TrackID input_tid) {
-  MOZ_MTLOG(ML_DEBUG, "MediaPipeline::NotifyQueuedTrackChanges()");
-
-  // ignore non-direct data if we're also getting direct data
-  if (!direct_connect_) {
-    NewData(graph, tid, offset, events, queued_media);
-  }
-}
-
-void MediaPipelineTransmit::PipelineListener::
 NotifyQueuedAudioData(MediaStreamGraph* graph, TrackID tid,
                       StreamTime offset,
                       const AudioSegment& queued_media,
                       MediaStream* input_stream,
                       TrackID input_tid) {
   if (!direct_connect_) {
     NewData(graph, tid, offset, 0, queued_media);
   }
--- a/media/webrtc/signaling/src/mediapipeline/MediaPipeline.h
+++ b/media/webrtc/signaling/src/mediapipeline/MediaPipeline.h
@@ -10,16 +10,17 @@
 
 #include "sigslot.h"
 
 #ifdef USE_FAKE_MEDIA_STREAMS
 #include "FakeMediaStreams.h"
 #else
 #include "DOMMediaStream.h"
 #include "MediaStreamGraph.h"
+#include "MediaStreamVideoSink.h"
 #include "VideoUtils.h"
 #endif
 #include "MediaConduitInterface.h"
 #include "MediaPipelineFilter.h"
 #include "AudioSegment.h"
 #include "mozilla/ReentrantMonitor.h"
 #include "mozilla/Atomics.h"
 #include "SrtpFlow.h"
@@ -386,16 +387,17 @@ public:
                         RefPtr<MediaSessionConduit> conduit,
                         RefPtr<TransportFlow> rtp_transport,
                         RefPtr<TransportFlow> rtcp_transport,
                         nsAutoPtr<MediaPipelineFilter> filter) :
       MediaPipeline(pc, TRANSMIT, main_thread, sts_thread,
                     domstream->GetOwnedStream(), track_id, level,
                     conduit, rtp_transport, rtcp_transport, filter),
       listener_(new PipelineListener(conduit)),
+      video_sink_(new PipelineVideoSink(conduit, listener_)),
       domstream_(domstream),
       is_video_(is_video)
   {}
 
   // Initialize (stuff here may fail)
   virtual nsresult Init() override;
 
   virtual void AttachToTrack(const std::string& track_id);
@@ -415,31 +417,31 @@ public:
 #endif
 
   // Called on the main thread.
   virtual void DetachMediaStream() override {
     ASSERT_ON_THREAD(main_thread_);
     domstream_->RemoveDirectListener(listener_);
     domstream_ = nullptr;
     stream_->RemoveListener(listener_);
+    stream_->RemoveVideoOutput(video_sink_);
     // Let the listener be destroyed with the pipeline (or later).
     stream_ = nullptr;
   }
 
   // Override MediaPipeline::TransportReady.
   virtual nsresult TransportReady_s(TransportInfo &info) override;
 
   // Replace a track with a different one
   // In non-compliance with the likely final spec, allow the new
   // track to be part of a different stream (since we don't support
   // multiple tracks of a type in a stream yet).  bug 1056650
   virtual nsresult ReplaceTrack(DOMMediaStream *domstream,
                                 const std::string& track_id);
 
-
   // Separate class to allow ref counting
   class PipelineListener : public MediaStreamDirectListener {
    friend class MediaPipelineTransmit;
    public:
     explicit PipelineListener(const RefPtr<MediaSessionConduit>& conduit)
       : conduit_(conduit),
         track_id_(TRACK_INVALID),
         mMutex("MediaPipelineTransmit::PipelineListener"),
@@ -479,17 +481,17 @@ public:
     }
 
     // Implement MediaStreamListener
     virtual void NotifyQueuedTrackChanges(MediaStreamGraph* graph, TrackID tid,
                                           StreamTime offset,
                                           uint32_t events,
                                           const MediaSegment& queued_media,
                                           MediaStream* input_stream,
-                                          TrackID input_tid) override;
+                                          TrackID input_tid) override {}
     virtual void NotifyQueuedAudioData(MediaStreamGraph* graph, TrackID tid,
                                        StreamTime offset,
                                        const AudioSegment& queued_media,
                                        MediaStream* input_stream,
                                        TrackID input_tid) override;
     virtual void NotifyPull(MediaStreamGraph* aGraph, StreamTime aDesiredTime) override {}
 
     // Implement MediaStreamDirectListener
@@ -533,18 +535,44 @@ public:
     bool direct_connect_;
 
     nsAutoPtr<AudioPacketizer<int16_t, int16_t>> packetizer_;
 #if !defined(MOZILLA_EXTERNAL_LINKAGE)
     int32_t last_img_; // serial number of last Image
 #endif // MOZILLA_INTERNAL_API
   };
 
+  class PipelineVideoSink : public MediaStreamVideoSink {
+  public:
+    explicit PipelineVideoSink(const RefPtr<MediaSessionConduit>& conduit, PipelineListener* listener)
+      : conduit_(conduit)
+      , pipelineListener_(listener)
+    {
+    }
+
+    virtual void SetCurrentFrames(const VideoSegment& aSegment) override;
+    virtual void ClearFrames() override {}
+
+  private:
+    ~PipelineVideoSink() {
+      // release conduit on mainthread.  Must use forget()!
+      nsresult rv = NS_DispatchToMainThread(new
+        ConduitDeleteEvent(conduit_.forget()));
+      MOZ_ASSERT(!NS_FAILED(rv),"Could not dispatch conduit shutdown to main");
+      if (NS_FAILED(rv)) {
+        MOZ_CRASH();
+      }
+    }
+    RefPtr<MediaSessionConduit> conduit_;
+    PipelineListener* pipelineListener_;
+  };
+
  private:
   RefPtr<PipelineListener> listener_;
+  RefPtr<PipelineVideoSink> video_sink_;
   DOMMediaStream *domstream_;
   bool is_video_;
 };
 
 
 // A specialization of pipeline for reading from the network and
 // rendering video.
 class MediaPipelineReceive : public MediaPipeline {
--- a/media/webrtc/signaling/test/FakeMediaStreams.h
+++ b/media/webrtc/signaling/test/FakeMediaStreams.h
@@ -15,16 +15,17 @@
 #include "nsIComponentManager.h"
 #include "nsIComponentRegistrar.h"
 #include "nsISupportsImpl.h"
 #include "nsServiceManagerUtils.h"
 
 // #includes from MediaStream.h
 #include "mozilla/Mutex.h"
 #include "AudioSegment.h"
+#include "VideoSegment.h"
 #include "MediaSegment.h"
 #include "StreamTracks.h"
 #include "nsTArray.h"
 #include "nsIRunnable.h"
 #include "nsISupportsImpl.h"
 
 class nsIDOMWindow;
 
@@ -106,16 +107,30 @@ protected:
 
 public:
   virtual void NotifyRealtimeData(mozilla::MediaStreamGraph* graph, mozilla::TrackID tid,
                                   mozilla::StreamTime offset,
                                   uint32_t events,
                                   const mozilla::MediaSegment& media) = 0;
 };
 
+class Fake_MediaStreamVideoSink {
+public:
+  NS_INLINE_DECL_THREADSAFE_REFCOUNTING(Fake_MediaStreamVideoSink)
+
+  Fake_MediaStreamVideoSink() {}
+
+  virtual void SetCurrentFrames(const mozilla::VideoSegment& aSegment) {};
+  virtual void ClearFrames() {};
+
+protected:
+  virtual ~Fake_MediaStreamVideoSink() {}
+};
+
+
 // Note: only one listener supported
 class Fake_MediaStream {
  protected:
   virtual ~Fake_MediaStream() { Stop(); }
 
  public:
   Fake_MediaStream () : mListeners(), mMutex("Fake MediaStream") {}
 
@@ -136,16 +151,19 @@ class Fake_MediaStream {
 
     mozilla::MutexAutoLock lock(mMutex);
     std::set<Fake_MediaStreamListener *>::iterator it;
     for (it = mListeners.begin(); it != mListeners.end(); ++it) {
       (*it)->NotifyPull(graph, aDesiredTime);
     }
   }
 
+  void AddVideoOutput(Fake_MediaStreamVideoSink* aSink) {}
+  void RemoveVideoOutput(Fake_MediaStreamVideoSink* aSink) {}
+
   virtual Fake_SourceMediaStream *AsSourceStream() { return nullptr; }
 
   virtual mozilla::MediaStreamGraphImpl *GraphImpl() { return nullptr; }
   virtual nsresult Start() { return NS_OK; }
   virtual nsresult Stop() { return NS_OK; }
   virtual void StopStream() {}
 
   virtual void Periodic() {}
@@ -495,11 +513,12 @@ class Fake_VideoStreamSource : public Fa
 
 namespace mozilla {
 typedef Fake_MediaStream MediaStream;
 typedef Fake_SourceMediaStream SourceMediaStream;
 typedef Fake_MediaStreamListener MediaStreamListener;
 typedef Fake_MediaStreamDirectListener MediaStreamDirectListener;
 typedef Fake_DOMMediaStream DOMMediaStream;
 typedef Fake_DOMMediaStream DOMLocalMediaStream;
+typedef Fake_MediaStreamVideoSink MediaStreamVideoSink;
 }
 
 #endif