Bug 1201363 - Add |MediaStreamListener::NotifyQueuedAudioData| and separate non TRACK_CREATE and TRACK_END part to NotifyQueuedAudioData. r?jesup,r?pehrsons draft
authorctai <ctai@mozilla.com>
Wed, 30 Dec 2015 08:29:06 +0800
changeset 344707 75d9356911b33b14375b3db4681858bded247abe
parent 344706 6d6c11003fa5d5ddfc0daf2d411491863d1a6f81
child 344708 5753e5929d58b48cdb65688018f7e6d1d0fc4f92
push id13910
push userbmo:ctai@mozilla.com
push dateFri, 25 Mar 2016 13:21:09 +0000
reviewersjesup, pehrsons
bugs1201363
milestone46.0a1
Bug 1201363 - Add |MediaStreamListener::NotifyQueuedAudioData| and separate non TRACK_CREATE and TRACK_END part to NotifyQueuedAudioData. r?jesup,r?pehrsons One of the goal of this bug is to simplified NotifyQueuedTrackChanges to notify TRACK_CREATE and TRACK_END event only. This changeset is trying to separate the audio data part first. MozReview-Commit-ID: JtziVEFdlPn
dom/media/MediaStreamGraph.cpp
dom/media/MediaStreamGraph.h
dom/media/TrackUnionStream.cpp
dom/media/encoder/MediaEncoder.cpp
dom/media/encoder/MediaEncoder.h
media/webrtc/signaling/src/mediapipeline/MediaPipeline.cpp
media/webrtc/signaling/src/mediapipeline/MediaPipeline.h
media/webrtc/signaling/test/FakeMediaStreams.h
--- a/dom/media/MediaStreamGraph.cpp
+++ b/dom/media/MediaStreamGraph.cpp
@@ -161,22 +161,43 @@ MediaStreamGraphImpl::ExtractPendingInpu
         }
       }
     }
     finished = aStream->mUpdateFinished;
     bool notifiedTrackCreated = false;
     for (int32_t i = aStream->mUpdateTracks.Length() - 1; i >= 0; --i) {
       SourceMediaStream::TrackData* data = &aStream->mUpdateTracks[i];
       aStream->ApplyTrackDisabling(data->mID, data->mData);
-      for (MediaStreamListener* l : aStream->mListeners) {
-        StreamTime offset = (data->mCommands & SourceMediaStream::TRACK_CREATE)
-            ? data->mStart : aStream->mTracks.FindTrack(data->mID)->GetSegment()->GetDuration();
-        l->NotifyQueuedTrackChanges(this, data->mID,
-                                    offset, data->mCommands, *data->mData);
+      // Dealing with NotifyQueuedTrackChanges and NotifyQueuedAudioData part.
+      // The logic is different from the manipulating of aStream->mTracks part.
+      // So it is not combined with the manipulating of aStream->mTracks part.
+      StreamTime offset = (data->mCommands & SourceMediaStream::TRACK_CREATE)
+        ? data->mStart : aStream->mTracks.FindTrack(data->mID)->GetSegment()->GetDuration();
+      if ((data->mCommands & SourceMediaStream::TRACK_CREATE) ||
+          (data->mCommands & SourceMediaStream::TRACK_END)) {
+        for (MediaStreamListener* l : aStream->mListeners) {
+            l->NotifyQueuedTrackChanges(this, data->mID,
+                                        offset, data->mCommands, *data->mData);
+        }
+      } else if (data->mData->GetType() == MediaSegment::AUDIO) {
+        for (MediaStreamListener* l : aStream->mListeners) {
+            l->NotifyQueuedAudioData(this, data->mID,
+                                     offset, *(static_cast<AudioSegment*>(data->mData.get())));
+        }
+      } else {
+        // Fixme: This part will be removed in the later changeset of this bug.
+        // It will be removed in changeset "Do not copy video segment to StreamTracks in TrackUnionStream."
+
+        // Dealing with video and not TRACK_CREATE and TRACK_END case.
+        for (MediaStreamListener* l : aStream->mListeners) {
+          l->NotifyQueuedTrackChanges(this, data->mID,
+                                      offset, data->mCommands, *data->mData);
+        }
       }
+
       if (data->mCommands & SourceMediaStream::TRACK_CREATE) {
         MediaSegment* segment = data->mData.forget();
         STREAM_LOG(LogLevel::Debug, ("SourceMediaStream %p creating track %d, start %lld, initial end %lld",
                                   aStream, data->mID, int64_t(data->mStart),
                                   int64_t(segment->GetDuration())));
 
         data->mEndOfFlushedData += segment->GetDuration();
         aStream->mTracks.AddTrack(data->mID, data->mStart, segment);
--- a/dom/media/MediaStreamGraph.h
+++ b/dom/media/MediaStreamGraph.h
@@ -171,16 +171,25 @@ public:
   virtual void NotifyQueuedTrackChanges(MediaStreamGraph* aGraph, TrackID aID,
                                         StreamTime aTrackOffset,
                                         uint32_t aTrackEvents,
                                         const MediaSegment& aQueuedMedia,
                                         MediaStream* aInputStream = nullptr,
                                         TrackID aInputTrackID = TRACK_INVALID) {}
 
   /**
+   * Notify queued audio data.
+   */
+  virtual void NotifyQueuedAudioData(MediaStreamGraph* aGraph, TrackID aID,
+                                     StreamTime aTrackOffset,
+                                     const AudioSegment& aQueuedMedia,
+                                     MediaStream* aInputStream = nullptr,
+                                     TrackID aInputTrackID = TRACK_INVALID) {}
+
+  /**
    * Notify that all new tracks this iteration have been created.
    * This is to ensure that tracks added atomically to MediaStreamGraph
    * are also notified of atomically to MediaStreamListeners.
    */
   virtual void NotifyFinishedTrackCreation(MediaStreamGraph* aGraph) {}
 };
 
 /**
--- a/dom/media/TrackUnionStream.cpp
+++ b/dom/media/TrackUnionStream.cpp
@@ -270,15 +270,21 @@ TrackUnionStream::TrackUnionStream(DOMMe
           segment->AppendSlice(*aInputTrack->GetSegment(),
                                std::min(inputTrackEndPoint, inputStart),
                                std::min(inputTrackEndPoint, inputEnd));
         }
       }
       ApplyTrackDisabling(outputTrack->GetID(), segment);
       for (uint32_t j = 0; j < mListeners.Length(); ++j) {
         MediaStreamListener* l = mListeners[j];
-        l->NotifyQueuedTrackChanges(Graph(), outputTrack->GetID(),
-                                    outputStart, 0, *segment);
+        // Separate Audio and Video.
+        if (segment->GetType() == MediaSegment::AUDIO) {
+          l->NotifyQueuedAudioData(Graph(), outputTrack->GetID(),
+                                   outputStart, *static_cast<AudioSegment*>(segment));
+        } else {
+          l->NotifyQueuedTrackChanges(Graph(), outputTrack->GetID(),
+                                      outputStart, 0, *segment);
+        }
       }
       outputTrack->GetSegment()->AppendFrom(segment);
     }
   }
 } // namespace mozilla
--- a/dom/media/encoder/MediaEncoder.cpp
+++ b/dom/media/encoder/MediaEncoder.cpp
@@ -48,23 +48,35 @@ MediaEncoder::NotifyQueuedTrackChanges(M
   // Process the incoming raw track data from MediaStreamGraph, called on the
   // thread of MediaStreamGraph.
   if (mAudioEncoder && aQueuedMedia.GetType() == MediaSegment::AUDIO) {
     mAudioEncoder->NotifyQueuedTrackChanges(aGraph, aID,
                                             aTrackOffset, aTrackEvents,
                                             aQueuedMedia);
 
   } else if (mVideoEncoder && aQueuedMedia.GetType() == MediaSegment::VIDEO) {
-      mVideoEncoder->NotifyQueuedTrackChanges(aGraph, aID,
-                                              aTrackOffset, aTrackEvents,
-                                              aQueuedMedia);
+    mVideoEncoder->NotifyQueuedTrackChanges(aGraph, aID,
+                                            aTrackOffset, aTrackEvents,
+                                            aQueuedMedia);
   }
 }
 
 void
+MediaEncoder::NotifyQueuedAudioData(MediaStreamGraph* aGraph, TrackID aID,
+                                    StreamTime aTrackOffset,
+                                    const AudioSegment& aQueuedMedia,
+                                    MediaStream* aInputStream,
+                                    TrackID aInputTrackID)
+{
+  mAudioEncoder->NotifyQueuedTrackChanges(aGraph, aID,
+                                          aTrackOffset, 0,
+                                          aQueuedMedia);
+}
+
+void
 MediaEncoder::NotifyEvent(MediaStreamGraph* aGraph,
                           MediaStreamListener::MediaStreamGraphEvent event)
 {
   // In case that MediaEncoder does not receive a TRACK_EVENT_ENDED event.
   LOG(LogLevel::Debug, ("NotifyRemoved in [MediaEncoder]."));
   if (mAudioEncoder) {
     mAudioEncoder->NotifyEvent(aGraph, event);
   }
--- a/dom/media/encoder/MediaEncoder.h
+++ b/dom/media/encoder/MediaEncoder.h
@@ -85,16 +85,26 @@ public :
   void NotifyQueuedTrackChanges(MediaStreamGraph* aGraph, TrackID aID,
                                 StreamTime aTrackOffset,
                                 uint32_t aTrackEvents,
                                 const MediaSegment& aQueuedMedia,
                                 MediaStream* aInputStream,
                                 TrackID aInputTrackID) override;
 
   /**
+   * Notifed by the control loop of MediaStreamGraph; aQueueMedia is the raw
+   * track data in form of AudioSegment.
+   */
+  virtual void NotifyQueuedAudioData(MediaStreamGraph* aGraph, TrackID aID,
+                                     StreamTime aTrackOffset,
+                                     const AudioSegment& aQueuedMedia,
+                                     MediaStream* aInputStream = nullptr,
+                                     TrackID aInputTrackID = TRACK_INVALID) override;
+
+  /**
    * Notified the stream is being removed.
    */
   void NotifyEvent(MediaStreamGraph* aGraph,
                    MediaStreamListener::MediaStreamGraphEvent event) override;
 
   /**
    * Creates an encoder with a given MIME type. Returns null if we are unable
    * to create the encoder. For now, default aMIMEType to "audio/ogg" and use
--- a/media/webrtc/signaling/src/mediapipeline/MediaPipeline.cpp
+++ b/media/webrtc/signaling/src/mediapipeline/MediaPipeline.cpp
@@ -870,26 +870,37 @@ NotifyRealtimeData(MediaStreamGraph* gra
   NewData(graph, tid, offset, events, media);
 }
 
 void MediaPipelineTransmit::PipelineListener::
 NotifyQueuedTrackChanges(MediaStreamGraph* graph, TrackID tid,
                          StreamTime offset,
                          uint32_t events,
                          const MediaSegment& queued_media,
-                         MediaStream* aInputStream,
-                         TrackID aInputTrackID) {
+                         MediaStream* input_stream,
+                         TrackID input_tid) {
   MOZ_MTLOG(ML_DEBUG, "MediaPipeline::NotifyQueuedTrackChanges()");
 
   // ignore non-direct data if we're also getting direct data
   if (!direct_connect_) {
     NewData(graph, tid, offset, events, queued_media);
   }
 }
 
+void MediaPipelineTransmit::PipelineListener::
+NotifyQueuedAudioData(MediaStreamGraph* graph, TrackID tid,
+                      StreamTime offset,
+                      const AudioSegment& queued_media,
+                      MediaStream* input_stream,
+                      TrackID input_tid) {
+  if (!direct_connect_) {
+    NewData(graph, tid, offset, 0, queued_media);
+  }
+}
+
 // I420 buffer size macros
 #define YSIZE(x,y) ((x)*(y))
 #define CRSIZE(x,y) ((((x)+1) >> 1) * (((y)+1) >> 1))
 #define I420SIZE(x,y) (YSIZE((x),(y)) + 2 * CRSIZE((x),(y)))
 
 // XXX NOTE: this code will have to change when we get support for multiple tracks of type
 // in a MediaStream and especially in a PeerConnection stream.  bug 1056650
 // It should be matching on the "correct" track for the pipeline, not just "any video track".
--- a/media/webrtc/signaling/src/mediapipeline/MediaPipeline.h
+++ b/media/webrtc/signaling/src/mediapipeline/MediaPipeline.h
@@ -480,16 +480,21 @@ public:
 
     // Implement MediaStreamListener
     virtual void NotifyQueuedTrackChanges(MediaStreamGraph* graph, TrackID tid,
                                           StreamTime offset,
                                           uint32_t events,
                                           const MediaSegment& queued_media,
                                           MediaStream* input_stream,
                                           TrackID input_tid) override;
+    virtual void NotifyQueuedAudioData(MediaStreamGraph* graph, TrackID tid,
+                                       StreamTime offset,
+                                       const AudioSegment& queued_media,
+                                       MediaStream* input_stream,
+                                       TrackID input_tid) override;
     virtual void NotifyPull(MediaStreamGraph* aGraph, StreamTime aDesiredTime) override {}
 
     // Implement MediaStreamDirectListener
     virtual void NotifyRealtimeData(MediaStreamGraph* graph, TrackID tid,
                                     StreamTime offset,
                                     uint32_t events,
                                     const MediaSegment& media) override;
 
@@ -630,17 +635,21 @@ class MediaPipelineReceiveAudio : public
     // Implement MediaStreamListener
     virtual void NotifyQueuedTrackChanges(MediaStreamGraph* graph, TrackID tid,
                                           StreamTime offset,
                                           uint32_t events,
                                           const MediaSegment& queued_media,
                                           MediaStream* input_stream,
                                           TrackID input_tid) override {}
     virtual void NotifyPull(MediaStreamGraph* graph, StreamTime desired_time) override;
-
+    virtual void NotifyQueuedAudioData(mozilla::MediaStreamGraph* aGraph, mozilla::TrackID aID,
+                                       mozilla::StreamTime aTrackOffset,
+                                       const mozilla::AudioSegment& aQueuedMedia,
+                                       MediaStream* aInputStream,
+                                       mozilla::TrackID aInputTrackID) override {}
    private:
     RefPtr<MediaSessionConduit> conduit_;
   };
 
   RefPtr<PipelineListener> listener_;
 };
 
 
@@ -741,16 +750,21 @@ class MediaPipelineReceiveVideo : public
     // Implement MediaStreamListener
     virtual void NotifyQueuedTrackChanges(MediaStreamGraph* graph, TrackID tid,
                                           StreamTime offset,
                                           uint32_t events,
                                           const MediaSegment& queued_media,
                                           MediaStream* input_stream,
                                           TrackID input_tid) override {}
     virtual void NotifyPull(MediaStreamGraph* graph, StreamTime desired_time) override;
+    virtual void NotifyQueuedAudioData(mozilla::MediaStreamGraph* aGraph, mozilla::TrackID aID,
+                                       mozilla::StreamTime aTrackOffset,
+                                       const mozilla::AudioSegment& aQueuedMedia,
+                                       MediaStream* aInputStream,
+                                       mozilla::TrackID aInputTrackID) override {}
 
     // Accessors for external writes from the renderer
     void FrameSizeChange(unsigned int width,
                          unsigned int height,
                          unsigned int number_of_streams) {
       ReentrantMonitorAutoEnter enter(monitor_);
 
       width_ = width;
--- a/media/webrtc/signaling/test/FakeMediaStreams.h
+++ b/media/webrtc/signaling/test/FakeMediaStreams.h
@@ -85,16 +85,21 @@ protected:
 public:
   virtual void NotifyQueuedTrackChanges(mozilla::MediaStreamGraph* aGraph, mozilla::TrackID aID,
                                         mozilla::StreamTime aTrackOffset,
                                         uint32_t aTrackEvents,
                                         const mozilla::MediaSegment& aQueuedMedia,
                                         Fake_MediaStream* aInputStream,
                                         mozilla::TrackID aInputTrackID) = 0;
   virtual void NotifyPull(mozilla::MediaStreamGraph* aGraph, mozilla::StreamTime aDesiredTime) = 0;
+  virtual void NotifyQueuedAudioData(mozilla::MediaStreamGraph* aGraph, mozilla::TrackID aID,
+                                     mozilla::StreamTime aTrackOffset,
+                                     const mozilla::AudioSegment& aQueuedMedia,
+                                     Fake_MediaStream* aInputStream,
+                                     mozilla::TrackID aInputTrackID) = 0;
 
   NS_INLINE_DECL_THREADSAFE_REFCOUNTING(Fake_MediaStreamListener)
 };
 
 class Fake_MediaStreamDirectListener : public Fake_MediaStreamListener
 {
 protected:
   virtual ~Fake_MediaStreamDirectListener() {}