Bug 1266647 - Clean NotifyQueuedTrackChange to only notify when command is track create and track end. r=jesup,pehrsons draft
authorctai <ctai@mozilla.com>
Wed, 04 May 2016 16:08:44 +0800
changeset 369088 d4631350825904fa0e9d9c9791aa48fc49c3755f
parent 369022 c67dc1f9fab86d4f2cf3224307809c44fe3ce820
child 521458 50cc0bc94e6a7cb052a32c94fb5b74ddccb81c09
push id18733
push userbmo:ctai@mozilla.com
push dateFri, 20 May 2016 08:08:55 +0000
reviewersjesup, pehrsons
bugs1266647
milestone49.0a1
Bug 1266647 - Clean NotifyQueuedTrackChange to only notify when command is track create and track end. r=jesup,pehrsons MozReview-Commit-ID: 4Pp3xHeBIW4
dom/media/MediaStreamGraph.cpp
dom/media/MediaStreamGraph.h
dom/media/TrackUnionStream.cpp
dom/media/encoder/MediaEncoder.cpp
dom/media/encoder/MediaEncoder.h
dom/media/webspeech/recognition/SpeechStreamListener.cpp
dom/media/webspeech/recognition/SpeechStreamListener.h
media/webrtc/signaling/src/mediapipeline/MediaPipeline.cpp
media/webrtc/signaling/test/FakeMediaStreams.h
--- a/dom/media/MediaStreamGraph.cpp
+++ b/dom/media/MediaStreamGraph.cpp
@@ -180,22 +180,68 @@ MediaStreamGraphImpl::ExtractPendingInpu
         }
       }
     }
     finished = aStream->mUpdateFinished;
     bool shouldNotifyTrackCreated = false;
     for (int32_t i = aStream->mUpdateTracks.Length() - 1; i >= 0; --i) {
       SourceMediaStream::TrackData* data = &aStream->mUpdateTracks[i];
       aStream->ApplyTrackDisabling(data->mID, data->mData);
+      // Dealing with NotifyQueuedTrackChanges and NotifyQueuedAudioData part.
+
+      // The logic is different from the manipulating of aStream->mTracks part.
+      // So it is not combined with the manipulating of aStream->mTracks part.
       StreamTime offset = (data->mCommands & SourceMediaStream::TRACK_CREATE)
           ? data->mStart : aStream->mTracks.FindTrack(data->mID)->GetSegment()->GetDuration();
-      for (MediaStreamListener* l : aStream->mListeners) {
-        l->NotifyQueuedTrackChanges(this, data->mID,
-                                    offset, data->mCommands, *data->mData);
+
+      // Audio case.
+      if (data->mData->GetType() == MediaSegment::AUDIO) {
+        if (data->mCommands) {
+          MOZ_ASSERT(!(data->mCommands & SourceMediaStream::TRACK_UNUSED));
+          for (MediaStreamListener* l : aStream->mListeners) {
+            if (data->mCommands & SourceMediaStream::TRACK_END) {
+              l->NotifyQueuedAudioData(this, data->mID,
+                                       offset, *(static_cast<AudioSegment*>(data->mData.get())));
+            }
+            l->NotifyQueuedTrackChanges(this, data->mID,
+                                        offset, data->mCommands, *data->mData);
+            if (data->mCommands & SourceMediaStream::TRACK_CREATE) {
+              l->NotifyQueuedAudioData(this, data->mID,
+                                       offset, *(static_cast<AudioSegment*>(data->mData.get())));
+            }
+          }
+        } else {
+          for (MediaStreamListener* l : aStream->mListeners) {
+              l->NotifyQueuedAudioData(this, data->mID,
+                                       offset, *(static_cast<AudioSegment*>(data->mData.get())));
+          }
+        }
       }
+
+      // Video case.
+      if (data->mData->GetType() == MediaSegment::VIDEO) {
+        if (data->mCommands) {
+          MOZ_ASSERT(!(data->mCommands & SourceMediaStream::TRACK_UNUSED));
+          for (MediaStreamListener* l : aStream->mListeners) {
+            l->NotifyQueuedTrackChanges(this, data->mID,
+                                        offset, data->mCommands, *data->mData);
+          }
+        } else {
+          // Fixme: This part will be removed in the bug 1201363. It will be
+          // removed in changeset "Do not copy video segment to StreamTracks in
+          // TrackUnionStream."
+
+          // Dealing with video and not TRACK_CREATE and TRACK_END case.
+          for (MediaStreamListener* l : aStream->mListeners) {
+            l->NotifyQueuedTrackChanges(this, data->mID,
+                                        offset, data->mCommands, *data->mData);
+          }
+        }
+      }
+
       for (TrackBound<MediaStreamTrackListener>& b : aStream->mTrackListeners) {
         if (b.mTrackID != data->mID) {
           continue;
         }
         b.mListener->NotifyQueuedChanges(this, offset, *data->mData);
         if (data->mCommands & SourceMediaStream::TRACK_END) {
           b.mListener->NotifyEnded();
         }
--- a/dom/media/MediaStreamGraph.h
+++ b/dom/media/MediaStreamGraph.h
@@ -154,19 +154,21 @@ public:
     EVENT_HAS_NO_DIRECT_LISTENERS,  // transition to no direct listeners
   };
 
   /**
    * Notify that an event has occurred on the Stream
    */
   virtual void NotifyEvent(MediaStreamGraph* aGraph, MediaStreamGraphEvent aEvent) {}
 
+  // maskable flags, not a simple enumerated value
   enum {
     TRACK_EVENT_CREATED = 0x01,
-    TRACK_EVENT_ENDED = 0x02
+    TRACK_EVENT_ENDED = 0x02,
+    TRACK_EVENT_UNUSED = ~(TRACK_EVENT_ENDED | TRACK_EVENT_CREATED),
   };
   /**
    * Notify that changes to one of the stream tracks have been queued.
    * aTrackEvents can be any combination of TRACK_EVENT_CREATED and
    * TRACK_EVENT_ENDED. aQueuedMedia is the data being added to the track
    * at aTrackOffset (relative to the start of the stream).
    * aInputStream and aInputTrackID will be set if the changes originated
    * from an input stream's track. In practice they will only be used for
@@ -175,16 +177,26 @@ public:
   virtual void NotifyQueuedTrackChanges(MediaStreamGraph* aGraph, TrackID aID,
                                         StreamTime aTrackOffset,
                                         uint32_t aTrackEvents,
                                         const MediaSegment& aQueuedMedia,
                                         MediaStream* aInputStream = nullptr,
                                         TrackID aInputTrackID = TRACK_INVALID) {}
 
   /**
+   * Notify queued audio data. Only audio data need to be queued. The video data
+   * will be notified by MediaStreamVideoSink::SetCurrentFrame.
+   */
+  virtual void NotifyQueuedAudioData(MediaStreamGraph* aGraph, TrackID aID,
+                                     StreamTime aTrackOffset,
+                                     const AudioSegment& aQueuedMedia,
+                                     MediaStream* aInputStream = nullptr,
+                                     TrackID aInputTrackID = TRACK_INVALID) {}
+
+  /**
    * Notify that all new tracks this iteration have been created.
    * This is to ensure that tracks added atomically to MediaStreamGraph
    * are also notified of atomically to MediaStreamListeners.
    */
   virtual void NotifyFinishedTrackCreation(MediaStreamGraph* aGraph) {}
 };
 
 class AudioDataListenerInterface {
@@ -1074,17 +1086,18 @@ public:
 
   // XXX need a Reset API
 
   friend class MediaStreamGraphImpl;
 
 protected:
   enum TrackCommands {
     TRACK_CREATE = MediaStreamListener::TRACK_EVENT_CREATED,
-    TRACK_END = MediaStreamListener::TRACK_EVENT_ENDED
+    TRACK_END = MediaStreamListener::TRACK_EVENT_ENDED,
+    TRACK_UNUSED = MediaStreamListener::TRACK_EVENT_UNUSED,
   };
   /**
    * Data for each track that hasn't ended.
    */
   struct TrackData {
     TrackID mID;
     // Sample rate of the input data.
     TrackRate mInputRate;
--- a/dom/media/TrackUnionStream.cpp
+++ b/dom/media/TrackUnionStream.cpp
@@ -319,20 +319,27 @@ TrackUnionStream::TrackUnionStream(DOMMe
           segment->AppendSlice(*aInputTrack->GetSegment(),
                                std::min(inputTrackEndPoint, inputStart),
                                std::min(inputTrackEndPoint, inputEnd));
         }
       }
       ApplyTrackDisabling(outputTrack->GetID(), segment);
       for (uint32_t j = 0; j < mListeners.Length(); ++j) {
         MediaStreamListener* l = mListeners[j];
-        l->NotifyQueuedTrackChanges(Graph(), outputTrack->GetID(),
-                                    outputStart, 0, *segment,
-                                    map->mInputPort->GetSource(),
-                                    map->mInputTrackID);
+        // Separate Audio and Video.
+        if (segment->GetType() == MediaSegment::AUDIO) {
+          l->NotifyQueuedAudioData(Graph(), outputTrack->GetID(),
+                                   outputStart, *static_cast<AudioSegment*>(segment));
+        } else {
+          // This part will be removed in bug 1201363.
+          l->NotifyQueuedTrackChanges(Graph(), outputTrack->GetID(),
+                                      outputStart, 0, *segment,
+                                      map->mInputPort->GetSource(),
+                                      map->mInputTrackID);
+        }
       }
       for (TrackBound<MediaStreamTrackListener>& b : mTrackListeners) {
         if (b.mTrackID != outputTrack->GetID()) {
           continue;
         }
         b.mListener->NotifyQueuedChanges(Graph(), outputStart, *segment);
       }
       outputTrack->GetSegment()->AppendFrom(segment);
--- a/dom/media/encoder/MediaEncoder.cpp
+++ b/dom/media/encoder/MediaEncoder.cpp
@@ -101,16 +101,27 @@ MediaEncoder::NotifyQueuedTrackChanges(M
       } else {
         mSuspended = RECORD_NOT_SUSPENDED; // no video
       }
     }
   }
 }
 
 void
+MediaEncoder::NotifyQueuedAudioData(MediaStreamGraph* aGraph, TrackID aID,
+                                    StreamTime aTrackOffset,
+                                    const AudioSegment& aQueuedMedia,
+                                    MediaStream* aInputStream,
+                                    TrackID aInputTrackID)
+{
+  mAudioEncoder->NotifyQueuedTrackChanges(aGraph, aID, aTrackOffset, 0,
+                                          aQueuedMedia);
+}
+
+void
 MediaEncoder::NotifyEvent(MediaStreamGraph* aGraph,
                           MediaStreamListener::MediaStreamGraphEvent event)
 {
   // In case that MediaEncoder does not receive a TRACK_EVENT_ENDED event.
   LOG(LogLevel::Debug, ("NotifyRemoved in [MediaEncoder]."));
   if (mAudioEncoder) {
     mAudioEncoder->NotifyEvent(aGraph, event);
   }
--- a/dom/media/encoder/MediaEncoder.h
+++ b/dom/media/encoder/MediaEncoder.h
@@ -125,17 +125,27 @@ public :
   void NotifyQueuedTrackChanges(MediaStreamGraph* aGraph, TrackID aID,
                                 StreamTime aTrackOffset,
                                 uint32_t aTrackEvents,
                                 const MediaSegment& aQueuedMedia,
                                 MediaStream* aInputStream,
                                 TrackID aInputTrackID) override;
 
   /**
-   * Notified the stream is being removed.
+   * Notifed by the control loop of MediaStreamGraph; aQueueMedia is the audio
+   * data in the form of an AudioSegment.
+   */
+  void NotifyQueuedAudioData(MediaStreamGraph* aGraph, TrackID aID,
+                             StreamTime aTrackOffset,
+                             const AudioSegment& aQueuedMedia,
+                             MediaStream* aInputStream,
+                             TrackID aInputTrackID) override;
+
+  /**
+   * * Notified the stream is being removed.
    */
   void NotifyEvent(MediaStreamGraph* aGraph,
                    MediaStreamListener::MediaStreamGraphEvent event) override;
 
   /**
    * Creates an encoder with a given MIME type. Returns null if we are unable
    * to create the encoder. For now, default aMIMEType to "audio/ogg" and use
    * Ogg+Opus if it is empty.
--- a/dom/media/webspeech/recognition/SpeechStreamListener.cpp
+++ b/dom/media/webspeech/recognition/SpeechStreamListener.cpp
@@ -21,23 +21,21 @@ SpeechStreamListener::~SpeechStreamListe
 {
   nsCOMPtr<nsIThread> mainThread;
   NS_GetMainThread(getter_AddRefs(mainThread));
 
   NS_ProxyRelease(mainThread, mRecognition.forget());
 }
 
 void
-SpeechStreamListener::NotifyQueuedTrackChanges(MediaStreamGraph* aGraph,
-                                               TrackID aID,
-                                               StreamTime aTrackOffset,
-                                               uint32_t aTrackEvents,
-                                               const MediaSegment& aQueuedMedia,
-                                               MediaStream* aInputStream,
-                                               TrackID aInputTrackID)
+SpeechStreamListener::NotifyQueuedAudioData(MediaStreamGraph* aGraph, TrackID aID,
+                                            StreamTime aTrackOffset,
+                                            const AudioSegment& aQueuedMedia,
+                                            MediaStream* aInputStream,
+                                            TrackID aInputTrackID)
 {
   AudioSegment* audio = const_cast<AudioSegment*>(
     static_cast<const AudioSegment*>(&aQueuedMedia));
 
   AudioSegment::ChunkIterator iterator(*audio);
   while (!iterator.IsEnded()) {
     // Skip over-large chunks so we don't crash!
     if (iterator->GetDuration() > INT_MAX) {
--- a/dom/media/webspeech/recognition/SpeechStreamListener.h
+++ b/dom/media/webspeech/recognition/SpeechStreamListener.h
@@ -19,22 +19,21 @@ namespace dom {
 class SpeechRecognition;
 
 class SpeechStreamListener : public MediaStreamListener
 {
 public:
   explicit SpeechStreamListener(SpeechRecognition* aRecognition);
   ~SpeechStreamListener();
 
-  void NotifyQueuedTrackChanges(MediaStreamGraph* aGraph, TrackID aID,
-                                StreamTime aTrackOffset,
-                                uint32_t aTrackEvents,
-                                const MediaSegment& aQueuedMedia,
-                                MediaStream* aInputStream,
-                                TrackID aInputTrackID) override;
+  void NotifyQueuedAudioData(MediaStreamGraph* aGraph, TrackID aID,
+                             StreamTime aTrackOffset,
+                             const AudioSegment& aQueuedMedia,
+                             MediaStream* aInputStream,
+                             TrackID aInputTrackID) override;
 
   void NotifyEvent(MediaStreamGraph* aGraph,
                    MediaStreamListener::MediaStreamGraphEvent event) override;
 
 private:
   template<typename SampleFormatType>
   void ConvertAndDispatchAudioChunk(int aDuration, float aVolume, SampleFormatType* aData, TrackRate aTrackRate);
   RefPtr<SpeechRecognition> mRecognition;
--- a/media/webrtc/signaling/src/mediapipeline/MediaPipeline.cpp
+++ b/media/webrtc/signaling/src/mediapipeline/MediaPipeline.cpp
@@ -1958,23 +1958,16 @@ public:
         MOZ_CRASH();
       }
     } else {
       conduit_ = nullptr;
     }
   }
 
   // Implement MediaStreamListener
-  void NotifyQueuedTrackChanges(MediaStreamGraph* graph, TrackID tid,
-                                StreamTime offset,
-                                uint32_t events,
-                                const MediaSegment& queued_media,
-                                MediaStream* input_stream,
-                                TrackID input_tid) override {}
-
   void NotifyPull(MediaStreamGraph* graph, StreamTime desired_time) override
   {
     MOZ_ASSERT(source_);
     if (!source_) {
       MOZ_MTLOG(ML_ERROR, "NotifyPull() called from a non-SourceMediaStream");
       return;
     }
 
@@ -2121,24 +2114,17 @@ public:
       monitor_("Video PipelineListener")
   {
 #if !defined(MOZILLA_EXTERNAL_LINKAGE)
     image_container_ =
       LayerManager::CreateImageContainer(ImageContainer::ASYNCHRONOUS);
 #endif
   }
 
-
   // Implement MediaStreamListener
-  void NotifyQueuedTrackChanges(MediaStreamGraph* graph, TrackID tid,
-                                StreamTime offset,
-                                uint32_t events,
-                                const MediaSegment& queued_media,
-                                MediaStream* input_stream,
-                                TrackID input_tid) override {}
   void NotifyPull(MediaStreamGraph* graph, StreamTime desired_time) override
   {
     ReentrantMonitorAutoEnter enter(monitor_);
 
   #if defined(MOZILLA_INTERNAL_API)
     RefPtr<Image> image = image_;
     // our constructor sets track_rate_ to the graph rate
     MOZ_ASSERT(track_rate_ == source_->GraphRate());
--- a/media/webrtc/signaling/test/FakeMediaStreams.h
+++ b/media/webrtc/signaling/test/FakeMediaStreams.h
@@ -87,18 +87,23 @@ protected:
   virtual ~Fake_MediaStreamListener() {}
 
 public:
   virtual void NotifyQueuedTrackChanges(mozilla::MediaStreamGraph* aGraph, mozilla::TrackID aID,
                                         mozilla::StreamTime aTrackOffset,
                                         uint32_t aTrackEvents,
                                         const mozilla::MediaSegment& aQueuedMedia,
                                         Fake_MediaStream* aInputStream,
-                                        mozilla::TrackID aInputTrackID) = 0;
+                                        mozilla::TrackID aInputTrackID) {}
   virtual void NotifyPull(mozilla::MediaStreamGraph* aGraph, mozilla::StreamTime aDesiredTime) = 0;
+  virtual void NotifyQueuedAudioData(mozilla::MediaStreamGraph* aGraph, mozilla::TrackID aID,
+                                       mozilla::StreamTime aTrackOffset,
+                                       const mozilla::AudioSegment& aQueuedMedia,
+                                       Fake_MediaStream* aInputStream,
+                                       mozilla::TrackID aInputTrackID) {}
 
   NS_INLINE_DECL_THREADSAFE_REFCOUNTING(Fake_MediaStreamListener)
 };
 
 class Fake_MediaStreamDirectListener : public Fake_MediaStreamListener
 {
 protected:
   virtual ~Fake_MediaStreamDirectListener() {}