Bug 1266647 - Clean NotifyQueuedTrackChange to only notify when command is track create and track end. r=jesup,pehrsons
MozReview-Commit-ID: 4Pp3xHeBIW4
--- a/dom/media/MediaStreamGraph.cpp
+++ b/dom/media/MediaStreamGraph.cpp
@@ -180,22 +180,68 @@ MediaStreamGraphImpl::ExtractPendingInpu
}
}
}
finished = aStream->mUpdateFinished;
bool shouldNotifyTrackCreated = false;
for (int32_t i = aStream->mUpdateTracks.Length() - 1; i >= 0; --i) {
SourceMediaStream::TrackData* data = &aStream->mUpdateTracks[i];
aStream->ApplyTrackDisabling(data->mID, data->mData);
+ // Dealing with NotifyQueuedTrackChanges and NotifyQueuedAudioData part.
+
+ // The logic is different from the manipulating of aStream->mTracks part.
+ // So it is not combined with the manipulating of aStream->mTracks part.
StreamTime offset = (data->mCommands & SourceMediaStream::TRACK_CREATE)
? data->mStart : aStream->mTracks.FindTrack(data->mID)->GetSegment()->GetDuration();
- for (MediaStreamListener* l : aStream->mListeners) {
- l->NotifyQueuedTrackChanges(this, data->mID,
- offset, data->mCommands, *data->mData);
+
+ // Audio case.
+ if (data->mData->GetType() == MediaSegment::AUDIO) {
+ if (data->mCommands) {
+ MOZ_ASSERT(!(data->mCommands & SourceMediaStream::TRACK_UNUSED));
+ for (MediaStreamListener* l : aStream->mListeners) {
+ if (data->mCommands & SourceMediaStream::TRACK_END) {
+ l->NotifyQueuedAudioData(this, data->mID,
+ offset, *(static_cast<AudioSegment*>(data->mData.get())));
+ }
+ l->NotifyQueuedTrackChanges(this, data->mID,
+ offset, data->mCommands, *data->mData);
+ if (data->mCommands & SourceMediaStream::TRACK_CREATE) {
+ l->NotifyQueuedAudioData(this, data->mID,
+ offset, *(static_cast<AudioSegment*>(data->mData.get())));
+ }
+ }
+ } else {
+ for (MediaStreamListener* l : aStream->mListeners) {
+ l->NotifyQueuedAudioData(this, data->mID,
+ offset, *(static_cast<AudioSegment*>(data->mData.get())));
+ }
+ }
}
+
+ // Video case.
+ if (data->mData->GetType() == MediaSegment::VIDEO) {
+ if (data->mCommands) {
+ MOZ_ASSERT(!(data->mCommands & SourceMediaStream::TRACK_UNUSED));
+ for (MediaStreamListener* l : aStream->mListeners) {
+ l->NotifyQueuedTrackChanges(this, data->mID,
+ offset, data->mCommands, *data->mData);
+ }
+ } else {
+ // Fixme: This part will be removed in the bug 1201363. It will be
+ // removed in changeset "Do not copy video segment to StreamTracks in
+ // TrackUnionStream."
+
+ // Dealing with video and not TRACK_CREATE and TRACK_END case.
+ for (MediaStreamListener* l : aStream->mListeners) {
+ l->NotifyQueuedTrackChanges(this, data->mID,
+ offset, data->mCommands, *data->mData);
+ }
+ }
+ }
+
for (TrackBound<MediaStreamTrackListener>& b : aStream->mTrackListeners) {
if (b.mTrackID != data->mID) {
continue;
}
b.mListener->NotifyQueuedChanges(this, offset, *data->mData);
if (data->mCommands & SourceMediaStream::TRACK_END) {
b.mListener->NotifyEnded();
}
--- a/dom/media/MediaStreamGraph.h
+++ b/dom/media/MediaStreamGraph.h
@@ -154,19 +154,21 @@ public:
EVENT_HAS_NO_DIRECT_LISTENERS, // transition to no direct listeners
};
/**
* Notify that an event has occurred on the Stream
*/
virtual void NotifyEvent(MediaStreamGraph* aGraph, MediaStreamGraphEvent aEvent) {}
+ // maskable flags, not a simple enumerated value
enum {
TRACK_EVENT_CREATED = 0x01,
- TRACK_EVENT_ENDED = 0x02
+ TRACK_EVENT_ENDED = 0x02,
+ TRACK_EVENT_UNUSED = ~(TRACK_EVENT_ENDED | TRACK_EVENT_CREATED),
};
/**
* Notify that changes to one of the stream tracks have been queued.
* aTrackEvents can be any combination of TRACK_EVENT_CREATED and
* TRACK_EVENT_ENDED. aQueuedMedia is the data being added to the track
* at aTrackOffset (relative to the start of the stream).
* aInputStream and aInputTrackID will be set if the changes originated
* from an input stream's track. In practice they will only be used for
@@ -175,16 +177,26 @@ public:
virtual void NotifyQueuedTrackChanges(MediaStreamGraph* aGraph, TrackID aID,
StreamTime aTrackOffset,
uint32_t aTrackEvents,
const MediaSegment& aQueuedMedia,
MediaStream* aInputStream = nullptr,
TrackID aInputTrackID = TRACK_INVALID) {}
/**
+ * Notify queued audio data. Only audio data need to be queued. The video data
+ * will be notified by MediaStreamVideoSink::SetCurrentFrame.
+ */
+ virtual void NotifyQueuedAudioData(MediaStreamGraph* aGraph, TrackID aID,
+ StreamTime aTrackOffset,
+ const AudioSegment& aQueuedMedia,
+ MediaStream* aInputStream = nullptr,
+ TrackID aInputTrackID = TRACK_INVALID) {}
+
+ /**
* Notify that all new tracks this iteration have been created.
* This is to ensure that tracks added atomically to MediaStreamGraph
* are also notified of atomically to MediaStreamListeners.
*/
virtual void NotifyFinishedTrackCreation(MediaStreamGraph* aGraph) {}
};
class AudioDataListenerInterface {
@@ -1074,17 +1086,18 @@ public:
// XXX need a Reset API
friend class MediaStreamGraphImpl;
protected:
enum TrackCommands {
TRACK_CREATE = MediaStreamListener::TRACK_EVENT_CREATED,
- TRACK_END = MediaStreamListener::TRACK_EVENT_ENDED
+ TRACK_END = MediaStreamListener::TRACK_EVENT_ENDED,
+ TRACK_UNUSED = MediaStreamListener::TRACK_EVENT_UNUSED,
};
/**
* Data for each track that hasn't ended.
*/
struct TrackData {
TrackID mID;
// Sample rate of the input data.
TrackRate mInputRate;
--- a/dom/media/TrackUnionStream.cpp
+++ b/dom/media/TrackUnionStream.cpp
@@ -319,20 +319,27 @@ TrackUnionStream::TrackUnionStream(DOMMe
segment->AppendSlice(*aInputTrack->GetSegment(),
std::min(inputTrackEndPoint, inputStart),
std::min(inputTrackEndPoint, inputEnd));
}
}
ApplyTrackDisabling(outputTrack->GetID(), segment);
for (uint32_t j = 0; j < mListeners.Length(); ++j) {
MediaStreamListener* l = mListeners[j];
- l->NotifyQueuedTrackChanges(Graph(), outputTrack->GetID(),
- outputStart, 0, *segment,
- map->mInputPort->GetSource(),
- map->mInputTrackID);
+ // Separate Audio and Video.
+ if (segment->GetType() == MediaSegment::AUDIO) {
+ l->NotifyQueuedAudioData(Graph(), outputTrack->GetID(),
+ outputStart, *static_cast<AudioSegment*>(segment));
+ } else {
+ // This part will be removed in bug 1201363.
+ l->NotifyQueuedTrackChanges(Graph(), outputTrack->GetID(),
+ outputStart, 0, *segment,
+ map->mInputPort->GetSource(),
+ map->mInputTrackID);
+ }
}
for (TrackBound<MediaStreamTrackListener>& b : mTrackListeners) {
if (b.mTrackID != outputTrack->GetID()) {
continue;
}
b.mListener->NotifyQueuedChanges(Graph(), outputStart, *segment);
}
outputTrack->GetSegment()->AppendFrom(segment);
--- a/dom/media/encoder/MediaEncoder.cpp
+++ b/dom/media/encoder/MediaEncoder.cpp
@@ -101,16 +101,27 @@ MediaEncoder::NotifyQueuedTrackChanges(M
} else {
mSuspended = RECORD_NOT_SUSPENDED; // no video
}
}
}
}
void
+MediaEncoder::NotifyQueuedAudioData(MediaStreamGraph* aGraph, TrackID aID,
+ StreamTime aTrackOffset,
+ const AudioSegment& aQueuedMedia,
+ MediaStream* aInputStream,
+ TrackID aInputTrackID)
+{
+ mAudioEncoder->NotifyQueuedTrackChanges(aGraph, aID, aTrackOffset, 0,
+ aQueuedMedia);
+}
+
+void
MediaEncoder::NotifyEvent(MediaStreamGraph* aGraph,
MediaStreamListener::MediaStreamGraphEvent event)
{
// In case that MediaEncoder does not receive a TRACK_EVENT_ENDED event.
LOG(LogLevel::Debug, ("NotifyRemoved in [MediaEncoder]."));
if (mAudioEncoder) {
mAudioEncoder->NotifyEvent(aGraph, event);
}
--- a/dom/media/encoder/MediaEncoder.h
+++ b/dom/media/encoder/MediaEncoder.h
@@ -125,17 +125,27 @@ public :
void NotifyQueuedTrackChanges(MediaStreamGraph* aGraph, TrackID aID,
StreamTime aTrackOffset,
uint32_t aTrackEvents,
const MediaSegment& aQueuedMedia,
MediaStream* aInputStream,
TrackID aInputTrackID) override;
/**
- * Notified the stream is being removed.
+ * Notifed by the control loop of MediaStreamGraph; aQueueMedia is the audio
+ * data in the form of an AudioSegment.
+ */
+ void NotifyQueuedAudioData(MediaStreamGraph* aGraph, TrackID aID,
+ StreamTime aTrackOffset,
+ const AudioSegment& aQueuedMedia,
+ MediaStream* aInputStream,
+ TrackID aInputTrackID) override;
+
+ /**
+ * * Notified the stream is being removed.
*/
void NotifyEvent(MediaStreamGraph* aGraph,
MediaStreamListener::MediaStreamGraphEvent event) override;
/**
* Creates an encoder with a given MIME type. Returns null if we are unable
* to create the encoder. For now, default aMIMEType to "audio/ogg" and use
* Ogg+Opus if it is empty.
--- a/dom/media/webspeech/recognition/SpeechStreamListener.cpp
+++ b/dom/media/webspeech/recognition/SpeechStreamListener.cpp
@@ -21,23 +21,21 @@ SpeechStreamListener::~SpeechStreamListe
{
nsCOMPtr<nsIThread> mainThread;
NS_GetMainThread(getter_AddRefs(mainThread));
NS_ProxyRelease(mainThread, mRecognition.forget());
}
void
-SpeechStreamListener::NotifyQueuedTrackChanges(MediaStreamGraph* aGraph,
- TrackID aID,
- StreamTime aTrackOffset,
- uint32_t aTrackEvents,
- const MediaSegment& aQueuedMedia,
- MediaStream* aInputStream,
- TrackID aInputTrackID)
+SpeechStreamListener::NotifyQueuedAudioData(MediaStreamGraph* aGraph, TrackID aID,
+ StreamTime aTrackOffset,
+ const AudioSegment& aQueuedMedia,
+ MediaStream* aInputStream,
+ TrackID aInputTrackID)
{
AudioSegment* audio = const_cast<AudioSegment*>(
static_cast<const AudioSegment*>(&aQueuedMedia));
AudioSegment::ChunkIterator iterator(*audio);
while (!iterator.IsEnded()) {
// Skip over-large chunks so we don't crash!
if (iterator->GetDuration() > INT_MAX) {
--- a/dom/media/webspeech/recognition/SpeechStreamListener.h
+++ b/dom/media/webspeech/recognition/SpeechStreamListener.h
@@ -19,22 +19,21 @@ namespace dom {
class SpeechRecognition;
class SpeechStreamListener : public MediaStreamListener
{
public:
explicit SpeechStreamListener(SpeechRecognition* aRecognition);
~SpeechStreamListener();
- void NotifyQueuedTrackChanges(MediaStreamGraph* aGraph, TrackID aID,
- StreamTime aTrackOffset,
- uint32_t aTrackEvents,
- const MediaSegment& aQueuedMedia,
- MediaStream* aInputStream,
- TrackID aInputTrackID) override;
+ void NotifyQueuedAudioData(MediaStreamGraph* aGraph, TrackID aID,
+ StreamTime aTrackOffset,
+ const AudioSegment& aQueuedMedia,
+ MediaStream* aInputStream,
+ TrackID aInputTrackID) override;
void NotifyEvent(MediaStreamGraph* aGraph,
MediaStreamListener::MediaStreamGraphEvent event) override;
private:
template<typename SampleFormatType>
void ConvertAndDispatchAudioChunk(int aDuration, float aVolume, SampleFormatType* aData, TrackRate aTrackRate);
RefPtr<SpeechRecognition> mRecognition;
--- a/media/webrtc/signaling/src/mediapipeline/MediaPipeline.cpp
+++ b/media/webrtc/signaling/src/mediapipeline/MediaPipeline.cpp
@@ -1958,23 +1958,16 @@ public:
MOZ_CRASH();
}
} else {
conduit_ = nullptr;
}
}
// Implement MediaStreamListener
- void NotifyQueuedTrackChanges(MediaStreamGraph* graph, TrackID tid,
- StreamTime offset,
- uint32_t events,
- const MediaSegment& queued_media,
- MediaStream* input_stream,
- TrackID input_tid) override {}
-
void NotifyPull(MediaStreamGraph* graph, StreamTime desired_time) override
{
MOZ_ASSERT(source_);
if (!source_) {
MOZ_MTLOG(ML_ERROR, "NotifyPull() called from a non-SourceMediaStream");
return;
}
@@ -2121,24 +2114,17 @@ public:
monitor_("Video PipelineListener")
{
#if !defined(MOZILLA_EXTERNAL_LINKAGE)
image_container_ =
LayerManager::CreateImageContainer(ImageContainer::ASYNCHRONOUS);
#endif
}
-
// Implement MediaStreamListener
- void NotifyQueuedTrackChanges(MediaStreamGraph* graph, TrackID tid,
- StreamTime offset,
- uint32_t events,
- const MediaSegment& queued_media,
- MediaStream* input_stream,
- TrackID input_tid) override {}
void NotifyPull(MediaStreamGraph* graph, StreamTime desired_time) override
{
ReentrantMonitorAutoEnter enter(monitor_);
#if defined(MOZILLA_INTERNAL_API)
RefPtr<Image> image = image_;
// our constructor sets track_rate_ to the graph rate
MOZ_ASSERT(track_rate_ == source_->GraphRate());
--- a/media/webrtc/signaling/test/FakeMediaStreams.h
+++ b/media/webrtc/signaling/test/FakeMediaStreams.h
@@ -87,18 +87,23 @@ protected:
virtual ~Fake_MediaStreamListener() {}
public:
virtual void NotifyQueuedTrackChanges(mozilla::MediaStreamGraph* aGraph, mozilla::TrackID aID,
mozilla::StreamTime aTrackOffset,
uint32_t aTrackEvents,
const mozilla::MediaSegment& aQueuedMedia,
Fake_MediaStream* aInputStream,
- mozilla::TrackID aInputTrackID) = 0;
+ mozilla::TrackID aInputTrackID) {}
virtual void NotifyPull(mozilla::MediaStreamGraph* aGraph, mozilla::StreamTime aDesiredTime) = 0;
+ virtual void NotifyQueuedAudioData(mozilla::MediaStreamGraph* aGraph, mozilla::TrackID aID,
+ mozilla::StreamTime aTrackOffset,
+ const mozilla::AudioSegment& aQueuedMedia,
+ Fake_MediaStream* aInputStream,
+ mozilla::TrackID aInputTrackID) {}
NS_INLINE_DECL_THREADSAFE_REFCOUNTING(Fake_MediaStreamListener)
};
class Fake_MediaStreamDirectListener : public Fake_MediaStreamListener
{
protected:
virtual ~Fake_MediaStreamDirectListener() {}