Bug 1201363 - Add |MediaStreamListener::NotifyQueuedAudioData| and separate non TRACK_CREATE and TRACK_END part to NotifyQueuedAudioData. r?jesup,r?pehrsons
One of the goal of this bug is to simplified NotifyQueuedTrackChanges to notify TRACK_CREATE and TRACK_END event only. This changeset is trying to separate the audio data part first.
MozReview-Commit-ID: JtziVEFdlPn
--- a/dom/media/MediaStreamGraph.cpp
+++ b/dom/media/MediaStreamGraph.cpp
@@ -161,22 +161,43 @@ MediaStreamGraphImpl::ExtractPendingInpu
}
}
}
finished = aStream->mUpdateFinished;
bool notifiedTrackCreated = false;
for (int32_t i = aStream->mUpdateTracks.Length() - 1; i >= 0; --i) {
SourceMediaStream::TrackData* data = &aStream->mUpdateTracks[i];
aStream->ApplyTrackDisabling(data->mID, data->mData);
- for (MediaStreamListener* l : aStream->mListeners) {
- StreamTime offset = (data->mCommands & SourceMediaStream::TRACK_CREATE)
- ? data->mStart : aStream->mTracks.FindTrack(data->mID)->GetSegment()->GetDuration();
- l->NotifyQueuedTrackChanges(this, data->mID,
- offset, data->mCommands, *data->mData);
+ // Dealing with NotifyQueuedTrackChanges and NotifyQueuedAudioData part.
+ // The logic is different from the manipulating of aStream->mTracks part.
+ // So it is not combined with the manipulating of aStream->mTracks part.
+ StreamTime offset = (data->mCommands & SourceMediaStream::TRACK_CREATE)
+ ? data->mStart : aStream->mTracks.FindTrack(data->mID)->GetSegment()->GetDuration();
+ if ((data->mCommands & SourceMediaStream::TRACK_CREATE) ||
+ (data->mCommands & SourceMediaStream::TRACK_END)) {
+ for (MediaStreamListener* l : aStream->mListeners) {
+ l->NotifyQueuedTrackChanges(this, data->mID,
+ offset, data->mCommands, *data->mData);
+ }
+ } else if (data->mData->GetType() == MediaSegment::AUDIO) {
+ for (MediaStreamListener* l : aStream->mListeners) {
+ l->NotifyQueuedAudioData(this, data->mID,
+ offset, *(static_cast<AudioSegment*>(data->mData.get())));
+ }
+ } else {
+ // Fixme: This part will be removed in the later changeset of this bug.
+ // It will be removed in changeset "Do not copy video segment to StreamTracks in TrackUnionStream."
+
+ // Dealing with video and not TRACK_CREATE and TRACK_END case.
+ for (MediaStreamListener* l : aStream->mListeners) {
+ l->NotifyQueuedTrackChanges(this, data->mID,
+ offset, data->mCommands, *data->mData);
+ }
}
+
if (data->mCommands & SourceMediaStream::TRACK_CREATE) {
MediaSegment* segment = data->mData.forget();
STREAM_LOG(LogLevel::Debug, ("SourceMediaStream %p creating track %d, start %lld, initial end %lld",
aStream, data->mID, int64_t(data->mStart),
int64_t(segment->GetDuration())));
data->mEndOfFlushedData += segment->GetDuration();
aStream->mTracks.AddTrack(data->mID, data->mStart, segment);
--- a/dom/media/MediaStreamGraph.h
+++ b/dom/media/MediaStreamGraph.h
@@ -171,16 +171,25 @@ public:
virtual void NotifyQueuedTrackChanges(MediaStreamGraph* aGraph, TrackID aID,
StreamTime aTrackOffset,
uint32_t aTrackEvents,
const MediaSegment& aQueuedMedia,
MediaStream* aInputStream = nullptr,
TrackID aInputTrackID = TRACK_INVALID) {}
/**
+ * Notify queued audio data.
+ */
+ virtual void NotifyQueuedAudioData(MediaStreamGraph* aGraph, TrackID aID,
+ StreamTime aTrackOffset,
+ const AudioSegment& aQueuedMedia,
+ MediaStream* aInputStream = nullptr,
+ TrackID aInputTrackID = TRACK_INVALID) {}
+
+ /**
* Notify that all new tracks this iteration have been created.
* This is to ensure that tracks added atomically to MediaStreamGraph
* are also notified of atomically to MediaStreamListeners.
*/
virtual void NotifyFinishedTrackCreation(MediaStreamGraph* aGraph) {}
};
/**
--- a/dom/media/TrackUnionStream.cpp
+++ b/dom/media/TrackUnionStream.cpp
@@ -270,15 +270,21 @@ TrackUnionStream::TrackUnionStream(DOMMe
segment->AppendSlice(*aInputTrack->GetSegment(),
std::min(inputTrackEndPoint, inputStart),
std::min(inputTrackEndPoint, inputEnd));
}
}
ApplyTrackDisabling(outputTrack->GetID(), segment);
for (uint32_t j = 0; j < mListeners.Length(); ++j) {
MediaStreamListener* l = mListeners[j];
- l->NotifyQueuedTrackChanges(Graph(), outputTrack->GetID(),
- outputStart, 0, *segment);
+ // Separate Audio and Video.
+ if (segment->GetType() == MediaSegment::AUDIO) {
+ l->NotifyQueuedAudioData(Graph(), outputTrack->GetID(),
+ outputStart, *static_cast<AudioSegment*>(segment));
+ } else {
+ l->NotifyQueuedTrackChanges(Graph(), outputTrack->GetID(),
+ outputStart, 0, *segment);
+ }
}
outputTrack->GetSegment()->AppendFrom(segment);
}
}
} // namespace mozilla
--- a/dom/media/encoder/MediaEncoder.cpp
+++ b/dom/media/encoder/MediaEncoder.cpp
@@ -48,23 +48,35 @@ MediaEncoder::NotifyQueuedTrackChanges(M
// Process the incoming raw track data from MediaStreamGraph, called on the
// thread of MediaStreamGraph.
if (mAudioEncoder && aQueuedMedia.GetType() == MediaSegment::AUDIO) {
mAudioEncoder->NotifyQueuedTrackChanges(aGraph, aID,
aTrackOffset, aTrackEvents,
aQueuedMedia);
} else if (mVideoEncoder && aQueuedMedia.GetType() == MediaSegment::VIDEO) {
- mVideoEncoder->NotifyQueuedTrackChanges(aGraph, aID,
- aTrackOffset, aTrackEvents,
- aQueuedMedia);
+ mVideoEncoder->NotifyQueuedTrackChanges(aGraph, aID,
+ aTrackOffset, aTrackEvents,
+ aQueuedMedia);
}
}
void
+MediaEncoder::NotifyQueuedAudioData(MediaStreamGraph* aGraph, TrackID aID,
+ StreamTime aTrackOffset,
+ const AudioSegment& aQueuedMedia,
+ MediaStream* aInputStream,
+ TrackID aInputTrackID)
+{
+ mAudioEncoder->NotifyQueuedTrackChanges(aGraph, aID,
+ aTrackOffset, 0,
+ aQueuedMedia);
+}
+
+void
MediaEncoder::NotifyEvent(MediaStreamGraph* aGraph,
MediaStreamListener::MediaStreamGraphEvent event)
{
// In case that MediaEncoder does not receive a TRACK_EVENT_ENDED event.
LOG(LogLevel::Debug, ("NotifyRemoved in [MediaEncoder]."));
if (mAudioEncoder) {
mAudioEncoder->NotifyEvent(aGraph, event);
}
--- a/dom/media/encoder/MediaEncoder.h
+++ b/dom/media/encoder/MediaEncoder.h
@@ -85,16 +85,26 @@ public :
void NotifyQueuedTrackChanges(MediaStreamGraph* aGraph, TrackID aID,
StreamTime aTrackOffset,
uint32_t aTrackEvents,
const MediaSegment& aQueuedMedia,
MediaStream* aInputStream,
TrackID aInputTrackID) override;
/**
+ * Notifed by the control loop of MediaStreamGraph; aQueueMedia is the raw
+ * track data in form of AudioSegment.
+ */
+ virtual void NotifyQueuedAudioData(MediaStreamGraph* aGraph, TrackID aID,
+ StreamTime aTrackOffset,
+ const AudioSegment& aQueuedMedia,
+ MediaStream* aInputStream = nullptr,
+ TrackID aInputTrackID = TRACK_INVALID) override;
+
+ /**
* Notified the stream is being removed.
*/
void NotifyEvent(MediaStreamGraph* aGraph,
MediaStreamListener::MediaStreamGraphEvent event) override;
/**
* Creates an encoder with a given MIME type. Returns null if we are unable
* to create the encoder. For now, default aMIMEType to "audio/ogg" and use
--- a/media/webrtc/signaling/src/mediapipeline/MediaPipeline.cpp
+++ b/media/webrtc/signaling/src/mediapipeline/MediaPipeline.cpp
@@ -870,26 +870,37 @@ NotifyRealtimeData(MediaStreamGraph* gra
NewData(graph, tid, offset, events, media);
}
void MediaPipelineTransmit::PipelineListener::
NotifyQueuedTrackChanges(MediaStreamGraph* graph, TrackID tid,
StreamTime offset,
uint32_t events,
const MediaSegment& queued_media,
- MediaStream* aInputStream,
- TrackID aInputTrackID) {
+ MediaStream* input_stream,
+ TrackID input_tid) {
MOZ_MTLOG(ML_DEBUG, "MediaPipeline::NotifyQueuedTrackChanges()");
// ignore non-direct data if we're also getting direct data
if (!direct_connect_) {
NewData(graph, tid, offset, events, queued_media);
}
}
+void MediaPipelineTransmit::PipelineListener::
+NotifyQueuedAudioData(MediaStreamGraph* graph, TrackID tid,
+ StreamTime offset,
+ const AudioSegment& queued_media,
+ MediaStream* input_stream,
+ TrackID input_tid) {
+ if (!direct_connect_) {
+ NewData(graph, tid, offset, 0, queued_media);
+ }
+}
+
// I420 buffer size macros
#define YSIZE(x,y) ((x)*(y))
#define CRSIZE(x,y) ((((x)+1) >> 1) * (((y)+1) >> 1))
#define I420SIZE(x,y) (YSIZE((x),(y)) + 2 * CRSIZE((x),(y)))
// XXX NOTE: this code will have to change when we get support for multiple tracks of type
// in a MediaStream and especially in a PeerConnection stream. bug 1056650
// It should be matching on the "correct" track for the pipeline, not just "any video track".
--- a/media/webrtc/signaling/src/mediapipeline/MediaPipeline.h
+++ b/media/webrtc/signaling/src/mediapipeline/MediaPipeline.h
@@ -480,16 +480,21 @@ public:
// Implement MediaStreamListener
virtual void NotifyQueuedTrackChanges(MediaStreamGraph* graph, TrackID tid,
StreamTime offset,
uint32_t events,
const MediaSegment& queued_media,
MediaStream* input_stream,
TrackID input_tid) override;
+ virtual void NotifyQueuedAudioData(MediaStreamGraph* graph, TrackID tid,
+ StreamTime offset,
+ const AudioSegment& queued_media,
+ MediaStream* input_stream,
+ TrackID input_tid) override;
virtual void NotifyPull(MediaStreamGraph* aGraph, StreamTime aDesiredTime) override {}
// Implement MediaStreamDirectListener
virtual void NotifyRealtimeData(MediaStreamGraph* graph, TrackID tid,
StreamTime offset,
uint32_t events,
const MediaSegment& media) override;
@@ -630,17 +635,21 @@ class MediaPipelineReceiveAudio : public
// Implement MediaStreamListener
virtual void NotifyQueuedTrackChanges(MediaStreamGraph* graph, TrackID tid,
StreamTime offset,
uint32_t events,
const MediaSegment& queued_media,
MediaStream* input_stream,
TrackID input_tid) override {}
virtual void NotifyPull(MediaStreamGraph* graph, StreamTime desired_time) override;
-
+ virtual void NotifyQueuedAudioData(mozilla::MediaStreamGraph* aGraph, mozilla::TrackID aID,
+ mozilla::StreamTime aTrackOffset,
+ const mozilla::AudioSegment& aQueuedMedia,
+ MediaStream* aInputStream,
+ mozilla::TrackID aInputTrackID) override {}
private:
RefPtr<MediaSessionConduit> conduit_;
};
RefPtr<PipelineListener> listener_;
};
@@ -741,16 +750,21 @@ class MediaPipelineReceiveVideo : public
// Implement MediaStreamListener
virtual void NotifyQueuedTrackChanges(MediaStreamGraph* graph, TrackID tid,
StreamTime offset,
uint32_t events,
const MediaSegment& queued_media,
MediaStream* input_stream,
TrackID input_tid) override {}
virtual void NotifyPull(MediaStreamGraph* graph, StreamTime desired_time) override;
+ virtual void NotifyQueuedAudioData(mozilla::MediaStreamGraph* aGraph, mozilla::TrackID aID,
+ mozilla::StreamTime aTrackOffset,
+ const mozilla::AudioSegment& aQueuedMedia,
+ MediaStream* aInputStream,
+ mozilla::TrackID aInputTrackID) override {}
// Accessors for external writes from the renderer
void FrameSizeChange(unsigned int width,
unsigned int height,
unsigned int number_of_streams) {
ReentrantMonitorAutoEnter enter(monitor_);
width_ = width;
--- a/media/webrtc/signaling/test/FakeMediaStreams.h
+++ b/media/webrtc/signaling/test/FakeMediaStreams.h
@@ -85,16 +85,21 @@ protected:
public:
virtual void NotifyQueuedTrackChanges(mozilla::MediaStreamGraph* aGraph, mozilla::TrackID aID,
mozilla::StreamTime aTrackOffset,
uint32_t aTrackEvents,
const mozilla::MediaSegment& aQueuedMedia,
Fake_MediaStream* aInputStream,
mozilla::TrackID aInputTrackID) = 0;
virtual void NotifyPull(mozilla::MediaStreamGraph* aGraph, mozilla::StreamTime aDesiredTime) = 0;
+ virtual void NotifyQueuedAudioData(mozilla::MediaStreamGraph* aGraph, mozilla::TrackID aID,
+ mozilla::StreamTime aTrackOffset,
+ const mozilla::AudioSegment& aQueuedMedia,
+ Fake_MediaStream* aInputStream,
+ mozilla::TrackID aInputTrackID) = 0;
NS_INLINE_DECL_THREADSAFE_REFCOUNTING(Fake_MediaStreamListener)
};
class Fake_MediaStreamDirectListener : public Fake_MediaStreamListener
{
protected:
virtual ~Fake_MediaStreamDirectListener() {}