Bug 1208371 - Add DirectTrackLister to MediaStreamGraph and MediaStreamTrack. r?roc,jesup
MozReview-Commit-ID: HRWa6A35FSC
--- a/dom/media/MediaStreamGraph.cpp
+++ b/dom/media/MediaStreamGraph.cpp
@@ -2292,16 +2292,72 @@ MediaStream::RemoveTrackListener(MediaSt
}
RefPtr<MediaStreamTrackListener> mListener;
TrackID mTrackID;
};
GraphImpl()->AppendMessage(MakeUnique<Message>(this, aListener, aTrackID));
}
void
+MediaStream::AddDirectTrackListenerImpl(already_AddRefed<MediaStreamTrackDirectListener> aListener,
+ TrackID aTrackID)
+{
+ // Base implementation, for streams that don't support direct track listeners.
+ RefPtr<MediaStreamTrackDirectListener> listener = aListener;
+ listener->NotifyDirectListenerInstalled(
+ MediaStreamTrackDirectListener::InstallationResult::STREAM_NOT_SUPPORTED);
+}
+
+void
+MediaStream::AddDirectTrackListener(MediaStreamTrackDirectListener* aListener,
+ TrackID aTrackID)
+{
+ class Message : public ControlMessage {
+ public:
+ Message(MediaStream* aStream, MediaStreamTrackDirectListener* aListener,
+ TrackID aTrackID) :
+ ControlMessage(aStream), mListener(aListener), mTrackID(aTrackID) {}
+ virtual void Run()
+ {
+ mStream->AddDirectTrackListenerImpl(mListener.forget(), mTrackID);
+ }
+ RefPtr<MediaStreamTrackDirectListener> mListener;
+ TrackID mTrackID;
+ };
+ GraphImpl()->AppendMessage(MakeUnique<Message>(this, aListener, aTrackID));
+}
+
+void
+MediaStream::RemoveDirectTrackListenerImpl(MediaStreamTrackDirectListener* aListener,
+ TrackID aTrackID)
+{
+ // Base implementation, the listener was never added so nothing to do.
+ RefPtr<MediaStreamTrackDirectListener> listener = aListener;
+}
+
+void
+MediaStream::RemoveDirectTrackListener(MediaStreamTrackDirectListener* aListener,
+ TrackID aTrackID)
+{
+ class Message : public ControlMessage {
+ public:
+ Message(MediaStream* aStream, MediaStreamTrackDirectListener* aListener,
+ TrackID aTrackID) :
+ ControlMessage(aStream), mListener(aListener), mTrackID(aTrackID) {}
+ virtual void Run()
+ {
+ mStream->RemoveDirectTrackListenerImpl(mListener, mTrackID);
+ }
+ RefPtr<MediaStreamTrackDirectListener> mListener;
+ TrackID mTrackID;
+ };
+ GraphImpl()->AppendMessage(MakeUnique<Message>(this, aListener, aTrackID));
+}
+
+void
MediaStream::RunAfterPendingUpdates(already_AddRefed<nsIRunnable> aRunnable)
{
MOZ_ASSERT(NS_IsMainThread());
MediaStreamGraphImpl* graph = GraphImpl();
nsCOMPtr<nsIRunnable> runnable(aRunnable);
// Special case when a non-realtime graph has not started, to ensure the
// runnable will run in finite time.
@@ -2552,25 +2608,34 @@ SourceMediaStream::AppendToTrack(TrackID
}
return appended;
}
void
SourceMediaStream::NotifyDirectConsumers(TrackData *aTrack,
MediaSegment *aSegment)
{
- // Call with mMutex locked
+ mMutex.AssertCurrentThreadOwns();
MOZ_ASSERT(aTrack);
for (uint32_t j = 0; j < mDirectListeners.Length(); ++j) {
MediaStreamDirectListener* l = mDirectListeners[j];
StreamTime offset = 0; // FIX! need a separate StreamTime.... or the end of the internal buffer
l->NotifyRealtimeData(static_cast<MediaStreamGraph*>(GraphImpl()), aTrack->mID,
offset, aTrack->mCommands, *aSegment);
}
+
+ for (const TrackBound<MediaStreamTrackDirectListener>& source
+ : mDirectTrackListeners) {
+ if (aTrack->mID != source.mTrackID) {
+ continue;
+ }
+ StreamTime offset = 0; // FIX! need a separate StreamTime.... or the end of the internal buffer
+ source.mListener->NotifyRealtimeTrackData(Graph(), offset, *aSegment);
+ }
}
// These handle notifying all the listeners of an event
void
SourceMediaStream::NotifyListenersEventImpl(MediaStreamListener::MediaStreamGraphEvent aEvent)
{
for (uint32_t j = 0; j < mListeners.Length(); ++j) {
MediaStreamListener* l = mListeners[j];
@@ -2621,16 +2686,73 @@ SourceMediaStream::RemoveDirectListener(
}
if (isEmpty) {
// Async
NotifyListenersEvent(MediaStreamListener::EVENT_HAS_NO_DIRECT_LISTENERS);
}
}
+void
+SourceMediaStream::AddDirectTrackListenerImpl(already_AddRefed<MediaStreamTrackDirectListener> aListener,
+ TrackID aTrackID)
+{
+ MOZ_ASSERT(IsTrackIDExplicit(aTrackID));
+ TrackData* data;
+ bool found;
+ bool isAudio;
+ RefPtr<MediaStreamTrackDirectListener> listener = aListener;
+ STREAM_LOG(LogLevel::Debug, ("Adding direct track listener %p bound to track %d to source stream %p",
+ listener.get(), aTrackID, this));
+ {
+ MutexAutoLock lock(mMutex);
+ data = FindDataForTrack(aTrackID);
+ found = !!data;
+ isAudio = found && data->mData->GetType() == MediaSegment::AUDIO;
+ if (found && isAudio) {
+ TrackBound<MediaStreamTrackDirectListener>* sourceListener =
+ mDirectTrackListeners.AppendElement();
+ sourceListener->mListener = listener;
+ sourceListener->mTrackID = aTrackID;
+ }
+ }
+ if (!found) {
+ STREAM_LOG(LogLevel::Warning, ("Couldn't find source track for direct track listener %p",
+ listener.get()));
+ listener->NotifyDirectListenerInstalled(
+ MediaStreamTrackDirectListener::InstallationResult::TRACK_NOT_FOUND_AT_SOURCE);
+ return;
+ }
+ if (!isAudio) {
+ STREAM_LOG(LogLevel::Warning, ("Source track for direct track listener %p is not audio",
+ listener.get()));
+ listener->NotifyDirectListenerInstalled(
+ MediaStreamTrackDirectListener::InstallationResult::TRACK_TYPE_NOT_SUPPORTED);
+ return;
+ }
+ STREAM_LOG(LogLevel::Debug, ("Added direct track listener %p", listener.get()));
+ listener->NotifyDirectListenerInstalled(
+ MediaStreamTrackDirectListener::InstallationResult::SUCCESS);
+}
+
+void
+SourceMediaStream::RemoveDirectTrackListenerImpl(MediaStreamTrackDirectListener* aListener,
+ TrackID aTrackID)
+{
+ MutexAutoLock lock(mMutex);
+ for (int32_t i = mDirectTrackListeners.Length() - 1; i >= 0; --i) {
+ const TrackBound<MediaStreamTrackDirectListener>& source =
+ mDirectTrackListeners[i];
+ if (source.mListener == aListener && source.mTrackID == aTrackID) {
+ aListener->NotifyDirectListenerUninstalled();
+ mDirectTrackListeners.RemoveElementAt(i);
+ }
+ }
+}
+
StreamTime
SourceMediaStream::GetEndOfAppendedData(TrackID aID)
{
MutexAutoLock lock(mMutex);
TrackData *track = FindDataForTrack(aID);
if (track) {
return track->mEndOfFlushedData + track->mData->GetDuration();
}
--- a/dom/media/MediaStreamGraph.h
+++ b/dom/media/MediaStreamGraph.h
@@ -270,16 +270,76 @@ public:
*/
virtual void NotifyRealtimeData(MediaStreamGraph* aGraph, TrackID aID,
StreamTime aTrackOffset,
uint32_t aTrackEvents,
const MediaSegment& aMedia) {}
};
/**
+ * This is a base class for media graph thread listener direct callbacks from
+ * within AppendToTrack(). It is bound to a certain track and can only be
+ * installed on audio tracks. Once added to a track on any stream in the graph,
+ * the graph will try to install it at that track's source of media data.
+ *
+ * This works for TrackUnionStreams, which will forward the listener to the
+ * track's input track if it exists, or wait for it to be created before
+ * forwarding if it doesn't.
+ * Once it reaches a SourceMediaStream, it can be successfully installed.
+ * Other types of streams will fail installation since they are not supported.
+ *
+ * Note that this listener and others for the same track will still get
+ * NotifyQueuedChanges() callbacks from the MSG tread, so you must be careful
+ * to ignore them if this listener was successfully installed.
+ */
+class MediaStreamTrackDirectListener : public MediaStreamTrackListener
+{
+public:
+ /*
+ * This will be called on any MediaStreamTrackDirectListener added to a
+ * SourceMediaStream when AppendToTrack() is called for the listener's bound
+ * track. The MediaSegment will be the RawSegment (unresampled) if available
+ * in AppendToTrack(). Note that NotifyQueuedTrackChanges() calls will also
+ * still occur.
+ */
+ virtual void NotifyRealtimeTrackData(MediaStreamGraph* aGraph,
+ StreamTime aTrackOffset,
+ const MediaSegment& aMedia) {}
+
+ /**
+ * When a direct listener is processed for installation by the
+ * MediaStreamGraph it will be notified with whether the installation was
+ * successful or not. The results of this installation are the following:
+ * TRACK_NOT_FOUND_AT_SOURCE
+ * We found the source stream of media data for this track, but the track
+ * didn't exist. This should only happen if you try to install the listener
+ * directly to a SourceMediaStream that doesn't contain the given TrackID.
+ * TRACK_TYPE_NOT_SUPPORTED
+ * This is the failure when you install the listener to a non-audio track.
+ * STREAM_NOT_SUPPORTED
+ * While looking for the data source of this track, we found a MediaStream
+ * that is not a SourceMediaStream or a TrackUnionStream.
+ * SUCCESS
+ * Installation was successful and this listener will start receiving
+ * NotifyRealtimeData on the next AppendToTrack().
+ */
+ enum class InstallationResult {
+ TRACK_NOT_FOUND_AT_SOURCE,
+ TRACK_TYPE_NOT_SUPPORTED,
+ STREAM_NOT_SUPPORTED,
+ SUCCESS
+ };
+ virtual void NotifyDirectListenerInstalled(InstallationResult aResult) {}
+ virtual void NotifyDirectListenerUninstalled() {}
+
+protected:
+ virtual ~MediaStreamTrackDirectListener() {}
+};
+
+/**
* This is a base class for main-thread listener callbacks.
* This callback is invoked on the main thread when the main-thread-visible
* state of a stream has changed.
*
* These methods are called with the media graph monitor held, so
* reentry into general media graph methods is not possible.
* You should do something non-blocking and non-reentrant (e.g. dispatch an
* event) and return. DispatchFromMainThreadAfterNextStreamStateUpdate
@@ -444,16 +504,39 @@ public:
virtual void Resume();
// Events will be dispatched by calling methods of aListener.
virtual void AddListener(MediaStreamListener* aListener);
virtual void RemoveListener(MediaStreamListener* aListener);
virtual void AddTrackListener(MediaStreamTrackListener* aListener,
TrackID aTrackID);
virtual void RemoveTrackListener(MediaStreamTrackListener* aListener,
TrackID aTrackID);
+
+ /**
+ * Adds aListener to the source stream of track aTrackID in this stream.
+ * When the MediaStreamGraph processes the added listener, it will traverse
+ * the graph and add it to the track's source stream (remapping the TrackID
+ * along the way).
+ * Note that the listener will be notified on the MediaStreamGraph thread
+ * with whether the installation of it at the source was successful or not.
+ */
+ virtual void AddDirectTrackListener(MediaStreamTrackDirectListener* aListener,
+ TrackID aTrackID);
+
+ /**
+ * Removes aListener from the source stream of track aTrackID in this stream.
+ * Note that the listener has already been removed if the link between the
+ * source of track aTrackID and this stream has been broken (and made track
+ * aTrackID end). The caller doesn't have to care about this, removing when
+ * the source cannot be found, or when the listener had already been removed
+ * does nothing.
+ */
+ virtual void RemoveDirectTrackListener(MediaStreamTrackDirectListener* aListener,
+ TrackID aTrackID);
+
// A disabled track has video replaced by black, and audio replaced by
// silence.
void SetTrackEnabled(TrackID aTrackID, bool aEnabled);
// Finish event will be notified by calling methods of aListener. It is the
// responsibility of the caller to remove aListener before it is destroyed.
void AddMainThreadListener(MainThreadMediaStreamListener* aListener);
// It's safe to call this even if aListener is not currently a listener;
@@ -544,16 +627,20 @@ public:
void RemoveVideoOutputImpl(VideoFrameContainer* aContainer);
void AddListenerImpl(already_AddRefed<MediaStreamListener> aListener);
void RemoveListenerImpl(MediaStreamListener* aListener);
void RemoveAllListenersImpl();
virtual void AddTrackListenerImpl(already_AddRefed<MediaStreamTrackListener> aListener,
TrackID aTrackID);
virtual void RemoveTrackListenerImpl(MediaStreamTrackListener* aListener,
TrackID aTrackID);
+ virtual void AddDirectTrackListenerImpl(already_AddRefed<MediaStreamTrackDirectListener> aListener,
+ TrackID aTrackID);
+ virtual void RemoveDirectTrackListenerImpl(MediaStreamTrackDirectListener* aListener,
+ TrackID aTrackID);
virtual void SetTrackEnabledImpl(TrackID aTrackID, bool aEnabled);
void AddConsumer(MediaInputPort* aPort)
{
mConsumers.AppendElement(aPort);
}
void RemoveConsumer(MediaInputPort* aPort)
{
@@ -967,16 +1054,21 @@ protected:
// this is cleared.
uint32_t mCommands;
};
bool NeedsMixing();
void ResampleAudioToGraphSampleRate(TrackData* aTrackData, MediaSegment* aSegment);
+ void AddDirectTrackListenerImpl(already_AddRefed<MediaStreamTrackDirectListener> aListener,
+ TrackID aTrackID) override;
+ void RemoveDirectTrackListenerImpl(MediaStreamTrackDirectListener* aListener,
+ TrackID aTrackID) override;
+
void AddTrackInternal(TrackID aID, TrackRate aRate,
StreamTime aStart, MediaSegment* aSegment,
uint32_t aFlags);
TrackData* FindDataForTrack(TrackID aID)
{
mMutex.AssertCurrentThreadOwns();
for (uint32_t i = 0; i < mUpdateTracks.Length(); ++i) {
@@ -1005,16 +1097,17 @@ protected:
// This must be acquired *before* MediaStreamGraphImpl's lock, if they are
// held together.
Mutex mMutex;
// protected by mMutex
StreamTime mUpdateKnownTracksTime;
nsTArray<TrackData> mUpdateTracks;
nsTArray<TrackData> mPendingTracks;
nsTArray<RefPtr<MediaStreamDirectListener> > mDirectListeners;
+ nsTArray<TrackBound<MediaStreamTrackDirectListener>> mDirectTrackListeners;
bool mPullEnabled;
bool mUpdateFinished;
bool mNeedsMixing;
};
/**
* Represents a connection between a ProcessedMediaStream and one of its
* input streams.
--- a/dom/media/MediaStreamTrack.cpp
+++ b/dom/media/MediaStreamTrack.cpp
@@ -252,16 +252,36 @@ void
MediaStreamTrack::RemoveListener(MediaStreamTrackListener* aListener)
{
LOG(LogLevel::Debug, ("MediaStreamTrack %p removing listener %p",
this, aListener));
GetOwnedStream()->RemoveTrackListener(aListener, mTrackID);
}
+void
+MediaStreamTrack::AddDirectListener(MediaStreamTrackDirectListener *aListener)
+{
+ LOG(LogLevel::Debug, ("MediaStreamTrack %p (%s) adding direct listener %p to "
+ "stream %p, track %d",
+ this, AsAudioStreamTrack() ? "audio" : "video",
+ aListener, GetOwnedStream(), mTrackID));
+
+ GetOwnedStream()->AddDirectTrackListener(aListener, mTrackID);
+}
+
+void
+MediaStreamTrack::RemoveDirectListener(MediaStreamTrackDirectListener *aListener)
+{
+ LOG(LogLevel::Debug, ("MediaStreamTrack %p removing direct listener %p from stream %p",
+ this, aListener, GetOwnedStream()));
+
+ GetOwnedStream()->RemoveDirectTrackListener(aListener, mTrackID);
+}
+
already_AddRefed<MediaInputPort>
MediaStreamTrack::ForwardTrackContentsTo(ProcessedMediaStream* aStream)
{
MOZ_ASSERT(NS_IsMainThread());
MOZ_RELEASE_ASSERT(aStream);
RefPtr<MediaInputPort> port =
aStream->AllocateInputPort(GetOwnedStream(), mTrackID);
return port.forget();
--- a/dom/media/MediaStreamTrack.h
+++ b/dom/media/MediaStreamTrack.h
@@ -17,16 +17,17 @@
namespace mozilla {
class DOMMediaStream;
class MediaEnginePhotoCallback;
class MediaInputPort;
class MediaStream;
class MediaStreamGraph;
class MediaStreamTrackListener;
+class MediaStreamTrackDirectListener;
class PeerConnectionImpl;
class PeerIdentity;
class ProcessedMediaStream;
class RemoteSourceStreamInfo;
namespace dom {
class AudioStreamTrack;
@@ -297,16 +298,25 @@ public:
/**
* Removes a MediaStreamTrackListener from the MediaStreamGraph representation
* of this track.
*/
void RemoveListener(MediaStreamTrackListener* aListener);
/**
+ * Attempts to add a direct track listener to this track.
+ * Callers must listen to the NotifyInstalled event to know if installing
+ * the listener succeeded (tracks originating from SourceMediaStreams) or
+ * failed (e.g., WebAudio originated tracks).
+ */
+ void AddDirectListener(MediaStreamTrackDirectListener *aListener);
+ void RemoveDirectListener(MediaStreamTrackDirectListener *aListener);
+
+ /**
* Sets up a MediaInputPort from the underlying track that this
* MediaStreamTrack represents, to aStream, and returns it.
*/
already_AddRefed<MediaInputPort> ForwardTrackContentsTo(ProcessedMediaStream* aStream);
/**
* Returns true if this track is connected to aPort and forwarded to aPort's
* output stream.
--- a/dom/media/TrackUnionStream.cpp
+++ b/dom/media/TrackUnionStream.cpp
@@ -214,16 +214,34 @@ TrackUnionStream::TrackUnionStream(DOMMe
TrackMapEntry* map = mTrackMap.AppendElement();
map->mEndOfConsumedInputTicks = 0;
map->mEndOfLastInputIntervalInInputStream = -1;
map->mEndOfLastInputIntervalInOutputStream = -1;
map->mInputPort = aPort;
map->mInputTrackID = aTrack->GetID();
map->mOutputTrackID = track->GetID();
map->mSegment = aTrack->GetSegment()->CreateEmptyClone();
+
+ for (int32_t i = mPendingDirectTrackListeners.Length() - 1; i >= 0; --i) {
+ TrackBound<MediaStreamTrackDirectListener>& bound =
+ mPendingDirectTrackListeners[i];
+ if (bound.mTrackID != map->mOutputTrackID) {
+ continue;
+ }
+ MediaStream* source = map->mInputPort->GetSource();
+ STREAM_LOG(LogLevel::Debug, ("TrackUnionStream %p adding direct listener "
+ "%p for track %d. Forwarding to input "
+ "stream %p track %d.",
+ this, bound.mListener.get(), bound.mTrackID,
+ source, map->mInputTrackID));
+ source->AddDirectTrackListenerImpl(bound.mListener.forget(),
+ map->mInputTrackID);
+ mPendingDirectTrackListeners.RemoveElementAt(i);
+ }
+
return mTrackMap.Length() - 1;
}
void TrackUnionStream::EndTrack(uint32_t aIndex)
{
StreamBuffer::Track* outputTrack = mBuffer.FindTrack(mTrackMap[aIndex].mOutputTrackID);
if (!outputTrack || outputTrack->IsEnded())
return;
@@ -311,9 +329,60 @@ TrackUnionStream::TrackUnionStream(DOMMe
if (b.mTrackID != outputTrack->GetID()) {
continue;
}
b.mListener->NotifyQueuedChanges(Graph(), outputStart, *segment);
}
outputTrack->GetSegment()->AppendFrom(segment);
}
}
+
+void
+TrackUnionStream::AddDirectTrackListenerImpl(already_AddRefed<MediaStreamTrackDirectListener> aListener,
+ TrackID aTrackID)
+{
+ RefPtr<MediaStreamTrackDirectListener> listener = aListener;
+
+ for (TrackMapEntry& entry : mTrackMap) {
+ if (entry.mOutputTrackID == aTrackID) {
+ MediaStream* source = entry.mInputPort->GetSource();
+ STREAM_LOG(LogLevel::Debug, ("TrackUnionStream %p adding direct listener "
+ "%p for track %d. Forwarding to input "
+ "stream %p track %d.",
+ this, listener.get(), aTrackID, source,
+ entry.mInputTrackID));
+ source->AddDirectTrackListenerImpl(listener.forget(),
+ entry.mInputTrackID);
+ return;
+ }
+ }
+
+ TrackBound<MediaStreamTrackDirectListener>* bound =
+ mPendingDirectTrackListeners.AppendElement();
+ bound->mListener = listener.forget();
+ bound->mTrackID = aTrackID;
+}
+
+void
+TrackUnionStream::RemoveDirectTrackListenerImpl(MediaStreamTrackDirectListener* aListener,
+ TrackID aTrackID)
+{
+ for (TrackMapEntry& entry : mTrackMap) {
+ // OutputTrackID is unique to this stream so we only need to do this once.
+ if (entry.mOutputTrackID != aTrackID) {
+ continue;
+ }
+ // Forward to the input
+ MediaStream* source = entry.mInputPort->GetSource();
+ source->RemoveDirectTrackListenerImpl(aListener, entry.mInputTrackID);
+ return;
+ }
+
+ for (size_t i = 0; i < mPendingDirectTrackListeners.Length(); ++i) {
+ TrackBound<MediaStreamTrackDirectListener>& bound =
+ mPendingDirectTrackListeners[i];
+ if (bound.mListener == aListener && bound.mTrackID == aTrackID) {
+ mPendingDirectTrackListeners.RemoveElementAt(i);
+ return;
+ }
+ }
+}
} // namespace mozilla
--- a/dom/media/TrackUnionStream.h
+++ b/dom/media/TrackUnionStream.h
@@ -50,21 +50,30 @@ protected:
// been previously used in this stream, allocating a new TrackID otherwise.
uint32_t AddTrack(MediaInputPort* aPort, StreamBuffer::Track* aTrack,
GraphTime aFrom);
void EndTrack(uint32_t aIndex);
void CopyTrackData(StreamBuffer::Track* aInputTrack,
uint32_t aMapIndex, GraphTime aFrom, GraphTime aTo,
bool* aOutputTrackFinished);
+ void AddDirectTrackListenerImpl(already_AddRefed<MediaStreamTrackDirectListener> aListener,
+ TrackID aTrackID) override;
+ void RemoveDirectTrackListenerImpl(MediaStreamTrackDirectListener* aListener,
+ TrackID aTrackID) override;
+
nsTArray<TrackMapEntry> mTrackMap;
// The next available TrackID, starting at 1 and progressing upwards.
// All TrackIDs in [1, mNextAvailableTrackID) have implicitly been used.
TrackID mNextAvailableTrackID;
// Sorted array of used TrackIDs that require manual tracking.
nsTArray<TrackID> mUsedTracks;
+
+ // Direct track listeners that have not been forwarded to their input stream
+ // yet. We'll forward these as their inputs become available.
+ nsTArray<TrackBound<MediaStreamTrackDirectListener>> mPendingDirectTrackListeners;
};
} // namespace mozilla
#endif /* MOZILLA_MEDIASTREAMGRAPH_H_ */