Bug 1208371 - Ensure DOMMediaStream principals reflect what could reside in their playback streams. r?mt,jesup
Calculating a principal when adding a track is easy - just combine the new
track principal into the stream's principal.
When removing a track it's a bit trickier. The DOMMediaStream has to wait until
the MediaStreamGraph has removed the track from the underlying playback stream.
We do this by letting the MediaStreamGraph return a Pledge (single threaded
Promise) when blocking a track in a stream (the way we end removed tracks).
The pledge gets passed to the MediaStreamGraph and when the block has been
applied it is passed back to the main thread where it is finally resolved
and the DOMMediaStream may recompute its principal once all outstanding
track removals have been applied.
MozReview-Commit-ID: 3QP0YcDyfGf
--- a/dom/media/DOMMediaStream.cpp
+++ b/dom/media/DOMMediaStream.cpp
@@ -13,28 +13,30 @@
#include "mozilla/dom/LocalMediaStreamBinding.h"
#include "mozilla/dom/AudioNode.h"
#include "AudioChannelAgent.h"
#include "mozilla/dom/AudioTrack.h"
#include "mozilla/dom/AudioTrackList.h"
#include "mozilla/dom/VideoTrack.h"
#include "mozilla/dom/VideoTrackList.h"
#include "mozilla/dom/HTMLCanvasElement.h"
+#include "mozilla/media/MediaUtils.h"
#include "MediaStreamGraph.h"
#include "AudioStreamTrack.h"
#include "VideoStreamTrack.h"
#include "Layers.h"
#ifdef LOG
#undef LOG
#endif
using namespace mozilla;
using namespace mozilla::dom;
using namespace mozilla::layers;
+using namespace mozilla::media;
static LazyLogModule gMediaStreamLog("MediaStream");
#define LOG(type, msg) MOZ_LOG(gMediaStreamLog, type, msg)
const TrackID TRACK_VIDEO_PRIMARY = 1;
DOMMediaStream::TrackPort::TrackPort(MediaInputPort* aInputPort,
@@ -78,22 +80,25 @@ DOMMediaStream::TrackPort::GetSource() c
}
TrackID
DOMMediaStream::TrackPort::GetSourceTrackId() const
{
return mInputPort ? mInputPort->GetSourceTrackId() : TRACK_INVALID;
}
-void
+already_AddRefed<Pledge<bool>>
DOMMediaStream::TrackPort::BlockTrackId(TrackID aTrackId)
{
if (mInputPort) {
- mInputPort->BlockTrackId(aTrackId);
+ return mInputPort->BlockTrackId(aTrackId);
}
+ RefPtr<Pledge<bool>> rejected = new Pledge<bool>();
+ rejected->Reject(NS_ERROR_FAILURE);
+ return rejected.forget();
}
NS_IMPL_CYCLE_COLLECTION(DOMMediaStream::TrackPort, mTrack)
NS_IMPL_CYCLE_COLLECTION_ROOT_NATIVE(DOMMediaStream::TrackPort, AddRef)
NS_IMPL_CYCLE_COLLECTION_UNROOT_NATIVE(DOMMediaStream::TrackPort, Release)
NS_IMPL_CYCLE_COLLECTING_ADDREF(MediaStreamTrackSourceGetter)
NS_IMPL_CYCLE_COLLECTING_RELEASE(MediaStreamTrackSourceGetter)
@@ -287,25 +292,29 @@ NS_IMPL_CYCLE_COLLECTION_CLASS(DOMMediaS
NS_IMPL_CYCLE_COLLECTION_UNLINK_BEGIN_INHERITED(DOMMediaStream,
DOMEventTargetHelper)
tmp->Destroy();
NS_IMPL_CYCLE_COLLECTION_UNLINK(mWindow)
NS_IMPL_CYCLE_COLLECTION_UNLINK(mOwnedTracks)
NS_IMPL_CYCLE_COLLECTION_UNLINK(mTracks)
NS_IMPL_CYCLE_COLLECTION_UNLINK(mConsumersToKeepAlive)
NS_IMPL_CYCLE_COLLECTION_UNLINK(mTrackSourceGetter)
+ NS_IMPL_CYCLE_COLLECTION_UNLINK(mPrincipal)
+ NS_IMPL_CYCLE_COLLECTION_UNLINK(mVideoPrincipal)
NS_IMPL_CYCLE_COLLECTION_UNLINK_END
NS_IMPL_CYCLE_COLLECTION_TRAVERSE_BEGIN_INHERITED(DOMMediaStream,
DOMEventTargetHelper)
NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mWindow)
NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mOwnedTracks)
NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mTracks)
NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mConsumersToKeepAlive)
NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mTrackSourceGetter)
+ NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mPrincipal)
+ NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mVideoPrincipal)
NS_IMPL_CYCLE_COLLECTION_TRAVERSE_END
NS_IMPL_ADDREF_INHERITED(DOMMediaStream, DOMEventTargetHelper)
NS_IMPL_RELEASE_INHERITED(DOMMediaStream, DOMEventTargetHelper)
NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION_INHERITED(DOMMediaStream)
NS_INTERFACE_MAP_ENTRY(DOMMediaStream)
NS_INTERFACE_MAP_END_INHERITING(DOMEventTargetHelper)
@@ -325,18 +334,18 @@ NS_IMPL_RELEASE_INHERITED(DOMAudioNodeMe
NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION_INHERITED(DOMAudioNodeMediaStream)
NS_INTERFACE_MAP_END_INHERITING(DOMMediaStream)
DOMMediaStream::DOMMediaStream(nsPIDOMWindowInner* aWindow,
MediaStreamTrackSourceGetter* aTrackSourceGetter)
: mLogicalStreamStartTime(0), mWindow(aWindow),
mInputStream(nullptr), mOwnedStream(nullptr), mPlaybackStream(nullptr),
- mTrackSourceGetter(aTrackSourceGetter), mTracksCreated(false),
- mNotifiedOfMediaStreamGraphShutdown(false)
+ mTracksPendingRemoval(0), mTrackSourceGetter(aTrackSourceGetter),
+ mTracksCreated(false), mNotifiedOfMediaStreamGraphShutdown(false)
{
nsresult rv;
nsCOMPtr<nsIUUIDGenerator> uuidgen =
do_GetService("@mozilla.org/uuid-generator;1", &rv);
if (NS_SUCCEEDED(rv) && uuidgen) {
nsID uuid;
memset(&uuid, 0, sizeof(uuid));
@@ -572,17 +581,17 @@ DOMMediaStream::RemoveTrack(MediaStreamT
LOG(LogLevel::Debug, ("DOMMediaStream %p does not contain track %p", this, &aTrack));
return;
}
// If the track comes from a TRACK_ANY input port (i.e., mOwnedPort), we need
// to block it in the port. Doing this for a locked track is still OK as it
// will first block the track, then destroy the port. Both cause the track to
// end.
- toRemove->BlockTrackId(aTrack.mTrackID);
+ BlockPlaybackTrack(toRemove);
DebugOnly<bool> removed = mTracks.RemoveElement(toRemove);
MOZ_ASSERT(removed);
NotifyTrackRemoved(&aTrack);
LOG(LogLevel::Debug, ("DOMMediaStream %p Removed track %p", this, &aTrack));
}
@@ -872,28 +881,47 @@ DOMMediaStream::PrincipalChanged(MediaSt
RecomputePrincipal();
}
void
DOMMediaStream::RecomputePrincipal()
{
nsCOMPtr<nsIPrincipal> previousPrincipal = mPrincipal.forget();
nsCOMPtr<nsIPrincipal> previousVideoPrincipal = mVideoPrincipal.forget();
+
+ if (mTracksPendingRemoval > 0) {
+ LOG(LogLevel::Info, ("DOMMediaStream %p RecomputePrincipal() Cannot "
+ "recompute stream principal with tracks pending "
+ "removal.", this));
+ return;
+ }
+
+ LOG(LogLevel::Debug, ("DOMMediaStream %p Recomputing principal. "
+ "Old principal was %p.", this, previousPrincipal.get()));
+
+ // mPrincipal is recomputed based on all current tracks, and tracks that have
+ // not ended in our playback stream.
for (const RefPtr<TrackPort>& info : mTracks) {
if (info->GetTrack()->Ended()) {
continue;
}
+ LOG(LogLevel::Debug, ("DOMMediaStream %p Taking live track %p with "
+ "principal %p into account.", this,
+ info->GetTrack(), info->GetTrack()->GetPrincipal()));
nsContentUtils::CombineResourcePrincipals(&mPrincipal,
info->GetTrack()->GetPrincipal());
if (info->GetTrack()->AsVideoStreamTrack()) {
nsContentUtils::CombineResourcePrincipals(&mVideoPrincipal,
info->GetTrack()->GetPrincipal());
}
}
+ LOG(LogLevel::Debug, ("DOMMediaStream %p new principal is %p.",
+ this, mPrincipal.get()));
+
if (previousPrincipal != mPrincipal ||
previousVideoPrincipal != mVideoPrincipal) {
NotifyPrincipalChanged();
}
}
void
DOMMediaStream::NotifyPrincipalChanged()
@@ -1139,17 +1167,36 @@ DOMMediaStream::UnregisterTrackListener(
mTrackListeners.RemoveElement(aListener);
}
void
DOMMediaStream::NotifyTrackAdded(const RefPtr<MediaStreamTrack>& aTrack)
{
MOZ_ASSERT(NS_IsMainThread());
- RecomputePrincipal();
+ if (mTracksPendingRemoval > 0) {
+ // If there are tracks pending removal we may not degrade the current
+ // principals until those tracks have been confirmed removed from the
+ // playback stream. Instead combine with the new track and the (potentially)
+ // degraded principal will be calculated when it's safe.
+ nsContentUtils::CombineResourcePrincipals(&mPrincipal,
+ aTrack->GetPrincipal());
+ LOG(LogLevel::Debug, ("DOMMediaStream %p saw a track get added. Combining "
+ "its principal %p into our while waiting for pending "
+ "tracks to be removed. New principal is %p.",
+ this, aTrack->GetPrincipal(), mPrincipal.get()));
+ if (aTrack->AsVideoStreamTrack()) {
+ nsContentUtils::CombineResourcePrincipals(&mVideoPrincipal,
+ aTrack->GetPrincipal());
+ }
+ } else {
+ LOG(LogLevel::Debug, ("DOMMediaStream %p saw a track get added. "
+ "Recomputing principal.", this));
+ RecomputePrincipal();
+ }
aTrack->AddPrincipalChangeObserver(this);
for (int32_t i = mTrackListeners.Length() - 1; i >= 0; --i) {
mTrackListeners[i]->NotifyTrackAdded(aTrack);
}
}
@@ -1159,27 +1206,56 @@ DOMMediaStream::NotifyTrackRemoved(const
MOZ_ASSERT(NS_IsMainThread());
aTrack->RemovePrincipalChangeObserver(this);
for (int32_t i = mTrackListeners.Length() - 1; i >= 0; --i) {
mTrackListeners[i]->NotifyTrackRemoved(aTrack);
}
- RecomputePrincipal();
+ // Don't call RecomputePrincipal here as the track may still exist in the
+ // playback stream in the MediaStreamGraph. It will instead be called when the
+ // track has been confirmed removed by the graph. See BlockPlaybackTrack().
}
void
DOMMediaStream::CreateAndAddPlaybackStreamListener(MediaStream* aStream)
{
MOZ_ASSERT(GetCameraStream(), "I'm a hack. Only DOMCameraControl may use me.");
mPlaybackListener = new PlaybackStreamListener(this);
aStream->AddListener(mPlaybackListener);
}
+void
+DOMMediaStream::BlockPlaybackTrack(TrackPort* aTrack)
+{
+ MOZ_ASSERT(aTrack);
+ ++mTracksPendingRemoval;
+ RefPtr<Pledge<bool>> p = aTrack->BlockTrackId(aTrack->GetTrack()->mTrackID);
+ RefPtr<DOMMediaStream> self = this;
+ p->Then([self] (const bool& aIgnore) { self->NotifyPlaybackTrackBlocked(); },
+ [] (const nsresult& aIgnore) { NS_ERROR("Could not remove track from MSG"); }
+ );
+}
+
+void
+DOMMediaStream::NotifyPlaybackTrackBlocked()
+{
+ MOZ_ASSERT(mTracksPendingRemoval > 0,
+ "A track reported finished blocking more times than we asked for");
+ if (--mTracksPendingRemoval == 0) {
+ // The MediaStreamGraph has reported a track was blocked and we are not
+ // waiting for any further tracks to get blocked. It is now safe to
+ // recompute the principal based on our main thread track set state.
+ LOG(LogLevel::Debug, ("DOMMediaStream %p saw all tracks pending removal "
+ "finish. Recomputing principal.", this));
+ RecomputePrincipal();
+ }
+}
+
DOMLocalMediaStream::~DOMLocalMediaStream()
{
if (mInputStream) {
// Make sure Listeners of this stream know it's going away
StopImpl();
}
}
--- a/dom/media/DOMMediaStream.h
+++ b/dom/media/DOMMediaStream.h
@@ -54,16 +54,20 @@ class MediaTrackListListener;
struct MediaTrackConstraints;
} // namespace dom
namespace layers {
class ImageContainer;
class OverlayImage;
} // namespace layers
+namespace media {
+template<typename V, typename E> class Pledge;
+} // namespace media
+
#define NS_DOMMEDIASTREAM_IID \
{ 0x8cb65468, 0x66c0, 0x444e, \
{ 0x89, 0x9f, 0x89, 0x1d, 0x9e, 0xd2, 0xbe, 0x7c } }
class OnTracksAvailableCallback {
public:
virtual ~OnTracksAvailableCallback() {}
virtual void NotifyTracksAvailable(DOMMediaStream* aStream) = 0;
@@ -297,19 +301,20 @@ public:
*/
TrackID GetSourceTrackId() const;
MediaInputPort* GetInputPort() const { return mInputPort; }
MediaStreamTrack* GetTrack() const { return mTrack; }
/**
* Blocks aTrackId from going into mInputPort unless the port has been
- * destroyed.
+ * destroyed. Returns a pledge that gets resolved when the MediaStreamGraph
+ * has applied the block in the playback stream.
*/
- void BlockTrackId(TrackID aTrackId);
+ already_AddRefed<media::Pledge<bool, nsresult>> BlockTrackId(TrackID aTrackId);
private:
RefPtr<MediaInputPort> mInputPort;
RefPtr<MediaStreamTrack> mTrack;
// Defines if we've been given ownership of the input port or if it's owned
// externally. The owner is responsible for destroying the port.
const InputPortOwnership mOwnership;
@@ -600,16 +605,29 @@ protected:
friend class OwnedStreamListener;
class PlaybackStreamListener;
friend class PlaybackStreamListener;
// XXX Bug 1124630. Remove with CameraPreviewMediaStream.
void CreateAndAddPlaybackStreamListener(MediaStream*);
+ /**
+ * Block a track in our playback stream. Calls NotifyPlaybackTrackBlocked()
+ * after the MediaStreamGraph has applied the block and the track is no longer
+ * live.
+ */
+ void BlockPlaybackTrack(TrackPort* aTrack);
+
+ /**
+ * Called on main thread after MediaStreamGraph has applied a track block in
+ * our playback stream.
+ */
+ void NotifyPlaybackTrackBlocked();
+
// Recomputes the current principal of this stream based on the set of tracks
// it currently contains. PrincipalChangeObservers will be notified only if
// the principal changes.
void RecomputePrincipal();
// StreamTime at which the currentTime attribute would return 0.
StreamTime mLogicalStreamStartTime;
@@ -640,16 +658,20 @@ protected:
RefPtr<MediaInputPort> mPlaybackPort;
// MediaStreamTracks corresponding to tracks in our mOwnedStream.
AutoTArray<RefPtr<TrackPort>, 2> mOwnedTracks;
// MediaStreamTracks corresponding to tracks in our mPlaybackStream.
AutoTArray<RefPtr<TrackPort>, 2> mTracks;
+ // Number of MediaStreamTracks that have been removed on main thread but are
+ // waiting to be removed on MediaStreamGraph thread.
+ size_t mTracksPendingRemoval;
+
// The interface through which we can query the stream producer for
// track sources.
RefPtr<MediaStreamTrackSourceGetter> mTrackSourceGetter;
RefPtr<OwnedStreamListener> mOwnedListener;
RefPtr<PlaybackStreamListener> mPlaybackListener;
nsTArray<nsAutoPtr<OnTracksAvailableCallback> > mRunOnTracksAvailable;
--- a/dom/media/MediaStreamGraph.cpp
+++ b/dom/media/MediaStreamGraph.cpp
@@ -23,26 +23,28 @@
#include "AudioNodeStream.h"
#include "AudioNodeExternalInputStream.h"
#include "mozilla/dom/AudioContextBinding.h"
#include "mozilla/media/MediaUtils.h"
#include <algorithm>
#include "DOMMediaStream.h"
#include "GeckoProfiler.h"
#include "mozilla/unused.h"
+#include "mozilla/media/MediaUtils.h"
#ifdef MOZ_WEBRTC
#include "AudioOutputObserver.h"
#endif
#include "mtransport/runnable_utils.h"
#include "webaudio/blink/HRTFDatabaseLoader.h"
using namespace mozilla::layers;
using namespace mozilla::dom;
using namespace mozilla::gfx;
+using namespace mozilla::media;
namespace mozilla {
LazyLogModule gMediaStreamGraphLog("MediaStreamGraph");
#define STREAM_LOG(type, msg) MOZ_LOG(gMediaStreamGraphLog, type, msg)
// #define ENABLE_LIFECYCLE_LOG
@@ -3007,39 +3009,53 @@ MediaInputPort::SetGraphImpl(MediaStream
}
void
MediaInputPort::BlockTrackIdImpl(TrackID aTrackId)
{
mBlockedTracks.AppendElement(aTrackId);
}
-void
+already_AddRefed<Pledge<bool>>
MediaInputPort::BlockTrackId(TrackID aTrackId)
{
class Message : public ControlMessage {
public:
- explicit Message(MediaInputPort* aPort, TrackID aTrackId)
+ explicit Message(MediaInputPort* aPort,
+ TrackID aTrackId,
+ already_AddRefed<nsIRunnable> aRunnable)
: ControlMessage(aPort->GetDestination()),
- mPort(aPort), mTrackId(aTrackId) {}
+ mPort(aPort), mTrackId(aTrackId), mRunnable(aRunnable) {}
void Run() override
{
mPort->BlockTrackIdImpl(mTrackId);
+ if (mRunnable) {
+ mStream->Graph()->DispatchToMainThreadAfterStreamStateUpdate(mRunnable.forget());
+ }
}
void RunDuringShutdown() override
{
Run();
}
RefPtr<MediaInputPort> mPort;
TrackID mTrackId;
+ nsCOMPtr<nsIRunnable> mRunnable;
};
MOZ_ASSERT(IsTrackIDExplicit(aTrackId),
"Only explicit TrackID is allowed");
- GraphImpl()->AppendMessage(MakeUnique<Message>(this, aTrackId));
+
+ RefPtr<Pledge<bool>> pledge = new Pledge<bool>();
+ nsCOMPtr<nsIRunnable> runnable = NewRunnableFrom([pledge]() {
+ MOZ_ASSERT(NS_IsMainThread());
+ pledge->Resolve(true);
+ return NS_OK;
+ });
+ GraphImpl()->AppendMessage(MakeUnique<Message>(this, aTrackId, runnable.forget()));
+ return pledge.forget();
}
already_AddRefed<MediaInputPort>
ProcessedMediaStream::AllocateInputPort(MediaStream* aStream, TrackID aTrackID,
TrackID aDestTrackID,
uint16_t aInputNumber, uint16_t aOutputNumber,
nsTArray<TrackID>* aBlockedTracks)
{
--- a/dom/media/MediaStreamGraph.h
+++ b/dom/media/MediaStreamGraph.h
@@ -36,16 +36,20 @@ class nsAutoRefTraits<SpeexResamplerStat
namespace mozilla {
extern LazyLogModule gMediaStreamGraphLog;
namespace dom {
enum class AudioContextOperation;
}
+namespace media {
+ template<typename V, typename E> class Pledge;
+}
+
/*
* MediaStreamGraph is a framework for synchronized audio/video processing
* and playback. It is designed to be used by other browser components such as
* HTML media elements, media capture APIs, real-time media streaming APIs,
* multitrack media APIs, and advanced audio APIs.
*
* The MediaStreamGraph uses a dedicated thread to process media --- the media
* graph thread. This ensures that we can process media through the graph
@@ -1228,18 +1232,22 @@ public:
void Destroy();
// Any thread
MediaStream* GetSource() { return mSource; }
TrackID GetSourceTrackId() { return mSourceTrack; }
ProcessedMediaStream* GetDestination() { return mDest; }
TrackID GetDestinationTrackId() { return mDestTrack; }
- // Block aTrackId in the port. Consumers will interpret this track as ended.
- void BlockTrackId(TrackID aTrackId);
+ /**
+ * Block aTrackId in the port. Consumers will interpret this track as ended.
+ * Returns a pledge that resolves on the main thread after the track block has
+ * been applied by the MSG.
+ */
+ already_AddRefed<media::Pledge<bool, nsresult>> BlockTrackId(TrackID aTrackId);
private:
void BlockTrackIdImpl(TrackID aTrackId);
public:
// Returns true if aTrackId has not been blocked and this port has not
// been locked to another track.
bool PassTrackThrough(TrackID aTrackId) {
return !mBlockedTracks.Contains(aTrackId) &&