Bug 1208371 - Ensure a media element's ImageContainer is protected when playing a stream. r?mt,jesup
HTMLMediaElement needs special protection when playing a stream since its
ImageContainer can outlive the video track of a stream.
Consider for instance when a (cross-origin) video track is removed from a
DOMMediaStream by a user and the remaining video track (non-CORS) does not yet
contain any actual video frames. The HTMLMediaElement will display a frame from
the removed track but the DOMMediaStream's principal has been updated to not
include the principal from the removed track.
With this patch we handle this by letting VideoFrameContainer notify
HTMLMediaElement when it has flushed out all video frames belonging to a
certain PrincipalHandle. I.e., when a new PrincipalHandle has been applied to the
underlying ImageContainer.
MozReview-Commit-ID: LvIZPl6Rdgj
--- a/dom/html/HTMLMediaElement.cpp
+++ b/dom/html/HTMLMediaElement.cpp
@@ -4184,24 +4184,63 @@ HTMLMediaElement::PrincipalChanged(DOMMe
LOG(LogLevel::Info, ("HTMLMediaElement %p Stream principal changed.", this));
nsContentUtils::CombineResourcePrincipals(&mSrcStreamVideoPrincipal,
aStream->GetVideoPrincipal());
LOG(LogLevel::Debug, ("HTMLMediaElement %p Stream video principal changed to "
"%p. Waiting for it to reach VideoFrameContainer before "
"setting.", this, aStream->GetVideoPrincipal()));
if (mVideoFrameContainer) {
- UpdateSrcStreamVideoPrincipal(aStream->GetVideoPrincipal());
+ UpdateSrcStreamVideoPrincipal(mVideoFrameContainer->GetLastPrincipalHandle());
}
}
void
-HTMLMediaElement::UpdateSrcStreamVideoPrincipal(nsIPrincipal* aPrincipal)
-{
- mSrcStreamVideoPrincipal = aPrincipal;
+HTMLMediaElement::UpdateSrcStreamVideoPrincipal(const PrincipalHandle& aPrincipalHandle)
+{
+ nsTArray<RefPtr<VideoStreamTrack>> videoTracks;
+ mSrcStream->GetVideoTracks(videoTracks);
+
+ PrincipalHandle handle(aPrincipalHandle);
+ bool matchesTrackPrincipal = false;
+ for (const RefPtr<VideoStreamTrack>& track : videoTracks) {
+ if (PrincipalHandleMatches(handle,
+ track->GetPrincipal()) &&
+ !track->Ended()) {
+ // When the PrincipalHandle for the VideoFrameContainer changes to that of
+ // a track in mSrcStream we know that a removed track was displayed but
+ // is no longer so.
+ matchesTrackPrincipal = true;
+ LOG(LogLevel::Debug, ("HTMLMediaElement %p VideoFrameContainer's "
+ "PrincipalHandle matches track %p. That's all we "
+ "need.", this, track.get()));
+ break;
+ }
+ }
+
+ if (matchesTrackPrincipal) {
+ mSrcStreamVideoPrincipal = mSrcStream->GetVideoPrincipal();
+ }
+}
+
+void
+HTMLMediaElement::PrincipalHandleChangedForVideoFrameContainer(VideoFrameContainer* aContainer,
+ const PrincipalHandle& aNewPrincipalHandle)
+{
+ MOZ_ASSERT(NS_IsMainThread());
+
+ if (!mSrcStream) {
+ return;
+ }
+
+ LOG(LogLevel::Debug, ("HTMLMediaElement %p PrincipalHandle changed in "
+ "VideoFrameContainer.",
+ this));
+
+ UpdateSrcStreamVideoPrincipal(aNewPrincipalHandle);
}
nsresult HTMLMediaElement::DispatchEvent(const nsAString& aName)
{
LOG_EVENT(LogLevel::Debug, ("%p Dispatching event %s", this,
NS_ConvertUTF16toUTF8(aName).get()));
// Save events that occur while in the bfcache. These will be dispatched
--- a/dom/html/HTMLMediaElement.h
+++ b/dom/html/HTMLMediaElement.h
@@ -226,17 +226,22 @@ public:
// Called by the media decoder and the video frame to get the
// ImageContainer containing the video data.
B2G_ACL_EXPORT virtual VideoFrameContainer* GetVideoFrameContainer() final override;
layers::ImageContainer* GetImageContainer();
// From PrincipalChangeObserver<DOMMediaStream>.
void PrincipalChanged(DOMMediaStream* aStream) override;
- void UpdateSrcStreamVideoPrincipal(nsIPrincipal* aPrincipal);
+ void UpdateSrcStreamVideoPrincipal(const PrincipalHandle& aPrincipalHandle);
+
+ // Called after the MediaStream we're playing rendered a frame to aContainer
+ // with a different principalHandle than the previous frame.
+ void PrincipalHandleChangedForVideoFrameContainer(VideoFrameContainer* aContainer,
+ const PrincipalHandle& aNewPrincipalHandle);
// Dispatch events
virtual nsresult DispatchAsyncEvent(const nsAString& aName) final override;
// Triggers a recomputation of readyState.
void UpdateReadyState() override { UpdateReadyStateInternal(); }
// Dispatch events that were raised while in the bfcache
--- a/dom/media/MediaStreamGraph.cpp
+++ b/dom/media/MediaStreamGraph.cpp
@@ -875,16 +875,17 @@ MediaStreamGraphImpl::PlayVideo(MediaStr
if (aStream->mVideoOutputs.IsEmpty())
return;
TimeStamp currentTimeStamp = CurrentDriver()->GetCurrentTimeStamp();
// Collect any new frames produced in this iteration.
AutoTArray<ImageContainer::NonOwningImage,4> newImages;
+ PrincipalHandle lastPrincipalHandle = PRINCIPAL_HANDLE_NONE;
RefPtr<Image> blackImage;
MOZ_ASSERT(mProcessedTime >= aStream->mBufferStartTime, "frame position before buffer?");
// We only look at the non-blocking interval
StreamTime frameBufferTime = aStream->GraphTimeToStreamTime(mProcessedTime);
StreamTime bufferEndTime = aStream->GraphTimeToStreamTime(aStream->mStartBlocking);
StreamTime start;
const VideoChunk* chunk;
@@ -938,28 +939,34 @@ MediaStreamGraphImpl::PlayVideo(MediaStr
}
}
if (blackImage) {
image = blackImage;
}
}
newImages.AppendElement(ImageContainer::NonOwningImage(image, targetTime));
+ lastPrincipalHandle = chunk->GetPrincipalHandle();
+
aStream->mLastPlayedVideoFrame = *frame;
}
if (!aStream->mLastPlayedVideoFrame.GetImage())
return;
AutoTArray<ImageContainer::NonOwningImage,4> images;
bool haveMultipleImages = false;
for (uint32_t i = 0; i < aStream->mVideoOutputs.Length(); ++i) {
VideoFrameContainer* output = aStream->mVideoOutputs[i];
+ bool principalHandleChanged =
+ lastPrincipalHandle != PRINCIPAL_HANDLE_NONE &&
+ lastPrincipalHandle != output->GetLastPrincipalHandle();
+
// Find previous frames that may still be valid.
AutoTArray<ImageContainer::OwningImage,4> previousImages;
output->GetImageContainer()->GetCurrentImages(&previousImages);
uint32_t j = previousImages.Length();
if (j) {
// Re-use the most recent frame before currentTimeStamp and subsequent,
// always keeping at least one frame.
do {
@@ -989,16 +996,22 @@ MediaStreamGraphImpl::PlayVideo(MediaStr
image.mTimeStamp, image.mFrameID));
}
// Add the frames from this iteration.
for (auto& image : newImages) {
image.mFrameID = output->NewFrameID();
images.AppendElement(image);
}
+
+ if (principalHandleChanged) {
+ output->UpdatePrincipalHandleForFrameID(lastPrincipalHandle,
+ newImages.LastElement().mFrameID);
+ }
+
output->SetCurrentFrames(aStream->mLastPlayedVideoFrame.GetIntrinsicSize(),
images);
nsCOMPtr<nsIRunnable> event =
new VideoFrameContainerInvalidateRunnable(output);
DispatchToMainThreadAfterStreamStateUpdate(event.forget());
images.ClearAndRetainStorage();
--- a/dom/media/VideoFrameContainer.cpp
+++ b/dom/media/VideoFrameContainer.cpp
@@ -15,25 +15,43 @@ using namespace mozilla::layers;
namespace mozilla {
VideoFrameContainer::VideoFrameContainer(dom::HTMLMediaElement* aElement,
already_AddRefed<ImageContainer> aContainer)
: mElement(aElement),
mImageContainer(aContainer), mMutex("nsVideoFrameContainer"),
mFrameID(0),
- mIntrinsicSizeChanged(false), mImageSizeChanged(false)
+ mIntrinsicSizeChanged(false), mImageSizeChanged(false),
+ mPendingPrincipalHandle(PRINCIPAL_HANDLE_NONE), mFrameIDForPendingPrincipalHandle(0)
{
NS_ASSERTION(aElement, "aElement must not be null");
NS_ASSERTION(mImageContainer, "aContainer must not be null");
}
VideoFrameContainer::~VideoFrameContainer()
{}
+PrincipalHandle VideoFrameContainer::GetLastPrincipalHandle()
+{
+ MutexAutoLock lock(mMutex);
+ return mLastPrincipalHandle;
+}
+
+void VideoFrameContainer::UpdatePrincipalHandleForFrameID(const PrincipalHandle& aPrincipalHandle,
+ const ImageContainer::FrameID& aFrameID)
+{
+ MutexAutoLock lock(mMutex);
+ if (mPendingPrincipalHandle == aPrincipalHandle) {
+ return;
+ }
+ mPendingPrincipalHandle = aPrincipalHandle;
+ mFrameIDForPendingPrincipalHandle = aFrameID;
+}
+
void VideoFrameContainer::SetCurrentFrame(const gfx::IntSize& aIntrinsicSize,
Image* aImage,
const TimeStamp& aTargetTime)
{
if (aImage) {
MutexAutoLock lock(mMutex);
AutoTArray<ImageContainer::NonOwningImage,1> imageList;
imageList.AppendElement(
@@ -64,18 +82,43 @@ void VideoFrameContainer::SetCurrentFram
gfx::IntSize oldFrameSize = mImageContainer->GetCurrentSize();
// When using the OMX decoder, destruction of the current image can indirectly
// block on main thread I/O. If we let this happen while holding onto
// |mImageContainer|'s lock, then when the main thread then tries to
// composite it can then block on |mImageContainer|'s lock, causing a
// deadlock. We use this hack to defer the destruction of the current image
// until it is safe.
- nsTArray<ImageContainer::OwningImage> kungFuDeathGrip;
- mImageContainer->GetCurrentImages(&kungFuDeathGrip);
+ nsTArray<ImageContainer::OwningImage> oldImages;
+ mImageContainer->GetCurrentImages(&oldImages);
+
+ ImageContainer::FrameID lastFrameIDForOldPrincipalHandle =
+ mFrameIDForPendingPrincipalHandle - 1;
+ if (mPendingPrincipalHandle != PRINCIPAL_HANDLE_NONE &&
+ ((!oldImages.IsEmpty() &&
+ oldImages.LastElement().mFrameID >= lastFrameIDForOldPrincipalHandle) ||
+ (!aImages.IsEmpty() &&
+ aImages[0].mFrameID > lastFrameIDForOldPrincipalHandle))) {
+ // We are releasing the last FrameID prior to `lastFrameIDForOldPrincipalHandle`
+ // OR
+ // there are no FrameIDs prior to `lastFrameIDForOldPrincipalHandle` in the new
+ // set of images.
+ // This means that the old principal handle has been flushed out and we can
+ // notify our video element about this change.
+ RefPtr<VideoFrameContainer> self = this;
+ PrincipalHandle principalHandle = mPendingPrincipalHandle;
+ mLastPrincipalHandle = mPendingPrincipalHandle;
+ mPendingPrincipalHandle = PRINCIPAL_HANDLE_NONE;
+ mFrameIDForPendingPrincipalHandle = 0;
+ NS_DispatchToMainThread(NS_NewRunnableFunction([self, principalHandle]() {
+ if (self->mElement) {
+ self->mElement->PrincipalHandleChangedForVideoFrameContainer(self, principalHandle);
+ }
+ }));
+ }
if (aImages.IsEmpty()) {
mImageContainer->ClearAllImages();
} else {
mImageContainer->SetCurrentImages(aImages);
}
gfx::IntSize newFrameSize = mImageContainer->GetCurrentSize();
if (oldFrameSize != newFrameSize) {
--- a/dom/media/VideoFrameContainer.h
+++ b/dom/media/VideoFrameContainer.h
@@ -8,16 +8,17 @@
#define VIDEOFRAMECONTAINER_H_
#include "mozilla/Mutex.h"
#include "mozilla/TimeStamp.h"
#include "gfxPoint.h"
#include "nsCOMPtr.h"
#include "nsAutoPtr.h"
#include "ImageContainer.h"
+#include "MediaSegment.h"
namespace mozilla {
namespace dom {
class HTMLMediaElement;
} // namespace dom
/**
@@ -37,16 +38,23 @@ public:
typedef layers::Image Image;
NS_INLINE_DECL_THREADSAFE_REFCOUNTING(VideoFrameContainer)
VideoFrameContainer(dom::HTMLMediaElement* aElement,
already_AddRefed<ImageContainer> aContainer);
// Call on any thread
+ // Returns the last principalHandle we notified mElement about.
+ PrincipalHandle GetLastPrincipalHandle();
+ // We will notify mElement that aPrincipalHandle has been applied when all
+ // FrameIDs prior to aFrameID have been flushed out.
+ // aFrameID is ignored if aPrincipalHandle already is our pending principalHandle.
+ void UpdatePrincipalHandleForFrameID(const PrincipalHandle& aPrincipalHandle,
+ const ImageContainer::FrameID& aFrameID);
B2G_ACL_EXPORT void SetCurrentFrame(const gfx::IntSize& aIntrinsicSize, Image* aImage,
const TimeStamp& aTargetTime);
void SetCurrentFrames(const gfx::IntSize& aIntrinsicSize,
const nsTArray<ImageContainer::NonOwningImage>& aImages);
void ClearCurrentFrame(const gfx::IntSize& aIntrinsicSize)
{
SetCurrentFrames(aIntrinsicSize, nsTArray<ImageContainer::NonOwningImage>());
}
@@ -106,13 +114,20 @@ protected:
// and update the intrinsic size on the element, request a frame reflow and
// then reset this flag.
bool mIntrinsicSizeChanged;
// True when the Image size has changed since the last time Invalidate() was
// called. When set, the next call to Invalidate() will ensure that the
// frame is fully invalidated instead of just invalidating for the image change
// in the ImageLayer.
bool mImageSizeChanged;
+ // The last PrincipalHandle we notified mElement about.
+ PrincipalHandle mLastPrincipalHandle;
+ // The PrincipalHandle the client has notified us is changing with FrameID
+ // mFrameIDForPendingPrincipalHandle.
+ PrincipalHandle mPendingPrincipalHandle;
+ // The FrameID for which mPendingPrincipal is first valid.
+ ImageContainer::FrameID mFrameIDForPendingPrincipalHandle;
};
} // namespace mozilla
#endif /* VIDEOFRAMECONTAINER_H_ */