Bug 1208371 - Ensure a media element's ImageContainer is protected when playing a stream. r?mt,jesup draft
authorAndreas Pehrson <pehrsons@gmail.com>
Thu, 04 Feb 2016 09:27:09 +0800
changeset 347663 cc2748b1fe42e6420d936d810b9d8975ade0bbd4
parent 347662 6d032f4fcc64f0ad20100d3faed540a87ec71ae4
child 347664 f700a0e783df7a4c37ec80d6139bcc8d5d6e1edc
push id14642
push userpehrsons@gmail.com
push dateTue, 05 Apr 2016 16:45:34 +0000
reviewersmt, jesup
bugs1208371
milestone47.0a1
Bug 1208371 - Ensure a media element's ImageContainer is protected when playing a stream. r?mt,jesup HTMLMediaElement needs special protection when playing a stream since its ImageContainer can outlive the video track of a stream. Consider for instance when a (cross-origin) video track is removed from a DOMMediaStream by a user and the remaining video track (non-CORS) does not yet contain any actual video frames. The HTMLMediaElement will display a frame from the removed track but the DOMMediaStream's principal has been updated to not include the principal from the removed track. With this patch we handle this by letting VideoFrameContainer notify HTMLMediaElement when it has flushed out all video frames belonging to a certain PrincipalHandle. I.e., when a new PrincipalHandle has been applied to the underlying ImageContainer. MozReview-Commit-ID: LvIZPl6Rdgj
dom/html/HTMLMediaElement.cpp
dom/html/HTMLMediaElement.h
dom/media/MediaStreamGraph.cpp
dom/media/VideoFrameContainer.cpp
dom/media/VideoFrameContainer.h
--- a/dom/html/HTMLMediaElement.cpp
+++ b/dom/html/HTMLMediaElement.cpp
@@ -4184,24 +4184,63 @@ HTMLMediaElement::PrincipalChanged(DOMMe
   LOG(LogLevel::Info, ("HTMLMediaElement %p Stream principal changed.", this));
   nsContentUtils::CombineResourcePrincipals(&mSrcStreamVideoPrincipal,
                                             aStream->GetVideoPrincipal());
 
   LOG(LogLevel::Debug, ("HTMLMediaElement %p Stream video principal changed to "
                         "%p. Waiting for it to reach VideoFrameContainer before "
                         "setting.", this, aStream->GetVideoPrincipal()));
   if (mVideoFrameContainer) {
-    UpdateSrcStreamVideoPrincipal(aStream->GetVideoPrincipal());
+    UpdateSrcStreamVideoPrincipal(mVideoFrameContainer->GetLastPrincipalHandle());
   }
 }
 
 void
-HTMLMediaElement::UpdateSrcStreamVideoPrincipal(nsIPrincipal* aPrincipal)
-{
-  mSrcStreamVideoPrincipal = aPrincipal;
+HTMLMediaElement::UpdateSrcStreamVideoPrincipal(const PrincipalHandle& aPrincipalHandle)
+{
+  nsTArray<RefPtr<VideoStreamTrack>> videoTracks;
+  mSrcStream->GetVideoTracks(videoTracks);
+
+  PrincipalHandle handle(aPrincipalHandle);
+  bool matchesTrackPrincipal = false;
+  for (const RefPtr<VideoStreamTrack>& track : videoTracks) {
+    if (PrincipalHandleMatches(handle,
+                               track->GetPrincipal()) &&
+        !track->Ended()) {
+      // When the PrincipalHandle for the VideoFrameContainer changes to that of
+      // a track in mSrcStream we know that a removed track was displayed but
+      // is no longer so.
+      matchesTrackPrincipal = true;
+      LOG(LogLevel::Debug, ("HTMLMediaElement %p VideoFrameContainer's "
+                            "PrincipalHandle matches track %p. That's all we "
+                            "need.", this, track.get()));
+      break;
+    }
+  }
+
+  if (matchesTrackPrincipal) {
+    mSrcStreamVideoPrincipal = mSrcStream->GetVideoPrincipal();
+  }
+}
+
+void
+HTMLMediaElement::PrincipalHandleChangedForVideoFrameContainer(VideoFrameContainer* aContainer,
+                                                               const PrincipalHandle& aNewPrincipalHandle)
+{
+  MOZ_ASSERT(NS_IsMainThread());
+
+  if (!mSrcStream) {
+    return;
+  }
+
+  LOG(LogLevel::Debug, ("HTMLMediaElement %p PrincipalHandle changed in "
+                        "VideoFrameContainer.",
+                        this));
+
+  UpdateSrcStreamVideoPrincipal(aNewPrincipalHandle);
 }
 
 nsresult HTMLMediaElement::DispatchEvent(const nsAString& aName)
 {
   LOG_EVENT(LogLevel::Debug, ("%p Dispatching event %s", this,
                           NS_ConvertUTF16toUTF8(aName).get()));
 
   // Save events that occur while in the bfcache. These will be dispatched
--- a/dom/html/HTMLMediaElement.h
+++ b/dom/html/HTMLMediaElement.h
@@ -226,17 +226,22 @@ public:
   // Called by the media decoder and the video frame to get the
   // ImageContainer containing the video data.
   B2G_ACL_EXPORT virtual VideoFrameContainer* GetVideoFrameContainer() final override;
   layers::ImageContainer* GetImageContainer();
 
   // From PrincipalChangeObserver<DOMMediaStream>.
   void PrincipalChanged(DOMMediaStream* aStream) override;
 
-  void UpdateSrcStreamVideoPrincipal(nsIPrincipal* aPrincipal);
+  void UpdateSrcStreamVideoPrincipal(const PrincipalHandle& aPrincipalHandle);
+
+  // Called after the MediaStream we're playing rendered a frame to aContainer
+  // with a different principalHandle than the previous frame.
+  void PrincipalHandleChangedForVideoFrameContainer(VideoFrameContainer* aContainer,
+                                                    const PrincipalHandle& aNewPrincipalHandle);
 
   // Dispatch events
   virtual nsresult DispatchAsyncEvent(const nsAString& aName) final override;
 
   // Triggers a recomputation of readyState.
   void UpdateReadyState() override { UpdateReadyStateInternal(); }
 
   // Dispatch events that were raised while in the bfcache
--- a/dom/media/MediaStreamGraph.cpp
+++ b/dom/media/MediaStreamGraph.cpp
@@ -875,16 +875,17 @@ MediaStreamGraphImpl::PlayVideo(MediaStr
 
   if (aStream->mVideoOutputs.IsEmpty())
     return;
 
   TimeStamp currentTimeStamp = CurrentDriver()->GetCurrentTimeStamp();
 
   // Collect any new frames produced in this iteration.
   AutoTArray<ImageContainer::NonOwningImage,4> newImages;
+  PrincipalHandle lastPrincipalHandle = PRINCIPAL_HANDLE_NONE;
   RefPtr<Image> blackImage;
 
   MOZ_ASSERT(mProcessedTime >= aStream->mBufferStartTime, "frame position before buffer?");
   // We only look at the non-blocking interval
   StreamTime frameBufferTime = aStream->GraphTimeToStreamTime(mProcessedTime);
   StreamTime bufferEndTime = aStream->GraphTimeToStreamTime(aStream->mStartBlocking);
   StreamTime start;
   const VideoChunk* chunk;
@@ -938,28 +939,34 @@ MediaStreamGraphImpl::PlayVideo(MediaStr
         }
       }
       if (blackImage) {
         image = blackImage;
       }
     }
     newImages.AppendElement(ImageContainer::NonOwningImage(image, targetTime));
 
+    lastPrincipalHandle = chunk->GetPrincipalHandle();
+
     aStream->mLastPlayedVideoFrame = *frame;
   }
 
   if (!aStream->mLastPlayedVideoFrame.GetImage())
     return;
 
   AutoTArray<ImageContainer::NonOwningImage,4> images;
   bool haveMultipleImages = false;
 
   for (uint32_t i = 0; i < aStream->mVideoOutputs.Length(); ++i) {
     VideoFrameContainer* output = aStream->mVideoOutputs[i];
 
+    bool principalHandleChanged =
+      lastPrincipalHandle != PRINCIPAL_HANDLE_NONE &&
+      lastPrincipalHandle != output->GetLastPrincipalHandle();
+
     // Find previous frames that may still be valid.
     AutoTArray<ImageContainer::OwningImage,4> previousImages;
     output->GetImageContainer()->GetCurrentImages(&previousImages);
     uint32_t j = previousImages.Length();
     if (j) {
       // Re-use the most recent frame before currentTimeStamp and subsequent,
       // always keeping at least one frame.
       do {
@@ -989,16 +996,22 @@ MediaStreamGraphImpl::PlayVideo(MediaStr
                                           image.mTimeStamp, image.mFrameID));
     }
 
     // Add the frames from this iteration.
     for (auto& image : newImages) {
       image.mFrameID = output->NewFrameID();
       images.AppendElement(image);
     }
+
+    if (principalHandleChanged) {
+      output->UpdatePrincipalHandleForFrameID(lastPrincipalHandle,
+                                              newImages.LastElement().mFrameID);
+    }
+
     output->SetCurrentFrames(aStream->mLastPlayedVideoFrame.GetIntrinsicSize(),
                              images);
 
     nsCOMPtr<nsIRunnable> event =
       new VideoFrameContainerInvalidateRunnable(output);
     DispatchToMainThreadAfterStreamStateUpdate(event.forget());
 
     images.ClearAndRetainStorage();
--- a/dom/media/VideoFrameContainer.cpp
+++ b/dom/media/VideoFrameContainer.cpp
@@ -15,25 +15,43 @@ using namespace mozilla::layers;
 
 namespace mozilla {
 
 VideoFrameContainer::VideoFrameContainer(dom::HTMLMediaElement* aElement,
                                          already_AddRefed<ImageContainer> aContainer)
   : mElement(aElement),
     mImageContainer(aContainer), mMutex("nsVideoFrameContainer"),
     mFrameID(0),
-    mIntrinsicSizeChanged(false), mImageSizeChanged(false)
+    mIntrinsicSizeChanged(false), mImageSizeChanged(false),
+    mPendingPrincipalHandle(PRINCIPAL_HANDLE_NONE), mFrameIDForPendingPrincipalHandle(0)
 {
   NS_ASSERTION(aElement, "aElement must not be null");
   NS_ASSERTION(mImageContainer, "aContainer must not be null");
 }
 
 VideoFrameContainer::~VideoFrameContainer()
 {}
 
+PrincipalHandle VideoFrameContainer::GetLastPrincipalHandle()
+{
+  MutexAutoLock lock(mMutex);
+  return mLastPrincipalHandle;
+}
+
+void VideoFrameContainer::UpdatePrincipalHandleForFrameID(const PrincipalHandle& aPrincipalHandle,
+                                                          const ImageContainer::FrameID& aFrameID)
+{
+  MutexAutoLock lock(mMutex);
+  if (mPendingPrincipalHandle == aPrincipalHandle) {
+    return;
+  }
+  mPendingPrincipalHandle = aPrincipalHandle;
+  mFrameIDForPendingPrincipalHandle = aFrameID;
+}
+
 void VideoFrameContainer::SetCurrentFrame(const gfx::IntSize& aIntrinsicSize,
                                           Image* aImage,
                                           const TimeStamp& aTargetTime)
 {
   if (aImage) {
     MutexAutoLock lock(mMutex);
     AutoTArray<ImageContainer::NonOwningImage,1> imageList;
     imageList.AppendElement(
@@ -64,18 +82,43 @@ void VideoFrameContainer::SetCurrentFram
   gfx::IntSize oldFrameSize = mImageContainer->GetCurrentSize();
 
   // When using the OMX decoder, destruction of the current image can indirectly
   //  block on main thread I/O. If we let this happen while holding onto
   //  |mImageContainer|'s lock, then when the main thread then tries to
   //  composite it can then block on |mImageContainer|'s lock, causing a
   //  deadlock. We use this hack to defer the destruction of the current image
   //  until it is safe.
-  nsTArray<ImageContainer::OwningImage> kungFuDeathGrip;
-  mImageContainer->GetCurrentImages(&kungFuDeathGrip);
+  nsTArray<ImageContainer::OwningImage> oldImages;
+  mImageContainer->GetCurrentImages(&oldImages);
+
+  ImageContainer::FrameID lastFrameIDForOldPrincipalHandle =
+    mFrameIDForPendingPrincipalHandle - 1;
+  if (mPendingPrincipalHandle != PRINCIPAL_HANDLE_NONE &&
+       ((!oldImages.IsEmpty() &&
+          oldImages.LastElement().mFrameID >= lastFrameIDForOldPrincipalHandle) ||
+        (!aImages.IsEmpty() &&
+          aImages[0].mFrameID > lastFrameIDForOldPrincipalHandle))) {
+    // We are releasing the last FrameID prior to `lastFrameIDForOldPrincipalHandle`
+    // OR
+    // there are no FrameIDs prior to `lastFrameIDForOldPrincipalHandle` in the new
+    // set of images.
+    // This means that the old principal handle has been flushed out and we can
+    // notify our video element about this change.
+    RefPtr<VideoFrameContainer> self = this;
+    PrincipalHandle principalHandle = mPendingPrincipalHandle;
+    mLastPrincipalHandle = mPendingPrincipalHandle;
+    mPendingPrincipalHandle = PRINCIPAL_HANDLE_NONE;
+    mFrameIDForPendingPrincipalHandle = 0;
+    NS_DispatchToMainThread(NS_NewRunnableFunction([self, principalHandle]() {
+      if (self->mElement) {
+        self->mElement->PrincipalHandleChangedForVideoFrameContainer(self, principalHandle);
+      }
+    }));
+  }
 
   if (aImages.IsEmpty()) {
     mImageContainer->ClearAllImages();
   } else {
     mImageContainer->SetCurrentImages(aImages);
   }
   gfx::IntSize newFrameSize = mImageContainer->GetCurrentSize();
   if (oldFrameSize != newFrameSize) {
--- a/dom/media/VideoFrameContainer.h
+++ b/dom/media/VideoFrameContainer.h
@@ -8,16 +8,17 @@
 #define VIDEOFRAMECONTAINER_H_
 
 #include "mozilla/Mutex.h"
 #include "mozilla/TimeStamp.h"
 #include "gfxPoint.h"
 #include "nsCOMPtr.h"
 #include "nsAutoPtr.h"
 #include "ImageContainer.h"
+#include "MediaSegment.h"
 
 namespace mozilla {
 
 namespace dom {
 class HTMLMediaElement;
 } // namespace dom
 
 /**
@@ -37,16 +38,23 @@ public:
   typedef layers::Image Image;
 
   NS_INLINE_DECL_THREADSAFE_REFCOUNTING(VideoFrameContainer)
 
   VideoFrameContainer(dom::HTMLMediaElement* aElement,
                       already_AddRefed<ImageContainer> aContainer);
 
   // Call on any thread
+  // Returns the last principalHandle we notified mElement about.
+  PrincipalHandle GetLastPrincipalHandle();
+  // We will notify mElement that aPrincipalHandle has been applied when all
+  // FrameIDs prior to aFrameID have been flushed out.
+  // aFrameID is ignored if aPrincipalHandle already is our pending principalHandle.
+  void UpdatePrincipalHandleForFrameID(const PrincipalHandle& aPrincipalHandle,
+                                       const ImageContainer::FrameID& aFrameID);
   B2G_ACL_EXPORT void SetCurrentFrame(const gfx::IntSize& aIntrinsicSize, Image* aImage,
                        const TimeStamp& aTargetTime);
   void SetCurrentFrames(const gfx::IntSize& aIntrinsicSize,
                         const nsTArray<ImageContainer::NonOwningImage>& aImages);
   void ClearCurrentFrame(const gfx::IntSize& aIntrinsicSize)
   {
     SetCurrentFrames(aIntrinsicSize, nsTArray<ImageContainer::NonOwningImage>());
   }
@@ -106,13 +114,20 @@ protected:
   // and update the intrinsic size on the element, request a frame reflow and
   // then reset this flag.
   bool mIntrinsicSizeChanged;
   // True when the Image size has changed since the last time Invalidate() was
   // called. When set, the next call to Invalidate() will ensure that the
   // frame is fully invalidated instead of just invalidating for the image change
   // in the ImageLayer.
   bool mImageSizeChanged;
+  // The last PrincipalHandle we notified mElement about.
+  PrincipalHandle mLastPrincipalHandle;
+  // The PrincipalHandle the client has notified us is changing with FrameID
+  // mFrameIDForPendingPrincipalHandle.
+  PrincipalHandle mPendingPrincipalHandle;
+  // The FrameID for which mPendingPrincipal is first valid.
+  ImageContainer::FrameID mFrameIDForPendingPrincipalHandle;
 };
 
 } // namespace mozilla
 
 #endif /* VIDEOFRAMECONTAINER_H_ */