Bug 1330919 - Pass RefreshDriver timestamp to captured frames from canvas. r?jesup draft
authorAndreas Pehrson <pehrsons@gmail.com>
Fri, 13 Jan 2017 11:56:03 +0100
changeset 463234 3d70c7acdc21614ea5e9c27b9eda0a3757a308d7
parent 463233 262c84fca42c483306751d47b5aa25bf2d25c713
child 542615 df614136dbf7f3fa38a943d63d5ac55bac60d5dc
push id41998
push userbmo:pehrson@telenordigital.com
push dateWed, 18 Jan 2017 18:51:45 +0000
reviewersjesup
bugs1330919
milestone53.0a1
Bug 1330919 - Pass RefreshDriver timestamp to captured frames from canvas. r?jesup MozReview-Commit-ID: 75L94Y5VcsQ
dom/html/HTMLCanvasElement.cpp
dom/html/HTMLCanvasElement.h
dom/media/CanvasCaptureMediaStream.cpp
dom/media/CanvasCaptureMediaStream.h
--- a/dom/html/HTMLCanvasElement.cpp
+++ b/dom/html/HTMLCanvasElement.cpp
@@ -143,17 +143,17 @@ public:
       copy = CopySurface(snapshot);
       if (!copy) {
         return;
       }
     }
 
     {
       PROFILER_LABEL("HTMLCanvasElement", "SetFrame", js::ProfileEntry::Category::OTHER);
-      mOwningElement->SetFrameCapture(copy.forget());
+      mOwningElement->SetFrameCapture(copy.forget(), aTime);
       mOwningElement->MarkContextCleanForFrameCapture();
     }
   }
 
   void DetachFromRefreshDriver()
   {
     MOZ_ASSERT(mOwningElement);
     MOZ_ASSERT(mRefreshDriver);
@@ -1269,28 +1269,29 @@ HTMLCanvasElement::ProcessDestroyedFrame
   }
 
   if (mRequestedFrameListeners.IsEmpty()) {
     mRequestedFrameRefreshObserver->Unregister();
   }
 }
 
 void
-HTMLCanvasElement::SetFrameCapture(already_AddRefed<SourceSurface> aSurface)
+HTMLCanvasElement::SetFrameCapture(already_AddRefed<SourceSurface> aSurface,
+                                   const TimeStamp& aTime)
 {
   RefPtr<SourceSurface> surface = aSurface;
   RefPtr<SourceSurfaceImage> image = new SourceSurfaceImage(surface->GetSize(), surface);
 
   for (WeakPtr<FrameCaptureListener> listener : mRequestedFrameListeners) {
     if (!listener) {
       continue;
     }
 
     RefPtr<Image> imageRefCopy = image.get();
-    listener->NewFrame(imageRefCopy.forget());
+    listener->NewFrame(imageRefCopy.forget(), aTime);
   }
 }
 
 already_AddRefed<SourceSurface>
 HTMLCanvasElement::GetSurfaceSnapshot(bool* aPremultAlpha)
 {
   if (!mCurrentContext)
     return nullptr;
--- a/dom/html/HTMLCanvasElement.h
+++ b/dom/html/HTMLCanvasElement.h
@@ -100,17 +100,18 @@ public:
    * Indicates to the canvas whether or not this listener has requested a frame.
    */
   bool FrameCaptureRequested() const { return mFrameCaptureRequested; }
 
   /*
    * Interface through which new video frames will be provided while
    * `mFrameCaptureRequested` is `true`.
    */
-  virtual void NewFrame(already_AddRefed<layers::Image> aImage) = 0;
+  virtual void NewFrame(already_AddRefed<layers::Image> aImage,
+                        const TimeStamp& aTime) = 0;
 
 protected:
   virtual ~FrameCaptureListener() {}
 
   bool mFrameCaptureRequested;
 };
 
 class HTMLCanvasElement final : public nsGenericHTMLElement,
@@ -278,17 +279,18 @@ public:
    */
   void ProcessDestroyedFrameListeners();
 
   /*
    * Called by the RefreshDriver hook when a frame has been captured.
    * Makes a copy of the provided surface and hands it to all
    * FrameCaptureListeners having requested frame capture.
    */
-  void SetFrameCapture(already_AddRefed<gfx::SourceSurface> aSurface);
+  void SetFrameCapture(already_AddRefed<gfx::SourceSurface> aSurface,
+                       const TimeStamp& aTime);
 
   virtual bool ParseAttribute(int32_t aNamespaceID,
                                 nsIAtom* aAttribute,
                                 const nsAString& aValue,
                                 nsAttrValue& aResult) override;
   nsChangeHint GetAttributeChangeHint(const nsIAtom* aAttribute, int32_t aModType) const override;
 
   // SetAttr override.  C++ is stupid, so have to override both
--- a/dom/media/CanvasCaptureMediaStream.cpp
+++ b/dom/media/CanvasCaptureMediaStream.cpp
@@ -26,43 +26,44 @@ public:
                           TrackID aTrackId,
                           PrincipalHandle aPrincipalHandle,
                           SourceMediaStream* aSourceStream)
     : mEnded(false)
     , mSourceStream(aSourceStream)
     , mTrackId(aTrackId)
     , mPrincipalHandle(aPrincipalHandle)
     , mMutex("CanvasCaptureMediaStream OutputStreamDriver::StreamListener")
-    , mImage(nullptr)
   {
     MOZ_ASSERT(mSourceStream);
   }
 
   void EndStream() {
     mEnded = true;
   }
 
-  void SetImage(const RefPtr<layers::Image>& aImage)
+  void SetImage(const RefPtr<layers::Image>& aImage, const TimeStamp& aTime)
   {
     MutexAutoLock lock(mMutex);
     mImage = aImage;
+    mImageTime = aTime;
   }
 
   void NotifyPull(MediaStreamGraph* aGraph, StreamTime aDesiredTime) override
   {
     // Called on the MediaStreamGraph thread.
+    MOZ_ASSERT(mSourceStream);
     StreamTime delta = aDesiredTime - mSourceStream->GetEndOfAppendedData(mTrackId);
     if (delta > 0) {
       MutexAutoLock lock(mMutex);
-      MOZ_ASSERT(mSourceStream);
 
       RefPtr<Image> image = mImage;
       IntSize size = image ? image->GetSize() : IntSize(0, 0);
       VideoSegment segment;
-      segment.AppendFrame(image.forget(), delta, size, mPrincipalHandle);
+      segment.AppendFrame(image.forget(), delta, size, mPrincipalHandle, false,
+                          mImageTime);
 
       mSourceStream->AppendToTrack(mTrackId, &segment);
     }
 
     if (mEnded) {
       mSourceStream->EndAllTrackAndFinish();
     }
   }
@@ -74,16 +75,17 @@ private:
   Atomic<bool> mEnded;
   const RefPtr<SourceMediaStream> mSourceStream;
   const TrackID mTrackId;
   const PrincipalHandle mPrincipalHandle;
 
   Mutex mMutex;
   // The below members are protected by mMutex.
   RefPtr<layers::Image> mImage;
+  TimeStamp mImageTime;
 };
 
 OutputStreamDriver::OutputStreamDriver(SourceMediaStream* aSourceStream,
                                        const TrackID& aTrackId,
                                        const PrincipalHandle& aPrincipalHandle)
   : FrameCaptureListener()
   , mSourceStream(aSourceStream)
   , mStreamListener(new StreamListener(this, aTrackId, aPrincipalHandle,
@@ -106,20 +108,21 @@ OutputStreamDriver::~OutputStreamDriver(
   if (mStreamListener) {
     // MediaStreamGraph will keep the listener alive until it can finish the
     // stream on the next NotifyPull().
     mStreamListener->EndStream();
   }
 }
 
 void
-OutputStreamDriver::SetImage(const RefPtr<layers::Image>& aImage)
+OutputStreamDriver::SetImage(const RefPtr<layers::Image>& aImage,
+                             const TimeStamp& aTime)
 {
   if (mStreamListener) {
-    mStreamListener->SetImage(aImage);
+    mStreamListener->SetImage(aImage, aTime);
   }
 }
 
 // ----------------------------------------------------------------------
 
 class TimerDriver : public OutputStreamDriver
 {
 public:
@@ -145,26 +148,26 @@ public:
   static void TimerTick(nsITimer* aTimer, void* aClosure)
   {
     MOZ_ASSERT(aClosure);
     TimerDriver* driver = static_cast<TimerDriver*>(aClosure);
 
     driver->RequestFrameCapture();
   }
 
-  void NewFrame(already_AddRefed<Image> aImage) override
+  void NewFrame(already_AddRefed<Image> aImage, const TimeStamp& aTime) override
   {
     RefPtr<Image> image = aImage;
 
     if (!mFrameCaptureRequested) {
       return;
     }
 
     mFrameCaptureRequested = false;
-    SetImage(image.forget());
+    SetImage(image.forget(), aTime);
   }
 
   void Forget() override
   {
     if (mTimer) {
       mTimer->Cancel();
       mTimer = nullptr;
     }
@@ -183,25 +186,25 @@ private:
 class AutoDriver : public OutputStreamDriver
 {
 public:
   explicit AutoDriver(SourceMediaStream* aSourceStream,
                       const TrackID& aTrackId,
                       const PrincipalHandle& aPrincipalHandle)
     : OutputStreamDriver(aSourceStream, aTrackId, aPrincipalHandle) {}
 
-  void NewFrame(already_AddRefed<Image> aImage) override
+  void NewFrame(already_AddRefed<Image> aImage, const TimeStamp& aTime) override
   {
     // Don't reset `mFrameCaptureRequested` since AutoDriver shall always have
     // `mFrameCaptureRequested` set to true.
     // This also means we should accept every frame as NewFrame is called only
     // after something changed.
 
     RefPtr<Image> image = aImage;
-    SetImage(image.forget());
+    SetImage(image.forget(), aTime);
   }
 
 protected:
   virtual ~AutoDriver() {}
 };
 
 // ----------------------------------------------------------------------
 
--- a/dom/media/CanvasCaptureMediaStream.h
+++ b/dom/media/CanvasCaptureMediaStream.h
@@ -74,17 +74,17 @@ public:
                      const PrincipalHandle& aPrincipalHandle);
 
   NS_INLINE_DECL_THREADSAFE_REFCOUNTING(OutputStreamDriver);
 
   /*
    * Sub classes can SetImage() to update the image being appended to the
    * output stream. It will be appended on the next NotifyPull from MSG.
    */
-  void SetImage(const RefPtr<layers::Image>& aImage);
+  void SetImage(const RefPtr<layers::Image>& aImage, const TimeStamp& aTime);
 
   /*
    * Makes sure any internal resources this driver is holding that may create
    * reference cycles are released.
    */
   virtual void Forget() {}
 
 protected: