Bug 1201363 - MediaStreamVideoSink for ImageCapture case. r?jesup draft
authorctai <ctai@mozilla.com>
Tue, 31 May 2016 13:53:49 +0800
changeset 396583 aa1b44201d8fd09a5b9c7e66a01ce9d7eeaee693
parent 396582 3daa00cea35356358f8496ace2d910ec85074edf
child 396584 d3d25d0f92474ccf42ad900e81f66ff9c6353dbb
child 396713 e9224290d6cf081258d1e2179099b2912442ea1d
push id25058
push userbmo:ctai@mozilla.com
push dateThu, 04 Aug 2016 05:45:45 +0000
reviewersjesup
bugs1201363
milestone51.0a1
Bug 1201363 - MediaStreamVideoSink for ImageCapture case. r?jesup Make CaptureTask to inherite from MediaStreamVideoSink. The main change is to move the logic of |NotifyQueuedTrackChanges| to |SetCurrentFrames|. The original image capture is not modified for support multiple video MediaStreamTracks. The design still used the track id in owned media stream. The should be fixed in the following bug if we still want to support ImageCapture in multiple video tracks case. MozReview-Commit-ID: Od4tHoR8Ef
dom/media/imagecapture/CaptureTask.cpp
dom/media/imagecapture/CaptureTask.h
dom/media/imagecapture/ImageCapture.cpp
--- a/dom/media/imagecapture/CaptureTask.cpp
+++ b/dom/media/imagecapture/CaptureTask.cpp
@@ -10,16 +10,42 @@
 #include "mozilla/dom/ImageEncoder.h"
 #include "mozilla/dom/MediaStreamTrack.h"
 #include "mozilla/dom/VideoStreamTrack.h"
 #include "gfxUtils.h"
 #include "nsThreadUtils.h"
 
 namespace mozilla {
 
+class CaptureTask::MediaStreamEventListener : public MediaStreamTrackListener
+{
+public:
+  explicit MediaStreamEventListener(CaptureTask* aCaptureTask)
+    : mCaptureTask(aCaptureTask) {};
+
+  // MediaStreamListener methods.
+  void NotifyEnded() override
+  {
+    if(!mCaptureTask->mImageGrabbedOrTrackEnd) {
+      mCaptureTask->PostTrackEndEvent();
+    }
+  }
+
+private:
+  CaptureTask* mCaptureTask;
+};
+
+CaptureTask::CaptureTask(dom::ImageCapture* aImageCapture)
+  : mImageCapture(aImageCapture)
+  , mEventListener(new MediaStreamEventListener(this))
+  , mImageGrabbedOrTrackEnd(false)
+  , mPrincipalChanged(false)
+{
+}
+
 nsresult
 CaptureTask::TaskComplete(already_AddRefed<dom::Blob> aBlob, nsresult aRv)
 {
   MOZ_ASSERT(NS_IsMainThread());
 
   DetachTrack();
 
   nsresult rv;
@@ -50,47 +76,45 @@ CaptureTask::TaskComplete(already_AddRef
 
 void
 CaptureTask::AttachTrack()
 {
   MOZ_ASSERT(NS_IsMainThread());
 
   dom::VideoStreamTrack* track = mImageCapture->GetVideoStreamTrack();
   track->AddPrincipalChangeObserver(this);
-  track->AddListener(this);
+  track->AddListener(mEventListener.get());
+  track->AddDirectListener(this);
 }
 
 void
 CaptureTask::DetachTrack()
 {
   MOZ_ASSERT(NS_IsMainThread());
 
   dom::VideoStreamTrack* track = mImageCapture->GetVideoStreamTrack();
   track->RemovePrincipalChangeObserver(this);
-  track->RemoveListener(this);
+  track->RemoveListener(mEventListener.get());
+  track->RemoveDirectListener(this);
 }
 
 void
 CaptureTask::PrincipalChanged(dom::MediaStreamTrack* aMediaStreamTrack)
 {
   MOZ_ASSERT(NS_IsMainThread());
   mPrincipalChanged = true;
 }
 
 void
-CaptureTask::NotifyQueuedChanges(MediaStreamGraph* aGraph,
-                                 StreamTime aTrackOffset,
-                                 const MediaSegment& aQueuedMedia)
+CaptureTask::SetCurrentFrames(const VideoSegment& aSegment)
 {
   if (mImageGrabbedOrTrackEnd) {
     return;
   }
 
-  MOZ_ASSERT(aQueuedMedia.GetType() == MediaSegment::VIDEO);
-
   // Callback for encoding complete, it calls on main thread.
   class EncodeComplete : public dom::EncodeCompleteCallback
   {
   public:
     explicit EncodeComplete(CaptureTask* aTask) : mTask(aTask) {}
 
     nsresult ReceiveBlob(already_AddRefed<dom::Blob> aBlob) override
     {
@@ -99,21 +123,23 @@ CaptureTask::NotifyQueuedChanges(MediaSt
       mTask = nullptr;
       return NS_OK;
     }
 
   protected:
     RefPtr<CaptureTask> mTask;
   };
 
-  VideoSegment* video =
-    const_cast<VideoSegment*> (static_cast<const VideoSegment*>(&aQueuedMedia));
-  VideoSegment::ChunkIterator iter(*video);
+  VideoSegment::ConstChunkIterator iter(aSegment);
+
+
+
   while (!iter.IsEnded()) {
     VideoChunk chunk = *iter;
+
     // Extract the first valid video frame.
     VideoFrame frame;
     if (!chunk.IsNull()) {
       RefPtr<layers::Image> image;
       if (chunk.mFrame.GetForceBlack()) {
         // Create a black image.
         image = VideoFrame::CreateBlackImage(chunk.mFrame.GetIntrinsicSize());
       } else {
@@ -137,24 +163,16 @@ CaptureTask::NotifyQueuedChanges(MediaSt
       }
       return;
     }
     iter.Next();
   }
 }
 
 void
-CaptureTask::NotifyEnded()
-{
-  if(!mImageGrabbedOrTrackEnd) {
-    PostTrackEndEvent();
-  }
-}
-
-void
 CaptureTask::PostTrackEndEvent()
 {
   mImageGrabbedOrTrackEnd = true;
 
   // Got track end or finish event, stop the task.
   class TrackEndRunnable : public Runnable
   {
   public:
--- a/dom/media/imagecapture/CaptureTask.h
+++ b/dom/media/imagecapture/CaptureTask.h
@@ -5,16 +5,17 @@
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 #ifndef CAPTURETASK_H
 #define CAPTURETASK_H
 
 #include "MediaStreamGraph.h"
 #include "MediaStreamListener.h"
 #include "PrincipalChangeObserver.h"
+#include "MediaStreamVideoSink.h"
 
 namespace mozilla {
 
 namespace dom {
 class Blob;
 class ImageCapture;
 class MediaStreamTrack;
 } // namespace dom
@@ -24,26 +25,25 @@ class MediaStreamTrack;
  * ImageEncoder. The whole procedures start at AttachTrack(), it will add this
  * class into MediaStream and retrieves an image in MediaStreamGraph thread.
  * Once the image is retrieved, it will be sent to ImageEncoder and the encoded
  * blob will be sent out via encoder callback in main thread.
  *
  * CaptureTask holds a reference of ImageCapture to ensure ImageCapture won't be
  * released during the period of the capturing process described above.
  */
-class CaptureTask : public MediaStreamTrackListener,
+class CaptureTask : public MediaStreamVideoSink,
                     public dom::PrincipalChangeObserver<dom::MediaStreamTrack>
 {
 public:
-  // MediaStreamTrackListener methods.
-  void NotifyQueuedChanges(MediaStreamGraph* aGraph,
-                           StreamTime aTrackOffset,
-                           const MediaSegment& aQueuedMedia) override;
+  class MediaStreamEventListener;
 
-  void NotifyEnded() override;
+  // MediaStreamVideoSink methods.
+  void SetCurrentFrames(const VideoSegment& aSegment) override;
+  void ClearFrames() override {}
 
   // PrincipalChangeObserver<MediaStreamTrack> method.
   void PrincipalChanged(dom::MediaStreamTrack* aMediaStreamTrack) override;
 
   // CaptureTask methods.
 
   // It is called when aBlob is ready to post back to script in company with
   // aRv == NS_OK. If aRv is not NS_OK, it will post an error event to script.
@@ -56,33 +56,32 @@ public:
   // It should be on main thread only.
   void AttachTrack();
 
   // Remove listeners from MediaStreamTrack and PrincipalChangeObserver.
   // It should be on main thread only.
   void DetachTrack();
 
   // CaptureTask should be created on main thread.
-  explicit CaptureTask(dom::ImageCapture* aImageCapture)
-    : mImageCapture(aImageCapture)
-    , mImageGrabbedOrTrackEnd(false)
-    , mPrincipalChanged(false) {}
+  explicit CaptureTask(dom::ImageCapture* aImageCapture);
 
 protected:
   virtual ~CaptureTask() {}
 
   // Post a runnable on main thread to end this task and call TaskComplete to post
   // error event to script. It is called off-main-thread.
   void PostTrackEndEvent();
 
   // The ImageCapture associates with this task. This reference count should not
   // change in this class unless it clears this reference after a blob or error
   // event to script.
   RefPtr<dom::ImageCapture> mImageCapture;
 
+  RefPtr<MediaStreamEventListener> mEventListener;
+
   // True when an image is retrieved from MediaStreamGraph or MediaStreamGraph
   // sends a track finish, end, or removed event.
   bool mImageGrabbedOrTrackEnd;
 
   // True after MediaStreamTrack principal changes while waiting for a photo
   // to finish and we should raise a security error.
   bool mPrincipalChanged;
 };
--- a/dom/media/imagecapture/ImageCapture.cpp
+++ b/dom/media/imagecapture/ImageCapture.cpp
@@ -142,18 +142,17 @@ ImageCapture::TakePhoto(ErrorResult& aRe
 
   // Try if MediaEngine supports taking photo.
   nsresult rv = TakePhotoByMediaEngine();
 
   // It falls back to MediaStreamGraph image capture if MediaEngine doesn't
   // support TakePhoto().
   if (rv == NS_ERROR_NOT_IMPLEMENTED) {
     IC_LOG("MediaEngine doesn't support TakePhoto(), it falls back to MediaStreamGraph.");
-    RefPtr<CaptureTask> task =
-      new CaptureTask(this);
+    RefPtr<CaptureTask> task = new CaptureTask(this);
 
     // It adds itself into MediaStreamGraph, so ImageCapture doesn't need to hold
     // the reference.
     task->AttachTrack();
   }
 }
 
 nsresult