Bug 1201363 - MediaStreamVideoSink for ImageCapture case. r?jesup,r?pehrsons draft
authorctai <ctai@mozilla.com>
Tue, 01 Mar 2016 10:47:46 +0800
changeset 344713 df338642019c7e4bfa18ef1b90c9c4291669fdd9
parent 344712 3e5c0b1092dc9179362c4f644e981378df632ee4
child 344714 6ddc21c179d84e943a7fa252962706f434af4a4b
push id13910
push userbmo:ctai@mozilla.com
push dateFri, 25 Mar 2016 13:21:09 +0000
reviewersjesup, pehrsons
bugs1201363
milestone46.0a1
Bug 1201363 - MediaStreamVideoSink for ImageCapture case. r?jesup,r?pehrsons Make CaptureTask to inherite from MediaStreamVideoSink. The main change is to move the logic of |NotifyQueuedTrackChanges| to |SetCurrentFrames|. The original image capture is not modified for support multiple video MediaStreamTracks. The design still used the track id in owned media stream. The should be fixed in the following bug if we still want to support ImageCapture in multiple video tracks case. MozReview-Commit-ID: JGrXg6ek9DQ
dom/media/imagecapture/CaptureTask.cpp
dom/media/imagecapture/CaptureTask.h
--- a/dom/media/imagecapture/CaptureTask.cpp
+++ b/dom/media/imagecapture/CaptureTask.cpp
@@ -9,16 +9,62 @@
 #include "mozilla/dom/ImageCaptureError.h"
 #include "mozilla/dom/ImageEncoder.h"
 #include "mozilla/dom/VideoStreamTrack.h"
 #include "gfxUtils.h"
 #include "nsThreadUtils.h"
 
 namespace mozilla {
 
+class CaptureTask::MediaStreamEventListener : public MediaStreamListener
+{
+public:
+  MediaStreamEventListener(CaptureTask* aCaptureTask)
+    : mCaptureTask(aCaptureTask) {};
+
+  // MediaStreamListener methods.
+  void NotifyEvent(MediaStreamGraph* aGraph,
+                   MediaStreamGraphEvent aEvent) override
+  {
+    if (((aEvent == EVENT_FINISHED) || (aEvent == EVENT_REMOVED)) &&
+        !mCaptureTask->mImageGrabbedOrTrackEnd) {
+      mCaptureTask->PostTrackEndEvent();
+    }
+  }
+
+  void NotifyQueuedTrackChanges(MediaStreamGraph* aGraph, TrackID aID,
+                                StreamTime aTrackOffset,
+                                uint32_t aTrackEvents,
+                                const MediaSegment& aQueuedMedia,
+                                MediaStream* aInputStream,
+                                TrackID aInputTrackID) override
+  {
+    if (mCaptureTask->mImageGrabbedOrTrackEnd) {
+      return;
+    }
+
+    if (aTrackEvents == MediaStreamListener::TRACK_EVENT_ENDED) {
+      mCaptureTask->PostTrackEndEvent();
+    }
+    return;
+  }
+
+private:
+  CaptureTask* mCaptureTask;
+};
+
+CaptureTask::CaptureTask(dom::ImageCapture* aImageCapture, TrackID aTrackID)
+  : mImageCapture(aImageCapture)
+  , mEventListener(new MediaStreamEventListener(this))
+  , mTrackID(aTrackID)
+  , mImageGrabbedOrTrackEnd(false)
+  , mPrincipalChanged(false)
+{
+}
+
 nsresult
 CaptureTask::TaskComplete(already_AddRefed<dom::Blob> aBlob, nsresult aRv)
 {
   MOZ_ASSERT(NS_IsMainThread());
 
   DetachStream();
 
   nsresult rv;
@@ -53,57 +99,49 @@ CaptureTask::AttachStream()
   MOZ_ASSERT(NS_IsMainThread());
 
   RefPtr<dom::VideoStreamTrack> track = mImageCapture->GetVideoStreamTrack();
 
   RefPtr<DOMMediaStream> domStream = track->GetStream();
   domStream->AddPrincipalChangeObserver(this);
 
   RefPtr<MediaStream> stream = domStream->GetPlaybackStream();
-  stream->AddListener(this);
+  stream->AddListener(mEventListener.get());
+  stream->AddVideoOutput(this, mTrackID);
 }
 
 void
 CaptureTask::DetachStream()
 {
   MOZ_ASSERT(NS_IsMainThread());
 
   RefPtr<dom::VideoStreamTrack> track = mImageCapture->GetVideoStreamTrack();
 
   RefPtr<DOMMediaStream> domStream = track->GetStream();
   domStream->RemovePrincipalChangeObserver(this);
 
   RefPtr<MediaStream> stream = domStream->GetPlaybackStream();
-  stream->RemoveListener(this);
+  stream->RemoveListener(mEventListener.get());
+  stream->RemoveVideoOutput(this, mTrackID);
 }
 
 void
 CaptureTask::PrincipalChanged(DOMMediaStream* aMediaStream)
 {
   MOZ_ASSERT(NS_IsMainThread());
   mPrincipalChanged = true;
 }
 
 void
-CaptureTask::NotifyQueuedTrackChanges(MediaStreamGraph* aGraph, TrackID aID,
-                                      StreamTime aTrackOffset,
-                                      uint32_t aTrackEvents,
-                                      const MediaSegment& aQueuedMedia,
-                                      MediaStream* aInputStream,
-                                      TrackID aInputTrackID)
+CaptureTask::SetCurrentFrames(const VideoSegment& aSegment)
 {
   if (mImageGrabbedOrTrackEnd) {
     return;
   }
 
-  if (aTrackEvents == MediaStreamListener::TRACK_EVENT_ENDED) {
-    PostTrackEndEvent();
-    return;
-  }
-
   // Callback for encoding complete, it calls on main thread.
   class EncodeComplete : public dom::EncodeCompleteCallback
   {
   public:
     explicit EncodeComplete(CaptureTask* aTask) : mTask(aTask) {}
 
     nsresult ReceiveBlob(already_AddRefed<dom::Blob> aBlob) override
     {
@@ -112,61 +150,49 @@ CaptureTask::NotifyQueuedTrackChanges(Me
       mTask = nullptr;
       return NS_OK;
     }
 
   protected:
     RefPtr<CaptureTask> mTask;
   };
 
-  if (aQueuedMedia.GetType() == MediaSegment::VIDEO && mTrackID == aID) {
-    VideoSegment* video =
-      const_cast<VideoSegment*> (static_cast<const VideoSegment*>(&aQueuedMedia));
-    VideoSegment::ChunkIterator iter(*video);
-    while (!iter.IsEnded()) {
-      VideoChunk chunk = *iter;
-      // Extract the first valid video frame.
-      VideoFrame frame;
-      if (!chunk.IsNull()) {
-        RefPtr<layers::Image> image;
-        if (chunk.mFrame.GetForceBlack()) {
-          // Create a black image.
-          image = VideoFrame::CreateBlackImage(chunk.mFrame.GetIntrinsicSize());
-        } else {
-          image = chunk.mFrame.GetImage();
-        }
-        MOZ_ASSERT(image);
-        mImageGrabbedOrTrackEnd = true;
+  VideoSegment::ConstChunkIterator iter(aSegment);
+  while (!iter.IsEnded()) {
+    VideoChunk chunk = *iter;
+
+    // Extract the first valid video frame.
+    VideoFrame frame;
+    if (!chunk.IsNull()) {
+      RefPtr<layers::Image> image;
+      if (chunk.mFrame.GetForceBlack()) {
+        // Create a black image.
+        image = VideoFrame::CreateBlackImage(chunk.mFrame.GetIntrinsicSize());
+      } else {
+        image = chunk.mFrame.GetImage();
+      }
+      MOZ_ASSERT(image);
+      mImageGrabbedOrTrackEnd = true;
 
-        // Encode image.
-        nsresult rv;
-        nsAutoString type(NS_LITERAL_STRING("image/jpeg"));
-        nsAutoString options;
-        rv = dom::ImageEncoder::ExtractDataFromLayersImageAsync(
-                                  type,
-                                  options,
-                                  false,
-                                  image,
-                                  new EncodeComplete(this));
-        if (NS_FAILED(rv)) {
-          PostTrackEndEvent();
-        }
-        return;
+      // Encode image.
+      nsresult rv;
+      nsAutoString type(NS_LITERAL_STRING("image/jpeg"));
+      nsAutoString options;
+      rv = dom::ImageEncoder::ExtractDataFromLayersImageAsync(
+                                type,
+                                options,
+                                false,
+                                image,
+                                new EncodeComplete(this));
+      if (NS_FAILED(rv)) {
+        PostTrackEndEvent();
       }
-      iter.Next();
+      return;
     }
-  }
-}
-
-void
-CaptureTask::NotifyEvent(MediaStreamGraph* aGraph, MediaStreamGraphEvent aEvent)
-{
-  if (((aEvent == EVENT_FINISHED) || (aEvent == EVENT_REMOVED)) &&
-      !mImageGrabbedOrTrackEnd) {
-    PostTrackEndEvent();
+    iter.Next();
   }
 }
 
 void
 CaptureTask::PostTrackEndEvent()
 {
   mImageGrabbedOrTrackEnd = true;
 
--- a/dom/media/imagecapture/CaptureTask.h
+++ b/dom/media/imagecapture/CaptureTask.h
@@ -4,16 +4,17 @@
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 #ifndef CAPTURETASK_H
 #define CAPTURETASK_H
 
 #include "DOMMediaStream.h"
 #include "MediaStreamGraph.h"
+#include "MediaStreamVideoSink.h"
 
 namespace mozilla {
 
 namespace dom {
 class Blob;
 class ImageCapture;
 } // namespace dom
 
@@ -22,30 +23,25 @@ class ImageCapture;
  * ImageEncoder. The whole procedures start at AttachStream(), it will add this
  * class into MediaStream and retrieves an image in MediaStreamGraph thread.
  * Once the image is retrieved, it will be sent to ImageEncoder and the encoded
  * blob will be sent out via encoder callback in main thread.
  *
  * CaptureTask holds a reference of ImageCapture to ensure ImageCapture won't be
  * released during the period of the capturing process described above.
  */
-class CaptureTask : public MediaStreamListener,
+class CaptureTask : public MediaStreamVideoSink,
                     public DOMMediaStream::PrincipalChangeObserver
 {
 public:
-  // MediaStreamListener methods.
-  void NotifyQueuedTrackChanges(MediaStreamGraph* aGraph, TrackID aID,
-                                StreamTime aTrackOffset,
-                                uint32_t aTrackEvents,
-                                const MediaSegment& aQueuedMedia,
-                                MediaStream* aInputStream,
-                                TrackID aInputTrackID) override;
+  class MediaStreamEventListener;
 
-  void NotifyEvent(MediaStreamGraph* aGraph,
-                   MediaStreamGraphEvent aEvent) override;
+  // MediaStreamVideoSink methods.
+  virtual void SetCurrentFrames(const VideoSegment& aSegment) override;
+  virtual void ClearFrames() override {}
 
   // DOMMediaStream::PrincipalChangeObserver method.
   void PrincipalChanged(DOMMediaStream* aMediaStream) override;
 
   // CaptureTask methods.
 
   // It is called when aBlob is ready to post back to script in company with
   // aRv == NS_OK. If aRv is not NS_OK, it will post an error event to script.
@@ -58,34 +54,32 @@ public:
   // main thread only.
   void AttachStream();
 
   // Remove listeners from MediaStream and PrincipalChangeObserver. It should be
   // on main thread only.
   void DetachStream();
 
   // CaptureTask should be created on main thread.
-  CaptureTask(dom::ImageCapture* aImageCapture, TrackID aTrackID)
-    : mImageCapture(aImageCapture)
-    , mTrackID(aTrackID)
-    , mImageGrabbedOrTrackEnd(false)
-    , mPrincipalChanged(false) {}
+  CaptureTask(dom::ImageCapture* aImageCapture, TrackID aTrackID);
 
 protected:
   virtual ~CaptureTask() {}
 
   // Post a runnable on main thread to end this task and call TaskComplete to post
   // error event to script. It is called off-main-thread.
   void PostTrackEndEvent();
 
   // The ImageCapture associates with this task. This reference count should not
   // change in this class unless it clears this reference after a blob or error
   // event to script.
   RefPtr<dom::ImageCapture> mImageCapture;
 
+  RefPtr<MediaStreamEventListener> mEventListener;
+
   TrackID mTrackID;
 
   // True when an image is retrieved from MediaStreamGraph or MediaStreamGraph
   // sends a track finish, end, or removed event.
   bool mImageGrabbedOrTrackEnd;
 
   // True when MediaStream principal is changed in the period of taking photo
   // and it causes a NS_ERROR_DOM_SECURITY_ERR error to script.