Bug 1201363 - Adding base class MediaStreamVideoSink and changing VideoFrameContainer to be inherited from MediaStreamVideoSink. r?jesup draft
authorctai <ctai@mozilla.com>
Thu, 24 Dec 2015 10:43:28 +0800
changeset 396577 0ac771f34d640c4f8c5d4890bb76709ef0dacdcb
parent 395624 6608e5864780589b25d5421c3d3673ab30c4c318
child 396578 00387ad857ee878ee92f2c5426eb766fae0939a6
push id25058
push userbmo:ctai@mozilla.com
push dateThu, 04 Aug 2016 05:45:45 +0000
reviewersjesup
bugs1201363
milestone51.0a1
Bug 1201363 - Adding base class MediaStreamVideoSink and changing VideoFrameContainer to be inherited from MediaStreamVideoSink. r?jesup MediaStreamVideoSink is the base class of VideoFrameContainer, CaptureTask(ImageCapture), MediaStreamVideoRecorderSink(MediaRecoreder) and PipelineVideoSink(WebRTC-MediaPipelineTransmit). In this patch, I change VideoFrameContainer only. The rest of cases will be changed in latter patches of this bug. MozReview-Commit-ID: JNUke3fyCoN
dom/media/MediaStreamListener.h
dom/media/MediaStreamVideoSink.cpp
dom/media/MediaStreamVideoSink.h
dom/media/VideoFrameContainer.cpp
dom/media/VideoFrameContainer.h
dom/media/VideoSegment.h
dom/media/moz.build
--- a/dom/media/MediaStreamListener.h
+++ b/dom/media/MediaStreamListener.h
@@ -2,20 +2,24 @@
 /* vim: set ts=8 sts=2 et sw=2 tw=80: */
 /* This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 #ifndef MOZILLA_MEDIASTREAMLISTENER_h_
 #define MOZILLA_MEDIASTREAMLISTENER_h_
 
+#include "StreamTracks.h"
+
 namespace mozilla {
 
+class AudioSegment;
 class MediaStream;
 class MediaStreamGraph;
+class VideoSegment;
 
 enum MediaStreamGraphEvent : uint32_t {
   EVENT_FINISHED,
   EVENT_REMOVED,
   EVENT_HAS_DIRECT_LISTENERS, // transition from no direct listeners
   EVENT_HAS_NO_DIRECT_LISTENERS,  // transition to no direct listeners
 };
 
new file mode 100644
--- /dev/null
+++ b/dom/media/MediaStreamVideoSink.cpp
@@ -0,0 +1,21 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-*/
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "MediaStreamVideoSink.h"
+
+#include "VideoSegment.h"
+
+namespace mozilla {
+void
+MediaStreamVideoSink::NotifyRealtimeTrackData(MediaStreamGraph* aGraph,
+                                              StreamTime aTrackOffset,
+                                              const MediaSegment& aMedia)
+{
+  if (aMedia.GetType() == MediaSegment::VIDEO) {
+    SetCurrentFrames(static_cast<const VideoSegment&>(aMedia));
+  }
+}
+
+} // namespace mozilla
new file mode 100644
--- /dev/null
+++ b/dom/media/MediaStreamVideoSink.h
@@ -0,0 +1,42 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et cindent: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef MEDIASTREAMVIDEOSINK_H_
+#define MEDIASTREAMVIDEOSINK_H_
+
+#include "mozilla/Pair.h"
+
+#include "gfxPoint.h"
+#include "MediaStreamListener.h"
+
+namespace mozilla {
+
+class VideoFrameContainer;
+
+/**
+ * Base class of MediaStreamVideoSink family. This is the output of MediaStream.
+ */
+class MediaStreamVideoSink : public DirectMediaStreamTrackListener {
+public:
+  // Method of DirectMediaStreamTrackListener.
+  void NotifyRealtimeTrackData(MediaStreamGraph* aGraph,
+                               StreamTime aTrackOffset,
+                               const MediaSegment& aMedia) override;
+
+  // Call on any thread
+  virtual void SetCurrentFrames(const VideoSegment& aSegment) = 0;
+  virtual void ClearFrames() = 0;
+
+  virtual VideoFrameContainer* AsVideoFrameContainer() { return nullptr; }
+  virtual void Invalidate() {}
+
+protected:
+  virtual ~MediaStreamVideoSink() {};
+};
+
+} // namespace mozilla
+
+#endif /* MEDIASTREAMVIDEOSINK_H_ */
--- a/dom/media/VideoFrameContainer.cpp
+++ b/dom/media/VideoFrameContainer.cpp
@@ -9,27 +9,33 @@
 #include "mozilla/dom/HTMLMediaElement.h"
 #include "nsIFrame.h"
 #include "nsDisplayList.h"
 #include "nsSVGEffects.h"
 
 using namespace mozilla::layers;
 
 namespace mozilla {
+PRLogModuleInfo* gVideoFrameContainerLog;
+#define CONTAINER_LOG(type, msg) MOZ_LOG(gVideoFrameContainerLog, type, msg)
 
 VideoFrameContainer::VideoFrameContainer(dom::HTMLMediaElement* aElement,
                                          already_AddRefed<ImageContainer> aContainer)
   : mElement(aElement),
     mImageContainer(aContainer), mMutex("nsVideoFrameContainer"),
+    mBlackImage(nullptr),
     mFrameID(0),
     mIntrinsicSizeChanged(false), mImageSizeChanged(false),
     mPendingPrincipalHandle(PRINCIPAL_HANDLE_NONE), mFrameIDForPendingPrincipalHandle(0)
 {
   NS_ASSERTION(aElement, "aElement must not be null");
   NS_ASSERTION(mImageContainer, "aContainer must not be null");
+  if (!gVideoFrameContainerLog) {
+    gVideoFrameContainerLog = PR_NewLogModule("VideoFrameContainer");
+  }
 }
 
 VideoFrameContainer::~VideoFrameContainer()
 {}
 
 PrincipalHandle VideoFrameContainer::GetLastPrincipalHandle()
 {
   MutexAutoLock lock(mMutex);
@@ -42,16 +48,136 @@ void VideoFrameContainer::UpdatePrincipa
   MutexAutoLock lock(mMutex);
   if (mPendingPrincipalHandle == aPrincipalHandle) {
     return;
   }
   mPendingPrincipalHandle = aPrincipalHandle;
   mFrameIDForPendingPrincipalHandle = aFrameID;
 }
 
+static void
+SetImageToBlackPixel(PlanarYCbCrImage* aImage)
+{
+  uint8_t blackPixel[] = { 0x10, 0x80, 0x80 };
+
+  PlanarYCbCrData data;
+  data.mYChannel = blackPixel;
+  data.mCbChannel = blackPixel + 1;
+  data.mCrChannel = blackPixel + 2;
+  data.mYStride = data.mCbCrStride = 1;
+  data.mPicSize = data.mYSize = data.mCbCrSize = gfx::IntSize(1, 1);
+  aImage->CopyData(data);
+}
+
+class VideoFrameContainerInvalidateRunnable : public Runnable {
+public:
+  explicit VideoFrameContainerInvalidateRunnable(VideoFrameContainer* aVideoFrameContainer)
+    : mVideoFrameContainer(aVideoFrameContainer)
+  {}
+  NS_IMETHOD Run()
+  {
+    MOZ_ASSERT(NS_IsMainThread());
+
+    mVideoFrameContainer->Invalidate();
+
+    return NS_OK;
+  }
+private:
+  RefPtr<VideoFrameContainer> mVideoFrameContainer;
+};
+
+void VideoFrameContainer::SetCurrentFrames(const VideoSegment& aSegment)
+{
+  if (aSegment.IsEmpty()) {
+    return;
+  }
+
+  MutexAutoLock lock(mMutex);
+
+  // Collect any new frames produced in this iteration.
+  AutoTArray<ImageContainer::NonOwningImage,4> newImages;
+  PrincipalHandle lastPrincipalHandle = PRINCIPAL_HANDLE_NONE;
+
+  VideoSegment::ConstChunkIterator iter(aSegment);
+  while (!iter.IsEnded()) {
+    VideoChunk chunk = *iter;
+
+    const VideoFrame* frame = &chunk.mFrame;
+    if (*frame == mLastPlayedVideoFrame) {
+      iter.Next();
+      continue;
+    }
+
+    Image* image = frame->GetImage();
+    CONTAINER_LOG(LogLevel::Verbose,
+                  ("VideoFrameContainer %p writing video frame %p (%d x %d)",
+                  this, image, frame->GetIntrinsicSize().width,
+                  frame->GetIntrinsicSize().height));
+
+    if (frame->GetForceBlack()) {
+      if (!mBlackImage) {
+        mBlackImage = GetImageContainer()->CreatePlanarYCbCrImage();
+        if (mBlackImage) {
+          // Sets the image to a single black pixel, which will be scaled to
+          // fill the rendered size.
+          SetImageToBlackPixel(mBlackImage->AsPlanarYCbCrImage());
+        }
+      }
+      if (mBlackImage) {
+        image = mBlackImage;
+      }
+    }
+    // Don't append null image to the newImages.
+    if (!image) {
+      iter.Next();
+      continue;
+    }
+    newImages.AppendElement(ImageContainer::NonOwningImage(image, chunk.mTimeStamp));
+
+    lastPrincipalHandle = chunk.GetPrincipalHandle();
+
+    mLastPlayedVideoFrame = *frame;
+    iter.Next();
+  }
+
+  // Don't update if there are no changes.
+  if (newImages.IsEmpty()) {
+    return;
+  }
+
+  AutoTArray<ImageContainer::NonOwningImage,4> images;
+
+  bool principalHandleChanged =
+     lastPrincipalHandle != PRINCIPAL_HANDLE_NONE &&
+     lastPrincipalHandle != GetLastPrincipalHandle();
+
+  // Add the frames from this iteration.
+  for (auto& image : newImages) {
+    image.mFrameID = NewFrameID();
+    images.AppendElement(image);
+  }
+
+  if (principalHandleChanged) {
+    UpdatePrincipalHandleForFrameID(lastPrincipalHandle,
+                                    newImages.LastElement().mFrameID);
+  }
+
+  SetCurrentFramesLocked(mLastPlayedVideoFrame.GetIntrinsicSize(), images);
+  nsCOMPtr<nsIRunnable> event =
+    new VideoFrameContainerInvalidateRunnable(this);
+  NS_DispatchToMainThread(event.forget());
+
+  images.ClearAndRetainStorage();
+}
+
+void VideoFrameContainer::ClearFrames()
+{
+  ClearFutureFrames();
+}
+
 void VideoFrameContainer::SetCurrentFrame(const gfx::IntSize& aIntrinsicSize,
                                           Image* aImage,
                                           const TimeStamp& aTargetTime)
 {
   if (aImage) {
     MutexAutoLock lock(mMutex);
     AutoTArray<ImageContainer::NonOwningImage,1> imageList;
     imageList.AppendElement(
--- a/dom/media/VideoFrameContainer.h
+++ b/dom/media/VideoFrameContainer.h
@@ -8,60 +8,63 @@
 #define VIDEOFRAMECONTAINER_H_
 
 #include "mozilla/Mutex.h"
 #include "mozilla/TimeStamp.h"
 #include "gfxPoint.h"
 #include "nsCOMPtr.h"
 #include "ImageContainer.h"
 #include "MediaSegment.h"
+#include "MediaStreamVideoSink.h"
+#include "VideoSegment.h"
 
 namespace mozilla {
 
 namespace dom {
 class HTMLMediaElement;
 } // namespace dom
 
 /**
  * This object is used in the decoder backend threads and the main thread
  * to manage the "current video frame" state. This state includes timing data
  * and an intrinsic size (see below).
  * This has to be a thread-safe object since it's accessed by resource decoders
  * and other off-main-thread components. So we can't put this state in the media
  * element itself ... well, maybe we could, but it could be risky and/or
  * confusing.
  */
-class VideoFrameContainer {
-  ~VideoFrameContainer();
+class VideoFrameContainer : public MediaStreamVideoSink {
+  virtual ~VideoFrameContainer();
 
 public:
   typedef layers::ImageContainer ImageContainer;
   typedef layers::Image Image;
 
-  NS_INLINE_DECL_THREADSAFE_REFCOUNTING(VideoFrameContainer)
-
   VideoFrameContainer(dom::HTMLMediaElement* aElement,
                       already_AddRefed<ImageContainer> aContainer);
 
   // Call on any thread
+  virtual void SetCurrentFrames(const VideoSegment& aSegment) override;
+  virtual void ClearFrames() override;
+  void SetCurrentFrame(const gfx::IntSize& aIntrinsicSize, Image* aImage,
+                       const TimeStamp& aTargetTime);
   // Returns the last principalHandle we notified mElement about.
   PrincipalHandle GetLastPrincipalHandle();
   // We will notify mElement that aPrincipalHandle has been applied when all
   // FrameIDs prior to aFrameID have been flushed out.
   // aFrameID is ignored if aPrincipalHandle already is our pending principalHandle.
   void UpdatePrincipalHandleForFrameID(const PrincipalHandle& aPrincipalHandle,
                                        const ImageContainer::FrameID& aFrameID);
-  void SetCurrentFrame(const gfx::IntSize& aIntrinsicSize, Image* aImage,
-                       const TimeStamp& aTargetTime);
   void SetCurrentFrames(const gfx::IntSize& aIntrinsicSize,
                         const nsTArray<ImageContainer::NonOwningImage>& aImages);
   void ClearCurrentFrame(const gfx::IntSize& aIntrinsicSize)
   {
     SetCurrentFrames(aIntrinsicSize, nsTArray<ImageContainer::NonOwningImage>());
   }
+  VideoFrameContainer* AsVideoFrameContainer() override { return this; }
 
   void ClearCurrentFrame();
   // Make the current frame the only frame in the container, i.e. discard
   // all future frames.
   void ClearFutureFrames();
   // Time in seconds by which the last painted video frame was late by.
   // E.g. if the last painted frame should have been painted at time t,
   // but was actually painted at t+n, this returns n in seconds. Threadsafe.
@@ -75,17 +78,17 @@ public:
     return ++mFrameID;
   }
 
   // Call on main thread
   enum {
     INVALIDATE_DEFAULT,
     INVALIDATE_FORCE
   };
-  void Invalidate() { InvalidateWithFlags(INVALIDATE_DEFAULT); }
+  void Invalidate() override { InvalidateWithFlags(INVALIDATE_DEFAULT); }
   void InvalidateWithFlags(uint32_t aFlags);
   ImageContainer* GetImageContainer();
   void ForgetElement() { mElement = nullptr; }
 
   uint32_t GetDroppedImageCount() { return mImageContainer->GetDroppedImageCount(); }
 
 protected:
   void SetCurrentFramesLocked(const gfx::IntSize& aIntrinsicSize,
@@ -93,25 +96,31 @@ protected:
 
   // Non-addreffed pointer to the element. The element calls ForgetElement
   // to clear this reference when the element is destroyed.
   dom::HTMLMediaElement* mElement;
   RefPtr<ImageContainer> mImageContainer;
 
   // mMutex protects all the fields below.
   Mutex mMutex;
+  // Once the frame is forced to black, we initialize mBlackImage for following
+  // frames.
+  RefPtr<Image> mBlackImage;
   // The intrinsic size is the ideal size which we should render the
   // ImageContainer's current Image at.
   // This can differ from the Image's actual size when the media resource
   // specifies that the Image should be stretched to have the correct aspect
   // ratio.
   gfx::IntSize mIntrinsicSize;
   // We maintain our own mFrameID which is auto-incremented at every
   // SetCurrentFrame() or NewFrameID() call.
   ImageContainer::FrameID mFrameID;
+  // We record the last played video frame to avoid playing the frame again
+  // with a different frame id.
+  VideoFrame mLastPlayedVideoFrame;
   // True when the intrinsic size has been changed by SetCurrentFrame() since
   // the last call to Invalidate().
   // The next call to Invalidate() will recalculate
   // and update the intrinsic size on the element, request a frame reflow and
   // then reset this flag.
   bool mIntrinsicSizeChanged;
   // True when the Image size has changed since the last time Invalidate() was
   // called. When set, the next call to Invalidate() will ensure that the
--- a/dom/media/VideoSegment.h
+++ b/dom/media/VideoSegment.h
@@ -132,13 +132,19 @@ public:
 
   // Segment-generic methods not in MediaSegmentBase
   static Type StaticType() { return VIDEO; }
 
   size_t SizeOfIncludingThis(MallocSizeOf aMallocSizeOf) const override
   {
     return aMallocSizeOf(this) + SizeOfExcludingThis(aMallocSizeOf);
   }
+
+  bool IsEmpty() const
+  {
+    return mChunks.IsEmpty();
+  }
+
 };
 
 } // namespace mozilla
 
 #endif /* MOZILLA_VIDEOSEGMENT_H_ */
--- a/dom/media/moz.build
+++ b/dom/media/moz.build
@@ -122,16 +122,17 @@ EXPORTS += [
     'MediaQueue.h',
     'MediaRecorder.h',
     'MediaResource.h',
     'MediaResourceCallback.h',
     'MediaSegment.h',
     'MediaStatistics.h',
     'MediaStreamGraph.h',
     'MediaStreamListener.h',
+    'MediaStreamVideoSink.h',
     'MediaTimer.h',
     'MediaTrack.h',
     'MediaTrackList.h',
     'MP3Decoder.h',
     'MP3Demuxer.h',
     'MP3FrameParser.h',
     'NextFrameSeekTask.h',
     'nsIDocumentActivity.h',
@@ -234,16 +235,17 @@ UNIFIED_SOURCES += [
     'MediaPrefs.cpp',
     'MediaRecorder.cpp',
     'MediaResource.cpp',
     'MediaShutdownManager.cpp',
     'MediaStreamError.cpp',
     'MediaStreamGraph.cpp',
     'MediaStreamListener.cpp',
     'MediaStreamTrack.cpp',
+    'MediaStreamVideoSink.cpp',
     'MediaTimer.cpp',
     'MediaTrack.cpp',
     'MediaTrackList.cpp',
     'MP3Decoder.cpp',
     'MP3Demuxer.cpp',
     'MP3FrameParser.cpp',
     'NextFrameSeekTask.cpp',
     'QueueObject.cpp',