Bug 1201363 - Adding base class MediaStreamVideoSink and changing VideoFrameContainer to be inherited from MediaStreamVideoSink. r?jesup
MediaStreamVideoSink is the base class of VideoFrameContainer, CaptureTask(ImageCapture), MediaStreamVideoRecorderSink(MediaRecoreder) and PipelineVideoSink(WebRTC-MediaPipelineTransmit). In this patch, I change VideoFrameContainer only. The rest of cases will be changed in latter patches of this bug.
MozReview-Commit-ID: JNUke3fyCoN
--- a/dom/media/MediaStreamListener.h
+++ b/dom/media/MediaStreamListener.h
@@ -2,20 +2,24 @@
/* vim: set ts=8 sts=2 et sw=2 tw=80: */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
#ifndef MOZILLA_MEDIASTREAMLISTENER_h_
#define MOZILLA_MEDIASTREAMLISTENER_h_
+#include "StreamTracks.h"
+
namespace mozilla {
+class AudioSegment;
class MediaStream;
class MediaStreamGraph;
+class VideoSegment;
enum MediaStreamGraphEvent : uint32_t {
EVENT_FINISHED,
EVENT_REMOVED,
EVENT_HAS_DIRECT_LISTENERS, // transition from no direct listeners
EVENT_HAS_NO_DIRECT_LISTENERS, // transition to no direct listeners
};
new file mode 100644
--- /dev/null
+++ b/dom/media/MediaStreamVideoSink.cpp
@@ -0,0 +1,21 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-*/
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "MediaStreamVideoSink.h"
+
+#include "VideoSegment.h"
+
+namespace mozilla {
+void
+MediaStreamVideoSink::NotifyRealtimeTrackData(MediaStreamGraph* aGraph,
+ StreamTime aTrackOffset,
+ const MediaSegment& aMedia)
+{
+ if (aMedia.GetType() == MediaSegment::VIDEO) {
+ SetCurrentFrames(static_cast<const VideoSegment&>(aMedia));
+ }
+}
+
+} // namespace mozilla
new file mode 100644
--- /dev/null
+++ b/dom/media/MediaStreamVideoSink.h
@@ -0,0 +1,42 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et cindent: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef MEDIASTREAMVIDEOSINK_H_
+#define MEDIASTREAMVIDEOSINK_H_
+
+#include "mozilla/Pair.h"
+
+#include "gfxPoint.h"
+#include "MediaStreamListener.h"
+
+namespace mozilla {
+
+class VideoFrameContainer;
+
+/**
+ * Base class of MediaStreamVideoSink family. This is the output of MediaStream.
+ */
+class MediaStreamVideoSink : public DirectMediaStreamTrackListener {
+public:
+ // Method of DirectMediaStreamTrackListener.
+ void NotifyRealtimeTrackData(MediaStreamGraph* aGraph,
+ StreamTime aTrackOffset,
+ const MediaSegment& aMedia) override;
+
+ // Call on any thread
+ virtual void SetCurrentFrames(const VideoSegment& aSegment) = 0;
+ virtual void ClearFrames() = 0;
+
+ virtual VideoFrameContainer* AsVideoFrameContainer() { return nullptr; }
+ virtual void Invalidate() {}
+
+protected:
+ virtual ~MediaStreamVideoSink() {};
+};
+
+} // namespace mozilla
+
+#endif /* MEDIASTREAMVIDEOSINK_H_ */
--- a/dom/media/VideoFrameContainer.cpp
+++ b/dom/media/VideoFrameContainer.cpp
@@ -9,27 +9,33 @@
#include "mozilla/dom/HTMLMediaElement.h"
#include "nsIFrame.h"
#include "nsDisplayList.h"
#include "nsSVGEffects.h"
using namespace mozilla::layers;
namespace mozilla {
+PRLogModuleInfo* gVideoFrameContainerLog;
+#define CONTAINER_LOG(type, msg) MOZ_LOG(gVideoFrameContainerLog, type, msg)
VideoFrameContainer::VideoFrameContainer(dom::HTMLMediaElement* aElement,
already_AddRefed<ImageContainer> aContainer)
: mElement(aElement),
mImageContainer(aContainer), mMutex("nsVideoFrameContainer"),
+ mBlackImage(nullptr),
mFrameID(0),
mIntrinsicSizeChanged(false), mImageSizeChanged(false),
mPendingPrincipalHandle(PRINCIPAL_HANDLE_NONE), mFrameIDForPendingPrincipalHandle(0)
{
NS_ASSERTION(aElement, "aElement must not be null");
NS_ASSERTION(mImageContainer, "aContainer must not be null");
+ if (!gVideoFrameContainerLog) {
+ gVideoFrameContainerLog = PR_NewLogModule("VideoFrameContainer");
+ }
}
VideoFrameContainer::~VideoFrameContainer()
{}
PrincipalHandle VideoFrameContainer::GetLastPrincipalHandle()
{
MutexAutoLock lock(mMutex);
@@ -42,16 +48,136 @@ void VideoFrameContainer::UpdatePrincipa
MutexAutoLock lock(mMutex);
if (mPendingPrincipalHandle == aPrincipalHandle) {
return;
}
mPendingPrincipalHandle = aPrincipalHandle;
mFrameIDForPendingPrincipalHandle = aFrameID;
}
+static void
+SetImageToBlackPixel(PlanarYCbCrImage* aImage)
+{
+ uint8_t blackPixel[] = { 0x10, 0x80, 0x80 };
+
+ PlanarYCbCrData data;
+ data.mYChannel = blackPixel;
+ data.mCbChannel = blackPixel + 1;
+ data.mCrChannel = blackPixel + 2;
+ data.mYStride = data.mCbCrStride = 1;
+ data.mPicSize = data.mYSize = data.mCbCrSize = gfx::IntSize(1, 1);
+ aImage->CopyData(data);
+}
+
+class VideoFrameContainerInvalidateRunnable : public Runnable {
+public:
+ explicit VideoFrameContainerInvalidateRunnable(VideoFrameContainer* aVideoFrameContainer)
+ : mVideoFrameContainer(aVideoFrameContainer)
+ {}
+ NS_IMETHOD Run()
+ {
+ MOZ_ASSERT(NS_IsMainThread());
+
+ mVideoFrameContainer->Invalidate();
+
+ return NS_OK;
+ }
+private:
+ RefPtr<VideoFrameContainer> mVideoFrameContainer;
+};
+
+void VideoFrameContainer::SetCurrentFrames(const VideoSegment& aSegment)
+{
+ if (aSegment.IsEmpty()) {
+ return;
+ }
+
+ MutexAutoLock lock(mMutex);
+
+ // Collect any new frames produced in this iteration.
+ AutoTArray<ImageContainer::NonOwningImage,4> newImages;
+ PrincipalHandle lastPrincipalHandle = PRINCIPAL_HANDLE_NONE;
+
+ VideoSegment::ConstChunkIterator iter(aSegment);
+ while (!iter.IsEnded()) {
+ VideoChunk chunk = *iter;
+
+ const VideoFrame* frame = &chunk.mFrame;
+ if (*frame == mLastPlayedVideoFrame) {
+ iter.Next();
+ continue;
+ }
+
+ Image* image = frame->GetImage();
+ CONTAINER_LOG(LogLevel::Verbose,
+ ("VideoFrameContainer %p writing video frame %p (%d x %d)",
+ this, image, frame->GetIntrinsicSize().width,
+ frame->GetIntrinsicSize().height));
+
+ if (frame->GetForceBlack()) {
+ if (!mBlackImage) {
+ mBlackImage = GetImageContainer()->CreatePlanarYCbCrImage();
+ if (mBlackImage) {
+ // Sets the image to a single black pixel, which will be scaled to
+ // fill the rendered size.
+ SetImageToBlackPixel(mBlackImage->AsPlanarYCbCrImage());
+ }
+ }
+ if (mBlackImage) {
+ image = mBlackImage;
+ }
+ }
+ // Don't append null image to the newImages.
+ if (!image) {
+ iter.Next();
+ continue;
+ }
+ newImages.AppendElement(ImageContainer::NonOwningImage(image, chunk.mTimeStamp));
+
+ lastPrincipalHandle = chunk.GetPrincipalHandle();
+
+ mLastPlayedVideoFrame = *frame;
+ iter.Next();
+ }
+
+ // Don't update if there are no changes.
+ if (newImages.IsEmpty()) {
+ return;
+ }
+
+ AutoTArray<ImageContainer::NonOwningImage,4> images;
+
+ bool principalHandleChanged =
+ lastPrincipalHandle != PRINCIPAL_HANDLE_NONE &&
+ lastPrincipalHandle != GetLastPrincipalHandle();
+
+ // Add the frames from this iteration.
+ for (auto& image : newImages) {
+ image.mFrameID = NewFrameID();
+ images.AppendElement(image);
+ }
+
+ if (principalHandleChanged) {
+ UpdatePrincipalHandleForFrameID(lastPrincipalHandle,
+ newImages.LastElement().mFrameID);
+ }
+
+ SetCurrentFramesLocked(mLastPlayedVideoFrame.GetIntrinsicSize(), images);
+ nsCOMPtr<nsIRunnable> event =
+ new VideoFrameContainerInvalidateRunnable(this);
+ NS_DispatchToMainThread(event.forget());
+
+ images.ClearAndRetainStorage();
+}
+
+void VideoFrameContainer::ClearFrames()
+{
+ ClearFutureFrames();
+}
+
void VideoFrameContainer::SetCurrentFrame(const gfx::IntSize& aIntrinsicSize,
Image* aImage,
const TimeStamp& aTargetTime)
{
if (aImage) {
MutexAutoLock lock(mMutex);
AutoTArray<ImageContainer::NonOwningImage,1> imageList;
imageList.AppendElement(
--- a/dom/media/VideoFrameContainer.h
+++ b/dom/media/VideoFrameContainer.h
@@ -8,60 +8,63 @@
#define VIDEOFRAMECONTAINER_H_
#include "mozilla/Mutex.h"
#include "mozilla/TimeStamp.h"
#include "gfxPoint.h"
#include "nsCOMPtr.h"
#include "ImageContainer.h"
#include "MediaSegment.h"
+#include "MediaStreamVideoSink.h"
+#include "VideoSegment.h"
namespace mozilla {
namespace dom {
class HTMLMediaElement;
} // namespace dom
/**
* This object is used in the decoder backend threads and the main thread
* to manage the "current video frame" state. This state includes timing data
* and an intrinsic size (see below).
* This has to be a thread-safe object since it's accessed by resource decoders
* and other off-main-thread components. So we can't put this state in the media
* element itself ... well, maybe we could, but it could be risky and/or
* confusing.
*/
-class VideoFrameContainer {
- ~VideoFrameContainer();
+class VideoFrameContainer : public MediaStreamVideoSink {
+ virtual ~VideoFrameContainer();
public:
typedef layers::ImageContainer ImageContainer;
typedef layers::Image Image;
- NS_INLINE_DECL_THREADSAFE_REFCOUNTING(VideoFrameContainer)
-
VideoFrameContainer(dom::HTMLMediaElement* aElement,
already_AddRefed<ImageContainer> aContainer);
// Call on any thread
+ virtual void SetCurrentFrames(const VideoSegment& aSegment) override;
+ virtual void ClearFrames() override;
+ void SetCurrentFrame(const gfx::IntSize& aIntrinsicSize, Image* aImage,
+ const TimeStamp& aTargetTime);
// Returns the last principalHandle we notified mElement about.
PrincipalHandle GetLastPrincipalHandle();
// We will notify mElement that aPrincipalHandle has been applied when all
// FrameIDs prior to aFrameID have been flushed out.
// aFrameID is ignored if aPrincipalHandle already is our pending principalHandle.
void UpdatePrincipalHandleForFrameID(const PrincipalHandle& aPrincipalHandle,
const ImageContainer::FrameID& aFrameID);
- void SetCurrentFrame(const gfx::IntSize& aIntrinsicSize, Image* aImage,
- const TimeStamp& aTargetTime);
void SetCurrentFrames(const gfx::IntSize& aIntrinsicSize,
const nsTArray<ImageContainer::NonOwningImage>& aImages);
void ClearCurrentFrame(const gfx::IntSize& aIntrinsicSize)
{
SetCurrentFrames(aIntrinsicSize, nsTArray<ImageContainer::NonOwningImage>());
}
+ VideoFrameContainer* AsVideoFrameContainer() override { return this; }
void ClearCurrentFrame();
// Make the current frame the only frame in the container, i.e. discard
// all future frames.
void ClearFutureFrames();
// Time in seconds by which the last painted video frame was late by.
// E.g. if the last painted frame should have been painted at time t,
// but was actually painted at t+n, this returns n in seconds. Threadsafe.
@@ -75,17 +78,17 @@ public:
return ++mFrameID;
}
// Call on main thread
enum {
INVALIDATE_DEFAULT,
INVALIDATE_FORCE
};
- void Invalidate() { InvalidateWithFlags(INVALIDATE_DEFAULT); }
+ void Invalidate() override { InvalidateWithFlags(INVALIDATE_DEFAULT); }
void InvalidateWithFlags(uint32_t aFlags);
ImageContainer* GetImageContainer();
void ForgetElement() { mElement = nullptr; }
uint32_t GetDroppedImageCount() { return mImageContainer->GetDroppedImageCount(); }
protected:
void SetCurrentFramesLocked(const gfx::IntSize& aIntrinsicSize,
@@ -93,25 +96,31 @@ protected:
// Non-addreffed pointer to the element. The element calls ForgetElement
// to clear this reference when the element is destroyed.
dom::HTMLMediaElement* mElement;
RefPtr<ImageContainer> mImageContainer;
// mMutex protects all the fields below.
Mutex mMutex;
+ // Once the frame is forced to black, we initialize mBlackImage for following
+ // frames.
+ RefPtr<Image> mBlackImage;
// The intrinsic size is the ideal size which we should render the
// ImageContainer's current Image at.
// This can differ from the Image's actual size when the media resource
// specifies that the Image should be stretched to have the correct aspect
// ratio.
gfx::IntSize mIntrinsicSize;
// We maintain our own mFrameID which is auto-incremented at every
// SetCurrentFrame() or NewFrameID() call.
ImageContainer::FrameID mFrameID;
+ // We record the last played video frame to avoid playing the frame again
+ // with a different frame id.
+ VideoFrame mLastPlayedVideoFrame;
// True when the intrinsic size has been changed by SetCurrentFrame() since
// the last call to Invalidate().
// The next call to Invalidate() will recalculate
// and update the intrinsic size on the element, request a frame reflow and
// then reset this flag.
bool mIntrinsicSizeChanged;
// True when the Image size has changed since the last time Invalidate() was
// called. When set, the next call to Invalidate() will ensure that the
--- a/dom/media/VideoSegment.h
+++ b/dom/media/VideoSegment.h
@@ -132,13 +132,19 @@ public:
// Segment-generic methods not in MediaSegmentBase
static Type StaticType() { return VIDEO; }
size_t SizeOfIncludingThis(MallocSizeOf aMallocSizeOf) const override
{
return aMallocSizeOf(this) + SizeOfExcludingThis(aMallocSizeOf);
}
+
+ bool IsEmpty() const
+ {
+ return mChunks.IsEmpty();
+ }
+
};
} // namespace mozilla
#endif /* MOZILLA_VIDEOSEGMENT_H_ */
--- a/dom/media/moz.build
+++ b/dom/media/moz.build
@@ -122,16 +122,17 @@ EXPORTS += [
'MediaQueue.h',
'MediaRecorder.h',
'MediaResource.h',
'MediaResourceCallback.h',
'MediaSegment.h',
'MediaStatistics.h',
'MediaStreamGraph.h',
'MediaStreamListener.h',
+ 'MediaStreamVideoSink.h',
'MediaTimer.h',
'MediaTrack.h',
'MediaTrackList.h',
'MP3Decoder.h',
'MP3Demuxer.h',
'MP3FrameParser.h',
'NextFrameSeekTask.h',
'nsIDocumentActivity.h',
@@ -234,16 +235,17 @@ UNIFIED_SOURCES += [
'MediaPrefs.cpp',
'MediaRecorder.cpp',
'MediaResource.cpp',
'MediaShutdownManager.cpp',
'MediaStreamError.cpp',
'MediaStreamGraph.cpp',
'MediaStreamListener.cpp',
'MediaStreamTrack.cpp',
+ 'MediaStreamVideoSink.cpp',
'MediaTimer.cpp',
'MediaTrack.cpp',
'MediaTrackList.cpp',
'MP3Decoder.cpp',
'MP3Demuxer.cpp',
'MP3FrameParser.cpp',
'NextFrameSeekTask.cpp',
'QueueObject.cpp',