Bug 1315850 - Implement video decoding through CDM. r=jya
At this stage, I store video frames in memory in nsTArrays rather than in
shmems just so we can get this working. Once this is working, I'll follow up
with patches to switch to storing all large buffer traffic between the CDM and
other processes in shmems.
I'm not planning on preffing this new CDM path on until that's in place.
MozReview-Commit-ID: LSTb42msWQS
--- a/dom/media/gmp/ChromiumCDMChild.cpp
+++ b/dom/media/gmp/ChromiumCDMChild.cpp
@@ -1,16 +1,17 @@
/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
#include "ChromiumCDMChild.h"
#include "GMPContentChild.h"
#include "WidevineUtils.h"
+#include "WidevineVideoFrame.h"
#include "GMPLog.h"
#include "GMPPlatform.h"
#include "mozilla/Unused.h"
#include "nsPrintfCString.h"
#include "base/time.h"
namespace mozilla {
namespace gmp {
@@ -462,16 +463,64 @@ ChromiumCDMChild::RecvResetVideoDecoder(
}
mozilla::ipc::IPCResult
ChromiumCDMChild::RecvDecryptAndDecodeFrame(const CDMInputBuffer& aBuffer)
{
MOZ_ASSERT(IsOnMessageLoopThread());
GMP_LOG("ChromiumCDMChild::RecvDecryptAndDecodeFrame()");
MOZ_ASSERT(mDecoderInitialized);
+
+ // The output frame may not have the same timestamp as the frame we put in.
+ // We may need to input a number of frames before we receive output. The
+ // CDM's decoder reorders to ensure frames output are in presentation order.
+ // So we need to store the durations of the frames input, and retrieve them
+ // on output.
+ mFrameDurations.Insert(aBuffer.mTimestamp(), aBuffer.mDuration());
+
+ cdm::InputBuffer input;
+ nsTArray<cdm::SubsampleEntry> subsamples;
+ InitInputBuffer(aBuffer, subsamples, input);
+
+ WidevineVideoFrame frame;
+ cdm::Status rv = mCDM->DecryptAndDecodeFrame(input, &frame);
+ GMP_LOG("WidevineVideoDecoder::Decode(timestamp=%" PRId64 ") rv=%d",
+ input.timestamp,
+ rv);
+
+ if (rv == cdm::kSuccess) {
+ // TODO: WidevineBuffers should hold a shmem instead of a array, and we can
+ // send the handle instead of copying the array here.
+
+ gmp::CDMVideoFrame output;
+ output.mFormat() = static_cast<cdm::VideoFormat>(frame.Format());
+ output.mImageWidth() = frame.Size().width;
+ output.mImageHeight() = frame.Size().height;
+ output.mData() = Move(
+ reinterpret_cast<WidevineBuffer*>(frame.FrameBuffer())->ExtractBuffer());
+ output.mYPlane() = { frame.PlaneOffset(cdm::VideoFrame::kYPlane),
+ frame.Stride(cdm::VideoFrame::kYPlane) };
+ output.mUPlane() = { frame.PlaneOffset(cdm::VideoFrame::kUPlane),
+ frame.Stride(cdm::VideoFrame::kUPlane) };
+ output.mVPlane() = { frame.PlaneOffset(cdm::VideoFrame::kVPlane),
+ frame.Stride(cdm::VideoFrame::kVPlane) };
+ output.mTimestamp() = frame.Timestamp();
+
+ uint64_t duration = 0;
+ if (mFrameDurations.Find(frame.Timestamp(), duration)) {
+ output.mDuration() = duration;
+ }
+
+ Unused << SendDecoded(output);
+ } else if (rv == cdm::kNeedMoreData) {
+ Unused << SendDecoded(gmp::CDMVideoFrame());
+ } else {
+ Unused << SendDecodeFailed(rv);
+ }
+
return IPC_OK();
}
mozilla::ipc::IPCResult
ChromiumCDMChild::RecvDestroy()
{
MOZ_ASSERT(IsOnMessageLoopThread());
GMP_LOG("ChromiumCDMChild::RecvDestroy()");
--- a/dom/media/gmp/ChromiumCDMChild.h
+++ b/dom/media/gmp/ChromiumCDMChild.h
@@ -3,16 +3,17 @@
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
#ifndef ChromiumCDMChild_h_
#define ChromiumCDMChild_h_
#include "mozilla/gmp/PChromiumCDMChild.h"
#include "content_decryption_module.h"
+#include "SimpleMap.h"
namespace mozilla {
namespace gmp {
class GMPContentChild;
class ChromiumCDMChild : public PChromiumCDMChild
, public cdm::Host_8
@@ -103,15 +104,18 @@ protected:
const CDMInputBuffer& aBuffer) override;
ipc::IPCResult RecvDestroy() override;
void DecryptFailed(uint32_t aId, cdm::Status aStatus);
GMPContentChild* mPlugin = nullptr;
cdm::ContentDecryptionModule_8* mCDM = nullptr;
+ typedef SimpleMap<uint64_t> DurationMap;
+ DurationMap mFrameDurations;
+
bool mDecoderInitialized = false;
};
} // namespace gmp
} // namespace mozilla
#endif // ChromiumCDMChild_h_
--- a/dom/media/gmp/ChromiumCDMParent.cpp
+++ b/dom/media/gmp/ChromiumCDMParent.cpp
@@ -440,22 +440,77 @@ ChromiumCDMParent::RecvDecrypted(const u
}
}
return IPC_OK();
}
ipc::IPCResult
ChromiumCDMParent::RecvDecoded(const CDMVideoFrame& aFrame)
{
+ VideoData::YCbCrBuffer b;
+ nsTArray<uint8_t> data;
+ data = aFrame.mData();
+
+ if (data.IsEmpty()) {
+ mDecodePromise.ResolveIfExists(nsTArray<RefPtr<MediaData>>(), __func__);
+ return IPC_OK();
+ }
+
+ b.mPlanes[0].mData = data.Elements();
+ b.mPlanes[0].mWidth = aFrame.mImageWidth();
+ b.mPlanes[0].mHeight = aFrame.mImageHeight();
+ b.mPlanes[0].mStride = aFrame.mYPlane().mStride();
+ b.mPlanes[0].mOffset = aFrame.mYPlane().mPlaneOffset();
+ b.mPlanes[0].mSkip = 0;
+
+ b.mPlanes[1].mData = data.Elements();
+ b.mPlanes[1].mWidth = (aFrame.mImageWidth() + 1) / 2;
+ b.mPlanes[1].mHeight = (aFrame.mImageHeight() + 1) / 2;
+ b.mPlanes[1].mStride = aFrame.mUPlane().mStride();
+ b.mPlanes[1].mOffset = aFrame.mUPlane().mPlaneOffset();
+ b.mPlanes[1].mSkip = 0;
+
+ b.mPlanes[2].mData = data.Elements();
+ b.mPlanes[2].mWidth = (aFrame.mImageWidth() + 1) / 2;
+ b.mPlanes[2].mHeight = (aFrame.mImageHeight() + 1) / 2;
+ b.mPlanes[2].mStride = aFrame.mVPlane().mStride();
+ b.mPlanes[2].mOffset = aFrame.mVPlane().mPlaneOffset();
+ b.mPlanes[2].mSkip = 0;
+
+ gfx::IntRect pictureRegion(0, 0, aFrame.mImageWidth(), aFrame.mImageHeight());
+ RefPtr<VideoData> v = VideoData::CreateAndCopyData(mVideoInfo,
+ mImageContainer,
+ mLastStreamOffset,
+ aFrame.mTimestamp(),
+ aFrame.mDuration(),
+ b,
+ false,
+ -1,
+ pictureRegion);
+
+ RefPtr<ChromiumCDMParent> self = this;
+ if (v) {
+ mDecodePromise.ResolveIfExists({ Move(v) }, __func__);
+ } else {
+ mDecodePromise.RejectIfExists(
+ MediaResult(NS_ERROR_OUT_OF_MEMORY,
+ RESULT_DETAIL("CallBack::CreateAndCopyData")),
+ __func__);
+ }
+
return IPC_OK();
}
ipc::IPCResult
ChromiumCDMParent::RecvDecodeFailed(const uint32_t& aStatus)
{
+ mDecodePromise.RejectIfExists(
+ MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
+ RESULT_DETAIL("ChromiumCDMParent::RecvDecodeFailed")),
+ __func__);
return IPC_OK();
}
ipc::IPCResult
ChromiumCDMParent::RecvShutdown()
{
GMP_LOG("ChromiumCDMParent::RecvShutdown(this=%p)", this);
// TODO: SendDestroy(), call Terminated.
@@ -465,25 +520,30 @@ ChromiumCDMParent::RecvShutdown()
void
ChromiumCDMParent::ActorDestroy(ActorDestroyReason aWhy)
{
GMP_LOG("ChromiumCDMParent::ActorDestroy(this=%p, reason=%d)", this, aWhy);
}
RefPtr<MediaDataDecoder::InitPromise>
ChromiumCDMParent::InitializeVideoDecoder(
- const gmp::CDMVideoDecoderConfig& aConfig)
+ const gmp::CDMVideoDecoderConfig& aConfig,
+ const VideoInfo& aInfo,
+ RefPtr<layers::ImageContainer> aImageContainer)
{
if (!SendInitializeVideoDecoder(aConfig)) {
return MediaDataDecoder::InitPromise::CreateAndReject(
MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
RESULT_DETAIL("Failed to send init video decoder to CDM")),
__func__);
}
+ mImageContainer = aImageContainer;
+ mVideoInfo = aInfo;
+
return mInitVideoDecoderPromise.Ensure(__func__);
}
ipc::IPCResult
ChromiumCDMParent::RecvOnDecoderInitDone(const uint32_t& aStatus)
{
GMP_LOG("ChromiumCDMParent::RecvOnDecoderInitDone(this=%p, status=%u)",
this,
@@ -495,10 +555,36 @@ ChromiumCDMParent::RecvOnDecoderInitDone
MediaResult(
NS_ERROR_DOM_MEDIA_FATAL_ERR,
RESULT_DETAIL("CDM init decode failed with %" PRIu32, aStatus)),
__func__);
}
return IPC_OK();
}
+RefPtr<MediaDataDecoder::DecodePromise>
+ChromiumCDMParent::DecryptAndDecodeFrame(MediaRawData* aSample)
+{
+ CDMInputBuffer buffer;
+
+ if (!InitCDMInputBuffer(buffer, aSample)) {
+ return MediaDataDecoder::DecodePromise::CreateAndReject(
+ MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR, "Failed to init CDM buffer."),
+ __func__);
+ }
+
+ mLastStreamOffset = aSample->mOffset;
+
+ if (!SendDecryptAndDecodeFrame(buffer)) {
+ GMP_LOG(
+ "ChromiumCDMParent::Decrypt(this=%p) failed to send decrypt message.",
+ this);
+ return MediaDataDecoder::DecodePromise::CreateAndReject(
+ MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
+ "Failed to send decrypt to CDM process."),
+ __func__);
+ }
+
+ return mDecodePromise.Ensure(__func__);
+}
+
} // namespace gmp
} // namespace mozilla
--- a/dom/media/gmp/ChromiumCDMParent.h
+++ b/dom/media/gmp/ChromiumCDMParent.h
@@ -9,16 +9,17 @@
#include "DecryptJob.h"
#include "GMPCrashHelper.h"
#include "GMPCrashHelperHolder.h"
#include "GMPMessageUtils.h"
#include "mozilla/gmp/PChromiumCDMParent.h"
#include "mozilla/RefPtr.h"
#include "nsDataHashtable.h"
#include "PlatformDecoderModule.h"
+#include "ImageContainer.h"
namespace mozilla {
class MediaRawData;
class ChromiumCDMProxy;
namespace gmp {
@@ -56,17 +57,22 @@ public:
void RemoveSession(const nsCString& aSessionId, uint32_t aPromiseId);
RefPtr<DecryptPromise> Decrypt(MediaRawData* aSample);
// TODO: Add functions for clients to send data to CDM, and
// a Close() function.
RefPtr<MediaDataDecoder::InitPromise> InitializeVideoDecoder(
- const gmp::CDMVideoDecoderConfig& aConfig);
+ const gmp::CDMVideoDecoderConfig& aConfig,
+ const VideoInfo& aInfo,
+ RefPtr<layers::ImageContainer> aImageContainer);
+
+ RefPtr<MediaDataDecoder::DecodePromise> DecryptAndDecodeFrame(
+ MediaRawData* aSample);
protected:
~ChromiumCDMParent() {}
ipc::IPCResult Recv__delete__() override;
ipc::IPCResult RecvOnResolveNewSessionPromise(
const uint32_t& aPromiseId,
const nsCString& aSessionId) override;
@@ -109,14 +115,19 @@ protected:
// Note: this pointer is a weak reference because otherwise it would cause
// a cycle, as ChromiumCDMProxy has a strong reference to the
// ChromiumCDMParent.
ChromiumCDMProxy* mProxy = nullptr;
nsDataHashtable<nsUint32HashKey, uint32_t> mPromiseToCreateSessionToken;
nsTArray<RefPtr<DecryptJob>> mDecrypts;
MozPromiseHolder<MediaDataDecoder::InitPromise> mInitVideoDecoderPromise;
+ MozPromiseHolder<MediaDataDecoder::DecodePromise> mDecodePromise;
+
+ RefPtr<layers::ImageContainer> mImageContainer;
+ VideoInfo mVideoInfo;
+ uint64_t mLastStreamOffset = 0;
};
} // namespace gmp
} // namespace mozilla
#endif // ChromiumCDMParent_h_
--- a/dom/media/gmp/widevine-adapter/moz.build
+++ b/dom/media/gmp/widevine-adapter/moz.build
@@ -11,17 +11,18 @@ SOURCES += [
'WidevineFileIO.cpp',
'WidevineUtils.cpp',
'WidevineVideoDecoder.cpp',
'WidevineVideoFrame.cpp',
]
EXPORTS += [
'WidevineDecryptor.h',
- 'WidevineUtils.h'
+ 'WidevineUtils.h',
+ 'WidevineVideoFrame.h'
]
FINAL_LIBRARY = 'xul'
LOCAL_INCLUDES += [
'/dom/media/gmp',
]
--- a/dom/media/platforms/agnostic/eme/ChromiumCDMVideoDecoder.cpp
+++ b/dom/media/platforms/agnostic/eme/ChromiumCDMVideoDecoder.cpp
@@ -63,41 +63,60 @@ ChromiumCDMVideoDecoder::Init()
}
gmp::CDMVideoDecoderConfig config;
if (MP4Decoder::IsH264(mConfig.mMimeType)) {
config.mCodec() = cdm::VideoDecoderConfig::kCodecH264;
config.mProfile() =
ToCDMH264Profile(mConfig.mExtraData->SafeElementAt(1, 0));
config.mExtraData() = *mConfig.mExtraData;
+ mConvertToAnnexB = true;
} else if (VPXDecoder::IsVP8(mConfig.mMimeType)) {
config.mCodec() = cdm::VideoDecoderConfig::kCodecVp8;
config.mProfile() = cdm::VideoDecoderConfig::kProfileNotNeeded;
} else if (VPXDecoder::IsVP9(mConfig.mMimeType)) {
config.mCodec() = cdm::VideoDecoderConfig::kCodecVp9;
config.mProfile() = cdm::VideoDecoderConfig::kProfileNotNeeded;
} else {
return MediaDataDecoder::InitPromise::CreateAndReject(
NS_ERROR_DOM_MEDIA_FATAL_ERR, __func__);
}
config.mImageWidth() = mConfig.mImage.width;
config.mImageHeight() = mConfig.mImage.height;
RefPtr<gmp::ChromiumCDMParent> cdm = mCDMParent;
- return InvokeAsync(mGMPThread, __func__, [cdm, config]() {
- return cdm->InitializeVideoDecoder(config);
- });
+ VideoInfo info = mConfig;
+ RefPtr<layers::ImageContainer> imageContainer = mImageContainer;
+ return InvokeAsync(
+ mGMPThread, __func__, [cdm, config, info, imageContainer]() {
+ return cdm->InitializeVideoDecoder(config, info, imageContainer);
+ });
+}
+
+const char*
+ChromiumCDMVideoDecoder::GetDescriptionName() const
+{
+ return "Chromium CDM video decoder";
+}
+
+MediaDataDecoder::ConversionRequired
+ChromiumCDMVideoDecoder::NeedsConversion() const
+{
+ return mConvertToAnnexB ? ConversionRequired::kNeedAnnexB
+ : ConversionRequired::kNeedNone;
}
RefPtr<MediaDataDecoder::DecodePromise>
ChromiumCDMVideoDecoder::Decode(MediaRawData* aSample)
{
- return DecodePromise::CreateAndReject(
- MediaResult(NS_ERROR_DOM_MEDIA_DECODE_ERR, RESULT_DETAIL("Unimplemented")),
- __func__);
+ RefPtr<gmp::ChromiumCDMParent> cdm = mCDMParent;
+ RefPtr<MediaRawData> sample = aSample;
+ return InvokeAsync(mGMPThread, __func__, [cdm, sample]() {
+ return cdm->DecryptAndDecodeFrame(sample);
+ });
}
RefPtr<MediaDataDecoder::FlushPromise>
ChromiumCDMVideoDecoder::Flush()
{
return FlushPromise::CreateAndReject(
MediaResult(NS_ERROR_DOM_MEDIA_DECODE_ERR, RESULT_DETAIL("Unimplemented")),
__func__);
--- a/dom/media/platforms/agnostic/eme/ChromiumCDMVideoDecoder.h
+++ b/dom/media/platforms/agnostic/eme/ChromiumCDMVideoDecoder.h
@@ -21,27 +21,26 @@ public:
ChromiumCDMVideoDecoder(const GMPVideoDecoderParams& aParams,
CDMProxy* aCDMProxy);
RefPtr<InitPromise> Init() override;
RefPtr<DecodePromise> Decode(MediaRawData* aSample) override;
RefPtr<FlushPromise> Flush() override;
RefPtr<DecodePromise> Drain() override;
RefPtr<ShutdownPromise> Shutdown() override;
- const char* GetDescriptionName() const override
- {
- return "Chromium CDM video decoder";
- }
+ const char* GetDescriptionName() const override;
+ ConversionRequired NeedsConversion() const override;
private:
~ChromiumCDMVideoDecoder();
RefPtr<gmp::ChromiumCDMParent> mCDMParent;
const VideoInfo mConfig;
RefPtr<GMPCrashHelper> mCrashHelper;
RefPtr<AbstractThread> mGMPThread;
RefPtr<layers::ImageContainer> mImageContainer;
MozPromiseHolder<InitPromise> mInitPromise;
+ bool mConvertToAnnexB = false;
};
} // mozilla
#endif // ChromiumCDMVideoDecoder_h_