Bug 1372080 - Reorder frames decoded by Widevine CDM. r=jya
The next version of the Widevine CDM (970) has a new H.264 decoder and it does
not appear to be outputing frames it decodes in presentation order, so we need
to reorder the frames output by the CDM.
MozReview-Commit-ID: HMsQVN3NCIU
--- a/dom/media/gmp/ChromiumCDMParent.cpp
+++ b/dom/media/gmp/ChromiumCDMParent.cpp
@@ -1,25 +1,28 @@
/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
#include "ChromiumCDMParent.h"
-#include "mozilla/gmp/GMPTypes.h"
+
+#include "ChromiumCDMProxy.h"
+#include "content_decryption_module.h"
#include "GMPContentChild.h"
#include "GMPContentParent.h"
-#include "mozilla/Unused.h"
-#include "ChromiumCDMProxy.h"
+#include "GMPLog.h"
+#include "GMPUtils.h"
+#include "MediaPrefs.h"
#include "mozilla/dom/MediaKeyMessageEventBinding.h"
+#include "mozilla/gmp/GMPTypes.h"
#include "mozilla/Telemetry.h"
-#include "content_decryption_module.h"
-#include "GMPLog.h"
-#include "MediaPrefs.h"
-#include "GMPUtils.h"
+#include "mozilla/Unused.h"
+#include "mp4_demuxer/AnnexB.h"
+#include "mp4_demuxer/H264.h"
namespace mozilla {
namespace gmp {
using namespace eme;
ChromiumCDMParent::ChromiumCDMParent(GMPContentParent* aContentParent,
uint32_t aPluginId)
@@ -726,26 +729,30 @@ ChromiumCDMParent::RecvDecodedData(const
if (!v) {
mDecodePromise.RejectIfExists(
MediaResult(NS_ERROR_OUT_OF_MEMORY,
RESULT_DETAIL("Can't create VideoData")),
__func__);
return IPC_OK();
}
- mDecodePromise.ResolveIfExists({ Move(v) }, __func__);
+ ReorderAndReturnOutput(Move(v));
return IPC_OK();
}
ipc::IPCResult
ChromiumCDMParent::RecvDecodedShmem(const CDMVideoFrame& aFrame,
ipc::Shmem&& aShmem)
{
- GMP_LOG("ChromiumCDMParent::RecvDecodedShmem(this=%p)", this);
+ GMP_LOG("ChromiumCDMParent::RecvDecodedShmem(this=%p) time=%" PRId64
+ " duration=%" PRId64,
+ this,
+ aFrame.mTimestamp(),
+ aFrame.mDuration());
// On failure we need to deallocate the shmem we're to return to the
// CDM. On success we return it to the CDM to be reused.
auto autoDeallocateShmem =
MakeScopeExit([&, this] { this->DeallocShmem(aShmem); });
if (mIsShutdown || mDecodePromise.IsEmpty()) {
return IPC_OK();
@@ -770,21 +777,36 @@ ChromiumCDMParent::RecvDecodedShmem(cons
__func__);
return IPC_OK();
}
// Don't need to deallocate the shmem since the CDM process is responsible
// for it again.
autoDeallocateShmem.release();
- mDecodePromise.ResolveIfExists({ Move(v) }, __func__);
+ ReorderAndReturnOutput(Move(v));
return IPC_OK();
}
+void
+ChromiumCDMParent::ReorderAndReturnOutput(RefPtr<VideoData>&& aFrame)
+{
+ if (mMaxRefFrames == 0) {
+ mDecodePromise.ResolveIfExists({ Move(aFrame) }, __func__);
+ return;
+ }
+ mReorderQueue.Push(Move(aFrame));
+ MediaDataDecoder::DecodedData results;
+ while (mReorderQueue.Length() > mMaxRefFrames) {
+ results.AppendElement(mReorderQueue.Pop());
+ }
+ mDecodePromise.Resolve(Move(results), __func__);
+}
+
already_AddRefed<VideoData>
ChromiumCDMParent::CreateVideoFrame(const CDMVideoFrame& aFrame,
Span<uint8_t> aData)
{
VideoData::YCbCrBuffer b;
MOZ_ASSERT(aData.Length() > 0);
b.mPlanes[0].mData = aData.Elements();
@@ -910,16 +932,23 @@ ChromiumCDMParent::InitializeVideoDecode
if (!SendInitializeVideoDecoder(aConfig)) {
return MediaDataDecoder::InitPromise::CreateAndReject(
MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
RESULT_DETAIL("Failed to send init video decoder to CDM")),
__func__);
}
+ mMaxRefFrames =
+ (aConfig.mCodec() == cdm::VideoDecoderConfig::kCodecH264)
+ ? mp4_demuxer::AnnexB::HasSPS(aInfo.mExtraData)
+ ? mp4_demuxer::H264::ComputeMaxRefFrames(aInfo.mExtraData)
+ : 16
+ : 0;
+
mVideoDecoderInitialized = true;
mImageContainer = aImageContainer;
mVideoInfo = aInfo;
mVideoFrameBufferSize = bufferSize;
return mInitVideoDecoderPromise.Ensure(__func__);
}
@@ -982,34 +1011,38 @@ ChromiumCDMParent::DecryptAndDecodeFrame
return mDecodePromise.Ensure(__func__);
}
RefPtr<MediaDataDecoder::FlushPromise>
ChromiumCDMParent::FlushVideoDecoder()
{
if (mIsShutdown) {
+ MOZ_ASSERT(mReorderQueue.IsEmpty());
return MediaDataDecoder::FlushPromise::CreateAndReject(
MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
RESULT_DETAIL("ChromiumCDMParent is shutdown")),
__func__);
}
+ mReorderQueue.Clear();
+
mDecodePromise.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED, __func__);
if (!SendResetVideoDecoder()) {
return MediaDataDecoder::FlushPromise::CreateAndReject(
MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR, "Failed to send flush to CDM."),
__func__);
}
return mFlushDecoderPromise.Ensure(__func__);
}
ipc::IPCResult
ChromiumCDMParent::RecvResetVideoDecoderComplete()
{
+ MOZ_ASSERT(mReorderQueue.IsEmpty());
if (mIsShutdown) {
MOZ_ASSERT(mFlushDecoderPromise.IsEmpty());
return IPC_OK();
}
mFlushDecoderPromise.ResolveIfExists(true, __func__);
return IPC_OK();
}
@@ -1033,17 +1066,23 @@ ChromiumCDMParent::Drain()
ipc::IPCResult
ChromiumCDMParent::RecvDrainComplete()
{
if (mIsShutdown) {
MOZ_ASSERT(mDecodePromise.IsEmpty());
return IPC_OK();
}
- mDecodePromise.ResolveIfExists(MediaDataDecoder::DecodedData(), __func__);
+
+ MediaDataDecoder::DecodedData samples;
+ while (!mReorderQueue.IsEmpty()) {
+ samples.AppendElement(Move(mReorderQueue.Pop()));
+ }
+
+ mDecodePromise.ResolveIfExists(Move(samples), __func__);
return IPC_OK();
}
RefPtr<ShutdownPromise>
ChromiumCDMParent::ShutdownVideoDecoder()
{
if (mIsShutdown || !mVideoDecoderInitialized) {
return ShutdownPromise::CreateAndResolve(true, __func__);
}
@@ -1092,16 +1131,18 @@ ChromiumCDMParent::Shutdown()
NS_DispatchToMainThread(task.forget());
}
// We may be called from a task holding the last reference to the proxy, so
// let's clear our local weak pointer to ensure it will not be used afterward
// (including from an already-queued task, e.g.: ActorDestroy).
mProxy = nullptr;
+ mReorderQueue.Clear();
+
for (RefPtr<DecryptJob>& decrypt : mDecrypts) {
decrypt->PostResult(eme::AbortedErr);
}
mDecrypts.Clear();
if (mVideoDecoderInitialized && !mActorDestroyed) {
Unused << SendDeinitializeVideoDecoder();
mVideoDecoderInitialized = false;
--- a/dom/media/gmp/ChromiumCDMParent.h
+++ b/dom/media/gmp/ChromiumCDMParent.h
@@ -11,16 +11,17 @@
#include "GMPCrashHelperHolder.h"
#include "GMPMessageUtils.h"
#include "mozilla/gmp/PChromiumCDMParent.h"
#include "mozilla/RefPtr.h"
#include "nsDataHashtable.h"
#include "PlatformDecoderModule.h"
#include "ImageContainer.h"
#include "mozilla/Span.h"
+#include "ReorderQueue.h"
namespace mozilla {
class MediaRawData;
class ChromiumCDMProxy;
namespace gmp {
@@ -123,16 +124,18 @@ protected:
nsTArray<uint8_t>&& aData) override;
ipc::IPCResult RecvDecodeFailed(const uint32_t& aStatus) override;
ipc::IPCResult RecvShutdown() override;
ipc::IPCResult RecvResetVideoDecoderComplete() override;
ipc::IPCResult RecvDrainComplete() override;
void ActorDestroy(ActorDestroyReason aWhy) override;
bool SendBufferToCDM(uint32_t aSizeInBytes);
+ void ReorderAndReturnOutput(RefPtr<VideoData>&& aFrame);
+
void RejectPromise(uint32_t aPromiseId,
nsresult aError,
const nsCString& aErrorMessage);
void ResolvePromise(uint32_t aPromiseId);
bool InitCDMInputBuffer(gmp::CDMInputBuffer& aBuffer, MediaRawData* aSample);
@@ -167,14 +170,23 @@ protected:
// Maximum number of shmems to use to return decoded video frames.
uint32_t mVideoShmemLimit;
// High water mark for mVideoShmemsActive, reported via telemetry.
uint32_t mMaxVideoShmemsActive = 0;
bool mIsShutdown = false;
bool mVideoDecoderInitialized = false;
bool mActorDestroyed = false;
+
+ // The H.264 decoder in Widevine CDM versions 970 and later output in decode
+ // order rather than presentation order, so we reorder in presentation order
+ // before presenting. mMaxRefFrames is non-zero if we have an initialized
+ // decoder and we are decoding H.264. If so, it stores the maximum length of
+ // the reorder queue that we need. Note we may have multiple decoders for the
+ // life time of this object, but never more than one active at once.
+ uint32_t mMaxRefFrames = 0;
+ ReorderQueue mReorderQueue;
};
} // namespace gmp
} // namespace mozilla
#endif // ChromiumCDMParent_h_
--- a/dom/media/platforms/moz.build
+++ b/dom/media/platforms/moz.build
@@ -9,16 +9,17 @@ EXPORTS += [
'agnostic/DummyMediaDataDecoder.h',
'agnostic/OpusDecoder.h',
'agnostic/TheoraDecoder.h',
'agnostic/VorbisDecoder.h',
'agnostic/VPXDecoder.h',
'MediaTelemetryConstants.h',
'PDMFactory.h',
'PlatformDecoderModule.h',
+ 'ReorderQueue.h',
'SimpleMap.h',
'wrappers/H264Converter.h',
'wrappers/MediaDataDecoderProxy.h'
]
UNIFIED_SOURCES += [
'agnostic/AgnosticDecoderModule.cpp',