Bug 1372080 - Reorder frames decoded by Widevine CDM. r=jya draft
authorChris Pearce <cpearce@mozilla.com>
Mon, 12 Jun 2017 17:47:05 +1200
changeset 592441 68ef406556087434fa12b72ae5ed5c2e1bce2b64
parent 592285 981da978f1f686ad024fa958c9d27d2f8acc5ad0
child 594551 9310a6431bb7534cbd537f13ffc491932b839be3
push id63393
push userbmo:cpearce@mozilla.com
push dateMon, 12 Jun 2017 08:34:06 +0000
reviewersjya
bugs1372080
milestone55.0a1
Bug 1372080 - Reorder frames decoded by Widevine CDM. r=jya The next version of the Widevine CDM (970) has a new H.264 decoder and it does not appear to be outputing frames it decodes in presentation order, so we need to reorder the frames output by the CDM. MozReview-Commit-ID: HMsQVN3NCIU
dom/media/gmp/ChromiumCDMParent.cpp
dom/media/gmp/ChromiumCDMParent.h
dom/media/platforms/moz.build
--- a/dom/media/gmp/ChromiumCDMParent.cpp
+++ b/dom/media/gmp/ChromiumCDMParent.cpp
@@ -1,25 +1,28 @@
 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
 /* This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 #include "ChromiumCDMParent.h"
-#include "mozilla/gmp/GMPTypes.h"
+
+#include "ChromiumCDMProxy.h"
+#include "content_decryption_module.h"
 #include "GMPContentChild.h"
 #include "GMPContentParent.h"
-#include "mozilla/Unused.h"
-#include "ChromiumCDMProxy.h"
+#include "GMPLog.h"
+#include "GMPUtils.h"
+#include "MediaPrefs.h"
 #include "mozilla/dom/MediaKeyMessageEventBinding.h"
+#include "mozilla/gmp/GMPTypes.h"
 #include "mozilla/Telemetry.h"
-#include "content_decryption_module.h"
-#include "GMPLog.h"
-#include "MediaPrefs.h"
-#include "GMPUtils.h"
+#include "mozilla/Unused.h"
+#include "mp4_demuxer/AnnexB.h"
+#include "mp4_demuxer/H264.h"
 
 namespace mozilla {
 namespace gmp {
 
 using namespace eme;
 
 ChromiumCDMParent::ChromiumCDMParent(GMPContentParent* aContentParent,
                                      uint32_t aPluginId)
@@ -726,26 +729,30 @@ ChromiumCDMParent::RecvDecodedData(const
   if (!v) {
     mDecodePromise.RejectIfExists(
       MediaResult(NS_ERROR_OUT_OF_MEMORY,
                   RESULT_DETAIL("Can't create VideoData")),
       __func__);
     return IPC_OK();
   }
 
-  mDecodePromise.ResolveIfExists({ Move(v) }, __func__);
+  ReorderAndReturnOutput(Move(v));
 
   return IPC_OK();
 }
 
 ipc::IPCResult
 ChromiumCDMParent::RecvDecodedShmem(const CDMVideoFrame& aFrame,
                                     ipc::Shmem&& aShmem)
 {
-  GMP_LOG("ChromiumCDMParent::RecvDecodedShmem(this=%p)", this);
+  GMP_LOG("ChromiumCDMParent::RecvDecodedShmem(this=%p) time=%" PRId64
+          " duration=%" PRId64,
+          this,
+          aFrame.mTimestamp(),
+          aFrame.mDuration());
 
   // On failure we need to deallocate the shmem we're to return to the
   // CDM. On success we return it to the CDM to be reused.
   auto autoDeallocateShmem =
     MakeScopeExit([&, this] { this->DeallocShmem(aShmem); });
 
   if (mIsShutdown || mDecodePromise.IsEmpty()) {
     return IPC_OK();
@@ -770,21 +777,36 @@ ChromiumCDMParent::RecvDecodedShmem(cons
       __func__);
     return IPC_OK();
   }
 
   // Don't need to deallocate the shmem since the CDM process is responsible
   // for it again.
   autoDeallocateShmem.release();
 
-  mDecodePromise.ResolveIfExists({ Move(v) }, __func__);
+  ReorderAndReturnOutput(Move(v));
 
   return IPC_OK();
 }
 
+void
+ChromiumCDMParent::ReorderAndReturnOutput(RefPtr<VideoData>&& aFrame)
+{
+  if (mMaxRefFrames == 0) {
+    mDecodePromise.ResolveIfExists({ Move(aFrame) }, __func__);
+    return;
+  }
+  mReorderQueue.Push(Move(aFrame));
+  MediaDataDecoder::DecodedData results;
+  while (mReorderQueue.Length() > mMaxRefFrames) {
+    results.AppendElement(mReorderQueue.Pop());
+  }
+  mDecodePromise.Resolve(Move(results), __func__);
+}
+
 already_AddRefed<VideoData>
 ChromiumCDMParent::CreateVideoFrame(const CDMVideoFrame& aFrame,
                                     Span<uint8_t> aData)
 {
   VideoData::YCbCrBuffer b;
   MOZ_ASSERT(aData.Length() > 0);
 
   b.mPlanes[0].mData = aData.Elements();
@@ -910,16 +932,23 @@ ChromiumCDMParent::InitializeVideoDecode
 
   if (!SendInitializeVideoDecoder(aConfig)) {
     return MediaDataDecoder::InitPromise::CreateAndReject(
       MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
                   RESULT_DETAIL("Failed to send init video decoder to CDM")),
       __func__);
   }
 
+  mMaxRefFrames =
+    (aConfig.mCodec() == cdm::VideoDecoderConfig::kCodecH264)
+      ? mp4_demuxer::AnnexB::HasSPS(aInfo.mExtraData)
+          ? mp4_demuxer::H264::ComputeMaxRefFrames(aInfo.mExtraData)
+          : 16
+      : 0;
+
   mVideoDecoderInitialized = true;
   mImageContainer = aImageContainer;
   mVideoInfo = aInfo;
   mVideoFrameBufferSize = bufferSize;
 
   return mInitVideoDecoderPromise.Ensure(__func__);
 }
 
@@ -982,34 +1011,38 @@ ChromiumCDMParent::DecryptAndDecodeFrame
 
   return mDecodePromise.Ensure(__func__);
 }
 
 RefPtr<MediaDataDecoder::FlushPromise>
 ChromiumCDMParent::FlushVideoDecoder()
 {
   if (mIsShutdown) {
+    MOZ_ASSERT(mReorderQueue.IsEmpty());
     return MediaDataDecoder::FlushPromise::CreateAndReject(
       MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
                   RESULT_DETAIL("ChromiumCDMParent is shutdown")),
       __func__);
   }
 
+  mReorderQueue.Clear();
+
   mDecodePromise.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED, __func__);
   if (!SendResetVideoDecoder()) {
     return MediaDataDecoder::FlushPromise::CreateAndReject(
       MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR, "Failed to send flush to CDM."),
       __func__);
   }
   return mFlushDecoderPromise.Ensure(__func__);
 }
 
 ipc::IPCResult
 ChromiumCDMParent::RecvResetVideoDecoderComplete()
 {
+  MOZ_ASSERT(mReorderQueue.IsEmpty());
   if (mIsShutdown) {
     MOZ_ASSERT(mFlushDecoderPromise.IsEmpty());
     return IPC_OK();
   }
   mFlushDecoderPromise.ResolveIfExists(true, __func__);
   return IPC_OK();
 }
 
@@ -1033,17 +1066,23 @@ ChromiumCDMParent::Drain()
 
 ipc::IPCResult
 ChromiumCDMParent::RecvDrainComplete()
 {
   if (mIsShutdown) {
     MOZ_ASSERT(mDecodePromise.IsEmpty());
     return IPC_OK();
   }
-  mDecodePromise.ResolveIfExists(MediaDataDecoder::DecodedData(), __func__);
+
+  MediaDataDecoder::DecodedData samples;
+  while (!mReorderQueue.IsEmpty()) {
+    samples.AppendElement(Move(mReorderQueue.Pop()));
+  }
+
+  mDecodePromise.ResolveIfExists(Move(samples), __func__);
   return IPC_OK();
 }
 RefPtr<ShutdownPromise>
 ChromiumCDMParent::ShutdownVideoDecoder()
 {
   if (mIsShutdown || !mVideoDecoderInitialized) {
     return ShutdownPromise::CreateAndResolve(true, __func__);
   }
@@ -1092,16 +1131,18 @@ ChromiumCDMParent::Shutdown()
     NS_DispatchToMainThread(task.forget());
   }
 
   // We may be called from a task holding the last reference to the proxy, so
   // let's clear our local weak pointer to ensure it will not be used afterward
   // (including from an already-queued task, e.g.: ActorDestroy).
   mProxy = nullptr;
 
+  mReorderQueue.Clear();
+
   for (RefPtr<DecryptJob>& decrypt : mDecrypts) {
     decrypt->PostResult(eme::AbortedErr);
   }
   mDecrypts.Clear();
 
   if (mVideoDecoderInitialized && !mActorDestroyed) {
     Unused << SendDeinitializeVideoDecoder();
     mVideoDecoderInitialized = false;
--- a/dom/media/gmp/ChromiumCDMParent.h
+++ b/dom/media/gmp/ChromiumCDMParent.h
@@ -11,16 +11,17 @@
 #include "GMPCrashHelperHolder.h"
 #include "GMPMessageUtils.h"
 #include "mozilla/gmp/PChromiumCDMParent.h"
 #include "mozilla/RefPtr.h"
 #include "nsDataHashtable.h"
 #include "PlatformDecoderModule.h"
 #include "ImageContainer.h"
 #include "mozilla/Span.h"
+#include "ReorderQueue.h"
 
 namespace mozilla {
 
 class MediaRawData;
 class ChromiumCDMProxy;
 
 namespace gmp {
 
@@ -123,16 +124,18 @@ protected:
                                  nsTArray<uint8_t>&& aData) override;
   ipc::IPCResult RecvDecodeFailed(const uint32_t& aStatus) override;
   ipc::IPCResult RecvShutdown() override;
   ipc::IPCResult RecvResetVideoDecoderComplete() override;
   ipc::IPCResult RecvDrainComplete() override;
   void ActorDestroy(ActorDestroyReason aWhy) override;
   bool SendBufferToCDM(uint32_t aSizeInBytes);
 
+  void ReorderAndReturnOutput(RefPtr<VideoData>&& aFrame);
+
   void RejectPromise(uint32_t aPromiseId,
                      nsresult aError,
                      const nsCString& aErrorMessage);
 
   void ResolvePromise(uint32_t aPromiseId);
 
   bool InitCDMInputBuffer(gmp::CDMInputBuffer& aBuffer, MediaRawData* aSample);
 
@@ -167,14 +170,23 @@ protected:
   // Maximum number of shmems to use to return decoded video frames.
   uint32_t mVideoShmemLimit;
   // High water mark for mVideoShmemsActive, reported via telemetry.
   uint32_t mMaxVideoShmemsActive = 0;
 
   bool mIsShutdown = false;
   bool mVideoDecoderInitialized = false;
   bool mActorDestroyed = false;
+
+  // The H.264 decoder in Widevine CDM versions 970 and later output in decode
+  // order rather than presentation order, so we reorder in presentation order
+  // before presenting. mMaxRefFrames is non-zero if we have an initialized
+  // decoder and we are decoding H.264. If so, it stores the maximum length of
+  // the reorder queue that we need. Note we may have multiple decoders for the
+  // life time of this object, but never more than one active at once.
+  uint32_t mMaxRefFrames = 0;
+  ReorderQueue mReorderQueue;
 };
 
 } // namespace gmp
 } // namespace mozilla
 
 #endif // ChromiumCDMParent_h_
--- a/dom/media/platforms/moz.build
+++ b/dom/media/platforms/moz.build
@@ -9,16 +9,17 @@ EXPORTS += [
     'agnostic/DummyMediaDataDecoder.h',
     'agnostic/OpusDecoder.h',
     'agnostic/TheoraDecoder.h',
     'agnostic/VorbisDecoder.h',
     'agnostic/VPXDecoder.h',
     'MediaTelemetryConstants.h',
     'PDMFactory.h',
     'PlatformDecoderModule.h',
+    'ReorderQueue.h',
     'SimpleMap.h',
     'wrappers/H264Converter.h',
     'wrappers/MediaDataDecoderProxy.h'
 
 ]
 
 UNIFIED_SOURCES += [
     'agnostic/AgnosticDecoderModule.cpp',