Bug 1341894 - Correct encryption subsamples in AVCC -> AnnexB conversion. r=jya draft
authorChris Pearce <cpearce@mozilla.com>
Thu, 23 Feb 2017 11:00:28 +1300
changeset 493732 b840489edafa5dc981ba44f722d92083a40e34cd
parent 493537 eb23648534779c110f3a1f2baae1849ae4a9c570
child 547919 b641083bd8b0181f63411aeb48ca3a49927289d9
push id47829
push userbmo:cpearce@mozilla.com
push dateSun, 05 Mar 2017 21:12:35 +0000
reviewersjya
bugs1341894
milestone54.0a1
Bug 1341894 - Correct encryption subsamples in AVCC -> AnnexB conversion. r=jya This menas we can have GMPVideoDecoder's AVCC -> AnnexB conversion done by the H264Converter, and simplify the code in WidevineVideoDecoder. MozReview-Commit-ID: 3HT5VXth6LL
dom/media/gmp/widevine-adapter/WidevineVideoDecoder.cpp
dom/media/gmp/widevine-adapter/WidevineVideoDecoder.h
dom/media/platforms/agnostic/gmp/GMPVideoDecoder.cpp
dom/media/platforms/agnostic/gmp/GMPVideoDecoder.h
media/libstagefright/binding/AnnexB.cpp
--- a/dom/media/gmp/widevine-adapter/WidevineVideoDecoder.cpp
+++ b/dom/media/gmp/widevine-adapter/WidevineVideoDecoder.cpp
@@ -1,29 +1,28 @@
 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
 /* This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 #include "WidevineVideoDecoder.h"
 
-#include "mp4_demuxer/AnnexB.h"
 #include "WidevineUtils.h"
 #include "WidevineVideoFrame.h"
 #include "mozilla/Move.h"
+#include <inttypes.h>
 
 using namespace cdm;
 
 namespace mozilla {
 
 WidevineVideoDecoder::WidevineVideoDecoder(GMPVideoHost* aVideoHost,
                                            RefPtr<CDMWrapper> aCDMWrapper)
   : mVideoHost(aVideoHost)
   , mCDMWrapper(Move(aCDMWrapper))
-  , mExtraData(new MediaByteBuffer())
   , mSentInput(false)
   , mCodecType(kGMPVideoCodecInvalid)
   , mReturnOutputCallDepth(0)
   , mDrainPending(false)
   , mResetInProgress(false)
 {
   // Expect to start with a CDM wrapper, will release it in DecodingComplete().
   MOZ_ASSERT(mCDMWrapper);
@@ -75,26 +74,29 @@ WidevineVideoDecoder::InitDecode(const G
     config.codec = VideoDecoderConfig::kCodecVp9;
     config.profile = VideoDecoderConfig::kProfileNotNeeded;
   } else {
     mCallback->Error(GMPInvalidArgErr);
     return;
   }
   config.format = kYv12;
   config.coded_size = Size(aCodecSettings.mWidth, aCodecSettings.mHeight);
-  mExtraData->AppendElements(aCodecSpecific + 1, aCodecSpecificLength);
-  config.extra_data = mExtraData->Elements();
-  config.extra_data_size = mExtraData->Length();
+  nsTArray<uint8_t> extraData;
+  if (aCodecSpecificLength > 0) {
+    // The first byte is the WebRTC packetization mode, which can be ignored.
+    extraData.AppendElements(aCodecSpecific + 1, aCodecSpecificLength - 1);
+    config.extra_data = extraData.Elements();
+    config.extra_data_size = extraData.Length();
+  }
   Status rv = CDM()->InitializeVideoDecoder(config);
   if (rv != kSuccess) {
     mCallback->Error(ToGMPErr(rv));
     return;
   }
   CDM_LOG("WidevineVideoDecoder::InitDecode() rv=%d", rv);
-  mAnnexB = mp4_demuxer::AnnexB::ConvertExtraDataToAnnexB(mExtraData);
 }
 
 void
 WidevineVideoDecoder::Decode(GMPVideoEncodedFrame* aInputFrame,
                              bool aMissingFrames,
                              const uint8_t* aCodecSpecificInfo,
                              uint32_t aCodecSpecificInfoLength,
                              int64_t aRenderTimeMs)
@@ -105,45 +107,22 @@ WidevineVideoDecoder::Decode(GMPVideoEnc
   // may be some latency, i.e. we may need to input (say) 30 frames before
   // we receive output. So we need to store the durations of the frames input,
   // and retrieve them on output.
   mFrameDurations[aInputFrame->TimeStamp()] = aInputFrame->Duration();
 
   mSentInput = true;
   InputBuffer sample;
 
-  RefPtr<MediaRawData> raw(
-    new MediaRawData(aInputFrame->Buffer(), aInputFrame->Size()));
-  if (aInputFrame->Size() && !raw->Data()) {
-    // OOM.
-    mCallback->Error(GMPAllocErr);
-    return;
-  }
-  raw->mExtraData = mExtraData;
-  raw->mKeyframe = (aInputFrame->FrameType() == kGMPKeyFrame);
-  if (mCodecType == kGMPVideoCodecH264) {
-    // Convert input from AVCC, which GMPAPI passes in, to AnnexB, which
-    // Chromium uses internally.
-    mp4_demuxer::AnnexB::ConvertSampleToAnnexB(raw);
-  }
-
   const GMPEncryptedBufferMetadata* crypto = aInputFrame->GetDecryptionData();
   nsTArray<SubsampleEntry> subsamples;
-  InitInputBuffer(crypto, aInputFrame->TimeStamp(), raw->Data(), raw->Size(),
+  InitInputBuffer(crypto, aInputFrame->TimeStamp(),
+                  aInputFrame->Buffer(), aInputFrame->Size(),
                   sample, subsamples);
 
-  // For keyframes, ConvertSampleToAnnexB will stick the AnnexB extra data
-  // at the start of the input. So we need to account for that as clear data
-  // in the subsamples.
-  if (raw->mKeyframe
-      && !subsamples.IsEmpty()
-      && mCodecType == kGMPVideoCodecH264) {
-    subsamples[0].clear_bytes += mAnnexB->Length();
-  }
-
   WidevineVideoFrame frame;
   Status rv = CDM()->DecryptAndDecodeFrame(sample, &frame);
   CDM_LOG("WidevineVideoDecoder::Decode(timestamp=%" PRId64 ") rv=%d",
           sample.timestamp, rv);
 
   // Destroy frame, so that the shmem is now free to be used to return
   // output to the Gecko process.
   aInputFrame->Destroy();
--- a/dom/media/gmp/widevine-adapter/WidevineVideoDecoder.h
+++ b/dom/media/gmp/widevine-adapter/WidevineVideoDecoder.h
@@ -54,18 +54,16 @@ private:
     return mCDMWrapper->GetCDM();
   }
 
   bool ReturnOutput(WidevineVideoFrame& aFrame);
   void CompleteReset();
 
   GMPVideoHost* mVideoHost;
   RefPtr<CDMWrapper> mCDMWrapper;
-  RefPtr<MediaByteBuffer> mExtraData;
-  RefPtr<MediaByteBuffer> mAnnexB;
   GMPVideoDecoderCallback* mCallback = nullptr;
   std::map<uint64_t, uint64_t> mFrameDurations;
   bool mSentInput;
   GMPVideoCodecType mCodecType;
   // Frames waiting on allocation
   std::deque<WidevineVideoFrame> mFrameAllocationQueue;
   // Number of calls of ReturnOutput currently in progress.
   int32_t mReturnOutputCallDepth;
--- a/dom/media/platforms/agnostic/gmp/GMPVideoDecoder.cpp
+++ b/dom/media/platforms/agnostic/gmp/GMPVideoDecoder.cpp
@@ -6,16 +6,17 @@
 
 #include "GMPVideoDecoder.h"
 #include "GMPDecoderModule.h"
 #include "GMPVideoHost.h"
 #include "MediaData.h"
 #include "VPXDecoder.h"
 #include "mozilla/EndianUtils.h"
 #include "prsystem.h"
+#include "mp4_demuxer/AnnexB.h"
 
 namespace mozilla {
 
 #if defined(DEBUG)
 static bool IsOnGMPThread()
 {
   nsCOMPtr<mozIGeckoMediaPluginService> mps =
     do_GetService("@mozilla.org/gecko-media-plugin-service;1");
@@ -226,26 +227,31 @@ GMPVideoDecoder::GMPInitDone(GMPVideoDec
 
   if (mInitPromise.IsEmpty()) {
     // GMP must have been shutdown while we were waiting for Init operation
     // to complete.
     aGMP->Close();
     return;
   }
 
+  bool isOpenH264 = aGMP->GetDisplayName().EqualsLiteral("gmpopenh264");
+
   GMPVideoCodec codec;
   memset(&codec, 0, sizeof(codec));
 
   codec.mGMPApiVersion = kGMPVersion33;
   nsTArray<uint8_t> codecSpecific;
   if (MP4Decoder::IsH264(mConfig.mMimeType)) {
     codec.mCodecType = kGMPVideoCodecH264;
     codecSpecific.AppendElement(0); // mPacketizationMode.
     codecSpecific.AppendElements(mConfig.mExtraData->Elements(),
                                  mConfig.mExtraData->Length());
+    // OpenH264 expects pseudo-AVCC, but others must be passed
+    // AnnexB for H264.
+    mConvertToAnnexB = !isOpenH264;
   } else if (VPXDecoder::IsVP8(mConfig.mMimeType)) {
     codec.mCodecType = kGMPVideoCodecVP8;
   } else if (VPXDecoder::IsVP9(mConfig.mMimeType)) {
     codec.mCodecType = kGMPVideoCodecVP9;
   } else {
     // Unrecognized mime type
     aGMP->Close();
     mInitPromise.Reject(NS_ERROR_DOM_MEDIA_FATAL_ERR, __func__);
@@ -270,17 +276,17 @@ GMPVideoDecoder::GMPInitDone(GMPVideoDec
   // GMP implementations have interpreted the meaning of GMP_BufferLength32
   // differently.  The OpenH264 GMP expects GMP_BufferLength32 to behave as
   // specified in the GMP API, where each buffer is prefixed by a 32-bit
   // host-endian buffer length that includes the size of the buffer length
   // field.  Other existing GMPs currently expect GMP_BufferLength32 (when
   // combined with kGMPVideoCodecH264) to mean "like AVCC but restricted to
   // 4-byte NAL lengths" (i.e. buffer lengths are specified in big-endian
   // and do not include the length of the buffer length field.
-  mConvertNALUnitLengths = mGMP->GetDisplayName().EqualsLiteral("gmpopenh264");
+  mConvertNALUnitLengths = isOpenH264;
 
   mInitPromise.Resolve(TrackInfo::kVideoTrack, __func__);
 }
 
 RefPtr<MediaDataDecoder::InitPromise>
 GMPVideoDecoder::Init()
 {
   MOZ_ASSERT(IsOnGMPThread());
--- a/dom/media/platforms/agnostic/gmp/GMPVideoDecoder.h
+++ b/dom/media/platforms/agnostic/gmp/GMPVideoDecoder.h
@@ -39,17 +39,18 @@ public:
   RefPtr<FlushPromise> Flush() override;
   RefPtr<ShutdownPromise> Shutdown() override;
   const char* GetDescriptionName() const override
   {
     return "GMP video decoder";
   }
   ConversionRequired NeedsConversion() const override
   {
-    return ConversionRequired::kNeedAVCC;
+    return mConvertToAnnexB ? ConversionRequired::kNeedAnnexB
+                            : ConversionRequired::kNeedAVCC;
   }
 
   // GMPVideoDecoderCallbackProxy
   // All those methods are called on the GMP thread.
   void Decoded(GMPVideoi420Frame* aDecodedFrame) override;
   void ReceivedDecodedReferenceFrame(const uint64_t aPictureId) override;
   void ReceivedDecodedFrame(const uint64_t aPictureId) override;
   void InputDataExhausted() override;
@@ -95,13 +96,14 @@ private:
 
   int64_t mLastStreamOffset = 0;
   RefPtr<layers::ImageContainer> mImageContainer;
 
   MozPromiseHolder<DecodePromise> mDecodePromise;
   MozPromiseHolder<DecodePromise> mDrainPromise;
   MozPromiseHolder<FlushPromise> mFlushPromise;
   DecodedData mDecodedData;
+  bool mConvertToAnnexB = false;
 };
 
 } // namespace mozilla
 
 #endif // GMPVideoDecoder_h_
--- a/media/libstagefright/binding/AnnexB.cpp
+++ b/media/libstagefright/binding/AnnexB.cpp
@@ -64,16 +64,25 @@ AnnexB::ConvertSampleToAnnexB(mozilla::M
 
   // Prepend the Annex B NAL with SPS and PPS tables to keyframes.
   if (aAddSPS && aSample->mKeyframe) {
     RefPtr<MediaByteBuffer> annexB =
       ConvertExtraDataToAnnexB(aSample->mExtraData);
     if (!samplewriter->Prepend(annexB->Elements(), annexB->Length())) {
       return false;
     }
+
+    // Prepending the NAL with SPS/PPS will mess up the encryption subsample
+    // offsets. So we need to account for the extra bytes by increasing
+    // the length of the first clear data subsample. Otherwise decryption
+    // will fail.
+    if (aSample->mCrypto.mValid) {
+      MOZ_ASSERT(samplewriter->mCrypto.mPlainSizes.Length() > 0);
+      samplewriter->mCrypto.mPlainSizes[0] += annexB->Length();
+    }
   }
 
   return true;
 }
 
 already_AddRefed<mozilla::MediaByteBuffer>
 AnnexB::ConvertExtraDataToAnnexB(const mozilla::MediaByteBuffer* aExtraData)
 {