Bug 1274626 part 3 - make the blank video decoder return samples in PTS order; r=jya draft
authorKaku Kuo <tkuo@mozilla.com>
Wed, 03 Aug 2016 15:22:49 +0800
changeset 396018 05df0526b442cfce765fa73c39ed6f1bc6b01e68
parent 396017 6696ee41757de6286fc5153e1ba0d79a745449af
child 396019 e52b85b0d4925c3a28d2127d6de14ba77f5bc88a
push id24896
push usertkuo@mozilla.com
push dateWed, 03 Aug 2016 07:58:00 +0000
reviewersjya
bugs1274626
milestone51.0a1
Bug 1274626 part 3 - make the blank video decoder return samples in PTS order; r=jya MozReview-Commit-ID: EULu34WIcw0
dom/media/platforms/ReorderQueue.h
dom/media/platforms/agnostic/BlankDecoderModule.cpp
dom/media/platforms/apple/AppleVTDecoder.cpp
dom/media/platforms/apple/ReorderQueue.h
media/libstagefright/binding/H264.cpp
media/libstagefright/binding/include/mp4_demuxer/H264.h
rename from dom/media/platforms/apple/ReorderQueue.h
rename to dom/media/platforms/ReorderQueue.h
--- a/dom/media/platforms/agnostic/BlankDecoderModule.cpp
+++ b/dom/media/platforms/agnostic/BlankDecoderModule.cpp
@@ -6,34 +6,41 @@
 
 #include "ImageContainer.h"
 #include "MediaDecoderReader.h"
 #include "MediaInfo.h"
 #include "mozilla/CheckedInt.h"
 #include "mozilla/mozalloc.h" // for operator new, and new (fallible)
 #include "mozilla/RefPtr.h"
 #include "mozilla/TaskQueue.h"
+#include "mp4_demuxer/H264.h"
+#include "MP4Decoder.h"
 #include "nsAutoPtr.h"
 #include "nsRect.h"
 #include "PlatformDecoderModule.h"
+#include "ReorderQueue.h"
 #include "TimeUnits.h"
 #include "VideoUtils.h"
 
 namespace mozilla {
 
 // Decoder that uses a passed in object's Create function to create blank
 // MediaData objects.
 template<class BlankMediaDataCreator>
 class BlankMediaDataDecoder : public MediaDataDecoder {
 public:
 
   BlankMediaDataDecoder(BlankMediaDataCreator* aCreator,
                         const CreateDecoderParams& aParams)
     : mCreator(aCreator)
     , mCallback(aParams.mCallback)
+    , mMaxRefFrames(aParams.mConfig.GetType() == TrackInfo::kVideoTrack &&
+                    MP4Decoder::IsH264(aParams.mConfig.mMimeType)
+                    ? mp4_demuxer::H264::ComputeMaxRefFrames(aParams.VideoConfig().mExtraData)
+                    : 0)
     , mType(aParams.mConfig.GetType())
   {
   }
 
   RefPtr<InitPromise> Init() override {
     return InitPromise::CreateAndResolve(mType, __func__);
   }
 
@@ -42,41 +49,70 @@ public:
   }
 
   nsresult Input(MediaRawData* aSample) override
   {
     RefPtr<MediaData> data =
       mCreator->Create(media::TimeUnit::FromMicroseconds(aSample->mTime),
                        media::TimeUnit::FromMicroseconds(aSample->mDuration),
                        aSample->mOffset);
-    if (!data) {
-    mCallback->Error(MediaDataDecoderError::FATAL_ERROR);
-    } else {
-      mCallback->Output(data);
-    }
+
+    OutputFrame(data);
+
     return NS_OK;
   }
 
-  nsresult Flush() override {
+  nsresult Flush() override
+  {
+    mReorderQueue.Clear();
+
     return NS_OK;
   }
 
-  nsresult Drain() override {
+  nsresult Drain() override
+  {
+    while (!mReorderQueue.IsEmpty()) {
+      mCallback->Output(mReorderQueue.Pop().get());
+    }
+
     mCallback->DrainComplete();
     return NS_OK;
   }
 
   const char* GetDescriptionName() const override
   {
     return "blank media data decoder";
   }
 
 private:
+  void OutputFrame(MediaData* aData)
+  {
+    if (!aData) {
+      mCallback->Error(MediaDataDecoderError::FATAL_ERROR);
+      return;
+    }
+
+    // Frames come out in DTS order but we need to output them in PTS order.
+    mReorderQueue.Push(aData);
+
+    while (mReorderQueue.Length() > mMaxRefFrames) {
+      mCallback->Output(mReorderQueue.Pop().get());
+    }
+
+    if (mReorderQueue.Length() <= mMaxRefFrames) {
+      mCallback->InputExhausted();
+    }
+
+  }
+
+private:
   nsAutoPtr<BlankMediaDataCreator> mCreator;
   MediaDataDecoderCallback* mCallback;
+  const uint32_t mMaxRefFrames;
+  ReorderQueue mReorderQueue;
   TrackInfo::TrackType mType;
 };
 
 class BlankVideoDataCreator {
 public:
   BlankVideoDataCreator(uint32_t aFrameWidth,
                         uint32_t aFrameHeight,
                         layers::ImageContainer* aImageContainer)
@@ -228,17 +264,21 @@ public:
                    DecoderDoctorDiagnostics* aDiagnostics) const override
   {
     return true;
   }
 
   ConversionRequired
   DecoderNeedsConversion(const TrackInfo& aConfig) const override
   {
-    return kNeedNone;
+    if (aConfig.IsVideo() && MP4Decoder::IsH264(aConfig.mMimeType)) {
+      return kNeedAVCC;
+    } else {
+      return kNeedNone;
+    }
   }
 
 };
 
 already_AddRefed<PlatformDecoderModule> CreateBlankDecoderModule()
 {
   RefPtr<PlatformDecoderModule> pdm = new BlankDecoderModule();
   return pdm.forget();
--- a/dom/media/platforms/apple/AppleVTDecoder.cpp
+++ b/dom/media/platforms/apple/AppleVTDecoder.cpp
@@ -6,58 +6,42 @@
 
 #include <CoreFoundation/CFString.h>
 
 #include "AppleCMLinker.h"
 #include "AppleDecoderModule.h"
 #include "AppleUtils.h"
 #include "AppleVTDecoder.h"
 #include "AppleVTLinker.h"
-#include "mp4_demuxer/H264.h"
 #include "MediaData.h"
 #include "mozilla/ArrayUtils.h"
+#include "mp4_demuxer/H264.h"
 #include "nsAutoPtr.h"
 #include "nsThreadUtils.h"
 #include "mozilla/Logging.h"
 #include "VideoUtils.h"
 #include "gfxPlatform.h"
 
 #define LOG(...) MOZ_LOG(sPDMLog, mozilla::LogLevel::Debug, (__VA_ARGS__))
 
 namespace mozilla {
 
-static uint32_t ComputeMaxRefFrames(const MediaByteBuffer* aExtraData)
-{
-  uint32_t maxRefFrames = 4;
-  // Retrieve video dimensions from H264 SPS NAL.
-  mp4_demuxer::SPSData spsdata;
-  if (mp4_demuxer::H264::DecodeSPSFromExtraData(aExtraData, spsdata)) {
-    // max_num_ref_frames determines the size of the sliding window
-    // we need to queue that many frames in order to guarantee proper
-    // pts frames ordering. Use a minimum of 4 to ensure proper playback of
-    // non compliant videos.
-    maxRefFrames =
-      std::min(std::max(maxRefFrames, spsdata.max_num_ref_frames + 1), 16u);
-  }
-  return maxRefFrames;
-}
-
 AppleVTDecoder::AppleVTDecoder(const VideoInfo& aConfig,
                                TaskQueue* aTaskQueue,
                                MediaDataDecoderCallback* aCallback,
                                layers::ImageContainer* aImageContainer)
   : mExtraData(aConfig.mExtraData)
   , mCallback(aCallback)
   , mPictureWidth(aConfig.mImage.width)
   , mPictureHeight(aConfig.mImage.height)
   , mDisplayWidth(aConfig.mDisplay.width)
   , mDisplayHeight(aConfig.mDisplay.height)
   , mQueuedSamples(0)
   , mTaskQueue(aTaskQueue)
-  , mMaxRefFrames(ComputeMaxRefFrames(aConfig.mExtraData))
+  , mMaxRefFrames(mp4_demuxer::H264::ComputeMaxRefFrames(aConfig.mExtraData))
   , mImageContainer(aImageContainer)
   , mInputIncoming(0)
   , mIsShutDown(false)
 #ifdef MOZ_WIDGET_UIKIT
   , mUseSoftwareImages(true)
 #else
   , mUseSoftwareImages(false)
 #endif
--- a/media/libstagefright/binding/H264.cpp
+++ b/media/libstagefright/binding/H264.cpp
@@ -526,9 +526,26 @@ H264::EnsureSPSIsSane(SPSData& aSPS)
   }
   if (aSPS.max_num_ref_frames > 16) {
     aSPS.max_num_ref_frames = 16;
     valid = false;
   }
   return valid;
 }
 
+/* static */ uint32_t
+H264::ComputeMaxRefFrames(const mozilla::MediaByteBuffer* aExtraData)
+{
+  uint32_t maxRefFrames = 4;
+  // Retrieve video dimensions from H264 SPS NAL.
+  SPSData spsdata;
+  if (DecodeSPSFromExtraData(aExtraData, spsdata)) {
+    // max_num_ref_frames determines the size of the sliding window
+    // we need to queue that many frames in order to guarantee proper
+    // pts frames ordering. Use a minimum of 4 to ensure proper playback of
+    // non compliant videos.
+    maxRefFrames =
+      std::min(std::max(maxRefFrames, spsdata.max_num_ref_frames + 1), 16u);
+  }
+  return maxRefFrames;
+}
+
 } // namespace mp4_demuxer
--- a/media/libstagefright/binding/include/mp4_demuxer/H264.h
+++ b/media/libstagefright/binding/include/mp4_demuxer/H264.h
@@ -327,27 +327,34 @@ struct SPSData
 
   SPSData();
 };
 
 class H264
 {
 public:
   static bool DecodeSPSFromExtraData(const mozilla::MediaByteBuffer* aExtraData, SPSData& aDest);
+
   /* Extract RAW BYTE SEQUENCE PAYLOAD from NAL content.
      Returns nullptr if invalid content.
      This is compliant to ITU H.264 7.3.1 Syntax in tabular form NAL unit syntax
    */
   static already_AddRefed<mozilla::MediaByteBuffer> DecodeNALUnit(const mozilla::MediaByteBuffer* aNAL);
+
   /* Decode SPS NAL RBSP and fill SPSData structure */
   static bool DecodeSPS(const mozilla::MediaByteBuffer* aSPS, SPSData& aDest);
+
   // Ensure that SPS data makes sense, Return true if SPS data was, and false
   // otherwise. If false, then content will be adjusted accordingly.
   static bool EnsureSPSIsSane(SPSData& aSPS);
 
+  // If the given aExtraData is valid, return the aExtraData.max_num_ref_frames
+  // clamped to be in the range of [4, 16]; otherwise return 4.
+  static uint32_t ComputeMaxRefFrames(const mozilla::MediaByteBuffer* aExtraData);
+
 private:
   static void vui_parameters(BitReader& aBr, SPSData& aDest);
   // Read HRD parameters, all data is ignored.
   static void hrd_parameters(BitReader& aBr);
 };
 
 } // namespace mp4_demuxer