Bug 1014393 - WIP Sync frames in mediaencoder.
authorBryce Van Dyk <bvandyk@mozilla.com>
Tue, 18 Apr 2017 12:02:58 +1200
changeset 563902 d1b6c5c995c63da912e9a3be6ff9b0ffcc46e2fa
parent 563901 c6edd409b3ff346449ba39f8d0b6e698d9174a84
push id54457
push userbvandyk@mozilla.com
push dateTue, 18 Apr 2017 01:45:52 +0000
bugs1014393
milestone55.0a1
Bug 1014393 - WIP Sync frames in mediaencoder. MozReview-Commit-ID: 59vBAOide01
dom/media/encoder/MediaEncoder.cpp
dom/media/encoder/MediaEncoder.h
--- a/dom/media/encoder/MediaEncoder.cpp
+++ b/dom/media/encoder/MediaEncoder.cpp
@@ -331,17 +331,17 @@ MediaEncoder::EncodeData()
     if (NS_FAILED(rv)) {
       // Encoding might be canceled.
       LOG(LogLevel::Error, ("Error! Fail to get encoded data from video "
                             "encoder."));
       mState = ENCODE_ERROR;
       return rv;
     }
     for (uint32_t i = 0; i < encodedVideoData.GetEncodedFrames().Length(); i++) {
-      mEncodedVideoFrames.AppendElement(
+      mEncodedVideoFrames.Push(
         encodedVideoData.GetEncodedFrames().ElementAt(i));
     }
   }
 
   if (mAudioEncoder && !mAudioEncoder->IsEncodingComplete()) {
     EncodedFrameContainer encodedAudioData;
     rv = mAudioEncoder->GetEncodedTrack(encodedAudioData);
     if (NS_FAILED(rv)) {
@@ -354,67 +354,136 @@ MediaEncoder::EncodeData()
     for (uint32_t i = 0; i < encodedAudioData.GetEncodedFrames().Length(); i++) {
       RefPtr<EncodedFrame> frame =
         encodedAudioData.GetEncodedFrames().ElementAt(i);
       // Offset audio samples by appropriate amount. Do this before the mux
       // so we can ensure ordered timestamps.
       if (frame->mFrameType == EncodedFrame::FrameType::OPUS_AUDIO_FRAME) {
         frame->mTime += mAudioTimestampOffset;
       }
-      mEncodedAudioFrames.AppendElement(frame);
+      mEncodedAudioFrames.Push(frame);
     }
   }
   return rv;
 }
 
 nsresult
 MediaEncoder::WriteEncodedDataToMuxer()
 {
   PROFILER_LABEL("MediaEncoder", "WriteEncodedDataToMuxer",
     js::ProfileEntry::Category::OTHER);
 
   nsresult rv = NS_OK;
 
-  if(mVideoEncoder) {
+  // If we have both video and audio encoders, interleave the frames
+  if (mVideoEncoder && mAudioEncoder) {
+    EncodedFrameContainer encodedData;
+
+    // Interleave frames until we're out of audio or video
+    RefPtr<EncodedFrame> videoFrame;
+    RefPtr<EncodedFrame> audioFrame;
+    uint64_t expectedNextVideoTime = 0;
+    uint64_t expectedNextAudioTime = 0;
+    while (mEncodedVideoFrames.GetSize() > 0 &&
+      mEncodedAudioFrames.GetSize() > 0) {
+      videoFrame = mEncodedVideoFrames.PeekFront();
+      audioFrame = mEncodedAudioFrames.PeekFront();
+      // For any expected time our frames should occur at or after that time
+      MOZ_ASSERT(videoFrame->mTime >= expectedNextVideoTime);
+      MOZ_ASSERT(audioFrame->mTime >= expectedNextAudioTime);
+      if (videoFrame->mTime <= audioFrame->mTime) {
+        expectedNextVideoTime = videoFrame->GetEndTime();
+        encodedData.AppendEncodedFrame(mEncodedVideoFrames.PopFront().take());
+      } else {
+        expectedNextAudioTime = audioFrame->GetEndTime();
+        encodedData.AppendEncodedFrame(mEncodedAudioFrames.PopFront().take());
+      }
+    }
+
+    // If we're out of audio we still may be able to add more video...
+    if (mEncodedAudioFrames.GetSize() == 0) {
+      while (mEncodedVideoFrames.GetSize() > 0) {
+        videoFrame = mEncodedVideoFrames.PeekFront();
+        // If audio encoding is complete or if the video would precede the
+        // next audio sample we can add it
+        if (mAudioEncoder->IsEncodingComplete() ||
+          videoFrame->mTime < expectedNextAudioTime) {
+          encodedData.AppendEncodedFrame(
+            mEncodedVideoFrames.PopFront().take());
+        } else {
+          break;
+        }
+      }
+    }
+
+    // If we're out of video we still may be able to add more audio...
+    if (mEncodedVideoFrames.GetSize() == 0) {
+      while (mEncodedAudioFrames.GetSize() > 0) {
+        audioFrame = mEncodedAudioFrames.PeekFront();
+        // If video encoding is complete or if audio would precede the next
+        // video sample we can add it
+        if (mVideoEncoder->IsEncodingComplete() ||
+          audioFrame->mTime < expectedNextVideoTime) {
+          encodedData.AppendEncodedFrame(
+            mEncodedAudioFrames.PopFront().take());
+        } else {
+          break;
+        }
+      }
+    }
+
+    rv = mWriter->WriteEncodedTrack(encodedData, 0);
+    if (NS_FAILED(rv)) {
+      LOG(LogLevel::Error, ("Error! Fail to write encoded video + audio track "
+                            "to the media container."));
+      mState = ENCODE_ERROR;
+    }
+    return rv;
+  }
+
+  // If we reach here, we have only a single encoder and don't have to worry
+  // about any interleaving
+  if (mVideoEncoder) {
+    MOZ_ASSERT(!mAudioEncoder);
     EncodedFrameContainer encodedVideoData;
-    for (uint32_t i = 0; i < mEncodedVideoFrames.Length(); i++) {
-      encodedVideoData.AppendEncodedFrame(mEncodedVideoFrames.ElementAt(i));
+    while (mEncodedVideoFrames.GetSize() > 0) {
+      encodedVideoData.AppendEncodedFrame(
+        mEncodedVideoFrames.PopFront().take());
     }
-    mEncodedVideoFrames.Clear();
 
     rv = mWriter->WriteEncodedTrack(encodedVideoData,
                                     mVideoEncoder->IsEncodingComplete() ?
                                     ContainerWriter::END_OF_STREAM : 0);
     if (NS_FAILED(rv)) {
       LOG(LogLevel::Error, ("Error! Fail to write encoded video track to the "
                             "media container."));
       mState = ENCODE_ERROR;
-      return rv;
     }
   }
 
-  if(mAudioEncoder) {
+  if (mAudioEncoder) {
+    MOZ_ASSERT(!mVideoEncoder);
     EncodedFrameContainer encodedAudioData;
-    for (uint32_t i = 0; i < mEncodedAudioFrames.Length(); i++) {
-      encodedAudioData.AppendEncodedFrame(mEncodedAudioFrames.ElementAt(i));
+    while (mEncodedAudioFrames.GetSize() > 0) {
+      encodedAudioData.AppendEncodedFrame(
+        mEncodedAudioFrames.PopFront().take());
     }
-    mEncodedAudioFrames.Clear();
 
     rv = mWriter->WriteEncodedTrack(encodedAudioData,
                                     mAudioEncoder->IsEncodingComplete() ?
                                     ContainerWriter::END_OF_STREAM : 0);
     if (NS_FAILED(rv)) {
       LOG(LogLevel::Error, ("Error! Fail to write encoded audio track to the "
                             "media container."));
       mState = ENCODE_ERROR;
-      return rv;
     }
   }
 
   return rv;
+
 }
 
 nsresult
 MediaEncoder::CopyMetadataToMuxer(TrackEncoder *aTrackEncoder)
 {
   if (aTrackEncoder == nullptr) {
     return NS_OK;
   }
--- a/dom/media/encoder/MediaEncoder.h
+++ b/dom/media/encoder/MediaEncoder.h
@@ -12,16 +12,17 @@
 #include "CubebUtils.h"
 #include "MediaStreamGraph.h"
 #include "MediaStreamListener.h"
 #include "nsAutoPtr.h"
 #include "MediaStreamVideoSink.h"
 #include "nsIMemoryReporter.h"
 #include "mozilla/MemoryReporting.h"
 #include "mozilla/Atomics.h"
+#include "MediaQueue.h"
 
 namespace mozilla {
 
 class MediaStreamVideoRecorderSink : public MediaStreamVideoSink
 {
 public:
   explicit MediaStreamVideoRecorderSink(VideoTrackEncoder* aEncoder)
     : mVideoEncoder(aEncoder) {}
@@ -233,18 +234,18 @@ private:
   nsresult WriteEncodedDataToMuxer();
   // Get metadata from trackEncoder and copy to muxer
   nsresult CopyMetadataToMuxer(TrackEncoder* aTrackEncoder);
   // Process data pending in encoders
   nsresult EncodeData();
   nsAutoPtr<ContainerWriter> mWriter;
   nsAutoPtr<AudioTrackEncoder> mAudioEncoder;
   nsAutoPtr<VideoTrackEncoder> mVideoEncoder;
-  nsTArray<RefPtr<EncodedFrame>> mEncodedAudioFrames;
-  nsTArray<RefPtr<EncodedFrame>> mEncodedVideoFrames;
+  MediaQueue<EncodedFrame> mEncodedAudioFrames;
+  MediaQueue<EncodedFrame> mEncodedVideoFrames;
   // How much each audio time stamp should be shifted forward in time. Used to
   // adjust for opus codec delay.
   uint64_t mAudioTimestampOffset = 0;
   RefPtr<MediaStreamVideoRecorderSink> mVideoSink;
   TimeStamp mStartTime;
   nsString mMIMEType;
   int64_t mSizeOfBuffer;
   int mState;