Bug 1014393 - Separate MediaEncoders encode and mux steps. r?pehrsons draft
authorBryce Van Dyk <bvandyk@mozilla.com>
Mon, 13 Nov 2017 14:06:03 -0500
changeset 698381 7ea6b9e23e4368432e758d8da87037739c7c0edb
parent 696354 0e18448c495cb5b3da689a554e8137dbd8eddcc0
child 698382 7e26b6073a842357dad65b1b71ec565031235011
push id89276
push userbvandyk@mozilla.com
push dateWed, 15 Nov 2017 17:29:54 +0000
reviewerspehrsons
bugs1014393
milestone58.0a1
Bug 1014393 - Separate MediaEncoders encode and mux steps. r?pehrsons Separating the encode and mux steps allows for better control over interleaving audio and video data. If encode and mux are done in a single step it's possible to mux large amounts of audio or video data which should have been interleaved with the other data type to give correctly ordered time stamps in the target container. MozReview-Commit-ID: CBYemrkpyF5
dom/media/encoder/MediaEncoder.cpp
dom/media/encoder/MediaEncoder.h
--- a/dom/media/encoder/MediaEncoder.cpp
+++ b/dom/media/encoder/MediaEncoder.cpp
@@ -759,36 +759,25 @@ MediaEncoder::GetEncodedData(nsTArray<ns
 
   if (!mMetadataEncoded) {
     MOZ_ASSERT(false);
     return NS_ERROR_FAILURE;
   }
 
   nsresult rv;
   LOG(LogLevel::Verbose, ("GetEncodedData TimeStamp = %f", GetEncodeTimeStamp()));
-  EncodedFrameContainer encodedData;
 
-  if (mVideoEncoder) {
-    // We're most likely to actually wait for a video frame, so do that first
-    // to minimize capture offset/lipsync issues.
-    rv = WriteEncodedDataToMuxer(mVideoEncoder);
-    LOG(LogLevel::Verbose, ("Video encoded TimeStamp = %f", GetEncodeTimeStamp()));
-    if (NS_FAILED(rv)) {
-      LOG(LogLevel::Warning, ("Failed to write encoded video data to muxer"));
-      return rv;
-    }
+  rv = EncodeData();
+  if (NS_FAILED(rv)) {
+    return rv;
   }
 
-  if (mAudioEncoder) {
-    rv = WriteEncodedDataToMuxer(mAudioEncoder);
-    LOG(LogLevel::Verbose, ("Audio encoded TimeStamp = %f", GetEncodeTimeStamp()));
-    if (NS_FAILED(rv)) {
-      LOG(LogLevel::Warning, ("Failed to write encoded audio data to muxer"));
-      return rv;
-    }
+  rv = WriteEncodedDataToMuxer();
+  if (NS_FAILED(rv)) {
+    return rv;
   }
 
   // In audio only or video only case, let unavailable track's flag to be true.
   bool isAudioCompleted = !mAudioEncoder || mAudioEncoder->IsEncodingComplete();
   bool isVideoCompleted = !mVideoEncoder || mVideoEncoder->IsEncodingComplete();
   rv = mWriter->GetContainerData(aOutputBufs,
                                  isAudioCompleted && isVideoCompleted ?
                                  ContainerWriter::FLUSH_NEEDED : 0);
@@ -839,47 +828,103 @@ MediaEncoder::Shutdown()
     // a DataAvailable() handler.
     mEncoderThread->Dispatch(
       NewRunnableMethod("mozilla::MediaEncoderListener::Shutdown",
                         l, &MediaEncoderListener::Shutdown));
   }
 }
 
 nsresult
-MediaEncoder::WriteEncodedDataToMuxer(TrackEncoder *aTrackEncoder)
+MediaEncoder::EncodeData()
+{
+  AUTO_PROFILER_LABEL("MediaEncoder::EncodeData", OTHER);
+
+  MOZ_ASSERT(mEncoderThread->IsCurrentThreadIn());
+
+  MOZ_ASSERT(mVideoEncoder || mAudioEncoder, "Must have atleast one encoder");
+
+  nsresult rv;
+
+  if (mVideoEncoder && !mVideoEncoder->IsEncodingComplete()) {
+    EncodedFrameContainer encodedVideoData;
+    rv = mVideoEncoder->GetEncodedTrack(encodedVideoData);
+    if (NS_FAILED(rv)) {
+      // Encoding might be canceled.
+      LOG(LogLevel::Error, ("Failed to get encoded data from video encoder."));
+      return rv;
+    }
+    for (const RefPtr<EncodedFrame>& frame :
+         encodedVideoData.GetEncodedFrames()) {
+      mEncodedVideoFrames.AppendElement(frame);
+    }
+  }
+
+  if (mAudioEncoder && !mAudioEncoder->IsEncodingComplete()) {
+    EncodedFrameContainer encodedAudioData;
+    rv = mAudioEncoder->GetEncodedTrack(encodedAudioData);
+    if (NS_FAILED(rv)) {
+      // Encoding might be canceled.
+      LOG(LogLevel::Error, ("Failed to get encoded data from audio encoder."));
+      return rv;
+    }
+    for (const RefPtr<EncodedFrame>& frame :
+         encodedAudioData.GetEncodedFrames()) {
+      mEncodedAudioFrames.AppendElement(frame);
+    }
+  }
+
+  return rv;
+}
+
+nsresult
+MediaEncoder::WriteEncodedDataToMuxer()
 {
   AUTO_PROFILER_LABEL("MediaEncoder::WriteEncodedDataToMuxer", OTHER);
 
   MOZ_ASSERT(mEncoderThread->IsCurrentThreadIn());
 
-  if (!aTrackEncoder) {
-    NS_ERROR("No track encoder to get data from");
-    return NS_ERROR_FAILURE;
-  }
+  MOZ_ASSERT(mVideoEncoder || mAudioEncoder, "Must have atleast one encoder");
+
+  nsresult rv;
 
-  if (aTrackEncoder->IsEncodingComplete()) {
-    return NS_OK;
+  if (mVideoEncoder) {
+    EncodedFrameContainer encodedVideoData;
+    for (const RefPtr<EncodedFrame>& frame : mEncodedVideoFrames) {
+      encodedVideoData.AppendEncodedFrame(frame);
+    }
+    mEncodedVideoFrames.Clear();
+
+    rv = mWriter->WriteEncodedTrack(
+      encodedVideoData,
+      mVideoEncoder->IsEncodingComplete() ? ContainerWriter::END_OF_STREAM : 0);
+    if (NS_FAILED(rv)) {
+      LOG(LogLevel::Error,
+          ("Failed to write encoded video track to the muxer."));
+      return rv;
+    }
   }
 
-  EncodedFrameContainer encodedData;
-  nsresult rv = aTrackEncoder->GetEncodedTrack(encodedData);
-  if (NS_FAILED(rv)) {
-    // Encoding might be canceled.
-    LOG(LogLevel::Error, ("Failed to get encoded data from encoder."));
-    SetError();
-    return rv;
+  if (mAudioEncoder) {
+    EncodedFrameContainer encodedAudioData;
+    for (const RefPtr<EncodedFrame>& frame : mEncodedAudioFrames) {
+      encodedAudioData.AppendEncodedFrame(frame);
+    }
+    mEncodedAudioFrames.Clear();
+
+    rv = mWriter->WriteEncodedTrack(
+      encodedAudioData,
+      mAudioEncoder->IsEncodingComplete() ? ContainerWriter::END_OF_STREAM : 0);
+    if (NS_FAILED(rv)) {
+      LOG(LogLevel::Error,
+          ("Failed to write encoded audio track to the muxer."));
+      return rv;
+    }
   }
-  rv = mWriter->WriteEncodedTrack(encodedData,
-                                  aTrackEncoder->IsEncodingComplete() ?
-                                  ContainerWriter::END_OF_STREAM : 0);
-  if (NS_FAILED(rv)) {
-    LOG(LogLevel::Error, ("Failed to write encoded track to the media container."));
-    SetError();
-  }
-  return rv;
+
+  return NS_OK;
 }
 
 nsresult
 MediaEncoder::CopyMetadataToMuxer(TrackEncoder *aTrackEncoder)
 {
   AUTO_PROFILER_LABEL("MediaEncoder::CopyMetadataToMuxer", OTHER);
 
   MOZ_ASSERT(mEncoderThread->IsCurrentThreadIn());
--- a/dom/media/encoder/MediaEncoder.h
+++ b/dom/media/encoder/MediaEncoder.h
@@ -235,20 +235,22 @@ private:
   void Shutdown();
 
   /**
    * Sets mError to true, notifies listeners of the error if mError changed,
    * and stops encoding.
    */
   void SetError();
 
-  // Get encoded data from trackEncoder and write to muxer
-  nsresult WriteEncodedDataToMuxer(TrackEncoder *aTrackEncoder);
   // Get metadata from trackEncoder and copy to muxer
   nsresult CopyMetadataToMuxer(TrackEncoder* aTrackEncoder);
+  // Process data pending in encoder(s)
+  nsresult EncodeData();
+  // Write pending encoded data to muxer
+  nsresult WriteEncodedDataToMuxer();
 
   const RefPtr<TaskQueue> mEncoderThread;
 
   UniquePtr<ContainerWriter> mWriter;
   RefPtr<AudioTrackEncoder> mAudioEncoder;
   RefPtr<AudioTrackListener> mAudioListener;
   RefPtr<VideoTrackEncoder> mVideoEncoder;
   RefPtr<VideoTrackListener> mVideoListener;
@@ -265,16 +267,22 @@ private:
   // Will be null when input is media stream or destination node.
   RefPtr<MediaInputPort> mInputPort;
   // An audio track that we are encoding. Will be null if the input stream
   // doesn't contain audio on start() or if the input is an AudioNode.
   RefPtr<dom::AudioStreamTrack> mAudioTrack;
   // A video track that we are encoding. Will be null if the input stream
   // doesn't contain video on start() or if the input is an AudioNode.
   RefPtr<dom::VideoStreamTrack> mVideoTrack;
+
+  // Audio frames that have been encoded and are pending write to the muxer
+  nsTArray<RefPtr<EncodedFrame>> mEncodedAudioFrames;
+  // Video frames that have been encoded and are pending write to the muxer
+  nsTArray<RefPtr<EncodedFrame>> mEncodedVideoFrames;
+
   TimeStamp mStartTime;
   nsString mMIMEType;
   bool mInitialized;
   bool mMetadataEncoded;
   bool mCompleted;
   bool mError;
   bool mCanceled;
   bool mShutdown;