Bug 1014393 - WIP: shift responsibility of adjusting packets with opus codec delay to MediaEncoder.
authorBryce Van Dyk <bvandyk@mozilla.com>
Mon, 27 Mar 2017 14:39:26 +1300
changeset 563901 c6edd409b3ff346449ba39f8d0b6e698d9174a84
parent 563900 badd75f177caf7bfab1e2c4e673b129723648e89
child 563902 d1b6c5c995c63da912e9a3be6ff9b0ffcc46e2fa
push id54457
push userbvandyk@mozilla.com
push dateTue, 18 Apr 2017 01:45:52 +0000
bugs1014393
milestone55.0a1
Bug 1014393 - WIP: shift responsibility of adjusting packets with opus codec delay to MediaEncoder. MozReview-Commit-ID: EDAshLKUaxR
dom/media/encoder/MediaEncoder.cpp
dom/media/encoder/MediaEncoder.h
dom/media/webm/EbmlComposer.cpp
--- a/dom/media/encoder/MediaEncoder.cpp
+++ b/dom/media/encoder/MediaEncoder.cpp
@@ -347,18 +347,24 @@ MediaEncoder::EncodeData()
     if (NS_FAILED(rv)) {
       // Encoding might be canceled.
       LOG(LogLevel::Error, ("Error! Fail to get encoded data from audio "
                             "encoder."));
       mState = ENCODE_ERROR;
       return rv;
     }
     for (uint32_t i = 0; i < encodedAudioData.GetEncodedFrames().Length(); i++) {
-      mEncodedAudioFrames.AppendElement(
-        encodedAudioData.GetEncodedFrames().ElementAt(i));
+      RefPtr<EncodedFrame> frame =
+        encodedAudioData.GetEncodedFrames().ElementAt(i);
+      // Offset audio samples by appropriate amount. Do this before the mux
+      // so we can ensure ordered timestamps.
+      if (frame->mFrameType == EncodedFrame::FrameType::OPUS_AUDIO_FRAME) {
+        frame->mTime += mAudioTimestampOffset;
+      }
+      mEncodedAudioFrames.AppendElement(frame);
     }
   }
   return rv;
 }
 
 nsresult
 MediaEncoder::WriteEncodedDataToMuxer()
 {
@@ -418,16 +424,25 @@ MediaEncoder::CopyMetadataToMuxer(TrackE
 
   RefPtr<TrackMetadataBase> meta = aTrackEncoder->GetMetadata();
   if (meta == nullptr) {
     LOG(LogLevel::Error, ("Error! metadata = null"));
     mState = ENCODE_ERROR;
     return NS_ERROR_ABORT;
   }
 
+  // In the case of Opus we need to calculate the timestamp offset
+  if (meta->GetKind() == TrackMetadataBase::MetadataKind::METADATA_OPUS) {
+    // Calculate offset in microseconds
+    OpusMetadata* opusMeta = static_cast<OpusMetadata*>(meta.get());
+    mAudioTimestampOffset =
+      (uint64_t)LittleEndian::readUint16(opusMeta->mIdHeader.Elements() + 10)
+      * PR_USEC_PER_SEC / 48000;
+  }
+
   nsresult rv = mWriter->SetMetadata(meta);
   if (NS_FAILED(rv)) {
    LOG(LogLevel::Error, ("Error! SetMetadata fail"));
    mState = ENCODE_ERROR;
   }
   return rv;
 }
 
--- a/dom/media/encoder/MediaEncoder.h
+++ b/dom/media/encoder/MediaEncoder.h
@@ -235,16 +235,19 @@ private:
   nsresult CopyMetadataToMuxer(TrackEncoder* aTrackEncoder);
   // Process data pending in encoders
   nsresult EncodeData();
   nsAutoPtr<ContainerWriter> mWriter;
   nsAutoPtr<AudioTrackEncoder> mAudioEncoder;
   nsAutoPtr<VideoTrackEncoder> mVideoEncoder;
   nsTArray<RefPtr<EncodedFrame>> mEncodedAudioFrames;
   nsTArray<RefPtr<EncodedFrame>> mEncodedVideoFrames;
+  // How much each audio time stamp should be shifted forward in time. Used to
+  // adjust for opus codec delay.
+  uint64_t mAudioTimestampOffset = 0;
   RefPtr<MediaStreamVideoRecorderSink> mVideoSink;
   TimeStamp mStartTime;
   nsString mMIMEType;
   int64_t mSizeOfBuffer;
   int mState;
   bool mShutdown;
   bool mDirectConnected;
   Atomic<int> mSuspended;
--- a/dom/media/webm/EbmlComposer.cpp
+++ b/dom/media/webm/EbmlComposer.cpp
@@ -128,17 +128,16 @@ EbmlComposer::WriteSimpleBlock(EncodedFr
   bool flush = false;
   bool isVP8IFrame = (frameType == EncodedFrame::FrameType::VP8_I_FRAME);
   if (isVP8IFrame) {
     FinishCluster();
     flush = true;
   } else {
     // Force it to calculate timecode using signed math via cast
     int64_t timeCode = (aFrame->mTime / ((int) PR_USEC_PER_MSEC) - mClusterTimecode);
-                       (mCodecDelay / PR_NSEC_PER_MSEC);
     if (timeCode < SHRT_MIN || timeCode > SHRT_MAX ) {
       // We're probably going to overflow (or underflow) the timeCode value later!
       FinishCluster();
       flush = true;
     }
   }
 
   auto block = mClusterBuffs.AppendElement();
@@ -156,19 +155,16 @@ EbmlComposer::WriteSimpleBlock(EncodedFr
     mClusterTimecode = aFrame->mTime / PR_USEC_PER_MSEC;
     Ebml_SerializeUnsigned(&ebml, Timecode, mClusterTimecode);
     mFlushState |= FLUSH_CLUSTER;
   }
 
   bool isOpus = (frameType == EncodedFrame::FrameType::OPUS_AUDIO_FRAME);
   // Can't underflow/overflow now
   int64_t timeCode = aFrame->mTime / ((int) PR_USEC_PER_MSEC) - mClusterTimecode;
-  if (isOpus) {
-    timeCode += mCodecDelay / PR_NSEC_PER_MSEC;
-  }
   MOZ_ASSERT(timeCode >= SHRT_MIN && timeCode <= SHRT_MAX);
   writeSimpleBlock(&ebml, isOpus ? 0x2 : 0x1, static_cast<short>(timeCode), isVP8IFrame,
                    0, 0, (unsigned char*)aFrame->GetFrameData().Elements(),
                    aFrame->GetFrameData().Length());
   MOZ_ASSERT(ebml.offset <= DEFAULT_HEADER_SIZE +
              aFrame->GetFrameData().Length(),
              "write more data > EBML_BUFFER_SIZE");
   block->SetLength(ebml.offset);