--- a/dom/media/MediaDecoderReaderWrapper.cpp
+++ b/dom/media/MediaDecoderReaderWrapper.cpp
@@ -148,17 +148,17 @@ MediaDecoderReaderWrapper::MediaDecoderR
 MediaDecoderReaderWrapper::~MediaDecoderReaderWrapper()
 {}
 
 media::TimeUnit
 MediaDecoderReaderWrapper::StartTime() const
 {
   MOZ_ASSERT(mOwnerThread->IsCurrentThreadIn());
   MOZ_ASSERT(!mShutdown);
-  return media::TimeUnit::FromMicroseconds(mStartTimeRendezvous->StartTime());
+  return mStartTime.ref();
 }
 
 RefPtr<MediaDecoderReaderWrapper::MetadataPromise>
 MediaDecoderReaderWrapper::ReadMetadata()
 {
   MOZ_ASSERT(mOwnerThread->IsCurrentThreadIn());
   MOZ_ASSERT(!mShutdown);
   return InvokeAsync(mReader->OwnerThread(), mReader.get(), __func__,
@@ -181,23 +181,16 @@ void
 MediaDecoderReaderWrapper::RequestAudioData()
 {
   MOZ_ASSERT(mOwnerThread->IsCurrentThreadIn());
   MOZ_ASSERT(!mShutdown);
 
   auto p = InvokeAsync(mReader->OwnerThread(), mReader.get(), __func__,
                        &MediaDecoderReader::RequestAudioData);
 
-  if (!mStartTimeRendezvous->HaveStartTime()) {
-    p = p->Then(mOwnerThread, __func__, mStartTimeRendezvous.get(),
-                &StartTimeRendezvous::ProcessFirstSample<MediaData::AUDIO_DATA>,
-                &StartTimeRendezvous::FirstSampleRejected<MediaData::AUDIO_DATA>)
-         ->CompletionPromise();
-  }
-
   RefPtr<MediaDecoderReaderWrapper> self = this;
   mAudioDataRequest.Begin(p->Then(mOwnerThread, __func__,
     [self] (MediaData* aAudioSample) {
       self->mAudioDataRequest.Complete();
       aAudioSample->AdjustForStartTime(self->StartTime().ToMicroseconds());
       self->mAudioCallback.Notify(AsVariant(aAudioSample));
     },
     [self] (const MediaResult& aError) {
@@ -212,32 +205,24 @@ MediaDecoderReaderWrapper::RequestVideoD
 {
   MOZ_ASSERT(mOwnerThread->IsCurrentThreadIn());
   MOZ_ASSERT(!mShutdown);
 
   // Time the video decode and send this value back to callbacks who accept
   // a TimeStamp as its second parameter.
   TimeStamp videoDecodeStartTime = TimeStamp::Now();
 
-  if (aTimeThreshold.ToMicroseconds() > 0 &&
-      mStartTimeRendezvous->HaveStartTime()) {
+  if (aTimeThreshold.ToMicroseconds() > 0) {
     aTimeThreshold += StartTime();
   }
 
   auto p = InvokeAsync(mReader->OwnerThread(), mReader.get(), __func__,
                        &MediaDecoderReader::RequestVideoData,
                        aSkipToNextKeyframe, aTimeThreshold.ToMicroseconds());
 
-  if (!mStartTimeRendezvous->HaveStartTime()) {
-    p = p->Then(mOwnerThread, __func__, mStartTimeRendezvous.get(),
-                &StartTimeRendezvous::ProcessFirstSample<MediaData::VIDEO_DATA>,
-                &StartTimeRendezvous::FirstSampleRejected<MediaData::VIDEO_DATA>)
-         ->CompletionPromise();
-  }
-
   RefPtr<MediaDecoderReaderWrapper> self = this;
   mVideoDataRequest.Begin(p->Then(mOwnerThread, __func__,
     [self, videoDecodeStartTime] (MediaData* aVideoSample) {
       self->mVideoDataRequest.Complete();
       aVideoSample->AdjustForStartTime(self->StartTime().ToMicroseconds());
       self->mVideoCallback.Notify(AsVariant(MakeTuple(aVideoSample, videoDecodeStartTime)));
     },
     [self] (const MediaResult& aError) {
@@ -384,33 +369,23 @@ MediaDecoderReaderWrapper::Shutdown()
 
 void
 MediaDecoderReaderWrapper::OnMetadataRead(MetadataHolder* aMetadata)
 {
   MOZ_ASSERT(mOwnerThread->IsCurrentThreadIn());
   if (mShutdown) {
     return;
   }
-  // Set up the start time rendezvous if it doesn't already exist (which is
-  // generally the case, unless we're coming out of dormant mode).
-  if (!mStartTimeRendezvous) {
-    mStartTimeRendezvous = new StartTimeRendezvous(
-      mOwnerThread, aMetadata->mInfo.HasAudio(),
-      aMetadata->mInfo.HasVideo(), mForceZeroStartTime);
 
-    RefPtr<MediaDecoderReaderWrapper> self = this;
-    mStartTimeRendezvous->AwaitStartTime()->Then(
-      mOwnerThread, __func__,
-      [self] ()  {
-        NS_ENSURE_TRUE_VOID(!self->mShutdown);
-        self->mReader->DispatchSetStartTime(self->StartTime().ToMicroseconds());
-      },
-      [] () {
-        NS_WARNING("Setting start time on reader failed");
-      });
+  if (mStartTime.isNothing()) {
+    mStartTime.emplace(aMetadata->mInfo.mStartTime);
+    // Note: MFR should be able to setup its start time by itself without going
+    // through here. MediaDecoderReader::DispatchSetStartTime() will be removed
+    // once we remove all the legacy readers' code in the following bugs.
+    mReader->DispatchSetStartTime(StartTime().ToMicroseconds());
   }
 }
 
 void
 MediaDecoderReaderWrapper::SetVideoBlankDecode(bool aIsBlankDecode)
 {
   MOZ_ASSERT(mOwnerThread->IsCurrentThreadIn());
   nsCOMPtr<nsIRunnable> r =

--- a/dom/media/MediaDecoderReaderWrapper.h
+++ b/dom/media/MediaDecoderReaderWrapper.h
@@ -126,16 +126,17 @@ private:
   MozPromiseRequestHolder<WaitForDataPromise>& WaitRequestRef(MediaData::Type aType);
 
   const bool mForceZeroStartTime;
   const RefPtr<AbstractThread> mOwnerThread;
   const RefPtr<MediaDecoderReader> mReader;
 
   bool mShutdown = false;
   RefPtr<StartTimeRendezvous> mStartTimeRendezvous;
+  Maybe<media::TimeUnit> mStartTime;
 
   MozPromiseRequestHolder<MediaDataPromise> mAudioDataRequest;
   MozPromiseRequestHolder<MediaDataPromise> mVideoDataRequest;
   MozPromiseRequestHolder<WaitForDataPromise> mAudioWaitRequest;
   MozPromiseRequestHolder<WaitForDataPromise> mVideoWaitRequest;
 };
 
 } // namespace mozilla