--- a/dom/html/HTMLMediaElement.cpp
+++ b/dom/html/HTMLMediaElement.cpp
@@ -3161,53 +3161,48 @@ void HTMLMediaElement::UpdateSrcMediaStr
                         this, shouldPlay ? "Setting up" : "Removing",
                         mSrcStream.get()));
 
   if (shouldPlay) {
     mSrcStreamPausedCurrentTime = -1;
 
     mMediaStreamListener = new StreamListener(this,
         "HTMLMediaElement::mMediaStreamListener");
-    mMediaStreamSizeListener = new StreamSizeListener(this);
     stream->AddListener(mMediaStreamListener);
-    stream->AddListener(mMediaStreamSizeListener);
 
     mWatchManager.Watch(*mMediaStreamListener,
         &HTMLMediaElement::UpdateReadyStateInternal);
 
     stream->AddAudioOutput(this);
     SetVolumeInternal();
 
     VideoFrameContainer* container = GetVideoFrameContainer();
     if (container) {
       stream->AddVideoOutput(container);
     }
   } else {
     if (stream) {
       mSrcStreamPausedCurrentTime = CurrentTime();
 
       stream->RemoveListener(mMediaStreamListener);
-      stream->RemoveListener(mMediaStreamSizeListener);
 
       stream->RemoveAudioOutput(this);
       VideoFrameContainer* container = GetVideoFrameContainer();
       if (container) {
         stream->RemoveVideoOutput(container);
       }
     }
     // If stream is null, then DOMMediaStream::Destroy must have been
     // called and that will remove all listeners/outputs.
 
     mWatchManager.Unwatch(*mMediaStreamListener,
         &HTMLMediaElement::UpdateReadyStateInternal);
 
     mMediaStreamListener->Forget();
     mMediaStreamListener = nullptr;
-    mMediaStreamSizeListener->Forget();
-    mMediaStreamSizeListener = nullptr;
   }
 }
 
 void HTMLMediaElement::SetupSrcMediaStreamPlayback(DOMMediaStream* aStream)
 {
   NS_ASSERTION(!mSrcStream && !mMediaStreamListener && !mMediaStreamSizeListener,
                "Should have been ended already");
 
@@ -3216,16 +3211,19 @@ void HTMLMediaElement::SetupSrcMediaStre
   nsIDOMWindow* window = OwnerDoc()->GetInnerWindow();
   if (!window) {
     return;
   }
 
   RefPtr<MediaStream> stream = GetSrcMediaStream();
   if (stream) {
     stream->SetAudioChannelType(mAudioChannel);
+
+    mMediaStreamSizeListener = new StreamSizeListener(this);
+    stream->AddListener(mMediaStreamSizeListener);
   }
 
   UpdateSrcMediaStreamPlaying();
 
   // If we pause this media element, track changes in the underlying stream
   // will continue to fire events at this element and alter its track list.
   // That's simpler than delaying the events, but probably confusing...
   ConstructMediaTracks();
@@ -3242,16 +3240,23 @@ void HTMLMediaElement::SetupSrcMediaStre
 }
 
 void HTMLMediaElement::EndSrcMediaStreamPlayback()
 {
   MOZ_ASSERT(mSrcStream);
 
   UpdateSrcMediaStreamPlaying(REMOVING_SRC_STREAM);
 
+  RefPtr<MediaStream> stream = GetSrcMediaStream();
+  if (stream) {
+    stream->RemoveListener(mMediaStreamSizeListener);
+  }
+  mMediaStreamSizeListener->Forget();
+  mMediaStreamSizeListener = nullptr;
+
   mSrcStream->UnregisterTrackListener(mMediaStreamTrackListener);
   mMediaStreamTrackListener = nullptr;
 
   mSrcStream = nullptr;
 }
 
 static already_AddRefed<AudioTrack>
 CreateAudioTrack(AudioStreamTrack* aStreamTrack)