Bug 1258870 - Don't push late video frames to the compositor, drop them. r?jwwang
We can get out of A/V sync if the decode is struggling to keep up. This is
because of the loop in VideoSink::UpdateRenderedVideoFrames(). If this function
runs while there's only one frame in the video queue, we won't drop that one
frame if it's late. If we're struggling to keep up, it's increasingly likely
that we'll end up running this function with only one frame in the video queue.
That results in us entering VideoSink::RenderVideoFrames() with only 1 late
frame, which the compositor dutifully draws. Resulting in a late frame being
drawn, and thus broken A/V sync.
This change makes VideoSink::UpdateRenderedVideoFrames() drop all late
frames, even the last one in the video queue. We now keep A/V sync when the
decode is struggling to keep up. However, if I do this, we end up dropping
(and reporting that we drop) a lot more frames, and thus rendering a lot fewer.
But since we when we drop the frames we report them as dropped, a well written
MSE player can detect that we've failing miserably to keep up, and and lower
their bitrate.
MozReview-Commit-ID: ybkq48mKk2
--- a/dom/media/MediaQueue.h
+++ b/dom/media/MediaQueue.h
@@ -40,16 +40,17 @@ public:
ReentrantMonitorAutoEnter mon(mReentrantMonitor);
return nsDeque::GetSize();
}
inline void Push(T* aItem) {
ReentrantMonitorAutoEnter mon(mReentrantMonitor);
MOZ_ASSERT(aItem);
NS_ADDREF(aItem);
+ MOZ_ASSERT(aItem->GetEndTime() >= aItem->mTime);
nsDeque::Push(aItem);
mPushEvent.Notify(RefPtr<T>(aItem));
}
inline void PushFront(T* aItem) {
ReentrantMonitorAutoEnter mon(mReentrantMonitor);
MOZ_ASSERT(aItem);
NS_ADDREF(aItem);--- a/dom/media/mediasink/VideoSink.cpp
+++ b/dom/media/mediasink/VideoSink.cpp
@@ -382,65 +382,63 @@ VideoSink::RenderVideoFrames(int32_t aMa
}
void
VideoSink::UpdateRenderedVideoFrames()
{
AssertOwnerThread();
MOZ_ASSERT(mAudioSink->IsPlaying(), "should be called while playing.");
+ // Get the current playback position.
TimeStamp nowTime;
const int64_t clockTime = mAudioSink->GetPosition(&nowTime);
- // Skip frames up to the frame at the playback position, and figure out
- // the time remaining until it's time to display the next frame and drop
- // the current frame.
NS_ASSERTION(clockTime >= 0, "Should have positive clock time.");
- int64_t remainingTime = -1;
- if (VideoQueue().GetSize() > 0) {
- RefPtr<MediaData> currentFrame = VideoQueue().PopFront();
- int32_t framesRemoved = 0;
- while (VideoQueue().GetSize() > 0) {
- RefPtr<MediaData> nextFrame = VideoQueue().PeekFront();
- if (nextFrame->mTime > clockTime) {
- remainingTime = nextFrame->mTime - clockTime;
- break;
- }
- ++framesRemoved;
- if (!currentFrame->As<VideoData>()->mSentToCompositor) {
- mFrameStats.NotifyDecodedFrames({ 0, 0, 1 });
- VSINK_LOG_V("discarding video frame mTime=%lld clock_time=%lld",
- currentFrame->mTime, clockTime);
- }
- currentFrame = VideoQueue().PopFront();
- }
- VideoQueue().PushFront(currentFrame);
- if (framesRemoved > 0) {
- mVideoFrameEndTime = currentFrame->GetEndTime();
+ // Skip frames up to the playback position.
+ int64_t lastDisplayedFrameEndTime = 0;
+ while (VideoQueue().GetSize() > 0 &&
+ clockTime > VideoQueue().PeekFront()->GetEndTime()) {
+ RefPtr<MediaData> frame = VideoQueue().PopFront();
+ if (frame->As<VideoData>()->mSentToCompositor) {
+ lastDisplayedFrameEndTime = frame->GetEndTime();
mFrameStats.NotifyPresentedFrame();
+ } else {
+ mFrameStats.NotifyDecodedFrames({ 0, 0, 1 });
+ VSINK_LOG_V("discarding video frame mTime=%lld clock_time=%lld",
+ frame->mTime, clockTime);
}
}
+ // The presentation end time of the last video frame displayed is either
+ // the end time of the current frame, or if we dropped all frames in the
+ // queue, the end time of the last frame we removed from the queue.
+ RefPtr<MediaData> currentFrame = VideoQueue().PeekFront();
+ mVideoFrameEndTime = currentFrame ? currentFrame->GetEndTime() : lastDisplayedFrameEndTime;
+
// All frames are rendered, Let's resolve the promise.
if (VideoQueue().IsFinished() &&
VideoQueue().GetSize() <= 1 &&
!mVideoSinkEndRequest.Exists()) {
mEndPromiseHolder.ResolveIfExists(true, __func__);
}
RenderVideoFrames(mVideoQueueSendToCompositorSize, clockTime, nowTime);
- // No next fame to render. There is no need to schedule next render
- // loop. We will run render loops again upon incoming frames.
- if (remainingTime < 0) {
+ // Get the timestamp of the next frame. Schedule the next update at
+ // the start time of the next frame. If we don't have a next frame,
+ // we will run render loops again upon incoming frames.
+ nsTArray<RefPtr<MediaData>> frames;
+ VideoQueue().GetFirstElements(2, &frames);
+ if (frames.Length() < 2) {
return;
}
+ int64_t nextFrameTime = frames[1]->mTime;
TimeStamp target = nowTime + TimeDuration::FromMicroseconds(
- remainingTime / mAudioSink->GetPlaybackParams().mPlaybackRate);
+ (nextFrameTime - clockTime) / mAudioSink->GetPlaybackParams().mPlaybackRate);
RefPtr<VideoSink> self = this;
mUpdateScheduler.Ensure(target, [self] () {
self->UpdateRenderedVideoFramesByTimer();
}, [self] () {
self->UpdateRenderedVideoFramesByTimer();
});
}