Bug 1249859 - Optimize simulcast if two layers are too close to each other in resolution; r?jesup
MozReview-Commit-ID: 6chD9O7Ms36
--- a/dom/media/tests/mochitest/mochitest.ini
+++ b/dom/media/tests/mochitest/mochitest.ini
@@ -177,16 +177,17 @@ skip-if = android_version
[test_peerConnection_restartIceLocalRollback.html]
skip-if = android_version
[test_peerConnection_restartIceLocalAndRemoteRollback.html]
skip-if = android_version
[test_peerConnection_scaleResolution.html]
skip-if = (android_version == '18') # android(Bug 1189784, timeouts on 4.3 emulator)
[test_peerConnection_simulcastOffer.html]
skip-if = android_version # no simulcast support on android
+[test_peerConnection_simulcastOfferPruning.html]
#[test_peerConnection_relayOnly.html]
[test_peerConnection_callbacks.html]
skip-if = (android_version == '18' && debug) # android(Bug 1189784, timeouts on 4.3 emulator)
[test_peerConnection_replaceTrack.html]
skip-if = android_version == '18' # android(Bug 1189784, timeouts on 4.3 emulator)
[test_peerConnection_syncSetDescription.html]
skip-if = android_version == '18' # android(Bug 1189784, timeouts on 4.3 emulator)
[test_peerConnection_setLocalAnswerInHaveLocalOffer.html]
new file mode 100644
--- /dev/null
+++ b/dom/media/tests/mochitest/test_peerConnection_simulcastOfferPruning.html
@@ -0,0 +1,143 @@
+<!DOCTYPE HTML>
+<html>
+<head>
+ <script type="application/javascript" src="pc.js"></script>
+ <script type="application/javascript" src="/tests/dom/canvas/test/captureStream_common.js"></script>
+</head>
+<body>
+<pre id="test">
+<script type="application/javascript">
+ createHTML({
+ bug: "1249859",
+ title: "Test pruning simulcast resolutions that are too close together",
+ visible: true
+ });
+
+ var test;
+ var pushPrefs = (...p) => new Promise(r => SpecialPowers.pushPrefEnv({set: p}, r));
+
+ function selectRecvSsrc(pc, index) {
+ var receivers = pc._pc.getReceivers();
+ is(receivers.length, 1, "We have exactly one RTP receiver");
+ var receiver = receivers[0];
+
+ SpecialPowers.wrap(pc._pc).mozSelectSsrc(receiver, index);
+ }
+
+ runNetworkTest(() =>
+ pushPrefs(['media.peerconnection.video.min_bitrate_estimate', 100*1000]).then(() => {
+ SimpleTest.requestCompleteLog();
+ var helper;
+
+ test = new PeerConnectionTest({bundle: false});
+ test.setMediaConstraints([{video: true}], []);
+
+ test.chain.replace("PC_LOCAL_GUM", [
+ function PC_LOCAL_CANVAS_CAPTURESTREAM(test) {
+ helper = new VideoStreamHelper();
+ test.pcLocal.attachLocalStream(helper.stream());
+ }
+ ]);
+
+ test.chain.insertBefore('PC_LOCAL_CREATE_OFFER', [
+ function PC_LOCAL_SET_RIDS(test) {
+ var senders = test.pcLocal._pc.getSenders();
+ is(senders.length, 1, "We have exactly one RTP sender");
+ var sender = senders[0];
+ ok(sender.track, "Sender has a track");
+
+ // We expect the 1.25 and 3.0 scalings to be pruned, so the rest
+ // of this test is functionally equivalent to
+ // test_peerConnection_simulcastOffer.html.
+ return sender.setParameters({
+ encodings: [{ rid: "foo", maxBitrate: 40000 },
+ { rid: "bar", maxBitrate: 40000, scaleResolutionDownBy: 1.25 },
+ { rid: "baz", maxBitrate: 40000, scaleResolutionDownBy: 2.0 },
+ { rid: "bazier", maxBitrate: 40000, scaleResolutionDownBy: 3.0 }]
+ });
+ }
+ ]);
+
+ test.chain.insertAfter('PC_LOCAL_GET_ANSWER', [
+ function PC_LOCAL_ADD_RIDS_TO_ANSWER(test) {
+ test._remote_answer.sdp = sdputils.transferSimulcastProperties(
+ test.originalOffer.sdp, test._remote_answer.sdp);
+ info("Answer with RIDs: " + JSON.stringify(test._remote_answer));
+ ok(test._remote_answer.sdp.match(/a=simulcast:/), "Modified answer has simulcast");
+ ok(test._remote_answer.sdp.match(/a=rid:/), "Modified answer has rid");
+ }
+ ]);
+
+ test.chain.insertAfter('PC_REMOTE_WAIT_FOR_MEDIA_FLOW',[
+ function PC_REMOTE_SET_RTP_FIRST_RID(test) {
+ // Cause pcRemote to filter out everything but the first SSRC. This
+ // lets only one of the simulcast streams through.
+ selectRecvSsrc(test.pcRemote, 0);
+ }
+ ]);
+
+ test.chain.append([
+ function PC_REMOTE_WAIT_FOR_FRAMES() {
+ var vremote = test.pcRemote.remoteMediaElements[0];
+ ok(vremote, "Should have remote video element for pcRemote");
+ return helper.waitForFrames(vremote);
+ },
+ function PC_REMOTE_CHECK_SIZE_1() {
+ var vlocal = test.pcLocal.localMediaElements[0];
+ var vremote = test.pcRemote.remoteMediaElements[0];
+ ok(vlocal, "Should have local video element for pcLocal");
+ ok(vremote, "Should have remote video element for pcRemote");
+ ok(vlocal.videoWidth > 0, "source width is positive");
+ ok(vlocal.videoHeight > 0, "source height is positive");
+ is(vremote.videoWidth, vlocal.videoWidth, "sink is same width as source");
+ is(vremote.videoHeight, vlocal.videoHeight, "sink is same height as source");
+ },
+ function PC_REMOTE_SET_RTP_SECOND_RID(test) {
+ // Now, cause pcRemote to filter out everything but the second SSRC.
+ // This lets only the other simulcast stream through.
+ selectRecvSsrc(test.pcRemote, 1);
+ },
+ function PC_REMOTE_WAIT_FOR_SECOND_MEDIA_FLOW(test) {
+ return test.pcRemote.waitForMediaFlow();
+ },
+ function PC_REMOTE_WAIT_FOR_FRAMES_2() {
+ var vremote = test.pcRemote.remoteMediaElements[0];
+ ok(vremote, "Should have remote video element for pcRemote");
+ return helper.waitForFrames(vremote);
+ },
+ // For some reason, even though we're getting a 25x25 stream, sometimes
+ // the resolution isn't updated on the video element on the first frame.
+ function PC_REMOTE_WAIT_FOR_FRAMES_3() {
+ var vremote = test.pcRemote.remoteMediaElements[0];
+ ok(vremote, "Should have remote video element for pcRemote");
+ return helper.waitForFrames(vremote);
+ },
+ function PC_REMOTE_CHECK_SIZE_2() {
+ var vlocal = test.pcLocal.localMediaElements[0];
+ var vremote = test.pcRemote.remoteMediaElements[0];
+ ok(vlocal, "Should have local video element for pcLocal");
+ ok(vremote, "Should have remote video element for pcRemote");
+ ok(vlocal.videoWidth > 0, "source width is positive");
+ ok(vlocal.videoHeight > 0, "source height is positive");
+ is(vremote.videoWidth, vlocal.videoWidth / 2, "sink is 1/2 width of source");
+ is(vremote.videoHeight, vlocal.videoHeight / 2, "sink is 1/2 height of source");
+ },
+ function PC_REMOTE_SET_RTP_NONEXISTENT_RID(test) {
+ // Now, cause pcRemote to filter out everything, just to make sure
+ // selectRecvSsrc is working.
+ selectRecvSsrc(test.pcRemote, 2);
+ },
+ function PC_REMOTE_ENSURE_NO_FRAMES() {
+ var vremote = test.pcRemote.remoteMediaElements[0];
+ ok(vremote, "Should have remote video element for pcRemote");
+ return helper.verifyNoFrames(vremote);
+ },
+ ]);
+
+ return test.run();
+ })
+ .catch(e => ok(false, "unexpected failure: " + e)));
+</script>
+</pre>
+</body>
+</html>--- a/media/webrtc/signaling/src/media-conduit/VideoConduit.cpp
+++ b/media/webrtc/signaling/src/media-conduit/VideoConduit.cpp
@@ -44,16 +44,20 @@
namespace mozilla {
static const char* logTag ="WebrtcVideoSessionConduit";
// 32 bytes is what WebRTC CodecInst expects
const unsigned int WebrtcVideoConduit::CODEC_PLNAME_SIZE = 32;
+// A simulcast stream must be at least this many times smaller in resolution
+// than the previous stream or it will be pruned as an optimization.
+const unsigned int SIMULCAST_PRUNING_FACTOR=4;
+
/**
* Factory Method for VideoConduit
*/
RefPtr<VideoSessionConduit>
VideoSessionConduit::Create()
{
NS_ASSERTION(NS_IsMainThread(), "Only call on main thread");
CSFLogDebug(logTag, "%s ", __FUNCTION__);
@@ -1406,24 +1410,42 @@ WebrtcVideoConduit::ReconfigureSendCodec
// These are based on lowest-fidelity, because if there is insufficient
// bandwidth for all streams, only the lowest fidelity one will be sent.
uint32_t minMinBitrate = 0;
uint32_t minStartBitrate = 0;
// Total for all simulcast streams.
uint32_t totalMaxBitrate = 0;
+ uint32_t last_width = 0;
+ uint32_t last_height = 0;
+
for (size_t i = vie_codec.numberOfSimulcastStreams; i > 0; --i) {
webrtc::SimulcastStream& stream(vie_codec.simulcastStream[i - 1]);
stream.width = width;
stream.height = height;
MOZ_ASSERT(stream.jsScaleDownBy >= 1.0);
uint32_t new_width = uint32_t(width / stream.jsScaleDownBy);
uint32_t new_height = uint32_t(height / stream.jsScaleDownBy);
- // TODO: If two layers are similar, only alloc bits to one (Bug 1249859)
+
+ // This ensures we preserve the highest resolution stream, which is the
+ // first one we examine.
+ if (last_width != 0 && last_height != 0) {
+ if (SIMULCAST_PRUNING_FACTOR*new_width*new_height > last_width*last_height) {
+ for (size_t j = i - 1; j < vie_codec.numberOfSimulcastStreams - 1; ++j) {
+ vie_codec.simulcastStream[j] = vie_codec.simulcastStream[j + 1];
+ }
+ --vie_codec.numberOfSimulcastStreams;
+ continue;
+ }
+ }
+
+ last_width = new_width;
+ last_height = new_height;
+
if (new_width != width || new_height != height) {
if (vie_codec.numberOfSimulcastStreams == 1) {
// Use less strict scaling in unicast. That way 320x240 / 3 = 106x79.
ConstrainPreservingAspectRatio(new_width, new_height,
&stream.width, &stream.height);
} else {
// webrtc.org supposedly won't tolerate simulcast unless every stream
// is exactly the same aspect ratio. 320x240 / 3 = 80x60.