Bug 1420322 - Test for seamless looping; r?jwwang,padenot
MozReview-Commit-ID: KE3Qsql8BW1
--- a/dom/media/test/mochitest.ini
+++ b/dom/media/test/mochitest.ini
@@ -547,16 +547,17 @@ support-files =
seek.webm^headers^
seek-short.webm
seek-short.webm^headers^
seek_support.js
seekLies.sjs
seek_with_sound.ogg^headers^
sequential.vtt
short-cenc.mp4
+ sine-3000hz-1s.wav
sine.webm
sine.webm^headers^
sintel-short-clearkey-subsample-encrypted-audio.webm
sintel-short-clearkey-subsample-encrypted-audio.webm^headers^
sintel-short-clearkey-subsample-encrypted-video.webm
sintel-short-clearkey-subsample-encrypted-video.webm^headers^
short.mp4
short.mp4.gz
@@ -1046,16 +1047,20 @@ skip-if = toolkit == 'android' # bug 131
skip-if = toolkit == 'android' # android(bug 1232305)
[test_video_dimensions.html]
skip-if = toolkit == 'android' # bug 1298238, bug 1304535, android(bug 1232305)
[test_resolution_change.html]
skip-if = android_version == '19' # bug 1393866
tags=capturestream
[test_resume.html]
skip-if = true # bug 1021673
+[test_seamless_loop.html]
+skip-if = toolkit == 'android' # bug 1242112, android(bug 1232305)
+support-files =
+ test_seamless_loop_helper.js
[test_seek_negative.html]
skip-if = toolkit == 'android' # bug 1295443, bug 1306787, android(bug 1232305)
[test_seek_nosrc.html]
[test_seek_out_of_range.html]
skip-if = toolkit == 'android' # bug 1299382, android(bug 1232305)
[test_seek_promise_bug1344357.html]
skip-if = toolkit == 'android' # bug 1299382, android(bug 1232305)
[test_seek-1.html]
new file mode 100644
index 0000000000000000000000000000000000000000..a7bff68f663783eda8a13518e400a1a485925f08
GIT binary patch
literal 88278
zc%1Ff%WsTp00!`HX4+2Mxx9^7xda=I4UQ9ug%go5Oe!u#j}1+u5u3wVG**a+#o0)(
z6C!Ho5^P+8g@v6&Vue_UuoQW}nd!7M?X>F8ko@v&o;-P0zoBZS^7e2h`Dfekj^qE1
zuN+8{B(sq1_mkxRg*}@LB)h8z4lY^Pt9vSYs|P9vOACGX=;`B|Hm}>b?#~T>CCS+6
z_~@^9l1%h0Dh`Ja;e6?N@3HL7nTg4ZX(o=OBa^#k3fcd9_m$p+lVPx!_8d+gw$?UY
z$K`RT)@knSl)IMp50^XPb@@wwp?h%d)6|#R_IM_qtG}AwKcDHjQCJzihfk%!zL2ZT
zu5G?c^Kn`FxpBLFB=a%<y?8HN3(ty|^7+jE_C%u-i?N-aYF?im%U$W~E;m9hj1(Sq
zx97&EtMwD{P+VMJGktsRNcXD3w{k12FTd}5kxOSEH&@pd$Io%BG2eQSY|URNUJNh8
ziqh)d$C+k(#pK^<F5XSQO{O!?ve~`|rLSR2_)r+?xide|dQ!g{H^oP_%TtZc)2>_n
zx61S3Y<Z;rSl7)?W$JwGk9ar!s2^=nqC|-jB}$YiQKCeN5+zEMC{dzBi4rABlqgZ6
zM2Qk5N|Y#3qC|-jB}$YiQKCeN5+zEMC{dzBi4rABlqgZ6M2Qk5N|Y#3qC|-jB}$Yi
zQKCeN5+zEMC{dzBi4rABlqgZ6M2Qk5N|Y#3qC|-jB}$YiQKCeN5+zEMC{dzBi4rAB
zlqgZ6M2Qk5N|Y#3qC|-jB}$YiQKCeN5+zEMC{dzBi4rABlqgZ6M2Qk5N|Y#3qC|-j
zB}$YiQKCeN5+zEMC{dzBi4rABlqgZ6M2Qk5N|Y#3qC|-jB}$YiQKCeN5+zEMC{dzB
zi4rABlqgZ6M2Qk5N|Y#3qC|-jB}$YiQKCeN5+zEMC{dzBi4rABlqgZ6M2Qk5N|Y#3
zqC|-jB}$YiQKCeN5+zEMC{dzBi4rABlqgZ6M2Qk5N|Y#3qC|-jB}$YiQKCeN5+zEM
xC{dzBi4rABlqgZ6M2Qk5N|Y#3qC|-jB}$YiQKCeN5+zEMC{dzBiPG<@^b_%w-uM6j
new file mode 100644
--- /dev/null
+++ b/dom/media/test/test_seamless_loop.html
@@ -0,0 +1,214 @@
+<!DOCTYPE HTML>
+<html>
+<meta charset="utf-8">
+<head>
+ <title>Test for seamless looping</title>
+ <script type="text/javascript" src="/tests/SimpleTest/SimpleTest.js"></script>
+ <script type="text/javascript" src="./test_seamless_loop_helper.js"></script>
+ <link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css" />
+</head>
+<body>
+<pre id="test">
+<h2>Fast Fourier transform(FFT)</h2>
+<p>Show the frequency-domain data of the audio.</p>
+<div id="fft"></div>
+<h2>Variation of FFT data</h2>
+<p>Compare the current data with last one at the same frequency.</p>
+<p>The variation below is 40x of the original value.</p>
+<div id="variation"></div>
+<script class="testbody" type="text/javascript">
+SimpleTest.waitForExplicitFinish();
+// ============================================================================
+// How this test works
+// ============================================================================
+// 1) Prepare a high-frequency sine wave
+// 2) Loop the sine wave by an audio element
+// 3) Capture the media stream from the playing audio element while looping
+// 4) Get the frequency-domain data of the captured stream by FFT
+// 5) Check if there is noise in the frequency-domain data
+//
+// Ideally, the frequency-domain data for a seamless sine wave should be:
+//
+// |
+// |
+// |
+// |
+// ----------+--------------> Frequency
+// ^
+// Frequency of the sine wave
+//
+// If there is discontinuity in the sine wave, it may look like:
+//
+// |
+// | noise
+// | v
+// | | | | | | | | |
+// --+-+-+-+-+-+-+-+-+------> Frequency
+// ^
+// Frequency of the sine wave
+//
+// However, in practice, our FFT is not perfect. The frequency-domain data for
+// a seamless sine wave will be:
+//
+// |
+// |||
+// |||||
+// |||||
+// --------+++++------------> Frequency
+// ^
+// Frequency of the sine wave
+//
+// The higher the frequency is, the more stable of the FFT data is. Thus, we
+// set an acceptable tolerance for the variation of the FFT data to check if
+// there is a discontinuity while looping the sine wave.
+// ============================================================================
+// Helpers
+// ============================================================================
+// Resolve the Promise after the environment setup.
+function setupEnvironment() {
+ return new Promise((resolve, reject) => {
+ let prefs = {
+ 'set': [
+ // Uncomment below to check if the test fails.
+ // ['media.seamless-looping', false],
+ ]
+ };
+ SpecialPowers.pushPrefEnv(prefs, () => { resolve(); });
+ });
+}
+// ============================================================================
+// Test
+// ============================================================================
+// peak
+// v
+// |
+// ||| ^
+// ||||| | tolerance
+// ||||||| v
+// |||||||||
+// ------+++++++++----------> Frequency
+// ^
+// |---|
+// ^ |
+// range |
+// |
+// Frequency of the sine wave
+const FREQUENCY = 3000; // The frequency of the sine wave.
+const RANGE = 8; // A half of an acceptable width of the FFT pulse
+ // whose peak is FREQUENCY.
+const TOLERANCE = 5; // A tolerance for the variation of FFT data.
+const TIMES = 3; // The looping time for the test.
+
+var audioElement = document.createElement("audio");
+audioElement.src = "sine-3000hz-1s.wav";
+audioElement.loop = true;
+ok(audioElement.loop == TIMES > 0, "The loop setting is different.");
+
+var context = new AudioContext();
+// Create an AudioAnalyser to get the FFT data of the audio stream.
+// The FFT data will be used to check if there is discontinuity while looping.
+var analyser = new AudioAnalyser(context, audioElement.mozCaptureStream());
+analyser.node.connect(context.destination);
+
+// Open a canvas to show the frequency-domain data.
+analyser.openSpectrum(document.getElementById("fft"));
+// Open a canvas to show the variation of frequency-domain data.
+// The variation is |current fft value - previous fft value| at the same index.
+analyser.openVariation(document.getElementById("variation"));
+
+// Create a SeamlessSineDetector to check if the sine wave loops seamlessly
+// every 100ms.
+var seamlessSineDetector =
+ new SeamlessSineDetector(analyser, 100, FREQUENCY, RANGE, TOLERANCE);
+
+// Numbers of the received events.
+var eventCount = {
+ play: 0,
+ seeking: 0,
+ seeked: 0,
+ timeupdate: 0,
+};
+
+// Get how much time has passed.
+function getTimePass(ae) {
+ return ae.duration * eventCount.seeked + ae.currentTime;
+}
+
+// Event callback for play and seeking event.
+function eventCounter(evt) {
+ ++eventCount[evt.type];
+ let ae = evt.target;
+ info(ae.currentTime + "(" + getTimePass(ae).toFixed(6) + ")\t" +
+ evt.type + ": " + eventCount[evt.type]);
+}
+
+// Event callback for seeked event.
+function loopCounter(evt) {
+ eventCounter(evt);
+ if (eventCount.seeked >= TIMES) {
+ info("Cancel loop.");
+ evt.target.loop = false;
+ seamlessSineDetector.stop();
+ }
+}
+
+// Event callback for timeupdate event. It will be removed after starting
+// the seamlessSineDetector. The result of the FFT might be not stable at
+// the very beginning, so we start detecting after the audio is played for
+// a while.
+function waitToStartDetector(evt) {
+ eventCounter(evt);
+ let ae = evt.target;
+ let begin = ae.duration / 2;
+ let time = getTimePass(ae);
+ if (time < begin) {
+ return;
+ }
+ seamlessSineDetector.start();
+ ae.removeEventListener(evt.type, waitToStartDetector, false);
+}
+
+// Event callback for end event. It's the endpoint of the test.
+function terminator(evt) {
+ let ae = evt.target;
+ ok(ae.currentTime == ae.duration,
+ "The end time should be same as the duration.");
+ ok(eventCount.play == 1,
+ "The play event must be received only once.");
+ ok(eventCount.seeking == TIMES,
+ "The number of received seeking events must be same as looping-times.");
+ ok(eventCount.seeked == TIMES,
+ "The number of received seeked events must be same as looping-times.");
+ ok(seamlessSineDetector.seamlessness,
+ "FFT Noise Detected. The looping is not seamless.");
+
+ context.close();
+ // analyser.closeSpectrum();
+ // analyser.closeVariation();
+ SimpleTest.finish();
+}
+
+// Register all the event callbacks.
+function registerEventListeners(audio) {
+ let listeners = {
+ play: eventCounter,
+ seeking: eventCounter,
+ seeked: loopCounter,
+ ended: terminator,
+ timeupdate: waitToStartDetector,
+ };
+
+ for (evt in listeners) {
+ audio.addEventListener(evt, listeners[evt], false);
+ }
+}
+
+(async () => {
+ await setupEnvironment();
+ registerEventListeners(audioElement);
+ audioElement.play();
+})();
+</script>
+</pre>
+</body>
+</html>
\ No newline at end of file
new file mode 100644
--- /dev/null
+++ b/dom/media/test/test_seamless_loop_helper.js
@@ -0,0 +1,347 @@
+/**
+ * This class provides helpers around analysing the audio content in a stream
+ * using WebAudio AnalyserNodes.
+ *
+ * @constructor
+ * @param {AudioContext} audioContext
+ * AudioContext in which to create the AnalyserNode.
+ * @param {MediaStream} audioStream
+ * The audio stream that will be processed by the AnalyserNode.
+ */
+function AudioAnalyser(audioContext, audioStream) {
+ this.context = audioContext;
+ this.analyser = this.context.createAnalyser();
+ // this.analyser.smoothingTimeConstant = 0.2;
+ this.analyser.fftSize = 1024;
+ this.fftData = new Uint8Array(this.analyser.frequencyBinCount);
+
+ if (audioStream) {
+ this.stream = audioStream;
+ this.source = this.context.createMediaStreamSource(this.stream);
+ this.source.connect(this.analyser);
+ }
+}
+
+AudioAnalyser.prototype = {
+ /**
+ * Get the internal AnalyserNode for analysing the audio stream.
+ */
+ get node() {
+ return this.analyser;
+ },
+
+ /**
+ * Get an array of frequency domain data for our audio stream.
+ *
+ * @returns {array} A Uint8Array containing the frequency domain data.
+ */
+ getByteFrequencyData: function() {
+ this.analyser.getByteFrequencyData(this.fftData);
+ return this.fftData;
+ },
+
+ /**
+ * Get an array containing the indices of the peaks of pulses in the
+ * frequency domain data.
+ */
+ getPeaks: function() {
+ let peaks = [];
+ const Direction = {
+ Unknown: 0,
+ Up: 1,
+ Down: 2,
+ };
+ let dir = Direction.Unknown;
+ let data = this.getByteFrequencyData();
+ for (let i = 1 ; i < data.length ; ++i) {
+ if (data[i] > data[i-1]) { // The wave goes up.
+ dir = Direction.Up;
+ }
+ if (data[i] < data[i-1]) { // The wave goes down.
+ if (dir == Direction.Up) {
+ peaks.push(i-1);
+ }
+ dir = Direction.Down;
+ }
+ }
+ return peaks;
+ },
+
+ /**
+ * Return the FFT bin index for a given frequency.
+ *
+ * @param {double} frequency
+ * The frequency for whicht to return the bin number.
+ * @returns {integer} the index of the bin in the FFT array.
+ */
+ binIndexForFrequency: function(frequency) {
+ return 1 + Math.round(frequency * this.analyser.fftSize / this.context.sampleRate);
+ },
+
+ /**
+ * Reverse operation, get the frequency for a bin index.
+ *
+ * @param {integer} index an index in an FFT array
+ * @returns {double} the frequency for this bin
+ */
+ frequencyForBinIndex: function(index) {
+ return (index - 1) * this.context.sampleRate / this.analyser.fftSize;
+ },
+
+ /**
+ * Stop drawing of and remove the canvas from the DOM.
+ */
+ removeCanvas: function(cvs) {
+ if (!cvs || !cvs.parentElement) {
+ return;
+ }
+
+ cvs.stopDrawing = true;
+ cvs.parentElement.removeChild(cvs);
+ },
+
+ /**
+ * Append a canvas to the DOM where the frequency data are drawn.
+ */
+ openSpectrum: function(content) {
+ let cvs = this.fftCanvas = document.createElement("canvas");
+ content.insertBefore(cvs, content.children[0]);
+
+ // Easy: 1px per bin
+ cvs.width = this.analyser.frequencyBinCount;
+ cvs.height = 200;
+ cvs.style.border = "1px solid red";
+
+ let c = cvs.getContext('2d');
+ c.fillStyle = 'black';
+
+ let self = this;
+ function render() {
+ c.clearRect(0, 0, cvs.width, cvs.height);
+ let array = self.getByteFrequencyData();
+ for (let i = 0; i < array.length; ++i) {
+ c.fillRect(i, (cvs.height - (array[i] / 2)), 1, cvs.height);
+ // if (array[i]) {
+ // console.log(i + " (" + self.frequencyForBinIndex(i) + "hz): " + array[i]);
+ // }
+ }
+ if (!cvs.stopDrawing) {
+ requestAnimationFrame(render);
+ }
+ }
+ requestAnimationFrame(render);
+ },
+
+ /**
+ * Stop drawing frequency data.
+ */
+ closeSpectrum: function() {
+ this.removeCanvas(this.fftCanvas);
+ },
+
+ /**
+ * Append a canvas to the DOM where the variation of frequency data are drawn.
+ */
+ openVariation: function(content) {
+ let cvs = this.variationCanvas = document.createElement("canvas");
+ content.appendChild(cvs);
+
+ // Easy: 1px per bin
+ cvs.width = this.analyser.frequencyBinCount;
+ cvs.height = 200;
+ cvs.style.border = "1px solid red";
+
+ let c = cvs.getContext('2d');
+ c.fillStyle = 'black';
+
+ let lastArray = [];
+ let self = this;
+ function render() {
+ c.clearRect(0, 0, cvs.width, cvs.height);
+ let array = self.getByteFrequencyData();
+ if (lastArray.length) {
+ for (let i = 0; i < array.length; ++i) {
+ let diff = Math.abs(lastArray[i] - array[i]);
+ c.fillRect(i, cvs.height - 40 * diff, 1, cvs.height);
+ // if (diff) {
+ // console.log(i + " (" + self.frequencyForBinIndex(i) +
+ // "hz) - last: " + lastArray[i] +
+ // ", current: " + array[i]);
+ // }
+ }
+ }
+ if (!cvs.stopDrawing) {
+ requestAnimationFrame(render);
+ }
+ lastArray = array.slice(0);
+ }
+ requestAnimationFrame(render);
+ },
+
+ /**
+ * Stop drawing the variation of the frequency data.
+ */
+ closeVariation: function() {
+ this.removeCanvas(this.variationCanvas);
+ },
+};
+
+/**
+ * This class creates a timer to repeatedly check if the stream that is
+ * processed in the given analyser is a seamless sine wave or not.
+ *
+ * @constructor
+ * @param {AudioAnalyser} analyser
+ * An AudioAnalyser that is processing a sine wave stream.
+ * @param {integer} interval
+ * The interval in milliseconds for the timer to detect.
+ * @param {integer} frequency
+ * The frequency of the sine wave that is processed
+ * by the given analyser.
+ * @param {integer} range
+ * A half of an acceptable width of the pulse whose peak is the
+ * frequency of the sine wave that is processed by the given analyser.
+ * @param {integer} tolerance
+ * The acceptable range of the variation for the FFT data.
+ *
+ * peak
+ * v
+ * |
+ * ||| ^
+ * ||||| | tolerance
+ * ||||||| v
+ * |||||||||
+ * ------+++++++++----------> Frequency
+ * ^
+ * |---|
+ * ^ |
+ * range |
+ * |
+ * Frequency of the sine wave
+ *
+ */
+ function SeamlessSineDetector(analyser, interval, frequency, range = 0, tolerance = 0)
+{
+ this.analyser = analyser;
+ this.interval = interval;
+ this.frequency = frequency;
+ this.range = range;
+ this.tolerance = tolerance;
+ this.peak = this.analyser.binIndexForFrequency(this.frequency);
+ this.data = [];
+ this.seamlessness = true;
+}
+
+SeamlessSineDetector.prototype = {
+ /**
+ * Set the timer and start detecting.
+ */
+ start: function() {
+ this.detector = setInterval(this.detect.bind(this), this.interval);
+ },
+
+ /**
+ * Clear the timer and stop detecting.
+ */
+ stop: function() {
+ clearInterval(this.detector);
+ },
+
+ /**
+ * The function that will be called repeatedly by the timer to check if the
+ * stream in the given analyser is a seamless sine wave or not.
+ */
+ detect: function() {
+ // Do nothing if it has already failed in the test.
+ if (!this.seamlessness) {
+ return;
+ }
+
+ // Make sure all the peaks are in the available range.
+ let peaks = this.analyser.getPeaks();
+ for (let i = 0 ; i < peaks.length ; ++i) {
+ if (!this._isIndexInPulse(peaks[i])) {
+ this.seamlessness = false;
+ // console.log("There are peaks out of range!");
+ return;
+ }
+ }
+
+ // Sort the peaks by the distance to the ideal peak(this.peak).
+ // Make sure the nearer to the ideal peak is, the higher its FFT value is.
+ let sortedPeaks = this._getSortedPeaksByDistance(peaks);
+ for (let i = 1 ; i < sortedPeaks.length ; ++i) {
+ if (sortedPeaks[i-1].value < sortedPeaks[i].value) {
+ this.seamlessness = false;
+ // console.log("The farther peak should not be higher than the nearer peak.");
+ return;
+ }
+ }
+
+ let currentdata = this.analyser.getByteFrequencyData();
+ if (this.data.length) {
+ for (let i = 0 ; i < currentdata.length ; ++i) {
+ // If the data is not in the pulse of a sine wave, then it must be 0.
+ if (!this._isIndexInPulse(i)) {
+ if (currentdata[i] == 0) {
+ continue;
+ }
+ this.seamlessness = false;
+ // console.log("This FFT value should be 0!");
+ return;
+ }
+ // If the data is in the pulse of a sine wave, then check
+ // 1) The FFT value doesn't change from 0/non-0 to non-0/0.
+ if ((currentdata[i] != 0) != (this.data[i] != 0)) {
+ this.seamlessness = false;
+ // console.log("The data changes from 0 to non-0, or non-0 to 0!");
+ return;
+ }
+ // 2) The variation of the FFT data is acceptable.
+ let diff = Math.abs(currentdata[i] - this.data[i]);
+ if (diff > this.tolerance) {
+ this.seamlessness = false;
+ // console.log("The shape of wave is changed!");
+ }
+ }
+ }
+ this.data = currentdata.slice(0); // Copy data.
+ },
+
+ /**
+ * Return true if the index is in the predefined range of the pulse.
+ */
+ _isIndexInPulse: function(index) {
+ let min = this.peak - this.range;
+ if (min < 0) {
+ min = 0;
+ }
+
+ let max = this.peak + this.range;
+ if (max >= this.analyser.node.frequencyBinCount) {
+ max = this.analyser.node.frequencyBinCount - 1;
+ }
+
+ return min <= index && index <= max;
+ },
+
+ /**
+ * Return an array containing the FFT data, sorted by its distance to the
+ * index for the frequency of the sine wave.
+ */
+ _getSortedPeaksByDistance: function(peaks) {
+ let sorted = [];
+ let data = this.analyser.getByteFrequencyData();
+ for (let i = 0 ; i < peaks.length ; ++i) {
+ sorted.push({
+ index: peaks[i],
+ distance: Math.abs(this.peak - peaks[i]),
+ value: data[peaks[i]],
+ });
+ }
+ sorted.sort(function(x, y) {
+ return x.distance - y.distance;
+ });
+ return sorted;
+ },
+};