Bug 1363428 - Add reftest-specific endpoints to Marionette, r=ato draft
authorJames Graham <james@hoppipolla.co.uk>
Wed, 10 May 2017 10:51:10 +0100
changeset 600212 c96ffa60c0c47fd67926dfc1176e5a42a10f02a5
parent 600211 637b01884025ad9a7fb409082eabff52f4434af5
child 600213 d7be32e0ac3f4c076605e28a2d13b763b328735c
push id65688
push userbmo:james@hoppipolla.co.uk
push dateSat, 24 Jun 2017 11:04:46 +0000
reviewersato
bugs1363428
milestone56.0a1
Bug 1363428 - Add reftest-specific endpoints to Marionette, r=ato This adds commands to start a reftest session, run a test, and end the session. It as assumed that after you start a reftest session you will just run reftests until you end the session. When starting a session the user provides a string indicating when screenshots should be taken, and an object mapping urls to a count of the number of times that url is expected to be used in the session, to help with caching. Running the tests takes a url to a test, an expected status, a timeout, and a nested list of possible references, in which each entry at a specific level is combined by OR and nested references are combined by AND. The implementation is heavilly inspired by the existing reftest harness, starting a minimal window with no tabs, and loading the urls directly in there. In order to get a screenshot in the e10s case we have to pass the DRAW_VIEW and USE_WIDGET_LAYERS flags when taking the screenshot. For performance we heavily cache canvases; for references that will be repeated we cache the full canvas with image, and we also cache a single canvas to use for all other screenshots to avoid the overhead of repeatedly creating a new canvas element. MozReview-Commit-ID: JOFvtmH7tg
layout/tools/reftest/reftest-content.js
testing/marionette/driver.js
testing/marionette/jar.mn
testing/marionette/listener.js
testing/marionette/reftest.js
testing/marionette/reftest.xul
testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/firefox.py
--- a/layout/tools/reftest/reftest-content.js
+++ b/layout/tools/reftest/reftest-content.js
@@ -442,16 +442,17 @@ function FlushRendering() {
 }
 
 function WaitForTestEnd(contentRootElement, inPrintMode, spellCheckedElements) {
     var stopAfterPaintReceived = false;
     var currentDoc = content.document;
     var state = STATE_WAITING_TO_FIRE_INVALIDATE_EVENT;
 
     function AfterPaintListener(event) {
+        dump("AfterPaintListener\n");
         LogInfo("AfterPaintListener in " + event.target.document.location.href);
         if (event.target.document != currentDoc) {
             // ignore paint events for subframes or old documents in the window.
             // Invalidation in subframes will cause invalidation in the toplevel document anyway.
             return;
         }
 
         SendUpdateCanvasForEvent(event, contentRootElement);
--- a/testing/marionette/driver.js
+++ b/testing/marionette/driver.js
@@ -26,16 +26,17 @@ Cu.import("chrome://marionette/content/e
 Cu.import("chrome://marionette/content/error.js");
 Cu.import("chrome://marionette/content/evaluate.js");
 Cu.import("chrome://marionette/content/event.js");
 Cu.import("chrome://marionette/content/interaction.js");
 Cu.import("chrome://marionette/content/l10n.js");
 Cu.import("chrome://marionette/content/legacyaction.js");
 Cu.import("chrome://marionette/content/modal.js");
 Cu.import("chrome://marionette/content/proxy.js");
+Cu.import("chrome://marionette/content/reftest.js");
 Cu.import("chrome://marionette/content/session.js");
 Cu.import("chrome://marionette/content/wait.js");
 
 Cu.importGlobalProperties(["URL"]);
 
 this.EXPORTED_SYMBOLS = ["GeckoDriver", "Context"];
 
 var FRAME_SCRIPT = "chrome://marionette/content/listener.js";
@@ -3189,16 +3190,75 @@ GeckoDriver.prototype.localizeProperty =
   }
   if (typeof id != "string") {
     throw new InvalidArgumentError("Value of `id` should be of type 'string'");
   }
 
   resp.body.value = l10n.localizeProperty(urls, id);
 }
 
+/**
+ * Initialize the reftest mode
+ */
+GeckoDriver.prototype.setupReftest = function* (cmd, resp) {
+  if (this._reftest) {
+    throw new UnsupportedOperationError("Called reftest:setup with a reftest session already active");
+  }
+
+  if (this.context !== Context.CHROME) {
+    throw new UnsupportedOperationError("Must set chrome context before running reftests");
+  }
+
+  let {urlCount = {}, screenshot = "unexpected"} = cmd.parameters;
+  if (!["always", "fail", "unexpected"].includes(screenshot)) {
+    throw new InvalidArgumentError("Value of `screenshot` should be 'always', 'fail' or 'unexpected'");
+  }
+
+  this._reftest = new reftest.Runner(this);
+
+  yield this._reftest.setup(urlCount, screenshot);
+};
+
+
+/**
+ * Run a reftest
+ */
+GeckoDriver.prototype.runReftest = function* (cmd, resp) {
+  let {test, references, expected, timeout} = cmd.parameters;
+
+  if (!this._reftest) {
+    throw new UnsupportedOperationError("Called reftest:run before reftest:start");
+  }
+
+  assert.string(test);
+  assert.string(expected);
+  assert.array(references);
+
+  let result = yield this._reftest.run(test, references, expected, timeout);
+
+  resp.body.value = result;
+};
+
+/**
+ * End a reftest run
+ *
+ * Closes the reftest window (without changing the current window handle),
+ * and removes cached canvases.
+ */
+GeckoDriver.prototype.teardownReftest = function* (cmd, resp) {
+  if (!this._reftest) {
+    throw new UnsupportedOperationError("Called reftest:teardown before reftest:start");
+  }
+
+  this._reftest.abort();
+
+  this._reftest = null;
+};
+
+
 GeckoDriver.prototype.commands = {
   // Marionette service
   "Marionette:SetContext": GeckoDriver.prototype.setContext,
   "setContext": GeckoDriver.prototype.setContext,  // deprecated, remove in Firefox 60
   "Marionette:GetContext": GeckoDriver.prototype.getContext,
   "getContext": GeckoDriver.prototype.getContext,
   "Marionette:AcceptConnections": GeckoDriver.prototype.acceptConnections,
   "acceptConnections": GeckoDriver.prototype.acceptConnections,  // deprecated, remove in Firefox 60
@@ -3212,16 +3272,21 @@ GeckoDriver.prototype.commands = {
   "addon:uninstall": GeckoDriver.prototype.uninstallAddon,  // deprecated, remove in Firefox 60
 
   // L10n service
   "L10n:LocalizeEntity": GeckoDriver.prototype.localizeEntity,
   "localization:l10n:localizeEntity": GeckoDriver.prototype.localizeEntity,  // deprecated, remove in Firefox 60
   "L10n:LocalizeProperty": GeckoDriver.prototype.localizeProperty,
   "localization:l10n:localizeProperty": GeckoDriver.prototype.localizeProperty,  // deprecated, remove in Firefox 60
 
+  // Reftest service
+  "reftest:setup": GeckoDriver.prototype.setupReftest,
+  "reftest:run": GeckoDriver.prototype.runReftest,
+  "reftest:teardown": GeckoDriver.prototype.teardownReftest,
+
   // WebDriver service
   "WebDriver:AcceptDialog": GeckoDriver.prototype.acceptDialog,
   "WebDriver:AddCookie": GeckoDriver.prototype.addCookie,
   "WebDriver:Back": GeckoDriver.prototype.goBack,
   "WebDriver:CloseChromeWindow": GeckoDriver.prototype.closeChromeWindow,
   "WebDriver:CloseWindow": GeckoDriver.prototype.close,
   "WebDriver:DeleteAllCookies": GeckoDriver.prototype.deleteAllCookies,
   "WebDriver:DeleteCookie": GeckoDriver.prototype.deleteCookie,
--- a/testing/marionette/jar.mn
+++ b/testing/marionette/jar.mn
@@ -28,16 +28,18 @@ marionette.jar:
   content/navigate.js (navigate.js)
   content/l10n.js (l10n.js)
   content/assert.js (assert.js)
   content/addon.js (addon.js)
   content/session.js (session.js)
   content/transport.js (transport.js)
   content/packets.js (packets.js)
   content/stream-utils.js (stream-utils.js)
+  content/reftest.js (reftest.js)
+  content/reftest.xul (reftest.xul)
 #ifdef ENABLE_TESTS
   content/test.xul (chrome/test.xul)
   content/test2.xul (chrome/test2.xul)
   content/test_dialog.dtd (chrome/test_dialog.dtd)
   content/test_dialog.properties (chrome/test_dialog.properties)
   content/test_dialog.xul (chrome/test_dialog.xul)
   content/test_nested_iframe.xul (chrome/test_nested_iframe.xul)
   content/test_anonymous_content.xul (chrome/test_anonymous_content.xul)
--- a/testing/marionette/listener.js
+++ b/testing/marionette/listener.js
@@ -474,16 +474,17 @@ var singleTapFn = dispatch(singleTap);
 var takeScreenshotFn = dispatch(takeScreenshot);
 var performActionsFn = dispatch(performActions);
 var releaseActionsFn = dispatch(releaseActions);
 var actionChainFn = dispatch(actionChain);
 var multiActionFn = dispatch(multiAction);
 var executeFn = dispatch(execute);
 var executeInSandboxFn = dispatch(executeInSandbox);
 var sendKeysToElementFn = dispatch(sendKeysToElement);
+var reftestWaitFn = dispatch(reftestWait);
 
 /**
  * Start all message listeners
  */
 function startListeners() {
   addMessageListenerId("Marionette:newSession", newSession);
   addMessageListenerId("Marionette:execute", executeFn);
   addMessageListenerId("Marionette:executeInSandbox", executeInSandboxFn);
@@ -517,16 +518,17 @@ function startListeners() {
   addMessageListenerId("Marionette:clearElement", clearElementFn);
   addMessageListenerId("Marionette:switchToFrame", switchToFrame);
   addMessageListenerId("Marionette:switchToParentFrame", switchToParentFrame);
   addMessageListenerId("Marionette:switchToShadowRoot", switchToShadowRootFn);
   addMessageListenerId("Marionette:deleteSession", deleteSession);
   addMessageListenerId("Marionette:sleepSession", sleepSession);
   addMessageListenerId("Marionette:getAppCacheStatus", getAppCacheStatus);
   addMessageListenerId("Marionette:takeScreenshot", takeScreenshotFn);
+  addMessageListenerId("Marionette:reftestWait", reftestWaitFn);
 }
 
 /**
  * Called when we start a new session. It registers the
  * current environment, and resets all values
  */
 function newSession(msg) {
   capabilities = session.Capabilities.fromJSON(msg.json);
@@ -1036,24 +1038,25 @@ function waitForPageLoaded(msg) {
 
 /**
  * Navigate to the given URL.  The operation will be performed on the
  * current browsing context, which means it handles the case where we
  * navigate within an iframe.  All other navigation is handled by the
  * driver (in chrome space).
  */
 function get(msg) {
-  let {command_id, pageTimeout, url} = msg.json;
-  let loadEventExpected = true;
+  let {command_id, pageTimeout, url, loadEventExpected=null} = msg.json;
 
   try {
     if (typeof url == "string") {
       try {
         let requestedURL = new URL(url).toString();
-        loadEventExpected = navigate.isLoadEventExpected(requestedURL);
+        if (loadEventExpected === null) {
+          loadEventExpected = navigate.isLoadEventExpected(requestedURL);
+        }
       } catch (e) {
         sendError(new InvalidArgumentError("Malformed URL: " + e.message), command_id);
         return;
       }
     }
 
     // We need to move to the top frame before navigating
     sendSyncMessage("Marionette:switchedToFrame", {frameValue: null});
@@ -1642,10 +1645,121 @@ function takeScreenshot(format, opts = {
     case capture.Format.Hash:
       return capture.toHash(canvas);
 
     default:
       throw new TypeError("Unknown screenshot format: " + format);
   }
 }
 
+function flushRendering() {
+  let content = curContainer.frame;
+  let anyPendingPaintsGeneratedInDescendants = false;
+
+  let windowUtils = content.QueryInterface(Ci.nsIInterfaceRequestor)
+    .getInterface(Ci.nsIDOMWindowUtils);
+
+  function flushWindow(win) {
+    let utils = win.QueryInterface(Ci.nsIInterfaceRequestor)
+      .getInterface(Ci.nsIDOMWindowUtils);
+    let afterPaintWasPending = utils.isMozAfterPaintPending;
+
+    let root = win.document.documentElement;
+    if (root) {
+      try {
+        // Flush pending restyles and reflows for this window
+        root.getBoundingClientRect();
+      } catch (e) {
+        logger.warning(`flushWindow failed: ${e}`);
+      }
+    }
+
+    if (!afterPaintWasPending && utils.isMozAfterPaintPending) {
+      anyPendingPaintsGeneratedInDescendants = true;
+    }
+
+    for (let i = 0; i < win.frames.length; ++i) {
+      flushWindow(win.frames[i]);
+    }
+  }
+  flushWindow(content);
+
+  if (anyPendingPaintsGeneratedInDescendants &&
+      !windowUtils.isMozAfterPaintPending) {
+    logger.error("Internal error: descendant frame generated a MozAfterPaint event, but the root document doesn't have one!");
+  }
+
+  logger.debug(`flushRendering ${windowUtils.isMozAfterPaintPending}`);
+  return windowUtils.isMozAfterPaintPending;
+}
+
+function* reftestWait(url, remote) {
+  let win = curContainer.frame;
+  let document = curContainer.frame.document;
+
+  let windowUtils = content.QueryInterface(Ci.nsIInterfaceRequestor)
+      .getInterface(Ci.nsIDOMWindowUtils);
+
+
+  let reftestWait = false;
+
+  if (document.location.href !== url || document.readyState != "complete") {
+    logger.debug(`Waiting for page load of ${url}`);
+    yield new Promise(resolve => {
+      let maybeResolve = (event) => {
+        if (event.target === curContainer.frame.document &&
+            event.target.location.href === url) {
+          win = curContainer.frame;
+          document = curContainer.frame.document;
+          reftestWait = document.documentElement.classList.contains("reftest-wait");
+          removeEventListener("load", maybeResolve, {once: true});
+          win.setTimeout(resolve, 0);
+        }
+      };
+      addEventListener("load", maybeResolve, true);
+    });
+  } else {
+    // Ensure that the event loop has spun at least once since load,
+    // so that setTimeout(fn, 0) in the load event has run
+    reftestWait = document.documentElement.classList.contains("reftest-wait");
+    yield new Promise(resolve => win.setTimeout(resolve, 0));
+  };
+
+  let root = document.documentElement;
+  if (reftestWait) {
+    // Check again in case reftest-wait was removed since the load event
+    if (root.classList.contains("reftest-wait")) {
+      logger.debug("Waiting for reftest-wait removal");
+      yield new Promise(resolve => {
+        let observer = new win.MutationObserver(() => {
+          if (!root.classList.contains("reftest-wait")) {
+            observer.disconnect();
+            logger.debug("reftest-wait removed");
+            win.setTimeout(resolve, 0);
+          }
+        });
+        observer.observe(root, {attributes: true});
+      });
+    }
+
+    logger.debug("Waiting for rendering");
+
+    yield new Promise(resolve => {
+      let maybeResolve = () => {
+        if (flushRendering()) {
+          win.addEventListener("MozAfterPaint", maybeResolve, {once: true});
+        } else {
+          win.setTimeout(resolve, 0);
+        }
+      };
+      maybeResolve();
+    });
+  } else {
+    flushRendering();
+  }
+
+  if (remote) {
+    windowUtils.updateLayerTree();
+  }
+}
+
 // Call register self when we get loaded
 registerSelf();
new file mode 100644
--- /dev/null
+++ b/testing/marionette/reftest.js
@@ -0,0 +1,356 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+const {classes: Cc, interfaces: Ci, utils: Cu} = Components;
+
+Cu.import("resource://gre/modules/Log.jsm");
+Cu.import("resource://gre/modules/Preferences.jsm");
+Cu.import("resource://gre/modules/Task.jsm");
+
+Cu.import("chrome://marionette/content/assert.js");
+Cu.import("chrome://marionette/content/capture.js");
+
+this.EXPORTED_SYMBOLS = ["reftest"];
+
+const XUL_NS = "http://www.mozilla.org/keymaster/gatekeeper/there.is.only.xul";
+const PREF_E10S = "browser.tabs.remote.autostart";
+
+const logger = Log.repository.getLogger("Marionette");
+
+const SCREENSHOT_MODE = {
+  unexpected: 0,
+  fail: 1,
+  always: 2
+};
+
+const STATUS = {
+  PASS: "PASS",
+  FAIL: "FAIL",
+  ERROR: "ERROR",
+  TIMEOUT: "TIMEOUT",
+};
+
+/**
+ * Implements an fast runner for web-platform-tests format reftests
+ * c.f. http://web-platform-tests.org/writing-tests/reftests.html
+ */
+let reftest = {};
+
+reftest.Runner = class {
+  constructor(driver) {
+    this.driver = driver;
+    this.canvasCache = new Map([[null, []]]);
+    this.windowUtils = null;
+    this.lastUrl = null;
+    this.remote = Preferences.get(PREF_E10S);
+  }
+
+  /**
+   * Setup the required environment for running reftests.
+   *
+   * This will open a non-browser window in which the tests will
+   * be loaded, and set up various caches for the reftest run.
+   *
+   * @param {Object.<Number>} urlCount
+   *     Object holding a map of URL: number of times the URL
+   *     will be opened during the reftest run, where that's
+   *     greater than 1.
+   * @param {string} screenshotMode
+   *     String enum representing when screenshots should be taken
+   */
+  *setup(urlCount, screenshotMode) {
+    this.parentWindow =  assert.window(this.driver.getCurrentWindow());
+
+    this.screenshotMode = SCREENSHOT_MODE[screenshotMode] ||
+        SCREENSHOT_MODE["unexpected"];
+
+    this.urlCount = Object.keys(urlCount || {})
+        .reduce((map, key) => map.set(key, urlCount[key]), new Map());
+
+    yield this.ensureWindow();
+  };
+
+  *ensureWindow() {
+    if (this.reftestWin && !this.reftestWin.closed) {
+      return this.reftestWin;
+    }
+
+    let reftestWin = yield this.openWindow();
+
+    let found = this.driver.findWindow([reftestWin], () => true);
+    yield this.driver.setWindowHandle(found, true);
+
+    this.windowUtils = reftestWin.QueryInterface(Ci.nsIInterfaceRequestor)
+      .getInterface(Ci.nsIDOMWindowUtils);
+    this.reftestWin = reftestWin;
+    return reftestWin;
+  }
+
+  *openWindow() {
+    let reftestWin;
+    yield new Promise(resolve => {
+      reftestWin = this.parentWindow.openDialog("chrome://marionette/content/reftest.xul",
+                                                "reftest",
+                                                "chrome,dialog,height=600,width=600,all",
+                                                () => resolve());
+    });
+
+    let browser = reftestWin.document.createElementNS(XUL_NS, "xul:browser");
+    browser.permanentKey = {};
+    browser.setAttribute("id", "browser");
+    browser.setAttribute("anonid", "initialBrowser");
+    browser.setAttribute("type", "content");
+    browser.setAttribute("primary", "true");
+
+    if (this.remote) {
+      browser.setAttribute("remote", "true");
+      browser.setAttribute("remoteType", "web");
+    }
+    // Make sure the browser element is exactly 600x600, no matter
+    // what size our window is
+    const window_style = `padding: 0px; margin: 0px; border:none;
+min-width: 600px; min-height: 600px; max-width: 600px; max-height: 600px`;
+    browser.setAttribute("style", window_style);
+
+    let doc = reftestWin.document.documentElement;
+    while (doc.firstChild) {
+      doc.firstChild.remove();
+    }
+    doc.appendChild(browser);
+    reftestWin.gBrowser = browser;
+
+    return reftestWin;
+  }
+
+  abort() {
+    this.driver.close();
+    this.reftestWin = null;
+  }
+
+  /**
+   * Run a specific reftest.
+   *
+   * The assumed semantics are those of web-platform-tests where
+   * references form a tree and each test must meet all the conditions
+   * to reach one leaf node of the tree in order for the overall test
+   * to pass.
+   *
+   * @param {string} testUrl
+   *     URL of the test itself.
+   * @param {Array.<Array>} references
+   *     Array representing a tree of references to try. Each item in
+   *     the array represents a single reference node and has the form
+   *     [referenceUrl, references, relation], where referenceUrl is a
+   *     string to the url, relation is either "==" or "!=" depending on
+   *     the type of reftest, and references is another array containing
+   *     items of the same form, representing further comparisons treated
+   *     as AND with the current item. Sibling entries are treated as OR.
+   *     For example with testUrl of T:
+   *       references = [[A, [[B, [], ==]], ==]]
+   *       Must have T == A AND A == B to pass
+   *
+   *       references = [[A, [], ==], [B, [], !=]
+   *       Must have T == A OR T != B
+   *
+   *       references = [[A, [[B, [], ==], [C, [], ==]], ==], [D, [], ]]
+   *       Must have (T == A AND A == B) OR (T == A AND A == C) OR (T == D)
+   * @param {string} expected
+   *     Expected test outcome (e.g. PASS, FAIL).
+   * @param {number} timeout
+   *     Test timeout in ms
+   *
+   * @return {Object}
+   *     Result object with fields status, message and extra.
+   */
+  *run(testUrl, references, expected, timeout) {
+
+    let timeoutHandle;
+
+    let timeoutPromise = new Promise(resolve => {
+      timeoutHandle = this.parentWindow.setTimeout(() => {
+        resolve({status: STATUS.TIMEOUT, message: null, extra: {}});
+      }, timeout);
+    });
+
+    let testRunner = Task.spawn(function*() {
+      let result;
+      try {
+        result = yield this.runTest(testUrl, references, expected, timeout);
+      } catch (e) {
+        result = {status: STATUS.ERROR, message: e.stack, extra: {}};
+      }
+      return result;
+    }.bind(this));
+
+    let result = yield Promise.race([testRunner, timeoutPromise]);
+    this.parentWindow.clearTimeout(timeoutHandle);
+    if(result.status === STATUS.TIMEOUT) {
+      this.abort();
+    }
+
+    return result;
+  }
+
+  *runTest(testUrl, references, expected, timeout) {
+
+    let win = yield this.ensureWindow();
+
+    win.innerWidth = 600;
+    win.innerHeight = 600;
+
+    let message = "";
+
+    let screenshotData = [];
+
+    let stack = [];
+    for (let i = references.length-1; i >= 0; i--) {
+      let item = references[i];
+      stack.push([testUrl, item[0], item[1], item[2]]);
+    }
+
+    let status = STATUS.FAIL;
+
+    while (stack.length) {
+      let [lhsUrl, rhsUrl, references, relation] = stack.pop();
+      message += `Testing ${lhsUrl} ${relation} ${rhsUrl}\n`;
+
+      let comparison = yield this.compareUrls(win, lhsUrl, rhsUrl, relation, timeout);
+
+      function recordScreenshot() {
+        let toBase64 =  screenshot => screenshot.canvas.toDataURL().split(",")[1];
+        screenshotData.push([{url: lhsUrl, screenshot: toBase64(comparison.lhs)},
+                             relation,
+                             {url:rhsUrl, screenshot: toBase64(comparison.rhs)}]);
+      }
+
+      if (this.screenshotMode === SCREENSHOT_MODE.always) {
+        recordScreenshot();
+      }
+
+      if (comparison.passed) {
+        if (references.length) {
+          for (let i = references.length - 1; i >= 0; i--) {
+            let item = references[i];
+            stack.push([testUrl, item[0], item[1], item[2]]);
+          }
+        } else {
+          // Reached a leaf node so all of one reference chain passed
+          status = STATUS.PASS;
+          if (this.screenshotMode <= SCREENSHOT_MODE.fail && expected != status) {
+            recordScreenshot();
+          }
+          break;
+        }
+      } else if (!stack.length) {
+        // If we don't have any alternatives to try then this will be the last iteration,
+        // so save the failing screenshots if required
+        if (this.screenshotMode === SCREENSHOT_MODE.fail ||
+            (this.screenshotMode === SCREENSHOT_MODE.unexpected && expected != status)) {
+          recordScreenshot();
+        }
+      }
+
+      // Return any reusable canvases to the pool
+      let canvasPool = this.canvasCache.get(null);
+      [comparison.lhs, comparison.rhs].map(screenshot => {
+        if (screenshot.reuseCanvas) {
+          canvasPool.push(screenshot.canvas);
+        }
+      });
+      logger.debug(`Canvas pool is of length ${canvasPool.length}`);
+    }
+
+    let result = {status, message, extra: {}};
+    if (screenshotData.length) {
+      // For now the tbpl formatter only accepts one screenshot, so just return the
+      // last one we took.
+      result.extra.reftest_screenshots = screenshotData[screenshotData.length - 1];
+    }
+
+    return result;
+  };
+
+  *compareUrls(win, lhsUrl, rhsUrl, relation, timeout) {
+    logger.info(`Testing ${lhsUrl} ${relation} ${rhsUrl}`);
+
+    // Take the reference screenshot first so that if we pause
+    // we see the test rendering
+    let rhs = yield this.screenshot(win, rhsUrl, timeout);
+    let lhs = yield this.screenshot(win, lhsUrl, timeout);
+
+    let maxDifferences = {};
+
+    let differences = this.windowUtils.compareCanvases(lhs.canvas, rhs.canvas, maxDifferences);
+
+    let passed;
+    switch (relation) {
+    case "==":
+      passed = differences === 0;
+      if (!passed) {
+        logger.info(`Found ${differences} pixels different, maximum difference per channel ${maxDifferences.value}`);
+      }
+      break;
+    case "!=":
+      passed = differences !== 0;
+      break;
+    default:
+      throw new InvalidArgumentError("Reftest operator should be '==' or '!='");
+    }
+
+    return {lhs, rhs, passed};
+  }
+
+  *screenshot(win, url, timeout) {
+    let canvas = null;
+    let remainingCount = this.urlCount.get(url) || 1;
+    let cache = remainingCount > 1;
+    logger.debug(`screenshot ${url} remainingCount: ${remainingCount} cache: ${cache}`);
+    let reuseCanvas = false;
+    if (this.canvasCache.has(url)) {
+      logger.debug(`screenshot ${url} taken from cache`);
+      canvas = this.canvasCache.get(url);
+      if (!cache) {
+        this.canvasCache.delete(url);
+      }
+    } else {
+      let canvases = this.canvasCache.get(null);
+      if (canvases.length) {
+        canvas = canvases.pop();
+      } else {
+        canvas = null;
+      }
+      reuseCanvas = !cache;
+
+      let ctxInterface = win.CanvasRenderingContext2D;
+      let flags = ctxInterface.DRAWWINDOW_DRAW_CARET |
+          ctxInterface.DRAWWINDOW_USE_WIDGET_LAYERS |
+          ctxInterface.DRAWWINDOW_DRAW_VIEW;
+
+      logger.debug(`Starting load of ${url}`);
+      if (this.lastUrl === url) {
+        logger.debug(`Refreshing page`);
+        yield this.driver.listener.refresh({commandId: this.driver.listener.activeMessageId,
+                                            pageTimeout: timeout});
+      } else {
+        yield this.driver.listener.get({commandId: this.driver.listener.activeMessageId,
+                                        url: url,
+                                        pageTimeout: timeout,
+                                        loadEventExpected: false});
+        this.lastUrl = url;
+      }
+
+      this.driver.curBrowser.contentBrowser.focus();
+      yield this.driver.listener.reftestWait(url, this.remote);
+
+      canvas = capture.canvas(win, 0, 0, win.innerWidth, win.innerHeight, {canvas, flags});
+    }
+    if (cache) {
+      this.canvasCache.set(url, canvas);
+    };
+    this.urlCount.set(url, remainingCount - 1);
+    return {canvas, reuseCanvas};
+  }
+};
new file mode 100644
--- /dev/null
+++ b/testing/marionette/reftest.xul
@@ -0,0 +1,1 @@
+<window id="reftest" xmlns="http://www.mozilla.org/keymaster/gatekeeper/there.is.only.xul" onload="window.arguments[0]()"></window>
\ No newline at end of file
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/firefox.py
+++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/firefox.py
@@ -179,17 +179,19 @@ class FirefoxBrowser(Browser):
         preferences = self.load_prefs()
 
         self.profile = FirefoxProfile(locations=locations,
                                       preferences=preferences)
         self.profile.set_preferences({"marionette.port": self.marionette_port,
                                       "dom.disable_open_during_load": False,
                                       "network.dns.localDomains": ",".join(hostnames),
                                       "network.proxy.type": 0,
-                                      "places.history.enabled": False})
+                                      "places.history.enabled": False,
+                                      "dom.send_after_paint_to_content": True,
+                                      "layout.interruptible-reflow.enabled": False})
         if self.e10s:
             self.profile.set_preferences({"browser.tabs.remote.autostart": True})
 
         if self.leak_check and kwargs.get("check_leaks", True):
             self.leak_report_file = os.path.join(self.profile.profile, "runtests_leaks.log")
             if os.path.exists(self.leak_report_file):
                 os.remove(self.leak_report_file)
             env["XPCOM_MEM_BLOAT_LOG"] = self.leak_report_file