Bug 1412357 - Added mochitest asserts + logging to TestRunner.jsm. r?mconley draft
authorgrenewode <grenewodemiller@gmail.com>
Sat, 11 Nov 2017 18:55:53 -0500
changeset 701501 b133052aa32078b439100e0a33cc80a4f6cf2136
parent 701250 72ee4800d4156931c89b58bd807af4a3083702bb
child 741191 57faab525d8f38fec032c57a39b44d8da7d96f46
push id90189
push usermill2540@msu.edu
push dateTue, 21 Nov 2017 20:13:00 +0000
reviewersmconley
bugs1412357
milestone59.0a1
Bug 1412357 - Added mochitest asserts + logging to TestRunner.jsm. r?mconley Moved the call to TestRunner.initTest to head.js so that it only needs to be called once. Also, TestRunner._findBoundingbox now throws exceptions instead of returning null, which makes debugging and testing easier. MozReview-Commit-ID: LpJmEL3CfKd
browser/tools/mozscreenshots/browser_boundingbox.js
browser/tools/mozscreenshots/head.js
browser/tools/mozscreenshots/mozscreenshots/extension/TestRunner.jsm
--- a/browser/tools/mozscreenshots/browser_boundingbox.js
+++ b/browser/tools/mozscreenshots/browser_boundingbox.js
@@ -62,16 +62,19 @@ add_task(async function() {
 
   // Check width calculation on union
   is(rect.width, expectedRight - expectedLeft,
      "Checking _findBoundingBox union width calculation");
   // Check height calculation on union
   is(rect.height, expectedBottom - expectedTop,
      "Checking _findBoundingBox union height calculation");
 
-  rect = TestRunner._findBoundingBox(["#does_not_exist"]);
-  // Check that 'selector not found' returns null
-  is(rect, null, "Checking that selector not found returns null");
+    // Check that nonexistent selectors throws an exception
+  Assert.throws(() => {
+    TestRunner._findBoundingBox(["#does_not_exist"]);
+  }, "No element for '#does_not_exist' found.", "Checking that nonexistent selectors throws an exception");
 
-  rect = TestRunner._findBoundingBox([]);
-  // Check that no selectors returns null
-  is(rect, null, "Checking that no selectors returns null");
+  // Check that no selectors throws an exception
+  Assert.throws(() => {
+    rect = TestRunner._findBoundingBox([]);
+
+  }, "No selectors specified.", "Checking that no selectors throws an exception");
 });
--- a/browser/tools/mozscreenshots/head.js
+++ b/browser/tools/mozscreenshots/head.js
@@ -19,19 +19,20 @@ async function setup() {
 
   info("installing extension temporarily");
   let chromeURL = Services.io.newURI(EXTENSION_DIR);
   let dir = chromeRegistry.convertChromeURL(chromeURL).QueryInterface(Ci.nsIFileURL).file;
   await AddonManager.installTemporaryAddon(dir);
 
   info("Checking for mozscreenshots extension");
   return new Promise((resolve) => {
-    AddonManager.getAddonByID("mozscreenshots@mozilla.org", function(aAddon) {
+    AddonManager.getAddonByID("mozscreenshots@mozilla.org", (aAddon) => {
       isnot(aAddon, null, "The mozscreenshots extension should be installed");
       TestRunner = Cu.import("chrome://mozscreenshots/content/TestRunner.jsm", {}).TestRunner;
+      TestRunner.initTest(this);
       resolve();
     });
   });
 }
 
 /**
  * Used by pre-defined sets of configurations to decide whether to run for a build.
  * @note This is not used by browser_screenshots.js which handles when MOZSCREENSHOTS_SETS is set.
--- a/browser/tools/mozscreenshots/mozscreenshots/extension/TestRunner.jsm
+++ b/browser/tools/mozscreenshots/mozscreenshots/extension/TestRunner.jsm
@@ -18,73 +18,70 @@ Cu.import("resource://gre/modules/XPCOMU
 Cu.import("resource://gre/modules/osfile.jsm");
 Cu.import("resource://gre/modules/Geometry.jsm");
 
 XPCOMUtils.defineLazyModuleGetter(this, "BrowserTestUtils",
                                   "resource://testing-common/BrowserTestUtils.jsm");
 // Screenshot.jsm must be imported this way for xpcshell tests to work
 XPCOMUtils.defineLazyModuleGetter(this, "Screenshot", "chrome://mozscreenshots/content/Screenshot.jsm");
 
-// Create a new instance of the ConsoleAPI so we can control the maxLogLevel with a pref.
-// See LOG_LEVELS in Console.jsm. Common examples: "All", "Info", "Warn", & "Error".
-const PREF_LOG_LEVEL = "extensions.mozscreenshots@mozilla.org.loglevel";
-XPCOMUtils.defineLazyGetter(this, "log", () => {
-  let ConsoleAPI = Cu.import("resource://gre/modules/Console.jsm", {}).ConsoleAPI;
-  let consoleOptions = {
-    maxLogLevel: "info",
-    maxLogLevelPref: PREF_LOG_LEVEL,
-    prefix: "mozscreenshots",
-  };
-  return new ConsoleAPI(consoleOptions);
-});
-
 this.TestRunner = {
   combos: null,
   completedCombos: 0,
   currentComboIndex: 0,
   _lastCombo: null,
   _libDir: null,
   croppingPadding: 10,
+  mochitestScope: null,
 
   init(extensionPath) {
-    log.debug("init");
     this._extensionPath = extensionPath;
   },
 
   /**
+   * Initialize the mochitest interface. This allows TestRunner to integrate
+   * with mochitest functions like is(...) and ok(...). This must be called
+   * prior to invoking any of the TestRunner functions. Note that this should
+   * be properly setup in head.js, so you probably don't need to call it.
+   */
+  initTest(mochitestScope) {
+    this.mochitestScope = mochitestScope;
+  },
+
+  /**
    * Load specified sets, execute all combinations of them, and capture screenshots.
    */
   async start(setNames, jobName = null) {
     let subDirs = ["mozscreenshots",
                    (new Date()).toISOString().replace(/:/g, "-") + "_" + Services.appinfo.OS];
     let screenshotPath = FileUtils.getFile("TmpD", subDirs).path;
 
     const MOZ_UPLOAD_DIR = env.get("MOZ_UPLOAD_DIR");
     if (MOZ_UPLOAD_DIR) {
       screenshotPath = MOZ_UPLOAD_DIR;
     }
 
-    log.info("Saving screenshots to:", screenshotPath);
+    this.mochitestScope.info("Saving screenshots to:", screenshotPath);
 
     let screenshotPrefix = Services.appinfo.appBuildID;
     if (jobName) {
       screenshotPrefix += "-" + jobName;
     }
     screenshotPrefix += "_";
     Screenshot.init(screenshotPath, this._extensionPath, screenshotPrefix);
     this._libDir = this._extensionPath.QueryInterface(Ci.nsIFileURL).file.clone();
     this._libDir.append("chrome");
     this._libDir.append("mozscreenshots");
     this._libDir.append("lib");
 
     let sets = this.loadSets(setNames);
 
-    log.info(sets.length + " sets:", setNames);
+    this.mochitestScope.info(sets.length + " sets:", setNames);
     this.combos = new LazyProduct(sets);
-    log.info(this.combos.length + " combinations");
+    this.mochitestScope.info(this.combos.length + " combinations");
 
     this.currentComboIndex = this.completedCombos = 0;
     this._lastCombo = null;
 
     // Setup some prefs
     Services.prefs.setCharPref("browser.aboutHomeSnippets.updateUrl",
                                "data:text/html;charset=utf-8,Generated by mozscreenshots");
     Services.prefs.setCharPref("extensions.ui.lastCategory", "addons://list/extension");
@@ -105,18 +102,18 @@ this.TestRunner = {
     await BrowserTestUtils.loadURI(selectedBrowser, HOME_PAGE);
     await BrowserTestUtils.browserLoaded(selectedBrowser);
 
     for (let i = 0; i < this.combos.length; i++) {
       this.currentComboIndex = i;
       await this._performCombo(this.combos.item(this.currentComboIndex));
     }
 
-    log.info("Done: Completed " + this.completedCombos + " out of " +
-             this.combos.length + " configurations.");
+    this.mochitestScope.info("Done: Completed " + this.completedCombos + " out of " +
+                             this.combos.length + " configurations.");
     this.cleanup();
   },
 
   /**
    * Helper function for loadSets. This filters out the restricted configs from setName.
    * This was made a helper function to facilitate xpcshell unit testing.
    * @param {String} setName - set name to be filtered e.g. "Toolbars[onlyNavBar,allToolbars]"
    * @return {Object} Returns an object with two values: the filtered set name and a set of
@@ -144,48 +141,42 @@ this.TestRunner = {
     let sets = [];
     for (let setName of setNames) {
       let restrictions = null;
       if (setName.includes("[")) {
         let filteredData = this.filterRestrictions(setName);
         setName = filteredData.trimmedSetName;
         restrictions = filteredData.restrictions;
       }
-      try {
-        let imported = {};
-        Cu.import("chrome://mozscreenshots/content/configurations/" + setName + ".jsm",
-                  imported);
-        imported[setName].init(this._libDir);
-        let configurationNames = Object.keys(imported[setName].configurations);
-        if (!configurationNames.length) {
-          throw new Error(setName + " has no configurations for this environment");
-        }
-        // Checks to see if nonexistent configuration have been specified
-        if (restrictions) {
-          let incorrectConfigs = [...restrictions].filter(r => !configurationNames.includes(r));
-          if (incorrectConfigs.length) {
-            throw new Error("non existent configurations: " + incorrectConfigs);
-          }
+      let imported = {};
+      Cu.import("chrome://mozscreenshots/content/configurations/" + setName + ".jsm",
+                imported);
+      imported[setName].init(this._libDir);
+      let configurationNames = Object.keys(imported[setName].configurations);
+      if (!configurationNames.length) {
+        throw new Error(setName + " has no configurations for this environment");
+      }
+      // Checks to see if nonexistent configuration have been specified
+      if (restrictions) {
+        let incorrectConfigs = [...restrictions].filter(r => !configurationNames.includes(r));
+        if (incorrectConfigs.length) {
+          throw new Error("non existent configurations: " + incorrectConfigs);
         }
-        let configurations = {};
-        for (let config of configurationNames) {
-          // Automatically set the name property of the configuration object to
-          // its name from the configuration object.
-          imported[setName].configurations[config].name = config;
-          // Filter restricted configurations.
-          if (!restrictions || restrictions.has(config)) {
-            configurations[config] = imported[setName].configurations[config];
-          }
+      }
+      let configurations = {};
+      for (let config of configurationNames) {
+        // Automatically set the name property of the configuration object to
+        // its name from the configuration object.
+        imported[setName].configurations[config].name = config;
+        // Filter restricted configurations.
+        if (!restrictions || restrictions.has(config)) {
+          configurations[config] = imported[setName].configurations[config];
         }
-        sets.push(configurations);
-      } catch (ex) {
-        log.error("Error loading set: " + setName);
-        log.error(ex);
-        throw ex;
       }
+      sets.push(configurations);
     }
     return sets;
   },
 
   cleanup() {
     let browserWindow = Services.wm.getMostRecentWindow("navigator:browser");
     let gBrowser = browserWindow.gBrowser;
     while (gBrowser.tabs.length > 1) {
@@ -201,20 +192,18 @@ this.TestRunner = {
 
   /**
   * Calculate the bounding box based on CSS selector from config for cropping
   *
   * @param {String[]} selectors - array of CSS selectors for relevant DOM element
   * @return {Geometry.jsm Rect} Rect holding relevant x, y, width, height with padding
   **/
   _findBoundingBox(selectors, windowType) {
-    // No selectors provided
     if (!selectors.length) {
-      log.info("_findBoundingBox: selectors argument is empty");
-      return null;
+      throw "No selectors specified.";
     }
 
     // Set window type, default "navigator:browser"
     windowType = windowType || "navigator:browser";
     let browserWindow = Services.wm.getMostRecentWindow(windowType);
     // Scale for high-density displays
     const scale = browserWindow.QueryInterface(Ci.nsIInterfaceRequestor)
                         .getInterface(Ci.nsIDocShell).QueryInterface(Ci.nsIBaseWindow)
@@ -226,20 +215,18 @@ this.TestRunner = {
       let element;
       // Check for function to find anonymous content
       if (typeof(selector) == "function") {
         element = selector();
       } else {
         element = browserWindow.document.querySelector(selector);
       }
 
-      // Selector not found
       if (!element) {
-        log.info("_findBoundingBox: selector not found");
-        return null;
+        throw `No element for '${selector}' found.`;
       }
 
       // Calculate box region, convert to Rect
       let box = element.ownerDocument.getBoxObjectFor(element);
       let newRect = new Rect(box.screenX * scale, box.screenY * scale,
                              box.width * scale, box.height * scale);
 
       if (!finalRect) {
@@ -263,100 +250,112 @@ this.TestRunner = {
     finalRect.right = Math.min(finalRect.right, windowLeft + windowWidth);
     finalRect.bottom = Math.min(finalRect.bottom, windowTop + windowHeight);
 
     return finalRect;
   },
 
   async _performCombo(combo) {
     let paddedComboIndex = padLeft(this.currentComboIndex + 1, String(this.combos.length).length);
-    log.info("Combination " + paddedComboIndex + "/" + this.combos.length + ": " +
-             this._comboName(combo).substring(1));
+    this.mochitestScope.info(
+      `Combination ${paddedComboIndex}/${this.combos.length}: ${this._comboName(combo).substring(1)}`
+    );
 
-    function changeConfig(config) {
-      log.debug("calling " + config.name);
+    // Notice that this does need to be a closure, not a function, as otherwise
+    // "this" gets replaced and we lose access to this.mochitestScope.
+    const changeConfig = (config) => {
+      this.mochitestScope.info("calling " + config.name);
+
       let applyPromise = Promise.resolve(config.applyConfig());
       let timeoutPromise = new Promise((resolve, reject) => {
         setTimeout(reject, APPLY_CONFIG_TIMEOUT_MS, "Timed out");
       });
-      log.debug("called " + config.name);
+
+      this.mochitestScope.info("called " + config.name);
       // Add a default timeout of 500ms to avoid conflicts when configurations
       // try to apply at the same time. e.g WindowSize and TabsInTitlebar
       return Promise.race([applyPromise, timeoutPromise]).then(() => {
         return new Promise((resolve) => {
           setTimeout(resolve, 500);
         });
       });
-    }
+    };
 
     try {
       // First go through and actually apply all of the configs
       for (let i = 0; i < combo.length; i++) {
         let config = combo[i];
         if (!this._lastCombo || config !== this._lastCombo[i]) {
-          log.debug("promising", config.name);
+          this.mochitestScope.info(`promising ${config.name}`);
           await changeConfig(config);
         }
       }
 
       // Update the lastCombo since it's now been applied regardless of whether it's accepted below.
-      log.debug("fulfilled all applyConfig so setting lastCombo.");
+      this.mochitestScope.info("fulfilled all applyConfig so setting lastCombo.");
       this._lastCombo = combo;
 
       // Then ask configs if the current setup is valid. We can't can do this in
       // the applyConfig methods of the config since it doesn't know what configs
       // later in the loop will do that may invalidate the combo.
       for (let i = 0; i < combo.length; i++) {
         let config = combo[i];
         // A configuration can specify an optional verifyConfig method to indicate
         // if the current config is valid for a screenshot. This gets called even
         // if the this config was used in the lastCombo since another config may
         // have invalidated it.
         if (config.verifyConfig) {
-          log.debug("checking if the combo is valid with", config.name);
+          this.mochitestScope.info(`checking if the combo is valid with ${config.name}`);
           await config.verifyConfig();
         }
       }
     } catch (ex) {
-      log.warn("\tskipped configuration: " + ex);
+      this.mochitestScope.info(`\tskipped configuration [ ${combo.map((e) => e.name).join(", ")} ]`);
+      this.mochitestScope.info(`\treason: ${ex.toString()}`);
       // Don't set lastCombo here so that we properly know which configurations
       // need to be applied since the last screenshot
 
       // Return so we don't take a screenshot.
       return;
     }
 
     // Collect selectors from combo configs for cropping region
     let windowType;
     const finalSelectors = [];
     for (const obj of combo) {
       if (!windowType) {
         windowType = obj.windowType;
       } else if (windowType !== obj.windowType) {
-        log.warn("\tConfigurations with multiple window types are not allowed");
+        this.mochitestScope.ok(false, "All configurations in the combo have a single window type");
         return;
       }
       for (const selector of obj.selectors) {
         finalSelectors.push(selector);
       }
     }
 
     const rect = this._findBoundingBox(finalSelectors, windowType);
+    this.mochitestScope.ok(rect, "A valid bounding box was found");
+    if (!rect) {
+      return;
+    }
     await this._onConfigurationReady(combo, rect);
   },
 
   async _onConfigurationReady(combo, rect) {
     let filename = padLeft(this.currentComboIndex + 1,
                            String(this.combos.length).length) + this._comboName(combo);
     const imagePath = await Screenshot.captureExternal(filename);
 
     let browserWindow = Services.wm.getMostRecentWindow("navigator:browser");
-    await this._cropImage(browserWindow, OS.Path.toFileURI(imagePath), rect, imagePath);
+    await this._cropImage(browserWindow, OS.Path.toFileURI(imagePath), rect, imagePath).catch((msg) => {
+      throw `Cropping combo [${combo.map((e) => e.name).join(", ")}] failed: ${msg}`;
+    });
     this.completedCombos++;
-    log.debug("_onConfigurationReady");
+    this.mochitestScope.info("_onConfigurationReady");
   },
 
   _comboName(combo) {
     return combo.reduce(function(a, b) {
       return a + "_" + b.name;
     }, "");
   },