--- a/browser/tools/mozscreenshots/mozscreenshots/extension/TestRunner.jsm
+++ b/browser/tools/mozscreenshots/mozscreenshots/extension/TestRunner.jsm
@@ -274,16 +274,32 @@ this.TestRunner = {
       } else {
         bounds = bounds.union(rect);
       }
     }
 
     return {bounds, rects};
   },
 
+  _do_skip(reason, combo, config, func) {
+    const { todo } = reason;
+    if (todo) {
+      this.mochitestScope.todo(
+        false,
+        `Skipped configuration ` +
+        `[ ${combo.map((e) => e.name).join(", ")} ] for failure in ` +
+        `${config.name}.${func}: ${todo}`);
+    } else {
+      this.mochitestScope.info(
+        `\tSkipped configuration ` +
+        `[ ${combo.map((e) => e.name).join(", ")} ] ` +
+        `for "${reason}" in  ${config.name}.${func}`);
+    }
+  },
+
   async _performCombo(combo) {
     let paddedComboIndex = padLeft(this.currentComboIndex + 1, String(this.combos.length).length);
     this.mochitestScope.info(
       `Combination ${paddedComboIndex}/${this.combos.length}: ${this._comboName(combo).substring(1)}`
     );
 
     // Notice that this does need to be a closure, not a function, as otherwise
     // "this" gets replaced and we lose access to this.mochitestScope.
@@ -293,30 +309,34 @@ this.TestRunner = {
       let applyPromise = Promise.resolve(config.applyConfig());
       let timeoutPromise = new Promise((resolve, reject) => {
         setTimeout(reject, APPLY_CONFIG_TIMEOUT_MS, "Timed out");
       });
 
       this.mochitestScope.info("called " + config.name);
       // Add a default timeout of 500ms to avoid conflicts when configurations
       // try to apply at the same time. e.g WindowSize and TabsInTitlebar
-      return Promise.race([applyPromise, timeoutPromise]).then(() => {
+      return Promise.race([applyPromise, timeoutPromise]).then(result => {
         return new Promise((resolve) => {
-          setTimeout(resolve, 500);
+          setTimeout(() => resolve(result), 500);
         });
       });
     };
 
     try {
       // First go through and actually apply all of the configs
       for (let i = 0; i < combo.length; i++) {
         let config = combo[i];
         if (!this._lastCombo || config !== this._lastCombo[i]) {
           this.mochitestScope.info(`promising ${config.name}`);
-          await changeConfig(config);
+          const reason = await changeConfig(config);
+          if (reason) {
+            this._do_skip(reason, combo, config, "applyConfig");
+            return;
+          }
         }
       }
 
       // Update the lastCombo since it's now been applied regardless of whether it's accepted below.
       this.mochitestScope.info("fulfilled all applyConfig so setting lastCombo.");
       this._lastCombo = combo;
 
       // Then ask configs if the current setup is valid. We can't can do this in
@@ -325,28 +345,32 @@ this.TestRunner = {
       for (let i = 0; i < combo.length; i++) {
         let config = combo[i];
         // A configuration can specify an optional verifyConfig method to indicate
         // if the current config is valid for a screenshot. This gets called even
         // if the this config was used in the lastCombo since another config may
         // have invalidated it.
         if (config.verifyConfig) {
           this.mochitestScope.info(`checking if the combo is valid with ${config.name}`);
-          await config.verifyConfig();
+          const reason = await config.verifyConfig();
+          if (reason) {
+            this._do_skip(reason, combo, config, "applyConfig");
+            return;
+          }
         }
       }
     } catch (ex) {
-      this.mochitestScope.info(`\tskipped configuration [ ${combo.map((e) => e.name).join(", ")} ]`);
-      this.mochitestScope.info(`\treason: ${ex.toString()}`);
-      // Don't set lastCombo here so that we properly know which configurations
-      // need to be applied since the last screenshot
-
-      // Return so we don't take a screenshot.
+      this.mochitestScope.ok(false, `Unexpected exception in [ ${combo.map(({ name }) => name).join(", ")} ]: ${ex.toString()}`);
+      this.mochitestScope.info(`\t${ex}`);
+      if (ex.stack) {
+        this.mochitestScope.info(`\t${ex.stack}`);
+      }
       return;
     }
+    this.mochitestScope.info(`Configured UI for [ ${combo.map(({ name }) => name).join(", ")} ] successfully`);
 
     // Collect selectors from combo configs for cropping region
     let windowType;
     const finalSelectors = [];
     for (const obj of combo) {
       if (!windowType) {
         windowType = obj.windowType;
       } else if (windowType !== obj.windowType) {

--- a/browser/tools/mozscreenshots/mozscreenshots/extension/configurations/AppMenu.jsm
+++ b/browser/tools/mozscreenshots/mozscreenshots/extension/configurations/AppMenu.jsm
@@ -30,57 +30,57 @@ this.AppMenu = {
         let promise = browserWindow.PanelUI.show();
         browserWindow.PanelUI.showMainView();
         return promise;
       },
     },
 
     appMenuHistorySubview: {
       selectors: ["#appMenu-popup"],
-      applyConfig() {
+      async applyConfig() {
         // History has a footer
         if (isCustomizing()) {
-          return Promise.reject("Can't show subviews while customizing");
+          return "Can't show subviews while customizing";
         }
         let browserWindow = Services.wm.getMostRecentWindow("navigator:browser");
-        let promise = browserWindow.PanelUI.show();
-        return promise.then(() => {
-          browserWindow.PanelUI.showMainView();
-          browserWindow.document.getElementById("history-panelmenu").click();
-        });
+        await browserWindow.PanelUI.show();
+        browserWindow.PanelUI.showMainView();
+        browserWindow.document.getElementById("history-panelmenu").click();
+
+        return undefined;
       },
 
       verifyConfig: verifyConfigHelper,
     },
 
     appMenuHelpSubview: {
       selectors: ["#appMenu-popup"],
-      applyConfig() {
+      async applyConfig() {
         if (isCustomizing()) {
-          return Promise.reject("Can't show subviews while customizing");
+          return "Can't show subviews while customizing";
         }
         let browserWindow = Services.wm.getMostRecentWindow("navigator:browser");
-        let promise = browserWindow.PanelUI.show();
-        return promise.then(() => {
-          browserWindow.PanelUI.showMainView();
-          browserWindow.document.getElementById("PanelUI-help").click();
-        });
+        await browserWindow.PanelUI.show();
+        browserWindow.PanelUI.showMainView();
+        browserWindow.document.getElementById("PanelUI-help").click();
+
+        return undefined;
       },
 
       verifyConfig: verifyConfigHelper,
     },
 
   },
 };
 
 function verifyConfigHelper() {
   if (isCustomizing()) {
-    return Promise.reject("AppMenu verifyConfigHelper");
+    return "navigator:browser has the customizing attribute";
   }
-  return Promise.resolve("AppMenu verifyConfigHelper");
+  return undefined;
 }
 
 function isCustomizing() {
   let browserWindow = Services.wm.getMostRecentWindow("navigator:browser");
   if (browserWindow.document.documentElement.hasAttribute("customizing")) {
     return true;
   }
   return false;

--- a/browser/tools/mozscreenshots/mozscreenshots/extension/configurations/Buttons.jsm
+++ b/browser/tools/mozscreenshots/mozscreenshots/extension/configurations/Buttons.jsm
@@ -33,37 +33,37 @@ this.Buttons = {
     },
 
     menuPanelButtons: {
       selectors: ["#widget-overflow"],
       applyConfig: async () => {
         CustomizableUI.addWidgetToArea("screenshot-widget", CustomizableUI.AREA_FIXED_OVERFLOW_PANEL);
       },
 
-      verifyConfig() {
+      async verifyConfig() {
         let browserWindow = Services.wm.getMostRecentWindow("navigator:browser");
         if (browserWindow.PanelUI.panel.state == "closed") {
-          return Promise.reject("The button isn't shown when the panel isn't open.");
+          return "The button isn't shown when the panel isn't open.";
         }
-        return Promise.resolve("menuPanelButtons.verifyConfig");
+        return undefined;
       },
     },
 
     custPaletteButtons: {
       selectors: ["#customization-palette"],
       applyConfig: async () => {
         CustomizableUI.removeWidgetFromArea("screenshot-widget");
       },
 
-      verifyConfig() {
+      async verifyConfig() {
         let browserWindow = Services.wm.getMostRecentWindow("navigator:browser");
         if (browserWindow.document.documentElement.getAttribute("customizing") != "true") {
-          return Promise.reject("The button isn't shown when we're not in customize mode.");
+          return "The button isn't shown when we're not in customize mode.";
         }
-        return Promise.resolve("custPaletteButtons.verifyConfig");
+        return undefined;
       },
     },
   },
 };
 
 function createWidget() {
   let id = "screenshot-widget";
   let spec = {

--- a/browser/tools/mozscreenshots/mozscreenshots/extension/configurations/ControlCenter.jsm
+++ b/browser/tools/mozscreenshots/mozscreenshots/extension/configurations/ControlCenter.jsm
@@ -50,17 +50,17 @@ this.ControlCenter = {
         let browserWindow = Services.wm.getMostRecentWindow("navigator:browser");
         let gBrowser = browserWindow.gBrowser;
         BrowserTestUtils.loadURI(gBrowser.selectedBrowser, channel.file.path);
         await BrowserTestUtils.browserLoaded(gBrowser.selectedBrowser);
         await openIdentityPopup();
       },
 
       async verifyConfig() {
-        return Promise.reject("Bug 1373563: intermittent controlCenter_localFile on Taskcluster");
+        return { todo: "Bug 1373563: intermittent controlCenter_localFile on Taskcluster" };
       },
     },
 
     http: {
       selectors: ["#identity-popup"],
       async applyConfig() {
         await loadPage(HTTP_PAGE);
         await openIdentityPopup();

--- a/browser/tools/mozscreenshots/mozscreenshots/extension/configurations/PermissionPrompts.jsm
+++ b/browser/tools/mozscreenshots/mozscreenshots/extension/configurations/PermissionPrompts.jsm
@@ -107,18 +107,20 @@ this.PermissionPrompts = {
         let browserWindow = Services.wm.getMostRecentWindow("navigator:browser");
         let notification = browserWindow.document.getElementById("addon-install-confirmation-notification");
 
         await closeLastTab();
         await clickOn("#addons");
 
         // We want to skip the progress-notification, so we wait for
         // the install-confirmation screen to be "not hidden" = shown.
-        await BrowserTestUtils.waitForCondition(() => !notification.hasAttribute("hidden"),
-                                                "addon install confirmation did not show", 200);
+        return BrowserTestUtils.waitForCondition(() => !notification.hasAttribute("hidden"),
+                                                "addon install confirmation did not show", 200).catch((msg) => {
+                                                  return Promise.resolve({todo: msg});
+                                                });
       },
     },
   },
 };
 
 async function closeLastTab() {
   if (!lastTab) {
     return;

--- a/browser/tools/mozscreenshots/mozscreenshots/extension/configurations/Preferences.jsm
+++ b/browser/tools/mozscreenshots/mozscreenshots/extension/configurations/Preferences.jsm
@@ -31,30 +31,38 @@ this.Preferences = {
 
     for (let [primary, customFn] of panes) {
       let configName = primary.replace(/^pane/, "prefs");
       if (customFn) {
         configName += "-" + customFn.name;
       }
       this.configurations[configName] = {};
       this.configurations[configName].selectors = ["#browser"];
-      this.configurations[configName].applyConfig = prefHelper.bind(null, primary, customFn);
+      if (primary == "panePrivacy" && customFn) {
+        this.configurations[configName].applyConfig = async () => {
+          return {todo: `${configName} times out on the try server`};
+        };
+      } else {
+        this.configurations[configName].applyConfig = prefHelper.bind(null, primary, customFn);
+      }
     }
   },
 
   configurations: {},
 };
 
 let prefHelper = async function(primary, customFn = null) {
   let browserWindow = Services.wm.getMostRecentWindow("navigator:browser");
   let selectedBrowser = browserWindow.gBrowser.selectedBrowser;
 
   // close any dialog that might still be open
   await ContentTask.spawn(selectedBrowser, null, async function() {
-    if (!content.window.gSubDialog) {
+    // Check that gSubDialog is defined on the content window
+    // and that there is an open dialog to close
+    if (!content.window.gSubDialog || !content.window.gSubDialog._topDialog) {
       return;
     }
     content.window.gSubDialog.close();
   });
 
   let readyPromise = null;
   if (selectedBrowser.currentURI.specIgnoringRef == "about:preferences") {
     if (selectedBrowser.currentURI.spec == "about:preferences#" + primary.replace(/^pane/, "")) {
@@ -68,19 +76,21 @@ let prefHelper = async function(primary,
   }
 
   browserWindow.openPreferences(primary);
 
   await readyPromise;
 
   if (customFn) {
     let customPaintPromise = paintPromise(browserWindow);
-    await customFn(selectedBrowser);
+    let result = await customFn(selectedBrowser);
     await customPaintPromise;
+    return result;
   }
+  return undefined;
 };
 
 function paintPromise(browserWindow) {
   return new Promise((resolve) => {
     browserWindow.addEventListener("MozAfterPaint", function() {
       resolve();
     }, {once: true});
   });
@@ -94,18 +104,23 @@ async function browsingGroup(aBrowser) {
 
 async function cacheGroup(aBrowser) {
   await ContentTask.spawn(aBrowser, null, async function() {
     content.document.getElementById("cacheGroup").scrollIntoView();
   });
 }
 
 async function DNTDialog(aBrowser) {
-  await ContentTask.spawn(aBrowser, null, async function() {
-    content.document.getElementById("doNotTrackSettings").click();
+  return ContentTask.spawn(aBrowser, null, async function() {
+    const button = content.document.getElementById("doNotTrackSettings");
+    if (!button) {
+      return {todo: "The dialog may have exited before we could click the button"};
+    }
+    button.click();
+    return undefined;
   });
 }
 
 async function connectionDialog(aBrowser) {
   await ContentTask.spawn(aBrowser, null, async function() {
     content.document.getElementById("connectionSettings").click();
   });
 }

--- a/browser/tools/mozscreenshots/mozscreenshots/extension/configurations/Tabs.jsm
+++ b/browser/tools/mozscreenshots/mozscreenshots/extension/configurations/Tabs.jsm
@@ -46,17 +46,17 @@ this.Tabs = {
         tab = browserWindow.gBrowser.addTab("about:privatebrowsing");
         browserWindow.gBrowser.pinTab(tab);
         tab = browserWindow.gBrowser.addTab("about:home");
         browserWindow.gBrowser.pinTab(tab);
         browserWindow.gBrowser.selectTabAtIndex(5);
         hoverTab(browserWindow.gBrowser.tabs[2]);
         // also hover the new tab button
         let newTabButton = browserWindow.document.getAnonymousElementByAttribute(browserWindow.
-                           gBrowser.tabContainer, "class", "tabs-newtab-button");
+                           gBrowser.tabContainer, "anonid", "tabs-newtab-button");
         hoverTab(newTabButton);
         browserWindow.gBrowser.tabs[browserWindow.gBrowser.tabs.length - 1].
                       setAttribute("beforehovered", true);
 
         await new Promise((resolve, reject) => {
           setTimeout(resolve, 3000);
         });
         await allTabTitlesDisplayed(browserWindow);
@@ -127,16 +127,17 @@ this.Tabs = {
 
 /* helpers */
 
 async function allTabTitlesDisplayed(browserWindow) {
   let specToTitleMap = {
     "about:home": "New Tab",
     "about:newtab": "New Tab",
     "about:addons": "Add-ons Manager",
+    "about:privatebrowsing": "Open a private window?"
   };
   specToTitleMap[PREFS_TAB] = "browser/skin/settings.svg";
   specToTitleMap[CUST_TAB] = "browser/skin/customize.svg";
   specToTitleMap[DEFAULT_FAVICON_TAB] = "No favicon";
 
   let tabTitlePromises = [];
   for (let tab of browserWindow.gBrowser.tabs) {
     function tabTitleLoaded(spec) {

--- a/browser/tools/mozscreenshots/mozscreenshots/extension/configurations/TabsInTitlebar.jsm
+++ b/browser/tools/mozscreenshots/mozscreenshots/extension/configurations/TabsInTitlebar.jsm
@@ -16,17 +16,17 @@ this.TabsInTitlebar = {
 
   init(libDir) {},
 
   configurations: {
     tabsInTitlebar: {
       selectors: ["#navigator-toolbox"],
       async applyConfig() {
         if (Services.appinfo.OS == "Linux") {
-          return Promise.reject("TabsInTitlebar isn't supported on Linux");
+          return "TabsInTitlebar isn't supported on Linux";
         }
         Services.prefs.setBoolPref(PREF_TABS_IN_TITLEBAR, true);
         return undefined;
       },
     },
 
     tabsOutsideTitlebar: {
       selectors: ["#navigator-toolbox"].concat(Services.appinfo.OS == "Linux" ? [] : ["#titlebar"]),

--- a/browser/tools/mozscreenshots/mozscreenshots/extension/configurations/Toolbars.jsm
+++ b/browser/tools/mozscreenshots/mozscreenshots/extension/configurations/Toolbars.jsm
@@ -31,17 +31,17 @@ this.Toolbars = {
         let personalToolbar = browserWindow.document.getElementById("PersonalToolbar");
         browserWindow.setToolbarVisibility(personalToolbar, true);
         toggleMenubarIfNecessary(true);
       },
 
       async verifyConfig() {
         let browserWindow = Services.wm.getMostRecentWindow("navigator:browser");
         if (browserWindow.fullScreen) {
-          return Promise.reject("The bookmark toolbar and menubar are not shown in fullscreen.");
+          return "The bookmark toolbar and menubar are not shown in fullscreen.";
         }
         return undefined;
       },
     },
 
   },
 };