--- a/testing/mochitest/browser-test.js
+++ b/testing/mochitest/browser-test.js
@@ -160,44 +160,41 @@ function Tester(aTests, structuredLogger
SIMPLETEST_OVERRIDES.forEach(m => {
this.SimpleTestOriginal[m] = this.SimpleTest[m];
});
this._coverageCollector = null;
this._toleratedUncaughtRejections = null;
this._uncaughtErrorObserver = ({message, date, fileName, stack, lineNumber}) => {
- let error = message;
+ let ex = message;
if (fileName || lineNumber) {
- error = {
+ ex = {
fileName: fileName,
lineNumber: lineNumber,
message: message,
toString: function() {
return message;
}
};
}
// We may have a whitelist of rejections we wish to tolerate.
- let tolerate = this._toleratedUncaughtRejections &&
+ let pass = this._toleratedUncaughtRejections &&
this._toleratedUncaughtRejections.indexOf(message) != -1;
let name = "A promise chain failed to handle a rejection: ";
- if (tolerate) {
+ if (pass) {
name = "WARNING: (PLEASE FIX THIS AS PART OF BUG 1077403) " + name;
}
- this.currentTest.addResult(
- new testResult(
- /*success*/tolerate,
- /*name*/name,
- /*error*/error,
- /*known*/tolerate,
- /*stack*/stack));
- };
+ this.currentTest.addResult(new testResult({
+ pass, name, ex, todo: pass, stack,
+ allowFailure: this.currentTest.allowFailure,
+ }));
+ };
}
Tester.prototype = {
EventUtils: {},
SimpleTest: {},
Task: null,
ContentTask: null,
ExtensionTestUtils: null,
Assert: null,
@@ -276,19 +273,21 @@ Tester.prototype = {
let baseMsg = timedOut ? "Found a {elt} after previous test timed out"
: this.currentTest ? "Found an unexpected {elt} at the end of test run"
: "Found an unexpected {elt}";
// Remove stale tabs
if (this.currentTest && window.gBrowser && gBrowser.tabs.length > 1) {
while (gBrowser.tabs.length > 1) {
let lastTab = gBrowser.tabContainer.lastChild;
- let msg = baseMsg.replace("{elt}", "tab") +
- ": " + lastTab.linkedBrowser.currentURI.spec;
- this.currentTest.addResult(new testResult(false, msg, "", false));
+ this.currentTest.addResult(new testResult({
+ name: baseMsg.replace("{elt}", "tab") + ": " +
+ lastTab.linkedBrowser.currentURI.spec,
+ allowFailure: this.currentTest.allowFailure,
+ }));
gBrowser.removeTab(lastTab);
}
}
// Replace the last tab with a fresh one
if (window.gBrowser) {
let newTab = gBrowser.addTab("about:blank", { skipAnimation: true });
gBrowser.removeTab(gBrowser.selectedTab, { skipPermitUnload: true });
@@ -310,17 +309,20 @@ Tester.prototype = {
break;
case null:
type = "unknown window with document URI: " + win.document.documentURI +
" and title: " + win.document.title;
break;
}
let msg = baseMsg.replace("{elt}", type);
if (this.currentTest) {
- this.currentTest.addResult(new testResult(false, msg, "", false));
+ this.currentTest.addResult(new testResult({
+ name: msg,
+ allowFailure: this.currentTest.allowFailure,
+ }));
} else {
if (!createdFakeTestForLogging) {
createdFakeTestForLogging = true;
this.structuredLogger.testStart("browser-test.js");
}
this.failuresFromInitialWindowState++;
this.structuredLogger.testStatus("browser-test.js",
msg, "FAIL", false, "");
@@ -429,55 +431,69 @@ Tester.prototype = {
// next one.
let testScope = this.currentTest.scope;
while (testScope.__cleanupFunctions.length > 0) {
let func = testScope.__cleanupFunctions.shift();
try {
yield func.apply(testScope);
}
catch (ex) {
- this.currentTest.addResult(new testResult(false, "Cleanup function threw an exception", ex, false));
+ this.currentTest.addResult(new testResult({
+ name: "Cleanup function threw an exception",
+ ex,
+ allowFailure: this.currentTest.allowFailure,
+ }));
}
}
if (this.currentTest.passCount === 0 &&
this.currentTest.failCount === 0 &&
this.currentTest.todoCount === 0) {
- this.currentTest.addResult(new testResult(false, "This test contains no passes, no fails and no todos. Maybe it threw a silent exception? Make sure you use waitForExplicitFinish() if you need it.", "", false));
- }
-
- if (testScope.__expected == 'fail' && testScope.__num_failed <= 0) {
- this.currentTest.addResult(new testResult(false, "We expected at least one assertion to fail because this test file was marked as fail-if in the manifest!", "", true));
+ this.currentTest.addResult(new testResult({
+ name: "This test contains no passes, no fails and no todos. Maybe" +
+ " it threw a silent exception? Make sure you use" +
+ " waitForExplicitFinish() if you need it.",
+ }));
}
this.Promise.Debugging.flushUncaughtErrors();
let winUtils = window.QueryInterface(Ci.nsIInterfaceRequestor)
.getInterface(Ci.nsIDOMWindowUtils);
if (winUtils.isTestControllingRefreshes) {
- this.currentTest.addResult(new testResult(false, "test left refresh driver under test control", "", false));
+ this.currentTest.addResult(new testResult({
+ name: "test left refresh driver under test control",
+ }));
winUtils.restoreNormalRefresh();
}
if (this.SimpleTest.isExpectingUncaughtException()) {
- this.currentTest.addResult(new testResult(false, "expectUncaughtException was called but no uncaught exception was detected!", "", false));
+ this.currentTest.addResult(new testResult({
+ name: "expectUncaughtException was called but no uncaught" +
+ " exception was detected!",
+ allowFailure: this.currentTest.allowFailure,
+ }));
}
Object.keys(window).forEach(function (prop) {
if (parseInt(prop) == prop) {
// This is a string which when parsed as an integer and then
// stringified gives the original string. As in, this is in fact a
// string representation of an integer, so an index into
// window.frames. Skip those.
return;
}
if (this._globalProperties.indexOf(prop) == -1) {
this._globalProperties.push(prop);
- if (this._globalPropertyWhitelist.indexOf(prop) == -1)
- this.currentTest.addResult(new testResult(false, "test left unexpected property on window: " + prop, "", false));
+ if (this._globalPropertyWhitelist.indexOf(prop) == -1) {
+ this.currentTest.addResult(new testResult({
+ name: "test left unexpected property on window: " + prop,
+ allowFailure: this.currentTest.allowFailure,
+ }));
+ }
}
}, this);
// Clear document.popupNode. The test could have set it to a custom value
// for its own purposes, nulling it out it will go back to the default
// behavior of returning the last opened popup.
document.popupNode = null;
@@ -497,52 +513,82 @@ Tester.prototype = {
this.structuredLogger.info(msg);
}
}
}
}
// Notify a long running test problem if it didn't end up in a timeout.
if (this.currentTest.unexpectedTimeouts && !this.currentTest.timedOut) {
- let msg = "This test exceeded the timeout threshold. It should be " +
- "rewritten or split up. If that's not possible, use " +
- "requestLongerTimeout(N), but only as a last resort.";
- this.currentTest.addResult(new testResult(false, msg, "", false));
+ this.currentTest.addResult(new testResult({
+ name: "This test exceeded the timeout threshold. It should be" +
+ " rewritten or split up. If that's not possible, use" +
+ " requestLongerTimeout(N), but only as a last resort.",
+ }));
}
// If we're in a debug build, check assertion counts. This code
// is similar to the code in TestRunner.testUnloaded in
// TestRunner.js used for all other types of mochitests.
let debugsvc = Cc["@mozilla.org/xpcom/debug;1"].getService(Ci.nsIDebug2);
if (debugsvc.isDebugBuild) {
let newAssertionCount = debugsvc.assertionCount;
let numAsserts = newAssertionCount - this.lastAssertionCount;
this.lastAssertionCount = newAssertionCount;
let max = testScope.__expectedMaxAsserts;
let min = testScope.__expectedMinAsserts;
if (numAsserts > max) {
- let msg = "Assertion count " + numAsserts +
- " is greater than expected range " +
- min + "-" + max + " assertions.";
- // TEST-UNEXPECTED-FAIL (TEMPORARILY TEST-KNOWN-FAIL)
- //this.currentTest.addResult(new testResult(false, msg, "", false));
- this.currentTest.addResult(new testResult(true, msg, "", true));
+ // TEST-UNEXPECTED-FAIL
+ this.currentTest.addResult(new testResult({
+ name: "Assertion count " + numAsserts +
+ " is greater than expected range " +
+ min + "-" + max + " assertions.",
+ pass: true, // TEMPORARILY TEST-KNOWN-FAIL
+ todo: true,
+ allowFailure: this.currentTest.allowFailure,
+ }));
} else if (numAsserts < min) {
- let msg = "Assertion count " + numAsserts +
- " is less than expected range " +
- min + "-" + max + " assertions.";
// TEST-UNEXPECTED-PASS
- this.currentTest.addResult(new testResult(false, msg, "", true));
+ this.currentTest.addResult(new testResult({
+ name: "Assertion count " + numAsserts +
+ " is less than expected range " +
+ min + "-" + max + " assertions.",
+ todo: true,
+ allowFailure: this.currentTest.allowFailure,
+ }));
} else if (numAsserts > 0) {
- let msg = "Assertion count " + numAsserts +
- " is within expected range " +
- min + "-" + max + " assertions.";
// TEST-KNOWN-FAIL
- this.currentTest.addResult(new testResult(true, msg, "", true));
+ this.currentTest.addResult(new testResult({
+ name: "Assertion count " + numAsserts +
+ " is within expected range " +
+ min + "-" + max + " assertions.",
+ pass: true,
+ todo: true,
+ allowFailure: this.currentTest.allowFailure,
+ }));
+ }
+ }
+
+ if (this.currentTest.allowFailure) {
+ if (this.currentTest.expectedAllowedFailureCount) {
+ this.currentTest.addResult(new testResult({
+ name: "Expected " +
+ this.currentTest.expectedAllowedFailureCount +
+ " failures in this file, got " +
+ this.currentTest.allowedFailureCount + ".",
+ pass: this.currentTest.expectedAllowedFailureCount ==
+ this.currentTest.allowedFailureCount,
+ }));
+ } else if (this.currentTest.allowedFailureCount == 0) {
+ this.currentTest.addResult(new testResult({
+ name: "We expect at least one assertion to fail because this" +
+ " test file is marked as fail-if in the manifest.",
+ todo: true,
+ }));
}
}
// Dump memory stats for main thread.
if (Cc["@mozilla.org/xre/runtime;1"]
.getService(Ci.nsIXULRuntime)
.processType == Ci.nsIXULRuntime.PROCESS_TYPE_DEFAULT)
{
@@ -684,23 +730,27 @@ Tester.prototype = {
this.currentTest.scope.gTestPath = this.currentTest.path;
this.currentTest.scope.Task = this.Task;
this.currentTest.scope.ContentTask = this.ContentTask;
this.currentTest.scope.BrowserTestUtils = this.BrowserTestUtils;
this.currentTest.scope.TestUtils = this.TestUtils;
this.currentTest.scope.ExtensionTestUtils = this.ExtensionTestUtils;
// Pass a custom report function for mochitest style reporting.
this.currentTest.scope.Assert = new this.Assert(function(err, message, stack) {
- let res;
- if (err) {
- res = new testResult(false, err.message, err.stack, false, err.stack);
- } else {
- res = new testResult(true, message, "", false, stack);
- }
- currentTest.addResult(res);
+ currentTest.addResult(new testResult(err ? {
+ name: err.message,
+ ex: err.stack,
+ stack: err.stack,
+ allowFailure: currentTest.allowFailure,
+ } : {
+ name: message,
+ pass: true,
+ stack,
+ allowFailure: currentTest.allowFailure,
+ }));
});
this.ContentTask.setTestScope(currentScope);
// Allow Assert.jsm methods to be tacked to the current scope.
this.currentTest.scope.export_assertions = function() {
for (let func in this.Assert) {
this[func] = this.Assert[func].bind(this.Assert);
@@ -723,17 +773,20 @@ Tester.prototype = {
var headPath = currentTestDirPath + "/head.js";
try {
this._scriptLoader.loadSubScript(headPath, this.currentTest.scope);
} catch (ex) {
// Ignore if no head.js exists, but report all other errors. Note this
// will also ignore an existing head.js attempting to import a missing
// module - see bug 755558 for why this strategy is preferred anyway.
if (!/^Error opening input stream/.test(ex.toString())) {
- this.currentTest.addResult(new testResult(false, "head.js import threw an exception", ex, false));
+ this.currentTest.addResult(new testResult({
+ name: "head.js import threw an exception",
+ ex,
+ }));
}
}
// Import the test script.
try {
this._scriptLoader.loadSubScript(this.currentTest.path,
this.currentTest.scope);
this.Promise.Debugging.flushUncaughtErrors();
@@ -747,36 +800,42 @@ Tester.prototype = {
let Promise = this.Promise;
this.Task.spawn(function*() {
let task;
while ((task = this.__tasks.shift())) {
this.SimpleTest.info("Entering test " + task.name);
try {
yield task();
} catch (ex) {
- let isExpected = !!this.SimpleTest.isExpectingUncaughtException();
- let stack = (typeof ex == "object" && "stack" in ex)?ex.stack:null;
- let name = "Uncaught exception";
- let result = new testResult(isExpected, name, ex, false, stack);
- currentTest.addResult(result);
+ currentTest.addResult(new testResult({
+ name: "Uncaught exception",
+ pass: this.SimpleTest.isExpectingUncaughtException(),
+ ex,
+ stack: (typeof ex == "object" && "stack" in ex) ? ex.stack : null,
+ allowFailure: currentTest.allowFailure,
+ }));
}
Promise.Debugging.flushUncaughtErrors();
this.SimpleTest.info("Leaving test " + task.name);
}
this.finish();
}.bind(currentScope));
} else if (typeof this.currentTest.scope.test == "function") {
this.currentTest.scope.test();
} else {
throw "This test didn't call add_task, nor did it define a generatorTest() function, nor did it define a test() function, so we don't know how to run it.";
}
} catch (ex) {
- let isExpected = !!this.SimpleTest.isExpectingUncaughtException();
if (!this.SimpleTest.isIgnoringAllUncaughtExceptions()) {
- this.currentTest.addResult(new testResult(isExpected, "Exception thrown", ex, false));
+ this.currentTest.addResult(new testResult({
+ name: "Exception thrown",
+ pass: this.SimpleTest.isExpectingUncaughtException(),
+ ex,
+ allowFailure: this.currentTest.allowFailure,
+ }));
this.SimpleTest.expectUncaughtException(false);
} else {
this.currentTest.addResult(new testMessage("Exception thrown: " + ex));
}
this.currentTest.scope.finish();
}
// If the test ran synchronously, move to the next test, otherwise the test
@@ -819,123 +878,141 @@ Tester.prototype = {
const MAX_UNEXPECTED_TIMEOUTS = 10;
if (Date.now() - self.currentTest.lastOutputTime < (gTimeoutSeconds / 2) * 1000 &&
++self.currentTest.unexpectedTimeouts <= MAX_UNEXPECTED_TIMEOUTS) {
self.currentTest.scope.__waitTimer =
setTimeout(timeoutFn, gTimeoutSeconds * 1000);
return;
}
- self.currentTest.addResult(new testResult(false, "Test timed out", null, false));
+ self.currentTest.addResult(new testResult({ name: "Test timed out" }));
self.currentTest.timedOut = true;
self.currentTest.scope.__waitTimer = null;
self.nextTest();
}, gTimeoutSeconds * 1000]);
}
},
QueryInterface: function(aIID) {
if (aIID.equals(Ci.nsIConsoleListener) ||
aIID.equals(Ci.nsISupports))
return this;
throw Components.results.NS_ERROR_NO_INTERFACE;
}
};
-function testResult(aCondition, aName, aDiag, aIsTodo, aStack) {
- this.name = aName;
+/**
+ * Represents the result of one test assertion. This is described with a string
+ * in traditional logging, and has a "status" and "expected" property used in
+ * structured logging. Normally, results are mapped as follows:
+ *
+ * pass: todo: Added to: Described as: Status: Expected:
+ * true false passCount TEST-PASS PASS PASS
+ * true true todoCount TEST-KNOWN-FAIL FAIL FAIL
+ * false false failCount TEST-UNEXPECTED-FAIL FAIL PASS
+ * false true failCount TEST-UNEXPECTED-PASS PASS FAIL
+ *
+ * The "allowFailure" argument indicates that this is one of the assertions that
+ * should be allowed to fail, for example because "fail-if" is true for the
+ * current test file in the manifest. In this case, results are mapped this way:
+ *
+ * pass: todo: Added to: Described as: Status: Expected:
+ * true false passCount TEST-PASS PASS PASS
+ * true true todoCount TEST-KNOWN-FAIL FAIL FAIL
+ * false false todoCount TEST-KNOWN-FAIL FAIL FAIL
+ * false true todoCount TEST-KNOWN-FAIL FAIL FAIL
+ */
+function testResult({ name, pass, todo, ex, stack, allowFailure }) {
+ this.info = false;
+ this.name = name;
this.msg = "";
- this.info = false;
- this.pass = !!aCondition;
- this.todo = aIsTodo;
+ if (allowFailure && !pass) {
+ this.allowedFailure = true;
+ this.pass = true;
+ this.todo = true;
+ } else {
+ this.pass = !!pass;
+ this.todo = todo;
+ }
+
+ this.expected = this.todo ? "FAIL" : "PASS";
if (this.pass) {
- if (aIsTodo) {
- this.status = "FAIL";
- this.expected = "FAIL";
- } else {
- this.status = "PASS";
- this.expected = "PASS";
- }
+ this.status = this.expected;
+ return;
+ }
- } else {
- if (aDiag) {
- if (typeof aDiag == "object" && "fileName" in aDiag) {
- // we have an exception - print filename and linenumber information
- this.msg += "at " + aDiag.fileName + ":" + aDiag.lineNumber + " - ";
- }
- this.msg += String(aDiag);
+ this.status = this.todo ? "PASS" : "FAIL";
+
+ if (ex) {
+ if (typeof ex == "object" && "fileName" in ex) {
+ // we have an exception - print filename and linenumber information
+ this.msg += "at " + ex.fileName + ":" + ex.lineNumber + " - ";
}
- if (aStack) {
- this.msg += "\nStack trace:\n";
- let normalized;
- if (aStack instanceof Components.interfaces.nsIStackFrame) {
- let frames = [];
- for (let frame = aStack; frame; frame = frame.caller) {
- frames.push(frame.filename + ":" + frame.name + ":" + frame.lineNumber);
- }
- normalized = frames.join("\n");
- } else {
- normalized = "" + aStack;
+ this.msg += String(ex);
+ }
+
+ if (stack) {
+ this.msg += "\nStack trace:\n";
+ let normalized;
+ if (stack instanceof Components.interfaces.nsIStackFrame) {
+ let frames = [];
+ for (let frame = stack; frame; frame = frame.caller) {
+ frames.push(frame.filename + ":" + frame.name + ":" + frame.lineNumber);
}
- this.msg += Task.Debugging.generateReadableStack(normalized, " ");
+ normalized = frames.join("\n");
+ } else {
+ normalized = "" + stack;
}
- if (aIsTodo) {
- this.status = "PASS";
- this.expected = "FAIL";
- } else {
- this.status = "FAIL";
- this.expected = "PASS";
- }
+ this.msg += Task.Debugging.generateReadableStack(normalized, " ");
+ }
- if (gConfig.debugOnFailure) {
- // You've hit this line because you requested to break into the
- // debugger upon a testcase failure on your test run.
- debugger;
- }
+ if (gConfig.debugOnFailure) {
+ // You've hit this line because you requested to break into the
+ // debugger upon a testcase failure on your test run.
+ debugger;
}
}
-function testMessage(aName) {
- this.msg = aName || "";
+function testMessage(msg) {
+ this.msg = msg || "";
this.info = true;
}
// Need to be careful adding properties to this object, since its properties
// cannot conflict with global variables used in tests.
function testScope(aTester, aTest, expected) {
this.__tester = aTester;
- this.__expected = expected;
- this.__num_failed = 0;
+
+ aTest.allowFailure = expected == "fail";
var self = this;
- this.ok = function test_ok(condition, name, diag, stack) {
- if (self.__expected == 'fail') {
- if (!condition) {
- self.__num_failed++;
- condition = true;
- }
- }
-
- aTest.addResult(new testResult(condition, name, diag, false,
- stack ? stack : Components.stack.caller));
+ this.ok = function test_ok(condition, name, ex, stack) {
+ aTest.addResult(new testResult({
+ name, pass: condition, ex,
+ stack: stack || Components.stack.caller,
+ allowFailure: aTest.allowFailure,
+ }));
};
this.is = function test_is(a, b, name) {
self.ok(a == b, name, "Got " + a + ", expected " + b, false,
Components.stack.caller);
};
this.isnot = function test_isnot(a, b, name) {
self.ok(a != b, name, "Didn't expect " + a + ", but got it", false,
Components.stack.caller);
};
- this.todo = function test_todo(condition, name, diag, stack) {
- aTest.addResult(new testResult(!condition, name, diag, true,
- stack ? stack : Components.stack.caller));
+ this.todo = function test_todo(condition, name, ex, stack) {
+ aTest.addResult(new testResult({
+ name, pass: !condition, todo: true, ex,
+ stack: stack || Components.stack.caller,
+ allowFailure: aTest.allowFailure,
+ }));
};
this.todo_is = function test_todo_is(a, b, name) {
self.todo(a == b, name, "Got " + a + ", expected " + b,
Components.stack.caller);
};
this.todo_isnot = function test_todo_isnot(a, b, name) {
self.todo(a != b, name, "Didn't expect " + a + ", but got it",
Components.stack.caller);
@@ -1000,18 +1077,19 @@ function testScope(aTester, aTest, expec
if (typeof(min) != "number" || typeof(max) != "number" ||
min < 0 || max < min) {
throw "bad parameter to expectAssertions";
}
self.__expectedMinAsserts = min;
self.__expectedMaxAsserts = max;
};
- this.setExpected = function test_setExpected() {
- self.__expected = 'fail';
+ this.setExpectedFailuresForSelfTest = function test_setExpectedFailuresForSelfTest(expectedAllowedFailureCount) {
+ aTest.allowFailure = true;
+ aTest.expectedAllowedFailureCount = expectedAllowedFailureCount;
};
this.finish = function test_finish() {
self.__done = true;
if (self.__waitTimer) {
self.executeSoon(function() {
if (self.__done && self.__waitTimer) {
clearTimeout(self.__waitTimer);
@@ -1032,17 +1110,16 @@ function testScope(aTester, aTest, expec
testScope.prototype = {
__done: true,
__tasks: null,
__waitTimer: null,
__cleanupFunctions: [],
__timeoutFactor: 1,
__expectedMinAsserts: 0,
__expectedMaxAsserts: 0,
- __expected: 'pass',
EventUtils: {},
SimpleTest: {},
Task: null,
ContentTask: null,
BrowserTestUtils: null,
TestUtils: null,
ExtensionTestUtils: null,