Bug 1278649 - Add code coverage to xpcshell tests. r=chmanchester
This adds the ability to use the command line flag '--jscov-dir-prefix' to collect javascript code coverage from xpcshell tests and output it into the specified directory as a JSON file.
MozReview-Commit-ID: 3MZm73SNChL
--- a/testing/mozharness/configs/unittests/linux_unittest.py
+++ b/testing/mozharness/configs/unittests/linux_unittest.py
@@ -271,16 +271,21 @@ config = {
"tests": []
},
"xpcshell-addons": {
"options": ["--xpcshell=%(abs_app_dir)s/" + XPCSHELL_NAME,
"--tag=addons",
"--manifest=tests/xpcshell/tests/xpcshell.ini"],
"tests": []
},
+ "xpcshell-coverage": {
+ "options": ["--xpcshell=%(abs_app_dir)s/" + XPCSHELL_NAME,
+ "--manifest=tests/xpcshell/tests/xpcshell.ini"],
+ "tests": []
+ },
},
"all_cppunittest_suites": {
"cppunittest": {"tests": ["tests/cppunittest"]}
},
"all_gtest_suites": {
"gtest": []
},
"all_jittest_suites": {
--- a/testing/mozharness/scripts/desktop_unittest.py
+++ b/testing/mozharness/scripts/desktop_unittest.py
@@ -381,17 +381,17 @@ class DesktopUnittest(TestingMixin, Merc
# set pluginsPath
abs_res_plugins_dir = os.path.join(abs_res_dir, 'plugins')
str_format_values['test_plugin_path'] = abs_res_plugins_dir
if suite_category not in c["suite_definitions"]:
self.fatal("'%s' not defined in the config!")
- if suite == 'browser-chrome-coverage':
+ if suite in ('browser-chrome-coverage', 'xpcshell-coverage'):
base_cmd.append('--jscov-dir-prefix=%s' %
dirs['abs_blob_upload_dir'])
options = c["suite_definitions"][suite_category]["options"]
if options:
for option in options:
option = option % str_format_values
if not option.endswith('None'):
--- a/testing/xpcshell/head.js
+++ b/testing/xpcshell/head.js
@@ -497,16 +497,22 @@ function _execute_test() {
// Override idle service by default.
// Call do_get_idle() to restore the factory and get the service.
_fakeIdleService.activate();
_PromiseTestUtils.init();
_PromiseTestUtils.Assert = Assert;
+ let coverageCollector = null;
+ if (typeof _JSCOV_DIR === 'string') {
+ let _CoverageCollector = Components.utils.import("resource://testing-common/CoverageUtils.jsm", {}).CoverageCollector;
+ coverageCollector = new _CoverageCollector(_JSCOV_DIR);
+ }
+
// _HEAD_FILES is dynamically defined by <runxpcshelltests.py>.
_load_files(_HEAD_FILES);
// _TEST_FILE is dynamically defined by <runxpcshelltests.py>.
_load_files(_TEST_FILE);
// Tack Assert.jsm methods to the current scope.
this.Assert = Assert;
for (let func in Assert) {
@@ -524,26 +530,35 @@ function _execute_test() {
// Check if run_test() is defined. If defined, run it.
// Else, call run_next_test() directly to invoke tests
// added by add_test() and add_task().
if (typeof run_test === "function") {
run_test();
} else {
run_next_test();
}
+
+ if (coverageCollector != null) {
+ coverageCollector.recordTestCoverage(_TEST_FILE);
+ }
+
do_test_finished("MAIN run_test");
_do_main();
_PromiseTestUtils.assertNoUncaughtRejections();
} catch (e) {
_passed = false;
// do_check failures are already logged and set _quit to true and throw
// NS_ERROR_ABORT. If both of those are true it is likely this exception
// has already been logged so there is no need to log it again. It's
// possible that this will mask an NS_ERROR_ABORT that happens after a
// do_check failure though.
+ if (coverageCollector != null) {
+ coverageCollector.recordTestCoverage(_TEST_FILE);
+ }
+
if (!_quit || e != Components.results.NS_ERROR_ABORT) {
let extra = {};
if (e.fileName) {
extra.source_file = e.fileName;
if (e.lineNumber) {
extra.line_number = e.lineNumber;
}
} else {
@@ -552,16 +567,20 @@ function _execute_test() {
let message = _exception_message(e);
if (e.stack) {
extra.stack = _format_stack(e.stack);
}
_testLogger.error(message, extra);
}
}
+ if (coverageCollector != null) {
+ coverageCollector.finalize();
+ }
+
// _TAIL_FILES is dynamically defined by <runxpcshelltests.py>.
_load_files(_TAIL_FILES);
// Execute all of our cleanup functions.
let reportCleanupError = function(ex) {
let stack, filename;
if (ex && typeof ex == "object" && "stack" in ex) {
stack = ex.stack;
@@ -1244,16 +1263,20 @@ function do_load_child_test_harness()
+ "const _HEAD_FILES=" + uneval(_HEAD_FILES) + "; "
+ "const _MOZINFO_JS_PATH=" + uneval(_MOZINFO_JS_PATH) + "; "
+ "const _TAIL_FILES=" + uneval(_TAIL_FILES) + "; "
+ "const _TEST_NAME=" + uneval(_TEST_NAME) + "; "
// We'll need more magic to get the debugger working in the child
+ "const _JSDEBUGGER_PORT=0; "
+ "const _XPCSHELL_PROCESS='child';";
+ if (typeof _JSCOV_DIR === 'string') {
+ command += " const _JSCOV_DIR=" + uneval(_JSCOV_DIR) + ";";
+ }
+
if (_TESTING_MODULES_DIR) {
command += " const _TESTING_MODULES_DIR=" + uneval(_TESTING_MODULES_DIR) + ";";
}
command += " load(_HEAD_JS_PATH);";
sendCommand(command);
}
--- a/testing/xpcshell/runxpcshelltests.py
+++ b/testing/xpcshell/runxpcshelltests.py
@@ -127,16 +127,17 @@ class XPCShellTestThread(Thread):
self.profileName = kwargs.get('profileName')
self.singleFile = kwargs.get('singleFile')
self.env = copy.deepcopy(kwargs.get('env'))
self.symbolsPath = kwargs.get('symbolsPath')
self.logfiles = kwargs.get('logfiles')
self.xpcshell = kwargs.get('xpcshell')
self.xpcsRunArgs = kwargs.get('xpcsRunArgs')
self.failureManifest = kwargs.get('failureManifest')
+ self.jscovdir = kwargs.get('jscovdir')
self.stack_fixer_function = kwargs.get('stack_fixer_function')
self._rootTempDir = kwargs.get('tempDir')
self.app_dir_key = app_dir_key
self.interactive = interactive
self.verbose = verbose
self.pStdout = pStdout
self.pStderr = pStderr
@@ -626,17 +627,24 @@ class XPCShellTestThread(Thread):
cmdT = self.buildCmdTestFile(path)
args = self.xpcsRunArgs[:]
if 'debug' in self.test_object:
args.insert(0, '-d')
# The test name to log
cmdI = ['-e', 'const _TEST_NAME = "%s"' % name]
- self.complete_command = cmdH + cmdT + cmdI + args
+
+ # Directory for javascript code coverage output, null by default.
+ cmdC = ['-e', 'const _JSCOV_DIR = null']
+ if self.jscovdir:
+ cmdC = ['-e', 'const _JSCOV_DIR = "%s"' % self.jscovdir.replace('\\', '/')]
+ self.complete_command = cmdH + cmdT + cmdI + cmdC + args
+ else:
+ self.complete_command = cmdH + cmdT + cmdI + args
if self.test_object.get('dmd') == 'true':
if sys.platform.startswith('linux'):
preloadEnvVar = 'LD_PRELOAD'
libdmd = os.path.join(self.xrePath, 'libdmd.so')
elif sys.platform == 'osx' or sys.platform == 'darwin':
preloadEnvVar = 'DYLD_INSERT_LIBRARIES'
# self.xrePath is <prefix>/Contents/Resources.
@@ -1076,17 +1084,17 @@ class XPCShellTests(object):
interactive=False, verbose=False, keepGoing=False, logfiles=True,
thisChunk=1, totalChunks=1, debugger=None,
debuggerArgs=None, debuggerInteractive=False,
profileName=None, mozInfo=None, sequential=False, shuffle=False,
testingModulesDir=None, pluginsPath=None,
testClass=XPCShellTestThread, failureManifest=None,
log=None, stream=None, jsDebugger=False, jsDebuggerPort=0,
test_tags=None, dump_tests=None, utility_path=None,
- rerun_failures=False, failure_manifest=None, **otherOptions):
+ rerun_failures=False, failure_manifest=None, jscovdir=None, **otherOptions):
"""Run xpcshell tests.
|xpcshell|, is the xpcshell executable to use to run the tests.
|xrePath|, if provided, is the path to the XRE to use.
|appPath|, if provided, is the path to an application directory.
|symbolsPath|, if provided is the path to a directory containing
breakpad symbols for processing crashes in tests.
|manifest|, if provided, is a file containing a list of
@@ -1176,16 +1184,17 @@ class XPCShellTests(object):
self.totalChunks = totalChunks
self.thisChunk = thisChunk
self.profileName = profileName or "xpcshell"
self.mozInfo = mozInfo
self.testingModulesDir = testingModulesDir
self.pluginsPath = pluginsPath
self.sequential = sequential
self.failure_manifest = failure_manifest
+ self.jscovdir = jscovdir
self.testCount = 0
self.passCount = 0
self.failCount = 0
self.todoCount = 0
self.setAbsPath()
self.buildXpcsRunArgs()
@@ -1258,16 +1267,17 @@ class XPCShellTests(object):
'profileName': self.profileName,
'singleFile': self.singleFile,
'env': self.env, # making a copy of this in the testthreads
'symbolsPath': self.symbolsPath,
'logfiles': self.logfiles,
'xpcshell': self.xpcshell,
'xpcsRunArgs': self.xpcsRunArgs,
'failureManifest': self.failure_manifest,
+ 'jscovdir': self.jscovdir,
'harness_timeout': self.harness_timeout,
'stack_fixer_function': self.stack_fixer_function,
}
if self.sequential:
# Allow user to kill hung xpcshell subprocess with SIGINT
# when we are only running tests sequentially.
signal.signal(signal.SIGINT, markGotSIGINT)
--- a/testing/xpcshell/xpcshellcommandline.py
+++ b/testing/xpcshell/xpcshellcommandline.py
@@ -59,16 +59,20 @@ def add_common_arguments(parser):
action="store", type=str, dest="xrePath",
# individual scripts will set a sane default
default=None,
help="absolute path to directory containing XRE (probably xulrunner)")
parser.add_argument("--symbols-path",
action="store", type=str, dest="symbolsPath",
default=None,
help="absolute path to directory containing breakpad symbols, or the URL of a zip file containing symbols")
+ parser.add_argument("--jscov-dir-prefix",
+ action="store", type=str, dest="jscovdir",
+ default=argparse.SUPPRESS,
+ help="Directory to store per-test javascript line coverage data as json.")
parser.add_argument("--debugger",
action="store", dest="debugger",
help="use the given debugger to launch the application")
parser.add_argument("--debugger-args",
action="store", dest="debuggerArgs",
help="pass the given args to the debugger _before_ "
"the application on the command line")
parser.add_argument("--debugger-interactive",