Bug 1278649 - Add code coverage to xpcshell tests. r=chmanchester draft
authorGreg Mierzwinski <gmierz2@outlook.com>
Wed, 08 Jun 2016 09:41:04 -0400
changeset 381234 639bdea93377e71189be2de8d918bc127a0a0ee4
parent 381233 0e073f5ca38a002d43e92016ee40d686da4a0534
child 523928 5334534ae7a17d9fd1c8cd67b5a7586925257be6
push id21440
push userbmo:gmierz2@outlook.com
push dateFri, 24 Jun 2016 22:16:07 +0000
reviewerschmanchester
bugs1278649
milestone50.0a1
Bug 1278649 - Add code coverage to xpcshell tests. r=chmanchester This adds the ability to use the command line flag '--jscov-dir-prefix' to collect javascript code coverage from xpcshell tests and output it into the specified directory as a JSON file. MozReview-Commit-ID: 3MZm73SNChL
testing/mozharness/configs/unittests/linux_unittest.py
testing/mozharness/scripts/desktop_unittest.py
testing/xpcshell/head.js
testing/xpcshell/runxpcshelltests.py
testing/xpcshell/xpcshellcommandline.py
--- a/testing/mozharness/configs/unittests/linux_unittest.py
+++ b/testing/mozharness/configs/unittests/linux_unittest.py
@@ -271,16 +271,21 @@ config = {
             "tests": []
         },
         "xpcshell-addons": {
             "options": ["--xpcshell=%(abs_app_dir)s/" + XPCSHELL_NAME,
                         "--tag=addons",
                         "--manifest=tests/xpcshell/tests/xpcshell.ini"],
             "tests": []
         },
+        "xpcshell-coverage": {
+            "options": ["--xpcshell=%(abs_app_dir)s/" + XPCSHELL_NAME,
+                        "--manifest=tests/xpcshell/tests/xpcshell.ini"],
+            "tests": []
+        },
     },
     "all_cppunittest_suites": {
         "cppunittest": {"tests": ["tests/cppunittest"]}
     },
     "all_gtest_suites": {
         "gtest": []
     },
     "all_jittest_suites": {
--- a/testing/mozharness/scripts/desktop_unittest.py
+++ b/testing/mozharness/scripts/desktop_unittest.py
@@ -381,17 +381,17 @@ class DesktopUnittest(TestingMixin, Merc
 
             # set pluginsPath
             abs_res_plugins_dir = os.path.join(abs_res_dir, 'plugins')
             str_format_values['test_plugin_path'] = abs_res_plugins_dir
 
             if suite_category not in c["suite_definitions"]:
                 self.fatal("'%s' not defined in the config!")
 
-            if suite == 'browser-chrome-coverage':
+            if suite in ('browser-chrome-coverage', 'xpcshell-coverage'):
                 base_cmd.append('--jscov-dir-prefix=%s' %
                                 dirs['abs_blob_upload_dir'])
 
             options = c["suite_definitions"][suite_category]["options"]
             if options:
                 for option in options:
                     option = option % str_format_values
                     if not option.endswith('None'):
--- a/testing/xpcshell/head.js
+++ b/testing/xpcshell/head.js
@@ -497,16 +497,22 @@ function _execute_test() {
 
   // Override idle service by default.
   // Call do_get_idle() to restore the factory and get the service.
   _fakeIdleService.activate();
 
   _PromiseTestUtils.init();
   _PromiseTestUtils.Assert = Assert;
 
+  let coverageCollector = null;
+  if (typeof _JSCOV_DIR === 'string') {
+    let _CoverageCollector = Components.utils.import("resource://testing-common/CoverageUtils.jsm", {}).CoverageCollector;
+    coverageCollector = new _CoverageCollector(_JSCOV_DIR);
+  }
+
   // _HEAD_FILES is dynamically defined by <runxpcshelltests.py>.
   _load_files(_HEAD_FILES);
   // _TEST_FILE is dynamically defined by <runxpcshelltests.py>.
   _load_files(_TEST_FILE);
 
   // Tack Assert.jsm methods to the current scope.
   this.Assert = Assert;
   for (let func in Assert) {
@@ -524,26 +530,35 @@ function _execute_test() {
     // Check if run_test() is defined. If defined, run it.
     // Else, call run_next_test() directly to invoke tests
     // added by add_test() and add_task().
     if (typeof run_test === "function") {
       run_test();
     } else {
       run_next_test();
     }
+
+    if (coverageCollector != null) {
+      coverageCollector.recordTestCoverage(_TEST_FILE);
+    }
+
     do_test_finished("MAIN run_test");
     _do_main();
     _PromiseTestUtils.assertNoUncaughtRejections();
   } catch (e) {
     _passed = false;
     // do_check failures are already logged and set _quit to true and throw
     // NS_ERROR_ABORT. If both of those are true it is likely this exception
     // has already been logged so there is no need to log it again. It's
     // possible that this will mask an NS_ERROR_ABORT that happens after a
     // do_check failure though.
+    if (coverageCollector != null) {
+      coverageCollector.recordTestCoverage(_TEST_FILE);
+    }
+
     if (!_quit || e != Components.results.NS_ERROR_ABORT) {
       let extra = {};
       if (e.fileName) {
         extra.source_file = e.fileName;
         if (e.lineNumber) {
           extra.line_number = e.lineNumber;
         }
       } else {
@@ -552,16 +567,20 @@ function _execute_test() {
       let message = _exception_message(e);
       if (e.stack) {
         extra.stack = _format_stack(e.stack);
       }
       _testLogger.error(message, extra);
     }
   }
 
+  if (coverageCollector != null) {
+    coverageCollector.finalize();
+  }
+
   // _TAIL_FILES is dynamically defined by <runxpcshelltests.py>.
   _load_files(_TAIL_FILES);
 
   // Execute all of our cleanup functions.
   let reportCleanupError = function(ex) {
     let stack, filename;
     if (ex && typeof ex == "object" && "stack" in ex) {
       stack = ex.stack;
@@ -1244,16 +1263,20 @@ function do_load_child_test_harness()
       + "const _HEAD_FILES=" + uneval(_HEAD_FILES) + "; "
       + "const _MOZINFO_JS_PATH=" + uneval(_MOZINFO_JS_PATH) + "; "
       + "const _TAIL_FILES=" + uneval(_TAIL_FILES) + "; "
       + "const _TEST_NAME=" + uneval(_TEST_NAME) + "; "
       // We'll need more magic to get the debugger working in the child
       + "const _JSDEBUGGER_PORT=0; "
       + "const _XPCSHELL_PROCESS='child';";
 
+  if (typeof _JSCOV_DIR === 'string') {
+    command += " const _JSCOV_DIR=" + uneval(_JSCOV_DIR) + ";";
+  }
+
   if (_TESTING_MODULES_DIR) {
     command += " const _TESTING_MODULES_DIR=" + uneval(_TESTING_MODULES_DIR) + ";";
   }
 
   command += " load(_HEAD_JS_PATH);";
   sendCommand(command);
 }
 
--- a/testing/xpcshell/runxpcshelltests.py
+++ b/testing/xpcshell/runxpcshelltests.py
@@ -127,16 +127,17 @@ class XPCShellTestThread(Thread):
         self.profileName = kwargs.get('profileName')
         self.singleFile = kwargs.get('singleFile')
         self.env = copy.deepcopy(kwargs.get('env'))
         self.symbolsPath = kwargs.get('symbolsPath')
         self.logfiles = kwargs.get('logfiles')
         self.xpcshell = kwargs.get('xpcshell')
         self.xpcsRunArgs = kwargs.get('xpcsRunArgs')
         self.failureManifest = kwargs.get('failureManifest')
+        self.jscovdir = kwargs.get('jscovdir')
         self.stack_fixer_function = kwargs.get('stack_fixer_function')
         self._rootTempDir = kwargs.get('tempDir')
 
         self.app_dir_key = app_dir_key
         self.interactive = interactive
         self.verbose = verbose
         self.pStdout = pStdout
         self.pStderr = pStderr
@@ -626,17 +627,24 @@ class XPCShellTestThread(Thread):
         cmdT = self.buildCmdTestFile(path)
 
         args = self.xpcsRunArgs[:]
         if 'debug' in self.test_object:
             args.insert(0, '-d')
 
         # The test name to log
         cmdI = ['-e', 'const _TEST_NAME = "%s"' % name]
-        self.complete_command = cmdH + cmdT + cmdI + args
+
+        # Directory for javascript code coverage output, null by default.
+        cmdC = ['-e', 'const _JSCOV_DIR = null']
+        if self.jscovdir:
+            cmdC = ['-e', 'const _JSCOV_DIR = "%s"' % self.jscovdir.replace('\\', '/')]
+            self.complete_command = cmdH + cmdT + cmdI + cmdC + args
+        else:
+            self.complete_command = cmdH + cmdT + cmdI + args
 
         if self.test_object.get('dmd') == 'true':
             if sys.platform.startswith('linux'):
                 preloadEnvVar = 'LD_PRELOAD'
                 libdmd = os.path.join(self.xrePath, 'libdmd.so')
             elif sys.platform == 'osx' or sys.platform == 'darwin':
                 preloadEnvVar = 'DYLD_INSERT_LIBRARIES'
                 # self.xrePath is <prefix>/Contents/Resources.
@@ -1076,17 +1084,17 @@ class XPCShellTests(object):
                  interactive=False, verbose=False, keepGoing=False, logfiles=True,
                  thisChunk=1, totalChunks=1, debugger=None,
                  debuggerArgs=None, debuggerInteractive=False,
                  profileName=None, mozInfo=None, sequential=False, shuffle=False,
                  testingModulesDir=None, pluginsPath=None,
                  testClass=XPCShellTestThread, failureManifest=None,
                  log=None, stream=None, jsDebugger=False, jsDebuggerPort=0,
                  test_tags=None, dump_tests=None, utility_path=None,
-                 rerun_failures=False, failure_manifest=None, **otherOptions):
+                 rerun_failures=False, failure_manifest=None, jscovdir=None, **otherOptions):
         """Run xpcshell tests.
 
         |xpcshell|, is the xpcshell executable to use to run the tests.
         |xrePath|, if provided, is the path to the XRE to use.
         |appPath|, if provided, is the path to an application directory.
         |symbolsPath|, if provided is the path to a directory containing
           breakpad symbols for processing crashes in tests.
         |manifest|, if provided, is a file containing a list of
@@ -1176,16 +1184,17 @@ class XPCShellTests(object):
         self.totalChunks = totalChunks
         self.thisChunk = thisChunk
         self.profileName = profileName or "xpcshell"
         self.mozInfo = mozInfo
         self.testingModulesDir = testingModulesDir
         self.pluginsPath = pluginsPath
         self.sequential = sequential
         self.failure_manifest = failure_manifest
+        self.jscovdir = jscovdir
 
         self.testCount = 0
         self.passCount = 0
         self.failCount = 0
         self.todoCount = 0
 
         self.setAbsPath()
         self.buildXpcsRunArgs()
@@ -1258,16 +1267,17 @@ class XPCShellTests(object):
             'profileName': self.profileName,
             'singleFile': self.singleFile,
             'env': self.env, # making a copy of this in the testthreads
             'symbolsPath': self.symbolsPath,
             'logfiles': self.logfiles,
             'xpcshell': self.xpcshell,
             'xpcsRunArgs': self.xpcsRunArgs,
             'failureManifest': self.failure_manifest,
+            'jscovdir': self.jscovdir,
             'harness_timeout': self.harness_timeout,
             'stack_fixer_function': self.stack_fixer_function,
         }
 
         if self.sequential:
             # Allow user to kill hung xpcshell subprocess with SIGINT
             # when we are only running tests sequentially.
             signal.signal(signal.SIGINT, markGotSIGINT)
--- a/testing/xpcshell/xpcshellcommandline.py
+++ b/testing/xpcshell/xpcshellcommandline.py
@@ -59,16 +59,20 @@ def add_common_arguments(parser):
                         action="store", type=str, dest="xrePath",
                         # individual scripts will set a sane default
                         default=None,
                         help="absolute path to directory containing XRE (probably xulrunner)")
     parser.add_argument("--symbols-path",
                         action="store", type=str, dest="symbolsPath",
                         default=None,
                         help="absolute path to directory containing breakpad symbols, or the URL of a zip file containing symbols")
+    parser.add_argument("--jscov-dir-prefix",
+                        action="store", type=str, dest="jscovdir",
+                        default=argparse.SUPPRESS,
+                        help="Directory to store per-test javascript line coverage data as json.")
     parser.add_argument("--debugger",
                         action="store", dest="debugger",
                         help="use the given debugger to launch the application")
     parser.add_argument("--debugger-args",
                         action="store", dest="debuggerArgs",
                         help="pass the given args to the debugger _before_ "
                         "the application on the command line")
     parser.add_argument("--debugger-interactive",