Bug 1278402 - Add linux64-ccov and linux64-jsdcov as code coverage build platforms to taskcluster. r?dustin,jmaher draft
authorGreg Mierzwinski <gmierz2@outlook.com>
Sat, 23 Jul 2016 11:27:49 -0400
changeset 415768 f688bc0dac92e5db61c05985a905f4b3055a5941
parent 415495 62f79d676e0e11b3ad59a5425b3ebb3ec5bbefb5
child 531700 942f4e03cb0cc1da6cd01cd794d808af060f3e87
push id29966
push userbmo:gmierz2@outlook.com
push dateWed, 21 Sep 2016 01:34:10 +0000
reviewersdustin, jmaher
bugs1278402
milestone52.0a1
Bug 1278402 - Add linux64-ccov and linux64-jsdcov as code coverage build platforms to taskcluster. r?dustin,jmaher These builds can be run on taskcluster to obtain per-test (JSDebugger) code coverage with the linux64-jsdcov build and overall (GCOV) code coverage with the linux64-ccov build. The linux64-jsdcov build also needed to have leak checking disabled for debug mode. MozReview-Commit-ID: ASgrU2X7RQV
browser/config/mozconfigs/linux64/code-coverage
taskcluster/ci/build/linux.yml
taskcluster/ci/desktop-test/test-platforms.yml
taskcluster/ci/desktop-test/test-sets.yml
taskcluster/ci/desktop-test/tests.yml
taskcluster/taskgraph/transforms/gecko_v2_whitelist.py
taskcluster/taskgraph/transforms/tests/all_kinds.py
taskcluster/taskgraph/transforms/tests/make_task_description.py
taskcluster/taskgraph/transforms/tests/test_description.py
testing/mochitest/runtests.py
testing/mozharness/configs/builds/releng_sub_linux_configs/64_code_coverage.py
testing/mozharness/mozharness/mozilla/testing/codecoverage.py
--- a/browser/config/mozconfigs/linux64/code-coverage
+++ b/browser/config/mozconfigs/linux64/code-coverage
@@ -1,6 +1,13 @@
 . "$topsrcdir/browser/config/mozconfigs/linux64/nightly"
 
+TOOLTOOL_DIR=${TOOLTOOL_DIR:-$topsrcdir}
+
+ac_add_options --disable-install-strip
+ac_add_options --disable-jemalloc
+ac_add_options --disable-crashreporter
+ac_add_options --disable-elf-hack
+
 MOZ_CODE_COVERAGE=1
 export CFLAGS="-fprofile-arcs -ftest-coverage"
 export CXXFLAGS="-fprofile-arcs -ftest-coverage"
-export LDFLAGS="-fprofile-arcs -ftest-coverage -lgcov"
+export LDFLAGS="-fprofile-arcs -ftest-coverage -lgcov -L$TOOLTOOL_DIR/gtk3/usr/local/lib"
--- a/taskcluster/ci/build/linux.yml
+++ b/taskcluster/ci/build/linux.yml
@@ -169,8 +169,58 @@ linux64-asan/debug:
             - builds/releng_base_linux_64_builds.py
             - balrog/production.py
         script: "mozharness/scripts/fx_desktop_build.py"
         secrets: true
         custom-build-variant-cfg: asan-tc-and-debug
         tooltool-downloads: public
         need-xvfb: true
 
+linux64-jsdcov/opt:
+    description: "Linux64-JSDCov Opt"
+    index:
+        product: firefox
+        job-name: linux64-jsdcov-opt
+    treeherder:
+        platform: linux64/jsdcov
+        symbol: tc(B)
+        tier: 2
+    run-on-projects: [ ]
+    worker-type: aws-provisioner-v1/gecko-{level}-b-linux
+    worker:
+        implementation: docker-worker
+        max-run-time: 36000
+    run:
+        using: mozharness
+        actions: [get-secrets build check-test generate-build-stats update]
+        config:
+            - builds/releng_base_linux_64_builds.py
+            - balrog/production.py
+        script: "mozharness/scripts/fx_desktop_build.py"
+        secrets: true
+        tooltool-downloads: public
+        need-xvfb: true
+
+linux64-ccov/opt:
+    description: "Linux64-CCov Opt"
+    index:
+        product: firefox
+        job-name: linux64-ccov-opt
+    treeherder:
+        platform: linux64/ccov
+        symbol: tc(B)
+        tier: 2
+    run-on-projects: [ ]
+    worker-type: aws-provisioner-v1/gecko-{level}-b-linux
+    worker:
+        implementation: docker-worker
+        max-run-time: 36000
+    run:
+        using: mozharness
+        actions: [get-secrets build check-test generate-build-stats update]
+        config:
+            - builds/releng_base_linux_64_builds.py
+            - balrog/production.py
+        script: "mozharness/scripts/fx_desktop_build.py"
+        secrets: true
+        custom-build-variant-cfg: code-coverage
+        tooltool-downloads: public
+        need-xvfb: true
\ No newline at end of file
--- a/taskcluster/ci/desktop-test/test-platforms.yml
+++ b/taskcluster/ci/desktop-test/test-platforms.yml
@@ -20,8 +20,14 @@ linux64/opt:
 
 # TODO: use 'pgo' and 'asan' labels here, instead of -pgo/opt
 linux64-pgo/opt:
     build-platform: linux64-pgo/opt
     test-set: all-tests
 linux64-asan/opt:
     build-platform: linux64-asan/opt
     test-set: asan-tests
+linux64-ccov/opt:
+    build-platform: linux64-ccov/opt
+    test-set: ccov-code-coverage-tests
+linux64-jsdcov/opt:
+    build-platform: linux64-jsdcov/opt
+    test-set: jsdcov-code-coverage-tests
\ No newline at end of file
--- a/taskcluster/ci/desktop-test/test-sets.yml
+++ b/taskcluster/ci/desktop-test/test-sets.yml
@@ -51,8 +51,14 @@ asan-tests:
     - mochitest-devtools-chrome
     - mochitest-gpu
     - mochitest-jetpack
     - mochitest-media
     - mochitest-webgl
     - reftest
     - reftest-no-accel
     - xpcshell
+
+ccov-code-coverage-tests:
+    - mochitest-browser-chrome
+
+jsdcov-code-coverage-tests:
+    - mochitest-browser-chrome
\ No newline at end of file
--- a/taskcluster/ci/desktop-test/tests.yml
+++ b/taskcluster/ci/desktop-test/tests.yml
@@ -185,36 +185,65 @@ mochitest-a11y:
                 default:
                     - mozharness/configs/unittests/linux_unittest.py
                     - mozharness/configs/remove_executables.py
         extra-options:
             - --mochitest-suite=a11y
 
 mochitest-browser-chrome:
     description: "Mochitest browser-chrome run"
-    suite: mochitest/browser-chrome-chunked
+    suite:
+        by-test-platform:
+            linux64-jsdcov/opt: mochitest/browser-chrome-coverage
+            default: mochitest/browser-chrome-chunked
     treeherder-symbol: tc-M(bc)
     loopback-video: true
-    chunks: 7
+    run-on-projects:
+        by-test-platform:
+            linux64-jsdcov/opt: []
+            linux64-ccov/opt: []
+            default: ['all']
+    chunks:
+        by-test-platform:
+            linux64-jsdcov/opt: 35
+            default: 7
+    e10s:
+        by-test-platform:
+            linux64-jsdcov/opt: false
+            linux64-ccov/opt: false
+            default: both
     max-run-time:
         by-test-platform:
+            linux64-jsdcov/opt: 7200
+            linux64-ccov/opt: 7200
             linux64/debug: 5400
             default: 3600
     mozharness:
         script: mozharness/scripts/desktop_unittest.py
         no-read-buildbot-config: true
         config:
             by-test-platform:
                 default:
                     - mozharness/configs/unittests/linux_unittest.py
                     - mozharness/configs/remove_executables.py
         extra-options:
-            - --mochitest-suite=browser-chrome-chunked
+            by-test-platform:
+                linux64-jsdcov/opt:
+                    - --mochitest-suite=browser-chrome-coverage
+                linux64-ccov/opt:
+                    - --mochitest-suite=browser-chrome-chunked
+                    - --code-coverage
+                default:
+                    - --mochitest-suite=browser-chrome-chunked
     # Bug 1281241: migrating to m3.large instances
-    instance-size: legacy
+    instance-size:
+        by-test-platform:
+            linux64-jsdcov/opt: xlarge
+            linux64-ccov/opt: xlarge
+            default: legacy
     allow-software-gl-layers: false
 
 mochitest-chrome:
     description: "Mochitest chrome run"
     suite: mochitest/chrome
     treeherder-symbol: tc-M(c)
     loopback-video: true
     chunks: 3
--- a/taskcluster/taskgraph/transforms/gecko_v2_whitelist.py
+++ b/taskcluster/taskgraph/transforms/gecko_v2_whitelist.py
@@ -25,17 +25,19 @@ JOB_NAME_WHITELIST = set([
     'android-x86-opt',
     'aries-debug',
     'aries-eng-opt',
     'browser-haz-debug',
     'linux32-l10n-opt',
     'linux64-artifact-opt',
     'linux64-asan-debug',
     'linux64-asan-opt',
+    'linux64-ccov-opt',
     'linux64-debug',
+    'linux64-jsdcov-opt',
     'linux64-l10n-opt',
     'linux64-opt',
     'linux64-pgo-opt',
     'linux64-st-an-opt',
     'linux64-valgrind-opt',
     'linux-debug',
     'linux-opt',
     'macosx64-debug',
--- a/taskcluster/taskgraph/transforms/tests/all_kinds.py
+++ b/taskcluster/taskgraph/transforms/tests/all_kinds.py
@@ -77,24 +77,31 @@ def set_download_symbols(config, tests):
 
 @transforms.add
 def resolve_keyed_by(config, tests):
     """Resolve fields that can be keyed by platform, etc."""
     fields = [
         'instance-size',
         'max-run-time',
         'chunks',
+        'e10s',
+        'suite',
+        'run-on-projects',
     ]
     for test in tests:
         for field in fields:
             test[field] = get_keyed_by(item=test, field=field, item_name=test['test-name'])
         test['mozharness']['config'] = get_keyed_by(item=test,
                                                     field='mozharness',
                                                     subfield='config',
                                                     item_name=test['test-name'])
+        test['mozharness']['extra-options'] = get_keyed_by(item=test,
+                                                           field='mozharness',
+                                                           subfield='extra-options',
+                                                           item_name=test['test-name'])
         yield test
 
 
 @transforms.add
 def split_chunks(config, tests):
     """Based on the 'chunks' key, split tests up into chunks by duplicating
     them and assigning 'this-chunk' appropriately and updating the treeherder
     symbol."""
--- a/taskcluster/taskgraph/transforms/tests/make_task_description.py
+++ b/taskcluster/taskgraph/transforms/tests/make_task_description.py
@@ -77,16 +77,17 @@ def make_task_description(config, tests)
         taskdesc = {}
         taskdesc['label'] = label
         taskdesc['description'] = test['description']
         taskdesc['attributes'] = attributes
         taskdesc['dependencies'] = {'build': build_label}
         taskdesc['deadline-after'] = '1 day'
         taskdesc['expires-after'] = test['expires-after']
         taskdesc['routes'] = []
+        taskdesc['run-on-projects'] = test.get('run-on-projects', ['all'])
         taskdesc['scopes'] = []
         taskdesc['extra'] = {
             'chunks': {
                 'current': test['this-chunk'],
                 'total': test['chunks'],
             },
             'suite': {
                 'name': suite,
--- a/taskcluster/taskgraph/transforms/tests/test_description.py
+++ b/taskcluster/taskgraph/transforms/tests/test_description.py
@@ -27,17 +27,20 @@ from voluptuous import (
 # See the warnings in taskcluster/docs/how-tos.rst
 #
 # *****WARNING*****
 test_description_schema = Schema({
     # description of the suite, for the task metadata
     'description': basestring,
 
     # test suite name, or <suite>/<flavor>
-    'suite': basestring,
+    Required('suite'): Any(
+        basestring,
+        {'by-test-platform': {basestring: basestring}},
+    ),
 
     # the name by which this test suite is addressed in try syntax; defaults to
     # the test-name
     Optional('unittest-try-name'): basestring,
 
     # the symbol, or group(symbol), under which this task should appear in
     # treeherder.
     'treeherder-symbol': basestring,
@@ -46,16 +49,24 @@ test_description_schema = Schema({
     # this is the same as build-platform, and that is the default, but in
     # practice it's not always a match.
     Optional('treeherder-machine-platform'): basestring,
 
     # attributes to appear in the resulting task (later transforms will add the
     # common attributes)
     Optional('attributes'): {basestring: object},
 
+    # The `run_on_projects` attribute, defaulting to "all".  This dictates the
+    # projects on which this task should be included in the target task set.
+    # See the attributes documentation for details.
+    Optional('run-on-projects', default=['all']): Any(
+        [basestring],
+        {'by-test-platform': {basestring: [basestring]}},
+    ),
+
     # the sheriffing tier for this task (default: set based on test platform)
     Optional('tier'): int,
 
     # number of chunks to create for this task.  This can be keyed by test
     # platform by passing a dictionary in the `by-test-platform` key.  If the
     # test platform is not found, the key 'default' will be tried.
     Required('chunks', default=1): Any(
         int,
@@ -133,17 +144,20 @@ test_description_schema = Schema({
             {'by-test-platform': {basestring: [basestring]}},
         ),
 
         # any additional actions to pass to the mozharness command
         Optional('actions'): [basestring],
 
         # additional command-line options for mozharness, beyond those
         # automatically added
-        Required('extra-options', default=[]): [basestring],
+        Required('extra-options', default=[]): Any(
+            [basestring],
+            {'by-test-platform': {basestring: [basestring]}},
+        ),
 
         # the artifact name (including path) to test on the build task; this is
         # generally set in a per-kind transformation
         Optional('build-artifact-name'): basestring,
 
         # If true, tooltool downloads will be enabled via relengAPIProxy.
         Required('tooltool-downloads', default=False): bool,
 
--- a/testing/mochitest/runtests.py
+++ b/testing/mochitest/runtests.py
@@ -2375,19 +2375,22 @@ class MochitestDesktop(MochitestBase):
             # then again to actually run mochitest
             if options.timeout:
                 timeout = options.timeout + 30
             elif options.debugger or not options.autorun:
                 timeout = None
             else:
                 timeout = 330.0  # default JS harness timeout is 300 seconds
 
-            # detect shutdown leaks for m-bc runs
-            detectShutdownLeaks = mozinfo.info[
-                "debug"] and options.flavor == 'browser'
+            # Detect shutdown leaks for m-bc runs if
+            # code coverage is not enabled.
+            detectShutdownLeaks = False
+            if options.jscov_dir_prefix is None:
+                detectShutdownLeaks = mozinfo.info[
+                    "debug"] and options.flavor == 'browser'
 
             self.start_script_args.append(self.normflavor(options.flavor))
             marionette_args = {
                 'symbols_path': options.symbolsPath,
                 'socket_timeout': options.marionette_socket_timeout,
                 'port_timeout': options.marionette_port_timeout,
             }
 
@@ -2422,20 +2425,30 @@ class MochitestDesktop(MochitestBase):
         except:
             traceback.print_exc()
             self.log.error(
                 "Automation Error: Received unexpected exception while running application\n")
             status = 1
         finally:
             self.stopServers()
 
+        ignoreMissingLeaks = options.ignoreMissingLeaks
+        leakThresholds = options.leakThresholds
+
+        # Stop leak detection if m-bc code coverage is enabled
+        # by maxing out the leak threshold for all processes.
+        if options.jscov_dir_prefix:
+            for processType in leakThresholds:
+                ignoreMissingLeaks.append(processType)
+                leakThresholds[processType] = sys.maxsize
+
         mozleak.process_leak_log(
             self.leak_report_file,
-            leak_thresholds=options.leakThresholds,
-            ignore_missing_leaks=options.ignoreMissingLeaks,
+            leak_thresholds=leakThresholds,
+            ignore_missing_leaks=ignoreMissingLeaks,
             log=self.log,
             stack_fixer=get_stack_fixer_function(options.utilityPath,
                                                  options.symbolsPath),
         )
 
         self.log.info("runtests.py | Running tests: end.")
 
         if self.manifest is not None:
--- a/testing/mozharness/configs/builds/releng_sub_linux_configs/64_code_coverage.py
+++ b/testing/mozharness/configs/builds/releng_sub_linux_configs/64_code_coverage.py
@@ -10,17 +10,17 @@ config = {
         'setup-mock',
         'build',
         'upload-files',
         'sendchange',
         'check-test',
         # 'generate-build-stats',
         'update',  # decided by query_is_nightly()
     ],
-    'stage_platform': 'linux64-cc',
+    'stage_platform': 'linux64-ccov',
     'platform_supports_post_upload_to_latest': False,
     'enable_signing': False,
     'enable_talos_sendchange': False,
     'enable_count_ctors': False,
     #### 64 bit build specific #####
     'env': {
         'MOZBUILD_STATE_PATH': os.path.join(os.getcwd(), '.mozbuild'),
         'MOZ_AUTOMATION': '1',
--- a/testing/mozharness/mozharness/mozilla/testing/codecoverage.py
+++ b/testing/mozharness/mozharness/mozilla/testing/codecoverage.py
@@ -31,17 +31,17 @@ class CodeCoverageMixin(object):
 
     @property
     def code_coverage_enabled(self):
         try:
             if self.config.get('code_coverage'):
                 return True
 
             # XXX workaround because bug 1110465 is hard
-            return self.buildbot_config['properties']['stage_platform'] in ('linux64-cc',)
+            return self.buildbot_config['properties']['stage_platform'] in ('linux64-ccov',)
         except (AttributeError, KeyError, TypeError):
             return False
 
 
     @PreScriptAction('run-tests')
     def _set_gcov_prefix(self, action):
         if not self.code_coverage_enabled:
             return