Bug 1278402 - Add linux64-ccov and linux64-jsdcov as code coverage build platforms to taskcluster. r?dustin,jmaher
These builds can be run on taskcluster to obtain per-test (JSDebugger) code coverage with the linux64-jsdcov build and overall (GCOV) code coverage with the linux64-ccov build. The linux64-jsdcov build also needed to have leak checking disabled for debug mode.
MozReview-Commit-ID: ASgrU2X7RQV
--- a/browser/config/mozconfigs/linux64/code-coverage
+++ b/browser/config/mozconfigs/linux64/code-coverage
@@ -1,6 +1,13 @@
. "$topsrcdir/browser/config/mozconfigs/linux64/nightly"
+TOOLTOOL_DIR=${TOOLTOOL_DIR:-$topsrcdir}
+
+ac_add_options --disable-install-strip
+ac_add_options --disable-jemalloc
+ac_add_options --disable-crashreporter
+ac_add_options --disable-elf-hack
+
MOZ_CODE_COVERAGE=1
export CFLAGS="-fprofile-arcs -ftest-coverage"
export CXXFLAGS="-fprofile-arcs -ftest-coverage"
-export LDFLAGS="-fprofile-arcs -ftest-coverage -lgcov"
+export LDFLAGS="-fprofile-arcs -ftest-coverage -lgcov -L$TOOLTOOL_DIR/gtk3/usr/local/lib"
--- a/taskcluster/ci/build/linux.yml
+++ b/taskcluster/ci/build/linux.yml
@@ -169,8 +169,58 @@ linux64-asan/debug:
- builds/releng_base_linux_64_builds.py
- balrog/production.py
script: "mozharness/scripts/fx_desktop_build.py"
secrets: true
custom-build-variant-cfg: asan-tc-and-debug
tooltool-downloads: public
need-xvfb: true
+linux64-jsdcov/opt:
+ description: "Linux64-JSDCov Opt"
+ index:
+ product: firefox
+ job-name: linux64-jsdcov-opt
+ treeherder:
+ platform: linux64/jsdcov
+ symbol: tc(B)
+ tier: 2
+ run-on-projects: [ ]
+ worker-type: aws-provisioner-v1/gecko-{level}-b-linux
+ worker:
+ implementation: docker-worker
+ max-run-time: 36000
+ run:
+ using: mozharness
+ actions: [get-secrets build check-test generate-build-stats update]
+ config:
+ - builds/releng_base_linux_64_builds.py
+ - balrog/production.py
+ script: "mozharness/scripts/fx_desktop_build.py"
+ secrets: true
+ tooltool-downloads: public
+ need-xvfb: true
+
+linux64-ccov/opt:
+ description: "Linux64-CCov Opt"
+ index:
+ product: firefox
+ job-name: linux64-ccov-opt
+ treeherder:
+ platform: linux64/ccov
+ symbol: tc(B)
+ tier: 2
+ run-on-projects: [ ]
+ worker-type: aws-provisioner-v1/gecko-{level}-b-linux
+ worker:
+ implementation: docker-worker
+ max-run-time: 36000
+ run:
+ using: mozharness
+ actions: [get-secrets build check-test generate-build-stats update]
+ config:
+ - builds/releng_base_linux_64_builds.py
+ - balrog/production.py
+ script: "mozharness/scripts/fx_desktop_build.py"
+ secrets: true
+ custom-build-variant-cfg: code-coverage
+ tooltool-downloads: public
+ need-xvfb: true
\ No newline at end of file
--- a/taskcluster/ci/desktop-test/test-platforms.yml
+++ b/taskcluster/ci/desktop-test/test-platforms.yml
@@ -20,8 +20,14 @@ linux64/opt:
# TODO: use 'pgo' and 'asan' labels here, instead of -pgo/opt
linux64-pgo/opt:
build-platform: linux64-pgo/opt
test-set: all-tests
linux64-asan/opt:
build-platform: linux64-asan/opt
test-set: asan-tests
+linux64-ccov/opt:
+ build-platform: linux64-ccov/opt
+ test-set: ccov-code-coverage-tests
+linux64-jsdcov/opt:
+ build-platform: linux64-jsdcov/opt
+ test-set: jsdcov-code-coverage-tests
\ No newline at end of file
--- a/taskcluster/ci/desktop-test/test-sets.yml
+++ b/taskcluster/ci/desktop-test/test-sets.yml
@@ -51,8 +51,14 @@ asan-tests:
- mochitest-devtools-chrome
- mochitest-gpu
- mochitest-jetpack
- mochitest-media
- mochitest-webgl
- reftest
- reftest-no-accel
- xpcshell
+
+ccov-code-coverage-tests:
+ - mochitest-browser-chrome
+
+jsdcov-code-coverage-tests:
+ - mochitest-browser-chrome
\ No newline at end of file
--- a/taskcluster/ci/desktop-test/tests.yml
+++ b/taskcluster/ci/desktop-test/tests.yml
@@ -185,36 +185,65 @@ mochitest-a11y:
default:
- mozharness/configs/unittests/linux_unittest.py
- mozharness/configs/remove_executables.py
extra-options:
- --mochitest-suite=a11y
mochitest-browser-chrome:
description: "Mochitest browser-chrome run"
- suite: mochitest/browser-chrome-chunked
+ suite:
+ by-test-platform:
+ linux64-jsdcov/opt: mochitest/browser-chrome-coverage
+ default: mochitest/browser-chrome-chunked
treeherder-symbol: tc-M(bc)
loopback-video: true
- chunks: 7
+ run-on-projects:
+ by-test-platform:
+ linux64-jsdcov/opt: []
+ linux64-ccov/opt: []
+ default: ['all']
+ chunks:
+ by-test-platform:
+ linux64-jsdcov/opt: 35
+ default: 7
+ e10s:
+ by-test-platform:
+ linux64-jsdcov/opt: false
+ linux64-ccov/opt: false
+ default: both
max-run-time:
by-test-platform:
+ linux64-jsdcov/opt: 7200
+ linux64-ccov/opt: 7200
linux64/debug: 5400
default: 3600
mozharness:
script: mozharness/scripts/desktop_unittest.py
no-read-buildbot-config: true
config:
by-test-platform:
default:
- mozharness/configs/unittests/linux_unittest.py
- mozharness/configs/remove_executables.py
extra-options:
- - --mochitest-suite=browser-chrome-chunked
+ by-test-platform:
+ linux64-jsdcov/opt:
+ - --mochitest-suite=browser-chrome-coverage
+ linux64-ccov/opt:
+ - --mochitest-suite=browser-chrome-chunked
+ - --code-coverage
+ default:
+ - --mochitest-suite=browser-chrome-chunked
# Bug 1281241: migrating to m3.large instances
- instance-size: legacy
+ instance-size:
+ by-test-platform:
+ linux64-jsdcov/opt: xlarge
+ linux64-ccov/opt: xlarge
+ default: legacy
allow-software-gl-layers: false
mochitest-chrome:
description: "Mochitest chrome run"
suite: mochitest/chrome
treeherder-symbol: tc-M(c)
loopback-video: true
chunks: 3
--- a/taskcluster/taskgraph/transforms/gecko_v2_whitelist.py
+++ b/taskcluster/taskgraph/transforms/gecko_v2_whitelist.py
@@ -25,17 +25,19 @@ JOB_NAME_WHITELIST = set([
'android-x86-opt',
'aries-debug',
'aries-eng-opt',
'browser-haz-debug',
'linux32-l10n-opt',
'linux64-artifact-opt',
'linux64-asan-debug',
'linux64-asan-opt',
+ 'linux64-ccov-opt',
'linux64-debug',
+ 'linux64-jsdcov-opt',
'linux64-l10n-opt',
'linux64-opt',
'linux64-pgo-opt',
'linux64-st-an-opt',
'linux64-valgrind-opt',
'linux-debug',
'linux-opt',
'macosx64-debug',
--- a/taskcluster/taskgraph/transforms/tests/all_kinds.py
+++ b/taskcluster/taskgraph/transforms/tests/all_kinds.py
@@ -77,24 +77,31 @@ def set_download_symbols(config, tests):
@transforms.add
def resolve_keyed_by(config, tests):
"""Resolve fields that can be keyed by platform, etc."""
fields = [
'instance-size',
'max-run-time',
'chunks',
+ 'e10s',
+ 'suite',
+ 'run-on-projects',
]
for test in tests:
for field in fields:
test[field] = get_keyed_by(item=test, field=field, item_name=test['test-name'])
test['mozharness']['config'] = get_keyed_by(item=test,
field='mozharness',
subfield='config',
item_name=test['test-name'])
+ test['mozharness']['extra-options'] = get_keyed_by(item=test,
+ field='mozharness',
+ subfield='extra-options',
+ item_name=test['test-name'])
yield test
@transforms.add
def split_chunks(config, tests):
"""Based on the 'chunks' key, split tests up into chunks by duplicating
them and assigning 'this-chunk' appropriately and updating the treeherder
symbol."""
--- a/taskcluster/taskgraph/transforms/tests/make_task_description.py
+++ b/taskcluster/taskgraph/transforms/tests/make_task_description.py
@@ -77,16 +77,17 @@ def make_task_description(config, tests)
taskdesc = {}
taskdesc['label'] = label
taskdesc['description'] = test['description']
taskdesc['attributes'] = attributes
taskdesc['dependencies'] = {'build': build_label}
taskdesc['deadline-after'] = '1 day'
taskdesc['expires-after'] = test['expires-after']
taskdesc['routes'] = []
+ taskdesc['run-on-projects'] = test.get('run-on-projects', ['all'])
taskdesc['scopes'] = []
taskdesc['extra'] = {
'chunks': {
'current': test['this-chunk'],
'total': test['chunks'],
},
'suite': {
'name': suite,
--- a/taskcluster/taskgraph/transforms/tests/test_description.py
+++ b/taskcluster/taskgraph/transforms/tests/test_description.py
@@ -27,17 +27,20 @@ from voluptuous import (
# See the warnings in taskcluster/docs/how-tos.rst
#
# *****WARNING*****
test_description_schema = Schema({
# description of the suite, for the task metadata
'description': basestring,
# test suite name, or <suite>/<flavor>
- 'suite': basestring,
+ Required('suite'): Any(
+ basestring,
+ {'by-test-platform': {basestring: basestring}},
+ ),
# the name by which this test suite is addressed in try syntax; defaults to
# the test-name
Optional('unittest-try-name'): basestring,
# the symbol, or group(symbol), under which this task should appear in
# treeherder.
'treeherder-symbol': basestring,
@@ -46,16 +49,24 @@ test_description_schema = Schema({
# this is the same as build-platform, and that is the default, but in
# practice it's not always a match.
Optional('treeherder-machine-platform'): basestring,
# attributes to appear in the resulting task (later transforms will add the
# common attributes)
Optional('attributes'): {basestring: object},
+ # The `run_on_projects` attribute, defaulting to "all". This dictates the
+ # projects on which this task should be included in the target task set.
+ # See the attributes documentation for details.
+ Optional('run-on-projects', default=['all']): Any(
+ [basestring],
+ {'by-test-platform': {basestring: [basestring]}},
+ ),
+
# the sheriffing tier for this task (default: set based on test platform)
Optional('tier'): int,
# number of chunks to create for this task. This can be keyed by test
# platform by passing a dictionary in the `by-test-platform` key. If the
# test platform is not found, the key 'default' will be tried.
Required('chunks', default=1): Any(
int,
@@ -133,17 +144,20 @@ test_description_schema = Schema({
{'by-test-platform': {basestring: [basestring]}},
),
# any additional actions to pass to the mozharness command
Optional('actions'): [basestring],
# additional command-line options for mozharness, beyond those
# automatically added
- Required('extra-options', default=[]): [basestring],
+ Required('extra-options', default=[]): Any(
+ [basestring],
+ {'by-test-platform': {basestring: [basestring]}},
+ ),
# the artifact name (including path) to test on the build task; this is
# generally set in a per-kind transformation
Optional('build-artifact-name'): basestring,
# If true, tooltool downloads will be enabled via relengAPIProxy.
Required('tooltool-downloads', default=False): bool,
--- a/testing/mochitest/runtests.py
+++ b/testing/mochitest/runtests.py
@@ -2375,19 +2375,22 @@ class MochitestDesktop(MochitestBase):
# then again to actually run mochitest
if options.timeout:
timeout = options.timeout + 30
elif options.debugger or not options.autorun:
timeout = None
else:
timeout = 330.0 # default JS harness timeout is 300 seconds
- # detect shutdown leaks for m-bc runs
- detectShutdownLeaks = mozinfo.info[
- "debug"] and options.flavor == 'browser'
+ # Detect shutdown leaks for m-bc runs if
+ # code coverage is not enabled.
+ detectShutdownLeaks = False
+ if options.jscov_dir_prefix is None:
+ detectShutdownLeaks = mozinfo.info[
+ "debug"] and options.flavor == 'browser'
self.start_script_args.append(self.normflavor(options.flavor))
marionette_args = {
'symbols_path': options.symbolsPath,
'socket_timeout': options.marionette_socket_timeout,
'port_timeout': options.marionette_port_timeout,
}
@@ -2422,20 +2425,30 @@ class MochitestDesktop(MochitestBase):
except:
traceback.print_exc()
self.log.error(
"Automation Error: Received unexpected exception while running application\n")
status = 1
finally:
self.stopServers()
+ ignoreMissingLeaks = options.ignoreMissingLeaks
+ leakThresholds = options.leakThresholds
+
+ # Stop leak detection if m-bc code coverage is enabled
+ # by maxing out the leak threshold for all processes.
+ if options.jscov_dir_prefix:
+ for processType in leakThresholds:
+ ignoreMissingLeaks.append(processType)
+ leakThresholds[processType] = sys.maxsize
+
mozleak.process_leak_log(
self.leak_report_file,
- leak_thresholds=options.leakThresholds,
- ignore_missing_leaks=options.ignoreMissingLeaks,
+ leak_thresholds=leakThresholds,
+ ignore_missing_leaks=ignoreMissingLeaks,
log=self.log,
stack_fixer=get_stack_fixer_function(options.utilityPath,
options.symbolsPath),
)
self.log.info("runtests.py | Running tests: end.")
if self.manifest is not None:
--- a/testing/mozharness/configs/builds/releng_sub_linux_configs/64_code_coverage.py
+++ b/testing/mozharness/configs/builds/releng_sub_linux_configs/64_code_coverage.py
@@ -10,17 +10,17 @@ config = {
'setup-mock',
'build',
'upload-files',
'sendchange',
'check-test',
# 'generate-build-stats',
'update', # decided by query_is_nightly()
],
- 'stage_platform': 'linux64-cc',
+ 'stage_platform': 'linux64-ccov',
'platform_supports_post_upload_to_latest': False,
'enable_signing': False,
'enable_talos_sendchange': False,
'enable_count_ctors': False,
#### 64 bit build specific #####
'env': {
'MOZBUILD_STATE_PATH': os.path.join(os.getcwd(), '.mozbuild'),
'MOZ_AUTOMATION': '1',
--- a/testing/mozharness/mozharness/mozilla/testing/codecoverage.py
+++ b/testing/mozharness/mozharness/mozilla/testing/codecoverage.py
@@ -31,17 +31,17 @@ class CodeCoverageMixin(object):
@property
def code_coverage_enabled(self):
try:
if self.config.get('code_coverage'):
return True
# XXX workaround because bug 1110465 is hard
- return self.buildbot_config['properties']['stage_platform'] in ('linux64-cc',)
+ return self.buildbot_config['properties']['stage_platform'] in ('linux64-ccov',)
except (AttributeError, KeyError, TypeError):
return False
@PreScriptAction('run-tests')
def _set_gcov_prefix(self, action):
if not self.code_coverage_enabled:
return