Bug 1472236 - Add grcov code coverage builds by using a fetch task. r=marco,dustin
MozReview-Commit-ID: 3PZMNDRqxRM
--- a/taskcluster/ci/fetch/toolchains.yml
+++ b/taskcluster/ci/fetch/toolchains.yml
@@ -188,8 +188,38 @@ mpfr-3.1.5:
run:
using: fetch-url
url: http://www.mpfr.org/mpfr-3.1.5/mpfr-3.1.5.tar.bz2
sha256: ca498c1c7a74dd37a576f353312d1e68d490978de4395fa28f1cbd46a364e658
size: 1279489
gpg-signature:
sig-url: "{url}.asc"
key-path: build/unix/build-gcc/07F3DBBECC1A39605078094D980C197698C3739D.key
+
+grcov-linux-x86_64:
+ description: grcov binary release
+ treeherder:
+ symbol: grcov-linux-x86_64
+ run:
+ using: fetch-url
+ url: https://github.com/mozilla/grcov/releases/download/v0.2.3/grcov-linux-x86_64.tar.bz2
+ sha256: 5ed6ee891e0a1cbf1d84e27c615ec1f844c89f517f29cbec97379ded7c0894cd
+ size: 1858593
+
+grcov-osx-x86_64:
+ description: grcov binary release
+ treeherder:
+ symbol: grcov-osx-x86_64
+ run:
+ using: fetch-url
+ url: https://github.com/mozilla/grcov/releases/download/v0.2.3/grcov-osx-x86_64.tar.bz2
+ sha256: 3db4ef6c0dfaf35d39c9cacf19c60c482dae9413b5fe3bf343fa26667451c893
+ size: 919638
+
+grcov-win-i686:
+ description: grcov binary release
+ treeherder:
+ symbol: grcov-win-i686
+ run:
+ using: fetch-url
+ url: https://github.com/mozilla/grcov/releases/download/v0.2.3/grcov-win-i686.tar.bz2
+ sha256: 87c3c22cd1bb99d9c41b6a6f9e20b6c655a580021776f1a532297f5e8315ad9b
+ size: 919079
--- a/taskcluster/ci/test/kind.yml
+++ b/taskcluster/ci/test/kind.yml
@@ -1,16 +1,18 @@
loader: taskgraph.loader.test:loader
kind-dependencies:
- build
- build-signing
+ - fetch
transforms:
- taskgraph.transforms.tests:transforms
+ - taskgraph.transforms.use_fetches:transforms
- taskgraph.transforms.job:transforms
- taskgraph.transforms.coalesce:transforms
- taskgraph.transforms.task:transforms
# Each stanza in a file pointed to by 'jobs-from' describes a particular test
# suite or sub-suite. These are processed through the transformations described
# above to produce a bunch of tasks. See the schema in
# `taskcluster/taskgraph/transforms/tests.py` for a description of the fields
--- a/taskcluster/taskgraph/loader/test.py
+++ b/taskcluster/taskgraph/loader/test.py
@@ -13,22 +13,16 @@ from ..util.yaml import load_yaml
logger = logging.getLogger(__name__)
def loader(kind, path, config, params, loaded_tasks):
"""
Generate tasks implementing Gecko tests.
"""
- # the kind on which this one depends
- if len(config.get('kind-dependencies', [])) != 2:
- raise Exception(
- 'Test kinds must have exactly 2 items in kind-dependencies'
- )
-
builds_by_platform = get_builds_by_platform(dep_kind='build', loaded_tasks=loaded_tasks)
signed_builds_by_platform = get_builds_by_platform(
dep_kind='build-signing', loaded_tasks=loaded_tasks
)
# get the test platforms for those build tasks
test_platforms_cfg = load_yaml(path, 'test-platforms.yml')
test_platforms = get_test_platforms(
--- a/taskcluster/taskgraph/transforms/job/mozharness_test.py
+++ b/taskcluster/taskgraph/transforms/job/mozharness_test.py
@@ -90,25 +90,26 @@ def mozharness_test_on_docker(config, jo
worker['caches'] = [{
'type': 'persistent',
'name': 'level-{}-{}-test-workspace'.format(
config.params['level'], config.params['project']),
'mount-point': "{workdir}/workspace".format(**run),
}]
- env = worker['env'] = {
+ env = worker.setdefault('env', {})
+ env.update({
'MOZHARNESS_CONFIG': ' '.join(mozharness['config']),
'MOZHARNESS_SCRIPT': mozharness['script'],
'MOZILLA_BUILD_URL': {'task-reference': installer_url},
'NEED_PULSEAUDIO': 'true',
'NEED_WINDOW_MANAGER': 'true',
'ENABLE_E10S': str(bool(test.get('e10s'))).lower(),
'MOZ_AUTOMATION': '1',
- }
+ })
if mozharness.get('mochitest-flavor'):
env['MOCHITEST_FLAVOR'] = mozharness['mochitest-flavor']
if mozharness['set-moz-node-path']:
env['MOZ_NODE_PATH'] = '/usr/local/bin/node'
if 'actions' in mozharness:
--- a/taskcluster/taskgraph/transforms/tests.py
+++ b/taskcluster/taskgraph/transforms/tests.py
@@ -718,16 +718,24 @@ def enable_code_coverage(config, tests):
test['run-on-projects'] = 'built-projects'
# Ensure we don't optimize test suites out.
# We always want to run all test suites for coverage purposes.
test.pop('schedules-component', None)
test.pop('when', None)
test['optimization'] = None
+ # Add a fetch task for the grcov binary.
+ if 'linux' in test['build-platform']:
+ test['fetches'] = ['grcov-linux-x86_64']
+ elif 'osx' in test['build-platform']:
+ test['fetches'] = ['grcov-osx-x86_64']
+ elif 'win' in test['build-platform']:
+ test['fetches'] = ['grcov-win-i686']
+
if 'talos' in test['test-name']:
test['max-run-time'] = 7200
if 'linux' in test['build-platform']:
test['docker-image'] = {"in-tree": "desktop1604-test"}
test['mozharness']['extra-options'].append('--add-option')
test['mozharness']['extra-options'].append('--cycles,1')
test['mozharness']['extra-options'].append('--add-option')
test['mozharness']['extra-options'].append('--tppagecycles,1')
@@ -1043,16 +1051,18 @@ def make_job_description(config, tests):
# otherwise just use skip-unless-schedules
jobdesc['optimization'] = {'skip-unless-schedules': schedules}
run = jobdesc['run'] = {}
run['using'] = 'mozharness-test'
run['test'] = test
jobdesc['worker-type'] = test.pop('worker-type')
+ if test.get('fetches'):
+ jobdesc['fetches'] = test.pop('fetches')
yield jobdesc
def normpath(path):
return path.replace('/', '\\')
--- a/testing/mozharness/mozharness/mozilla/testing/codecoverage.py
+++ b/testing/mozharness/mozharness/mozilla/testing/codecoverage.py
@@ -106,44 +106,30 @@ class CodeCoverageMixin(SingleTestMixin)
raise Exception('Unexpected OS: {}'.format(mozinfo.os))
os.environ['GCOV_PREFIX_STRIP'] = str(strip_count)
# Install grcov on the test machine
# Get the path to the build machines gcno files.
self.url_to_gcno = self.query_build_dir_url('target.code-coverage-gcno.zip')
self.url_to_chrome_map = self.query_build_dir_url('chrome-map.json')
- dirs = self.query_abs_dirs()
-
- # Create the grcov directory, get the tooltool manifest, and finally
- # download and unpack the grcov binary.
- self.grcov_dir = tempfile.mkdtemp()
- if mozinfo.os == 'linux':
- platform = 'linux64'
- tar_file = 'grcov-linux-x86_64.tar.bz2'
- elif mozinfo.os == 'win':
- platform = 'win32'
- tar_file = 'grcov-win-i686.tar.bz2'
- elif mozinfo.os == 'mac':
- platform = 'macosx64'
- tar_file = 'grcov-osx-x86_64.tar.bz2'
+ # Create the grcov directory, then download it.
+ # TODO: use the fetch-content script to download artifacts.
+ self.grcov_dir = tempfile.mkdtemp()
+ ARTIFACT_URL = 'https://queue.taskcluster.net/v1/task/{task}/artifacts/{artifact}'
+ for word in os.getenv('MOZ_FETCHES').split():
+ artifact, task = word.split('@', 1)
+ filename = os.path.basename(artifact)
+ url = ARTIFACT_URL.format(artifact=artifact, task=task)
+ self.download_file(url, parent_dir=self.grcov_dir)
- manifest = os.path.join(dirs.get('abs_test_install_dir',
- os.path.join(dirs['abs_work_dir'], 'tests')),
- 'config/tooltool-manifests/%s/ccov.manifest' % platform)
-
- self.tooltool_fetch(
- manifest=manifest,
- output_dir=self.grcov_dir,
- cache=self.config.get('tooltool_cache')
- )
-
- with tarfile.open(os.path.join(self.grcov_dir, tar_file)) as tar:
- tar.extractall(self.grcov_dir)
+ with tarfile.open(os.path.join(self.grcov_dir, filename), 'r') as tar:
+ tar.extractall(self.grcov_dir)
+ os.remove(os.path.join(self.grcov_dir, filename))
# Download the gcno archive from the build machine.
self.download_file(self.url_to_gcno, parent_dir=self.grcov_dir)
# Download the chrome-map.json file from the build machine.
self.download_file(self.url_to_chrome_map, parent_dir=self.grcov_dir)
@PostScriptAction('download-and-extract')