--- a/taskcluster/taskgraph/transforms/job/toolchain.py
+++ b/taskcluster/taskgraph/transforms/job/toolchain.py
@@ -4,16 +4,18 @@
"""
Support for running toolchain-building jobs via dedicated scripts
"""
from __future__ import absolute_import, print_function, unicode_literals
import hashlib
+from mozbuild.shellutil import quote as shell_quote
+
from taskgraph.util.schema import Schema
from voluptuous import Optional, Required, Any
from taskgraph.transforms.job import run_job_using
from taskgraph.transforms.job.common import (
docker_worker_add_tc_vcs_cache,
docker_worker_add_gecko_vcs_env_vars,
docker_worker_add_public_artifacts,
@@ -29,16 +31,19 @@ TOOLCHAIN_INDEX = 'gecko.cache.level-{le
toolchain_run_schema = Schema({
Required('using'): 'toolchain-script',
# The script (in taskcluster/scripts/misc) to run.
# Python scripts are invoked with `mach python` so vendored libraries
# are available.
Required('script'): basestring,
+ # Arguments to pass to the script.
+ Optional('arguments'): [basestring],
+
# If not false, tooltool downloads will be enabled via relengAPIProxy
# for either just public files, or all files. Not supported on Windows
Required('tooltool-downloads', default=False): Any(
False,
'public',
'internal',
),
@@ -61,31 +66,36 @@ def add_optimization(config, run, taskde
files.append('taskcluster/taskgraph/transforms/job/toolchain.py')
# The script
files.append('taskcluster/scripts/misc/{}'.format(run['script']))
# Tooltool manifest if any is defined:
tooltool_manifest = taskdesc['worker']['env'].get('TOOLTOOL_MANIFEST')
if tooltool_manifest:
files.append(tooltool_manifest)
- digest = hash_paths(GECKO, files)
+ # Accumulate dependency hashes for index generation.
+ data = [hash_paths(GECKO, files)]
# If the task has dependencies, we need those dependencies to influence
# the index path. So take the digest from the files above, add the list
# of its dependencies, and hash the aggregate.
# If the task has no dependencies, just use the digest from above.
deps = taskdesc['dependencies']
if deps:
- data = [digest] + sorted(deps.values())
- digest = hashlib.sha256('\n'.join(data)).hexdigest()
+ data.extend(sorted(deps.values()))
+
+ # Likewise script arguments should influence the index.
+ args = run.get('arguments')
+ if args:
+ data.extend(args)
label = taskdesc['label']
subs = {
'name': label.replace('%s-' % config.kind, ''),
- 'digest': digest,
+ 'digest': hashlib.sha256('\n'.join(data)).hexdigest()
}
# We'll try to find a cached version of the toolchain at levels above
# and including the current level, starting at the highest level.
index_routes = []
for level in reversed(range(int(config.params['level']), 4)):
subs['level'] = level
index_routes.append(TOOLCHAIN_INDEX.format(**subs))
@@ -123,26 +133,30 @@ def docker_worker_toolchain(config, job,
docker_worker_add_tooltool(config, job, taskdesc, internal=internal)
# Use `mach` to invoke python scripts so in-tree libraries are available.
if run['script'].endswith('.py'):
wrapper = 'workspace/build/src/mach python '
else:
wrapper = ''
+ args = run.get('arguments', '')
+ if args:
+ args = ' ' + shell_quote(*args)
+
worker['command'] = [
'/builds/worker/bin/run-task',
'--vcs-checkout=/builds/worker/workspace/build/src',
'--sparse-profile', 'build/sparse-profiles/toolchain-build',
'--',
'bash',
'-c',
'cd /builds/worker && '
- '{}workspace/build/src/taskcluster/scripts/misc/{}'.format(
- wrapper, run['script'])
+ '{}workspace/build/src/taskcluster/scripts/misc/{}{}'.format(
+ wrapper, run['script'], args)
]
attributes = taskdesc.setdefault('attributes', {})
attributes['toolchain-artifact'] = run['toolchain-artifact']
if 'toolchain-alias' in run:
attributes['toolchain-alias'] = run['toolchain-alias']
add_optimization(config, run, taskdesc)
@@ -179,21 +193,26 @@ def windows_toolchain(config, job, taskd
hg_command.extend(['--revision', '%GECKO_HEAD_REV%'])
hg_command.append('%GECKO_HEAD_REPOSITORY%')
hg_command.append('.\\build\\src')
# Use `mach` to invoke python scripts so in-tree libraries are available.
if run['script'].endswith('.py'):
raise NotImplementedError("Python scripts don't work on Windows")
+ args = run.get('arguments', '')
+ if args:
+ args = ' ' + shell_quote(*args)
+
bash = r'c:\mozilla-build\msys\bin\bash'
worker['command'] = [
' '.join(hg_command),
# do something intelligent.
- r'{} -c ./build/src/taskcluster/scripts/misc/{}'.format(bash, run['script'])
+ r'{} build/src/taskcluster/scripts/misc/{}{}'.format(
+ bash, run['script'], args)
]
attributes = taskdesc.setdefault('attributes', {})
attributes['toolchain-artifact'] = run['toolchain-artifact']
if 'toolchain-alias' in run:
attributes['toolchain-alias'] = run['toolchain-alias']
add_optimization(config, run, taskdesc)