Bug 1391427 - Add script 'arguments' key to toolchain tasks. r?glandium draft
authorRalph Giles <giles@mozilla.com>
Tue, 12 Sep 2017 16:06:15 -0700
changeset 670741 5d257b4c2d914c0295e53dfa324874a3357d0b3c
parent 670740 5fe0cdfe715d011f3799f632c26f5dd5327327f5
child 670742 65ed166843e9fc8642dcab340dcb7bec2e4cfe8a
push id81697
push userbmo:giles@thaumas.net
push dateTue, 26 Sep 2017 19:16:56 +0000
reviewersglandium
bugs1391427
milestone58.0a1
Bug 1391427 - Add script 'arguments' key to toolchain tasks. r?glandium Add an optional 'arguments' key to the yaml description for toolchain tasks. This is a list of strings to be passed to the script invocation. This lets us set behaviour, e.g. selecting the version to build or selecting targets from the task description instead of having to hard-code those things in the build script itself. Where the same script otherwise works for multiple configurations, that is easier to update and simplifies supporting variants. MozReview-Commit-ID: 30oJYnQaZ7A
taskcluster/taskgraph/transforms/job/toolchain.py
--- a/taskcluster/taskgraph/transforms/job/toolchain.py
+++ b/taskcluster/taskgraph/transforms/job/toolchain.py
@@ -4,16 +4,18 @@
 """
 Support for running toolchain-building jobs via dedicated scripts
 """
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 import hashlib
 
+from mozbuild.shellutil import quote as shell_quote
+
 from taskgraph.util.schema import Schema
 from voluptuous import Optional, Required, Any
 
 from taskgraph.transforms.job import run_job_using
 from taskgraph.transforms.job.common import (
     docker_worker_add_tc_vcs_cache,
     docker_worker_add_gecko_vcs_env_vars,
     docker_worker_add_public_artifacts,
@@ -29,16 +31,19 @@ TOOLCHAIN_INDEX = 'gecko.cache.level-{le
 toolchain_run_schema = Schema({
     Required('using'): 'toolchain-script',
 
     # The script (in taskcluster/scripts/misc) to run.
     # Python scripts are invoked with `mach python` so vendored libraries
     # are available.
     Required('script'): basestring,
 
+    # Arguments to pass to the script.
+    Optional('arguments'): [basestring],
+
     # If not false, tooltool downloads will be enabled via relengAPIProxy
     # for either just public files, or all files.  Not supported on Windows
     Required('tooltool-downloads', default=False): Any(
         False,
         'public',
         'internal',
     ),
 
@@ -61,31 +66,36 @@ def add_optimization(config, run, taskde
     files.append('taskcluster/taskgraph/transforms/job/toolchain.py')
     # The script
     files.append('taskcluster/scripts/misc/{}'.format(run['script']))
     # Tooltool manifest if any is defined:
     tooltool_manifest = taskdesc['worker']['env'].get('TOOLTOOL_MANIFEST')
     if tooltool_manifest:
         files.append(tooltool_manifest)
 
-    digest = hash_paths(GECKO, files)
+    # Accumulate dependency hashes for index generation.
+    data = [hash_paths(GECKO, files)]
 
     # If the task has dependencies, we need those dependencies to influence
     # the index path. So take the digest from the files above, add the list
     # of its dependencies, and hash the aggregate.
     # If the task has no dependencies, just use the digest from above.
     deps = taskdesc['dependencies']
     if deps:
-        data = [digest] + sorted(deps.values())
-        digest = hashlib.sha256('\n'.join(data)).hexdigest()
+        data.extend(sorted(deps.values()))
+
+    # Likewise script arguments should influence the index.
+    args = run.get('arguments')
+    if args:
+        data.extend(args)
 
     label = taskdesc['label']
     subs = {
         'name': label.replace('%s-' % config.kind, ''),
-        'digest': digest,
+        'digest': hashlib.sha256('\n'.join(data)).hexdigest()
     }
 
     # We'll try to find a cached version of the toolchain at levels above
     # and including the current level, starting at the highest level.
     index_routes = []
     for level in reversed(range(int(config.params['level']), 4)):
         subs['level'] = level
         index_routes.append(TOOLCHAIN_INDEX.format(**subs))
@@ -123,26 +133,30 @@ def docker_worker_toolchain(config, job,
         docker_worker_add_tooltool(config, job, taskdesc, internal=internal)
 
     # Use `mach` to invoke python scripts so in-tree libraries are available.
     if run['script'].endswith('.py'):
         wrapper = 'workspace/build/src/mach python '
     else:
         wrapper = ''
 
+    args = run.get('arguments', '')
+    if args:
+        args = ' ' + shell_quote(*args)
+
     worker['command'] = [
         '/builds/worker/bin/run-task',
         '--vcs-checkout=/builds/worker/workspace/build/src',
         '--sparse-profile', 'build/sparse-profiles/toolchain-build',
         '--',
         'bash',
         '-c',
         'cd /builds/worker && '
-        '{}workspace/build/src/taskcluster/scripts/misc/{}'.format(
-            wrapper, run['script'])
+        '{}workspace/build/src/taskcluster/scripts/misc/{}{}'.format(
+            wrapper, run['script'], args)
     ]
 
     attributes = taskdesc.setdefault('attributes', {})
     attributes['toolchain-artifact'] = run['toolchain-artifact']
     if 'toolchain-alias' in run:
         attributes['toolchain-alias'] = run['toolchain-alias']
 
     add_optimization(config, run, taskdesc)
@@ -179,21 +193,26 @@ def windows_toolchain(config, job, taskd
     hg_command.extend(['--revision', '%GECKO_HEAD_REV%'])
     hg_command.append('%GECKO_HEAD_REPOSITORY%')
     hg_command.append('.\\build\\src')
 
     # Use `mach` to invoke python scripts so in-tree libraries are available.
     if run['script'].endswith('.py'):
         raise NotImplementedError("Python scripts don't work on Windows")
 
+    args = run.get('arguments', '')
+    if args:
+        args = ' ' + shell_quote(*args)
+
     bash = r'c:\mozilla-build\msys\bin\bash'
     worker['command'] = [
         ' '.join(hg_command),
         # do something intelligent.
-        r'{} -c ./build/src/taskcluster/scripts/misc/{}'.format(bash, run['script'])
+        r'{} build/src/taskcluster/scripts/misc/{}{}'.format(
+            bash, run['script'], args)
     ]
 
     attributes = taskdesc.setdefault('attributes', {})
     attributes['toolchain-artifact'] = run['toolchain-artifact']
     if 'toolchain-alias' in run:
         attributes['toolchain-alias'] = run['toolchain-alias']
 
     add_optimization(config, run, taskdesc)