Bug 1391427 - Execute python toolchain scripts with mach. r?glandium draft
authorRalph Giles <giles@mozilla.com>
Tue, 12 Sep 2017 16:01:26 -0700
changeset 670740 5fe0cdfe715d011f3799f632c26f5dd5327327f5
parent 670739 39aaf54972cb11a63815a96b532786133baa95bc
child 670741 5d257b4c2d914c0295e53dfa324874a3357d0b3c
push id81697
push userbmo:giles@thaumas.net
push dateTue, 26 Sep 2017 19:16:56 +0000
reviewersglandium
bugs1391427
milestone58.0a1
Bug 1391427 - Execute python toolchain scripts with mach. r?glandium Run scripts with a `.py` filename extension through `./mach python` so the normal enviroment with in-tree python libraries is available. This is helpful for the Rust upstream release download and repackaging steps, which are more easily expressed in python than in an sh-based build script like we use for clang and other tools. Invocation of `mach python` on Windows-hosted generic workers fails because of some missing environment pieces. For the purposes of this bug we can just run the script for Windows targets in a docker-worker so Python support was left unimplemented for generic workers. MozReview-Commit-ID: 4XUQ1XrVBc2
taskcluster/taskgraph/transforms/job/toolchain.py
--- a/taskcluster/taskgraph/transforms/job/toolchain.py
+++ b/taskcluster/taskgraph/transforms/job/toolchain.py
@@ -24,17 +24,19 @@ from taskgraph.util.hash import hash_pat
 from taskgraph import GECKO
 
 
 TOOLCHAIN_INDEX = 'gecko.cache.level-{level}.toolchains.v1.{name}.{digest}'
 
 toolchain_run_schema = Schema({
     Required('using'): 'toolchain-script',
 
-    # the script (in taskcluster/scripts/misc) to run
+    # The script (in taskcluster/scripts/misc) to run.
+    # Python scripts are invoked with `mach python` so vendored libraries
+    # are available.
     Required('script'): basestring,
 
     # If not false, tooltool downloads will be enabled via relengAPIProxy
     # for either just public files, or all files.  Not supported on Windows
     Required('tooltool-downloads', default=False): Any(
         False,
         'public',
         'internal',
@@ -115,26 +117,32 @@ def docker_worker_toolchain(config, job,
         'TOOLS_DISABLE': 'true',
         'MOZ_AUTOMATION': '1',
     })
 
     if run['tooltool-downloads']:
         internal = run['tooltool-downloads'] == 'internal'
         docker_worker_add_tooltool(config, job, taskdesc, internal=internal)
 
+    # Use `mach` to invoke python scripts so in-tree libraries are available.
+    if run['script'].endswith('.py'):
+        wrapper = 'workspace/build/src/mach python '
+    else:
+        wrapper = ''
+
     worker['command'] = [
         '/builds/worker/bin/run-task',
         '--vcs-checkout=/builds/worker/workspace/build/src',
         '--sparse-profile', 'build/sparse-profiles/toolchain-build',
         '--',
         'bash',
         '-c',
         'cd /builds/worker && '
-        './workspace/build/src/taskcluster/scripts/misc/{}'.format(
-            run['script'])
+        '{}workspace/build/src/taskcluster/scripts/misc/{}'.format(
+            wrapper, run['script'])
     ]
 
     attributes = taskdesc.setdefault('attributes', {})
     attributes['toolchain-artifact'] = run['toolchain-artifact']
     if 'toolchain-alias' in run:
         attributes['toolchain-alias'] = run['toolchain-alias']
 
     add_optimization(config, run, taskdesc)
@@ -167,16 +175,20 @@ def windows_toolchain(config, job, taskd
     hg_command.append('robustcheckout')
     hg_command.extend(['--sharebase', 'y:\\hg-shared'])
     hg_command.append('--purge')
     hg_command.extend(['--upstream', 'https://hg.mozilla.org/mozilla-unified'])
     hg_command.extend(['--revision', '%GECKO_HEAD_REV%'])
     hg_command.append('%GECKO_HEAD_REPOSITORY%')
     hg_command.append('.\\build\\src')
 
+    # Use `mach` to invoke python scripts so in-tree libraries are available.
+    if run['script'].endswith('.py'):
+        raise NotImplementedError("Python scripts don't work on Windows")
+
     bash = r'c:\mozilla-build\msys\bin\bash'
     worker['command'] = [
         ' '.join(hg_command),
         # do something intelligent.
         r'{} -c ./build/src/taskcluster/scripts/misc/{}'.format(bash, run['script'])
     ]
 
     attributes = taskdesc.setdefault('attributes', {})