Bug 1391427 - Execute python toolchain scripts with mach. r?glandium
Run scripts with a `.py` filename extension through `./mach python`
so the normal enviroment with in-tree python libraries is available.
This is helpful for the Rust upstream release download and repackaging
steps, which are more easily expressed in python than in an sh-based
build script like we use for clang and other tools.
Invocation of `mach python` on Windows-hosted generic workers fails
because of some missing environment pieces. For the purposes of this
bug we can just run the script for Windows targets in a docker-worker
so Python support was left unimplemented for generic workers.
MozReview-Commit-ID: 4XUQ1XrVBc2
--- a/taskcluster/taskgraph/transforms/job/toolchain.py
+++ b/taskcluster/taskgraph/transforms/job/toolchain.py
@@ -24,17 +24,19 @@ from taskgraph.util.hash import hash_pat
from taskgraph import GECKO
TOOLCHAIN_INDEX = 'gecko.cache.level-{level}.toolchains.v1.{name}.{digest}'
toolchain_run_schema = Schema({
Required('using'): 'toolchain-script',
- # the script (in taskcluster/scripts/misc) to run
+ # The script (in taskcluster/scripts/misc) to run.
+ # Python scripts are invoked with `mach python` so vendored libraries
+ # are available.
Required('script'): basestring,
# If not false, tooltool downloads will be enabled via relengAPIProxy
# for either just public files, or all files. Not supported on Windows
Required('tooltool-downloads', default=False): Any(
False,
'public',
'internal',
@@ -115,26 +117,32 @@ def docker_worker_toolchain(config, job,
'TOOLS_DISABLE': 'true',
'MOZ_AUTOMATION': '1',
})
if run['tooltool-downloads']:
internal = run['tooltool-downloads'] == 'internal'
docker_worker_add_tooltool(config, job, taskdesc, internal=internal)
+ # Use `mach` to invoke python scripts so in-tree libraries are available.
+ if run['script'].endswith('.py'):
+ wrapper = 'workspace/build/src/mach python '
+ else:
+ wrapper = ''
+
worker['command'] = [
'/builds/worker/bin/run-task',
'--vcs-checkout=/builds/worker/workspace/build/src',
'--sparse-profile', 'build/sparse-profiles/toolchain-build',
'--',
'bash',
'-c',
'cd /builds/worker && '
- './workspace/build/src/taskcluster/scripts/misc/{}'.format(
- run['script'])
+ '{}workspace/build/src/taskcluster/scripts/misc/{}'.format(
+ wrapper, run['script'])
]
attributes = taskdesc.setdefault('attributes', {})
attributes['toolchain-artifact'] = run['toolchain-artifact']
if 'toolchain-alias' in run:
attributes['toolchain-alias'] = run['toolchain-alias']
add_optimization(config, run, taskdesc)
@@ -167,16 +175,20 @@ def windows_toolchain(config, job, taskd
hg_command.append('robustcheckout')
hg_command.extend(['--sharebase', 'y:\\hg-shared'])
hg_command.append('--purge')
hg_command.extend(['--upstream', 'https://hg.mozilla.org/mozilla-unified'])
hg_command.extend(['--revision', '%GECKO_HEAD_REV%'])
hg_command.append('%GECKO_HEAD_REPOSITORY%')
hg_command.append('.\\build\\src')
+ # Use `mach` to invoke python scripts so in-tree libraries are available.
+ if run['script'].endswith('.py'):
+ raise NotImplementedError("Python scripts don't work on Windows")
+
bash = r'c:\mozilla-build\msys\bin\bash'
worker['command'] = [
' '.join(hg_command),
# do something intelligent.
r'{} -c ./build/src/taskcluster/scripts/misc/{}'.format(bash, run['script'])
]
attributes = taskdesc.setdefault('attributes', {})