Bug 1462791 - Add helper function for VCS checkout on generic-worker; r?dustin
This functionality was implemented at least 3 times. Let's consolidate
it to a central function.
Returning multiple command strings is kind of funky. I preserved
existing behavior and mozharness jobs are the only ones printing the
forensic logging. We should probably move this logging into
robustcheckout so we don't need to involve taskgraph with this. But
that can be deferred to another day.
MozReview-Commit-ID: I2LglJvfI6
--- a/taskcluster/taskgraph/transforms/job/common.py
+++ b/taskcluster/taskgraph/transforms/job/common.py
@@ -127,16 +127,44 @@ def support_vcs_checkout(config, job, ta
# for hg.mozilla.org.
taskdesc['scopes'].append('secrets:get:project/taskcluster/gecko/hgfingerprint')
# only some worker platforms have taskcluster-proxy enabled
if job['worker']['implementation'] in ('docker-worker', 'docker-engine'):
taskdesc['worker']['taskcluster-proxy'] = True
+def generic_worker_hg_commands(base_repo, head_repo, head_rev, path):
+ """Obtain commands needed to obtain a Mercurial checkout on generic-worker.
+
+ Returns two command strings. One performs the checkout. Another logs.
+ """
+ args = [
+ r'"c:\Program Files\Mercurial\hg.exe"',
+ 'robustcheckout',
+ '--sharebase', r'y:\hg-shared',
+ '--purge',
+ '--upstream', base_repo,
+ '--revision', head_rev,
+ head_repo,
+ path,
+ ]
+
+ logging_args = [
+ b":: TinderboxPrint:<a href={source_repo}/rev/{revision} "
+ b"title='Built from {repo_name} revision {revision}'>{revision}</a>"
+ b"\n".format(
+ revision=head_rev,
+ source_repo=head_repo,
+ repo_name=head_repo.split('/')[-1]),
+ ]
+
+ return ' '.join(args), ' '.join(logging_args)
+
+
def docker_worker_setup_secrets(config, job, taskdesc):
"""Set up access to secrets via taskcluster-proxy. The value of
run['secrets'] should be a boolean or a list of secret names that
can be accessed."""
if not job['run'].get('secrets'):
return
taskdesc['worker']['taskcluster-proxy'] = True
--- a/taskcluster/taskgraph/transforms/job/mozharness.py
+++ b/taskcluster/taskgraph/transforms/job/mozharness.py
@@ -19,16 +19,17 @@ from voluptuous import Required, Optiona
from taskgraph.transforms.job import run_job_using
from taskgraph.transforms.job.common import (
docker_worker_add_workspace_cache,
docker_worker_add_gecko_vcs_env_vars,
docker_worker_setup_secrets,
docker_worker_add_artifacts,
docker_worker_add_tooltool,
generic_worker_add_artifacts,
+ generic_worker_hg_commands,
support_vcs_checkout,
)
mozharness_run_schema = Schema({
Required('using'): 'mozharness',
# the mozharness script used to run this task, relative to the testing/
# directory and using forward slashes even on Windows
@@ -287,53 +288,30 @@ def mozharness_on_generic_worker(config,
for option in run.get('options', []):
assert ' ' not in option
mh_command.append('--' + option)
if run.get('custom-build-variant-cfg'):
mh_command.append('--custom-build-variant')
mh_command.append(run['custom-build-variant-cfg'])
- def checkout_repo(base_repo, head_repo, head_rev, path):
- hg_command = ['"c:\\Program Files\\Mercurial\\hg.exe"']
- hg_command.append('robustcheckout')
- hg_command.extend(['--sharebase', 'y:\\hg-shared'])
- hg_command.append('--purge')
- hg_command.extend(['--upstream', base_repo])
- hg_command.extend(['--revision', head_rev])
- hg_command.append(head_repo)
- hg_command.append(path)
-
- logging_command = [
- b":: TinderboxPrint:<a href={source_repo}/rev/{revision} "
- b"title='Built from {repo_name} revision {revision}'>{revision}</a>\n".format(
- revision=head_rev,
- source_repo=head_repo,
- repo_name=head_repo.split('/')[-1],
- )]
-
- return [
- ' '.join(hg_command),
- ' '.join(logging_command),
- ]
-
- hg_commands = checkout_repo(
+ hg_commands = generic_worker_hg_commands(
base_repo=env['GECKO_BASE_REPOSITORY'],
head_repo=env['GECKO_HEAD_REPOSITORY'],
head_rev=env['GECKO_HEAD_REV'],
- path='.\\build\\src')
+ path=r'.\build\src',
+ )
if run['comm-checkout']:
hg_commands.extend(
- checkout_repo(
+ generic_worker_hg_commands(
base_repo=env['COMM_BASE_REPOSITORY'],
head_repo=env['COMM_HEAD_REPOSITORY'],
head_rev=env['COMM_HEAD_REV'],
- path='.\\build\\src\\comm')
- )
+ path=r'.\build\src\comm'))
worker['command'] = []
if taskdesc.get('needs-sccache'):
worker['command'].extend([
# Make the comment part of the first command, as it will help users to
# understand what is going on, and why these steps are implemented.
dedent('''\
:: sccache currently uses the full compiler commandline as input to the
--- a/taskcluster/taskgraph/transforms/job/spidermonkey.py
+++ b/taskcluster/taskgraph/transforms/job/spidermonkey.py
@@ -9,16 +9,17 @@ from __future__ import absolute_import,
from taskgraph.util.schema import Schema
from voluptuous import Required, Any
from taskgraph.transforms.job import run_job_using
from taskgraph.transforms.job.common import (
docker_worker_add_artifacts,
generic_worker_add_artifacts,
+ generic_worker_hg_commands,
docker_worker_add_gecko_vcs_env_vars,
docker_worker_add_tooltool,
support_vcs_checkout,
)
sm_run_schema = Schema({
Required('using'): Any('spidermonkey', 'spidermonkey-package', 'spidermonkey-mozjs-crate',
'spidermonkey-rust-bindings'),
@@ -110,25 +111,22 @@ def generic_worker_spidermonkey(config,
script = "build-sm-mozjs-crate.sh"
# Don't allow untested configurations yet
raise Exception("spidermonkey-mozjs-crate is not a supported configuration")
elif run['using'] == 'spidermonkey-rust-bindings':
script = "build-sm-rust-bindings.sh"
# Don't allow untested configurations yet
raise Exception("spidermonkey-rust-bindings is not a supported configuration")
- hg_command = ['"c:\\Program Files\\Mercurial\\hg.exe"']
- hg_command.append('robustcheckout')
- hg_command.extend(['--sharebase', 'y:\\hg-shared'])
- hg_command.append('--purge')
- hg_command.extend(['--upstream', 'https://hg.mozilla.org/mozilla-unified'])
- hg_command.extend(['--revision', env['GECKO_HEAD_REV']])
- hg_command.append(env['GECKO_HEAD_REPOSITORY'])
- hg_command.append('.\\src')
+ hg_command = generic_worker_hg_commands(
+ 'https://hg.mozilla.org/mozilla-unified',
+ env['GECKO_HEAD_REPOSITORY'],
+ env['GECKO_HEAD_REV'],
+ r'.\src',
+ )[0]
command = ['c:\\mozilla-build\\msys\\bin\\bash.exe ' # string concat
'"./src/taskcluster/scripts/builder/%s"' % script]
- worker['command'] = []
- worker['command'].extend([
- ' '.join(hg_command),
- ' '.join(command)
- ])
+ worker['command'] = [
+ hg_command,
+ ' '.join(command),
+ ]
--- a/taskcluster/taskgraph/transforms/job/toolchain.py
+++ b/taskcluster/taskgraph/transforms/job/toolchain.py
@@ -12,16 +12,17 @@ from mozbuild.shellutil import quote as
from taskgraph.util.schema import Schema
from voluptuous import Optional, Required, Any
from taskgraph.transforms.job import run_job_using
from taskgraph.transforms.job.common import (
docker_worker_add_gecko_vcs_env_vars,
docker_worker_add_artifacts,
docker_worker_add_tooltool,
+ generic_worker_hg_commands,
support_vcs_checkout,
)
from taskgraph.util.hash import hash_paths
from taskgraph import GECKO
from taskgraph.util.cached_tasks import add_optimization
import taskgraph
@@ -200,37 +201,33 @@ def windows_toolchain(config, job, taskd
env = worker['env']
env.update({
'MOZ_BUILD_DATE': config.params['moz_build_date'],
'MOZ_SCM_LEVEL': config.params['level'],
'MOZ_AUTOMATION': '1',
})
- hg = r'c:\Program Files\Mercurial\hg.exe'
- hg_command = ['"{}"'.format(hg)]
- hg_command.append('robustcheckout')
- hg_command.extend(['--sharebase', 'y:\\hg-shared'])
- hg_command.append('--purge')
- hg_command.extend(['--upstream', 'https://hg.mozilla.org/mozilla-unified'])
- hg_command.extend(['--revision', '%GECKO_HEAD_REV%'])
- hg_command.append('%GECKO_HEAD_REPOSITORY%')
- hg_command.append('.\\build\\src')
+ hg_command = generic_worker_hg_commands(
+ 'https://hg.mozilla.org/mozilla-unified',
+ env['GECKO_HEAD_REPOSITORY'],
+ env['GECKO_HEAD_REV'],
+ r'.\build\src')[0]
# Use `mach` to invoke python scripts so in-tree libraries are available.
if run['script'].endswith('.py'):
raise NotImplementedError("Python scripts don't work on Windows")
args = run.get('arguments', '')
if args:
args = ' ' + shell_quote(*args)
bash = r'c:\mozilla-build\msys\bin\bash'
worker['command'] = [
- ' '.join(hg_command),
+ hg_command,
# do something intelligent.
r'{} build/src/taskcluster/scripts/misc/{}{}'.format(
bash, run['script'], args)
]
attributes = taskdesc.setdefault('attributes', {})
attributes['toolchain-artifact'] = run['toolchain-artifact']
if 'toolchain-alias' in run: