Bug 1462791 - Add helper function for VCS checkout on generic-worker; r?dustin draft
authorGregory Szorc <gps@mozilla.com>
Fri, 20 Apr 2018 13:34:45 -0700
changeset 799475 ebe1bd358ab1b3fafddd8a5413464328fc4390f6
parent 799392 ff8505d177b9fcba44b040ccd9b6bb709e238e84
child 799476 6d50aba8c093d8e9d37301275d7c6a6c3a958547
push id111074
push userbmo:gps@mozilla.com
push dateThu, 24 May 2018 19:19:07 +0000
reviewersdustin
bugs1462791
milestone62.0a1
Bug 1462791 - Add helper function for VCS checkout on generic-worker; r?dustin This functionality was implemented at least 3 times. Let's consolidate it to a central function. Returning multiple command strings is kind of funky. I preserved existing behavior and mozharness jobs are the only ones printing the forensic logging. We should probably move this logging into robustcheckout so we don't need to involve taskgraph with this. But that can be deferred to another day. MozReview-Commit-ID: I2LglJvfI6
taskcluster/taskgraph/transforms/job/common.py
taskcluster/taskgraph/transforms/job/mozharness.py
taskcluster/taskgraph/transforms/job/spidermonkey.py
taskcluster/taskgraph/transforms/job/toolchain.py
--- a/taskcluster/taskgraph/transforms/job/common.py
+++ b/taskcluster/taskgraph/transforms/job/common.py
@@ -127,16 +127,44 @@ def support_vcs_checkout(config, job, ta
     # for hg.mozilla.org.
     taskdesc['scopes'].append('secrets:get:project/taskcluster/gecko/hgfingerprint')
 
     # only some worker platforms have taskcluster-proxy enabled
     if job['worker']['implementation'] in ('docker-worker', 'docker-engine'):
         taskdesc['worker']['taskcluster-proxy'] = True
 
 
+def generic_worker_hg_commands(base_repo, head_repo, head_rev, path):
+    """Obtain commands needed to obtain a Mercurial checkout on generic-worker.
+
+    Returns two command strings. One performs the checkout. Another logs.
+    """
+    args = [
+        r'"c:\Program Files\Mercurial\hg.exe"',
+        'robustcheckout',
+        '--sharebase', r'y:\hg-shared',
+        '--purge',
+        '--upstream', base_repo,
+        '--revision', head_rev,
+        head_repo,
+        path,
+    ]
+
+    logging_args = [
+        b":: TinderboxPrint:<a href={source_repo}/rev/{revision} "
+        b"title='Built from {repo_name} revision {revision}'>{revision}</a>"
+        b"\n".format(
+            revision=head_rev,
+            source_repo=head_repo,
+            repo_name=head_repo.split('/')[-1]),
+    ]
+
+    return ' '.join(args), ' '.join(logging_args)
+
+
 def docker_worker_setup_secrets(config, job, taskdesc):
     """Set up access to secrets via taskcluster-proxy.  The value of
     run['secrets'] should be a boolean or a list of secret names that
     can be accessed."""
     if not job['run'].get('secrets'):
         return
 
     taskdesc['worker']['taskcluster-proxy'] = True
--- a/taskcluster/taskgraph/transforms/job/mozharness.py
+++ b/taskcluster/taskgraph/transforms/job/mozharness.py
@@ -19,16 +19,17 @@ from voluptuous import Required, Optiona
 from taskgraph.transforms.job import run_job_using
 from taskgraph.transforms.job.common import (
     docker_worker_add_workspace_cache,
     docker_worker_add_gecko_vcs_env_vars,
     docker_worker_setup_secrets,
     docker_worker_add_artifacts,
     docker_worker_add_tooltool,
     generic_worker_add_artifacts,
+    generic_worker_hg_commands,
     support_vcs_checkout,
 )
 
 mozharness_run_schema = Schema({
     Required('using'): 'mozharness',
 
     # the mozharness script used to run this task, relative to the testing/
     # directory and using forward slashes even on Windows
@@ -287,53 +288,30 @@ def mozharness_on_generic_worker(config,
 
     for option in run.get('options', []):
         assert ' ' not in option
         mh_command.append('--' + option)
     if run.get('custom-build-variant-cfg'):
         mh_command.append('--custom-build-variant')
         mh_command.append(run['custom-build-variant-cfg'])
 
-    def checkout_repo(base_repo, head_repo, head_rev, path):
-        hg_command = ['"c:\\Program Files\\Mercurial\\hg.exe"']
-        hg_command.append('robustcheckout')
-        hg_command.extend(['--sharebase', 'y:\\hg-shared'])
-        hg_command.append('--purge')
-        hg_command.extend(['--upstream', base_repo])
-        hg_command.extend(['--revision', head_rev])
-        hg_command.append(head_repo)
-        hg_command.append(path)
-
-        logging_command = [
-            b":: TinderboxPrint:<a href={source_repo}/rev/{revision} "
-            b"title='Built from {repo_name} revision {revision}'>{revision}</a>\n".format(
-                revision=head_rev,
-                source_repo=head_repo,
-                repo_name=head_repo.split('/')[-1],
-            )]
-
-        return [
-            ' '.join(hg_command),
-            ' '.join(logging_command),
-        ]
-
-    hg_commands = checkout_repo(
+    hg_commands = generic_worker_hg_commands(
         base_repo=env['GECKO_BASE_REPOSITORY'],
         head_repo=env['GECKO_HEAD_REPOSITORY'],
         head_rev=env['GECKO_HEAD_REV'],
-        path='.\\build\\src')
+        path=r'.\build\src',
+    )
 
     if run['comm-checkout']:
         hg_commands.extend(
-            checkout_repo(
+            generic_worker_hg_commands(
                 base_repo=env['COMM_BASE_REPOSITORY'],
                 head_repo=env['COMM_HEAD_REPOSITORY'],
                 head_rev=env['COMM_HEAD_REV'],
-                path='.\\build\\src\\comm')
-        )
+                path=r'.\build\src\comm'))
 
     worker['command'] = []
     if taskdesc.get('needs-sccache'):
         worker['command'].extend([
             # Make the comment part of the first command, as it will help users to
             # understand what is going on, and why these steps are implemented.
             dedent('''\
             :: sccache currently uses the full compiler commandline as input to the
--- a/taskcluster/taskgraph/transforms/job/spidermonkey.py
+++ b/taskcluster/taskgraph/transforms/job/spidermonkey.py
@@ -9,16 +9,17 @@ from __future__ import absolute_import, 
 
 from taskgraph.util.schema import Schema
 from voluptuous import Required, Any
 
 from taskgraph.transforms.job import run_job_using
 from taskgraph.transforms.job.common import (
     docker_worker_add_artifacts,
     generic_worker_add_artifacts,
+    generic_worker_hg_commands,
     docker_worker_add_gecko_vcs_env_vars,
     docker_worker_add_tooltool,
     support_vcs_checkout,
 )
 
 sm_run_schema = Schema({
     Required('using'): Any('spidermonkey', 'spidermonkey-package', 'spidermonkey-mozjs-crate',
                            'spidermonkey-rust-bindings'),
@@ -110,25 +111,22 @@ def generic_worker_spidermonkey(config, 
         script = "build-sm-mozjs-crate.sh"
         # Don't allow untested configurations yet
         raise Exception("spidermonkey-mozjs-crate is not a supported configuration")
     elif run['using'] == 'spidermonkey-rust-bindings':
         script = "build-sm-rust-bindings.sh"
         # Don't allow untested configurations yet
         raise Exception("spidermonkey-rust-bindings is not a supported configuration")
 
-    hg_command = ['"c:\\Program Files\\Mercurial\\hg.exe"']
-    hg_command.append('robustcheckout')
-    hg_command.extend(['--sharebase', 'y:\\hg-shared'])
-    hg_command.append('--purge')
-    hg_command.extend(['--upstream', 'https://hg.mozilla.org/mozilla-unified'])
-    hg_command.extend(['--revision', env['GECKO_HEAD_REV']])
-    hg_command.append(env['GECKO_HEAD_REPOSITORY'])
-    hg_command.append('.\\src')
+    hg_command = generic_worker_hg_commands(
+        'https://hg.mozilla.org/mozilla-unified',
+        env['GECKO_HEAD_REPOSITORY'],
+        env['GECKO_HEAD_REV'],
+        r'.\src',
+    )[0]
 
     command = ['c:\\mozilla-build\\msys\\bin\\bash.exe '  # string concat
                '"./src/taskcluster/scripts/builder/%s"' % script]
 
-    worker['command'] = []
-    worker['command'].extend([
-        ' '.join(hg_command),
-        ' '.join(command)
-    ])
+    worker['command'] = [
+        hg_command,
+        ' '.join(command),
+    ]
--- a/taskcluster/taskgraph/transforms/job/toolchain.py
+++ b/taskcluster/taskgraph/transforms/job/toolchain.py
@@ -12,16 +12,17 @@ from mozbuild.shellutil import quote as 
 from taskgraph.util.schema import Schema
 from voluptuous import Optional, Required, Any
 
 from taskgraph.transforms.job import run_job_using
 from taskgraph.transforms.job.common import (
     docker_worker_add_gecko_vcs_env_vars,
     docker_worker_add_artifacts,
     docker_worker_add_tooltool,
+    generic_worker_hg_commands,
     support_vcs_checkout,
 )
 from taskgraph.util.hash import hash_paths
 from taskgraph import GECKO
 from taskgraph.util.cached_tasks import add_optimization
 import taskgraph
 
 
@@ -200,37 +201,33 @@ def windows_toolchain(config, job, taskd
 
     env = worker['env']
     env.update({
         'MOZ_BUILD_DATE': config.params['moz_build_date'],
         'MOZ_SCM_LEVEL': config.params['level'],
         'MOZ_AUTOMATION': '1',
     })
 
-    hg = r'c:\Program Files\Mercurial\hg.exe'
-    hg_command = ['"{}"'.format(hg)]
-    hg_command.append('robustcheckout')
-    hg_command.extend(['--sharebase', 'y:\\hg-shared'])
-    hg_command.append('--purge')
-    hg_command.extend(['--upstream', 'https://hg.mozilla.org/mozilla-unified'])
-    hg_command.extend(['--revision', '%GECKO_HEAD_REV%'])
-    hg_command.append('%GECKO_HEAD_REPOSITORY%')
-    hg_command.append('.\\build\\src')
+    hg_command = generic_worker_hg_commands(
+        'https://hg.mozilla.org/mozilla-unified',
+        env['GECKO_HEAD_REPOSITORY'],
+        env['GECKO_HEAD_REV'],
+        r'.\build\src')[0]
 
     # Use `mach` to invoke python scripts so in-tree libraries are available.
     if run['script'].endswith('.py'):
         raise NotImplementedError("Python scripts don't work on Windows")
 
     args = run.get('arguments', '')
     if args:
         args = ' ' + shell_quote(*args)
 
     bash = r'c:\mozilla-build\msys\bin\bash'
     worker['command'] = [
-        ' '.join(hg_command),
+        hg_command,
         # do something intelligent.
         r'{} build/src/taskcluster/scripts/misc/{}{}'.format(
             bash, run['script'], args)
     ]
 
     attributes = taskdesc.setdefault('attributes', {})
     attributes['toolchain-artifact'] = run['toolchain-artifact']
     if 'toolchain-alias' in run: