Bug 1385055 - Add taskgraph support for checking out comm-central and related branches. draft
authorTom Prince <mozilla@hocat.ca>
Thu, 27 Jul 2017 12:39:08 -0600
changeset 676749 1ed7335f330de5a42f9fafc386518f7bb2900e3e
parent 676748 6741015b8a24d3a844b3dc43ee56d011ff47d5fb
child 735055 31b20a23bc5a3d6d5762f7fd4ed2922df4295d8f
push id83623
push userbmo:mozilla@hocat.ca
push dateMon, 09 Oct 2017 16:33:09 +0000
bugs1385055
milestone58.0a1
Bug 1385055 - Add taskgraph support for checking out comm-central and related branches. MozReview-Commit-ID: 1eU567Cuc8a
taskcluster/taskgraph/transforms/job/common.py
taskcluster/taskgraph/transforms/job/mach.py
taskcluster/taskgraph/transforms/job/mozharness.py
taskcluster/taskgraph/transforms/job/run_task.py
--- a/taskcluster/taskgraph/transforms/job/common.py
+++ b/taskcluster/taskgraph/transforms/job/common.py
@@ -67,16 +67,24 @@ def docker_worker_add_gecko_vcs_env_vars
     env = taskdesc['worker'].setdefault('env', {})
     env.update({
         'GECKO_BASE_REPOSITORY': config.params['base_repository'],
         'GECKO_HEAD_REF': config.params['head_rev'],
         'GECKO_HEAD_REPOSITORY': config.params['head_repository'],
         'GECKO_HEAD_REV': config.params['head_rev'],
     })
 
+    if 'comm_base_repository' in config.params:
+        taskdesc['worker']['env'].update({
+            'COMM_BASE_REPOSITORY': config.params['comm_base_repository'],
+            'COMM_HEAD_REF': config.params['comm_head_rev'],
+            'COMM_HEAD_REPOSITORY': config.params['comm_head_repository'],
+            'COMM_HEAD_REV': config.params['comm_head_rev'],
+        })
+
 
 def support_vcs_checkout(config, job, taskdesc, sparse=False):
     """Update a job/task with parameters to enable a VCS checkout.
 
     This can only be used with ``run-task`` tasks, as the cache name is
     reserved for ``run-task`` tasks.
     """
     level = config.params['level']
@@ -99,16 +107,25 @@ def support_vcs_checkout(config, job, ta
 
     taskdesc['worker'].setdefault('env', {}).update({
         'GECKO_BASE_REPOSITORY': config.params['base_repository'],
         'GECKO_HEAD_REPOSITORY': config.params['head_repository'],
         'GECKO_HEAD_REV': config.params['head_rev'],
         'HG_STORE_PATH': '/builds/worker/checkouts/hg-store',
     })
 
+    if 'comm_base_repository' in config.params:
+        taskdesc['worker']['env'].update({
+            'COMM_BASE_REPOSITORY': config.params['comm_base_repository'],
+            'COMM_HEAD_REPOSITORY': config.params['comm_head_repository'],
+            'COMM_HEAD_REV': config.params['comm_head_rev'],
+        })
+    elif job['run'].get('comm-checkout', False):
+        raise Exception("Can't checkout from comm-* repository if not given a repository.")
+
     # Give task access to hgfingerprint secret so it can pin the certificate
     # for hg.mozilla.org.
     taskdesc['scopes'].append('secrets:get:project/taskcluster/gecko/hgfingerprint')
 
     # only some worker platforms have taskcluster-proxy enabled
     if job['worker']['implementation'] in ('docker-worker', 'docker-engine'):
         taskdesc['worker']['taskcluster-proxy'] = True
 
--- a/taskcluster/taskgraph/transforms/job/mach.py
+++ b/taskcluster/taskgraph/transforms/job/mach.py
@@ -2,21 +2,17 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 """
 Support for running mach tasks (via run-task)
 """
 
 from __future__ import absolute_import, print_function, unicode_literals
 
-from taskgraph.transforms.job import run_job_using
-from taskgraph.transforms.job.run_task import (
-    docker_worker_run_task,
-    native_engine_run_task,
-)
+from taskgraph.transforms.job import run_job_using, configure_taskdesc_for_run
 from taskgraph.util.schema import Schema
 from voluptuous import Required
 
 mach_schema = Schema({
     Required('using'): 'mach',
 
     # The mach command (omitting `./mach`) to run
     Required('mach'): basestring,
@@ -25,15 +21,11 @@ mach_schema = Schema({
 
 @run_job_using("docker-worker", "mach", schema=mach_schema)
 @run_job_using("native-engine", "mach", schema=mach_schema)
 def docker_worker_mach(config, job, taskdesc):
     run = job['run']
 
     # defer to the run_task implementation
     run['command'] = 'cd /builds/worker/checkouts/gecko && ./mach ' + run['mach']
-    run['checkout'] = True
-    run['sparse-profile'] = None
+    run['using'] = 'run-task'
     del run['mach']
-    if job['worker']['implementation'] == 'docker-worker':
-        docker_worker_run_task(config, job, taskdesc)
-    else:
-        native_engine_run_task(config, job, taskdesc)
+    configure_taskdesc_for_run(config, job, taskdesc, job['worker']['implementation'])
--- a/taskcluster/taskgraph/transforms/job/mozharness.py
+++ b/taskcluster/taskgraph/transforms/job/mozharness.py
@@ -86,16 +86,20 @@ mozharness_run_schema = Schema({
 
     # If false, don't set MOZ_SIMPLE_PACKAGE_NAME
     # Only disableable on windows
     Required('use-simple-package', default=True): bool,
 
     # If false don't pass --branch or --skip-buildbot-actions to mozharness script
     # Only disableable on windows
     Required('use-magic-mh-args', default=True): bool,
+
+    # if true, perform a checkout of a comm-central based branch inside the
+    # gecko checkout
+    Required('comm-checkout', default=False): bool,
 })
 
 
 @run_job_using("docker-worker", "mozharness", schema=mozharness_run_schema)
 def mozharness_on_docker_worker_setup(config, job, taskdesc):
     run = job['run']
 
     worker = taskdesc['worker']
@@ -164,16 +168,21 @@ def mozharness_on_docker_worker_setup(co
     worker['retry-exit-status'] = 4
 
     docker_worker_setup_secrets(config, job, taskdesc)
 
     command = [
         '/builds/worker/bin/run-task',
         '--vcs-checkout', '/builds/worker/workspace/build/src',
         '--tools-checkout', '/builds/worker/workspace/build/tools',
+    ]
+    if run['comm-checkout']:
+        command.append('--comm-checkout=/builds/worker/workspace/build/src/comm')
+
+    command += [
         '--',
         '/builds/worker/workspace/build/src/{}'.format(
             run.get('job-script', 'taskcluster/scripts/builder/build-linux.sh')
         ),
     ]
 
     worker['command'] = command
 
--- a/taskcluster/taskgraph/transforms/job/run_task.py
+++ b/taskcluster/taskgraph/transforms/job/run_task.py
@@ -21,16 +21,20 @@ run_task_schema = Schema({
 
     # if true (the default), perform a checkout in /builds/worker/checkouts/gecko
     Required('checkout', default=True): bool,
 
     # The sparse checkout profile to use. Value is the filename relative to the
     # directory where sparse profiles are defined (build/sparse-profiles/).
     Required('sparse-profile', default=None): basestring,
 
+    # if true, perform a checkout of a comm-central based branch inside the
+    # gecko checkout
+    Required('comm-checkout', default=False): bool,
+
     # The command arguments to pass to the `run-task` script, after the
     # checkout arguments.  If a list, it will be passed directly; otherwise
     # it will be included in a single argument to `bash -cx`.
     Required('command'): Any([basestring], basestring),
 })
 
 
 def common_setup(config, job, taskdesc):
@@ -66,16 +70,18 @@ def docker_worker_run_task(config, job, 
         'skip-untrusted': True,
     })
 
     run_command = run['command']
     if isinstance(run_command, basestring):
         run_command = ['bash', '-cx', run_command]
     command = ['/builds/worker/bin/run-task']
     add_checkout_to_command(run, command)
+    if run['comm-checkout']:
+        command.append('--comm-checkout=/builds/worker/checkouts/gecko/comm')
     command.append('--fetch-hgfingerprint')
     command.append('--')
     command.extend(run_command)
     worker['command'] = command
 
 
 @run_job_using("native-engine", "run-task", schema=run_task_schema)
 def native_engine_run_task(config, job, taskdesc):