Bug 1289249 - Use run-task for mozharness jobs on docker-worker; r?dustin draft
authorGregory Szorc <gps@mozilla.com>
Thu, 29 Sep 2016 16:51:15 -0700
changeset 419541 be414101fa2915055caf5e623618dcc78ec6789e
parent 419540 a56bf75f2f8afe2573e26547215f073804589945
child 532599 2587c0a7a3f641b3755f58f0c965eb7908c158a1
push id30957
push userbmo:gps@mozilla.com
push dateFri, 30 Sep 2016 16:24:29 +0000
reviewersdustin
bugs1289249
milestone52.0a1
Bug 1289249 - Use run-task for mozharness jobs on docker-worker; r?dustin This transform appears to always be executed on desktop-build. build.sh always calls checkout-sources.sh and checkout-sources.sh always performs a Gecko checkout. It /may/ perform a build/tools checkout unless an environment variable is set. Now that desktop-build is ready to use run-task, we switch this transform to run-task. A gecko and tools checkout is *always* performed. There is room to make the tools checkout optional. But it completes nearly instantly. So I'm inclined to do this as a follow-up enhancement, if needed. Ideally, the checkout would be on a cache so subsequent tasks on the worker only need to perform a purge+update as opposed to a full checkout. However, many tasks make assumptions about the VCS behavior and modifying the checkout path is a non-trivial amount of work. This can be deferred to a follow-up bug. MozReview-Commit-ID: 5Auerl4OL9x
taskcluster/taskgraph/transforms/job/mozharness.py
--- a/taskcluster/taskgraph/transforms/job/mozharness.py
+++ b/taskcluster/taskgraph/transforms/job/mozharness.py
@@ -11,20 +11,20 @@ way, and certainly anything using mozhar
 from __future__ import absolute_import, print_function, unicode_literals
 
 import time
 from voluptuous import Schema, Required, Optional, Any
 
 from taskgraph.transforms.job import run_job_using
 from taskgraph.transforms.job.common import (
     docker_worker_add_workspace_cache,
-    docker_worker_add_tc_vcs_cache,
     docker_worker_add_gecko_vcs_env_vars,
     docker_worker_setup_secrets,
-    docker_worker_add_public_artifacts
+    docker_worker_add_public_artifacts,
+    docker_worker_support_vcs_checkout,
 )
 
 COALESCE_KEY = 'builds.{project}.{name}'
 
 mozharness_run_schema = Schema({
     Required('using'): 'mozharness',
 
     # the mozharness script used to run this task, relative to the testing/
@@ -83,19 +83,18 @@ def mozharness_on_docker_worker_setup(co
 
     # running via mozharness assumes desktop-build (which contains build.sh)
     taskdesc['worker']['docker-image'] = {"in-tree": "desktop-build"}
 
     worker['relengapi-proxy'] = False  # but maybe enabled for tooltool below
     worker['taskcluster-proxy'] = run.get('taskcluster-proxy')
 
     docker_worker_add_public_artifacts(config, job, taskdesc)
-    docker_worker_add_tc_vcs_cache(config, job, taskdesc)
     docker_worker_add_workspace_cache(config, job, taskdesc)
-    docker_worker_add_gecko_vcs_env_vars(config, job, taskdesc)
+    docker_worker_support_vcs_checkout(config, job, taskdesc)
 
     env = worker.setdefault('env', {})
     env.update({
         'MOZHARNESS_CONFIG': ' '.join(run['config']),
         'MOZHARNESS_SCRIPT': run['script'],
         'MH_BRANCH': config.params['project'],
         'MH_BUILD_POOL': 'taskcluster',
         'MOZ_BUILD_DATE': time.strftime("%Y%m%d%H%M%S", time.gmtime(config.params['pushdate'])),
@@ -138,17 +137,28 @@ def mozharness_on_docker_worker_setup(co
             taskdesc['scopes'].append(
                 'docker-worker:relengapi-proxy:tooltool.download.internal')
         env['TOOLTOOL_CACHE'] = '/home/worker/tooltool-cache'
         env['TOOLTOOL_REPO'] = 'https://github.com/mozilla/build-tooltool'
         env['TOOLTOOL_REV'] = 'master'
 
     docker_worker_setup_secrets(config, job, taskdesc)
 
-    worker['command'] = ["/bin/bash", "bin/build.sh"]
+    command = [
+        '/home/worker/bin/run-task',
+        # Various caches/volumes are default owned by root:root.
+        '--chown-recursive', '/home/worker/workspace',
+        '--chown-recursive', '/home/worker/tooltool-cache',
+        '--vcs-checkout', '/home/worker/workspace/build/src',
+        '--tools-checkout', '/home/worker/workspace/build/tools',
+        '--',
+        '/home/worker/workspace/build/src/taskcluster/scripts/builder/build-linux.sh',
+    ]
+
+    worker['command'] = command
 
 
 # We use the generic worker to run tasks on Windows
 @run_job_using("generic-worker", "mozharness", schema=mozharness_run_schema)
 def mozharness_on_windows(config, job, taskdesc):
     run = job['run']
 
     # fail if invalid run options are included