Bug 1048446 - [taskcluster] Move 'require-build' out of run_task and into source_test, r?dustin draft
authorAndrew Halberstadt <ahalberstadt@mozilla.com>
Mon, 29 May 2017 15:34:54 -0400
changeset 591744 e9e0846331c43bfc1b508a16cfd73049578640ea
parent 591743 3c9703d47928a5f30ea1d9925d3ffa2881ed6273
child 591745 7414d042db01b87d4845ac4da506c623f4905362
push id63159
push userahalberstadt@mozilla.com
push dateFri, 09 Jun 2017 15:00:30 +0000
reviewersdustin
bugs1048446
milestone55.0a1
Bug 1048446 - [taskcluster] Move 'require-build' out of run_task and into source_test, r?dustin The 'platform' key was recently moved out of 'job' tasks and into the 'source-test' kind. Since the concept of requiring a build depends on this key, let's move that back to source-test as well. MozReview-Commit-ID: 4bs8G4wN5OH
taskcluster/taskgraph/transforms/job/common.py
taskcluster/taskgraph/transforms/job/mach.py
taskcluster/taskgraph/transforms/job/run_task.py
taskcluster/taskgraph/transforms/source_test.py
--- a/taskcluster/taskgraph/transforms/job/common.py
+++ b/taskcluster/taskgraph/transforms/job/common.py
@@ -4,20 +4,16 @@
 """
 Common support for various job types.  These functions are all named after the
 worker implementation they operate on, and take the same three parameters, for
 consistency.
 """
 
 from __future__ import absolute_import, print_function, unicode_literals
 
-from taskgraph.util.attributes import keymatch
-
-
-ARTIFACT_URL = 'https://queue.taskcluster.net/v1/task/{}/artifacts/{}'
 SECRET_SCOPE = 'secrets:get:project/releng/gecko/{}/level-{}/{}'
 
 
 def docker_worker_add_workspace_cache(config, job, taskdesc):
     """Add the workspace cache based on the build platform/type and level,
     except on try where workspace caches are not used."""
     if config.params['project'] == 'try':
         return
@@ -56,40 +52,16 @@ def docker_worker_add_gecko_vcs_env_vars
     env.update({
         'GECKO_BASE_REPOSITORY': config.params['base_repository'],
         'GECKO_HEAD_REF': config.params['head_rev'],
         'GECKO_HEAD_REPOSITORY': config.params['head_repository'],
         'GECKO_HEAD_REV': config.params['head_rev'],
     })
 
 
-def add_build_dependency(config, job, taskdesc):
-    """Add build dependency to the task description and installer_url to env."""
-    key = job['platform']
-    build_labels = config.config.get('dependent-build-platforms', {})
-    matches = keymatch(build_labels, key)
-    if not matches:
-        raise Exception("No build platform found for '{}'. "
-                        "Define 'dependent-build-platforms' in the kind config.".format(key))
-
-    if len(matches) > 1:
-        raise Exception("More than one build platform found for '{}'.".format(key))
-
-    label = matches[0]['label']
-    target = matches[0]['target-name']
-    deps = taskdesc.setdefault('dependencies', {})
-    deps.update({'build': label})
-
-    build_artifact = 'public/build/{}'.format(target)
-    installer_url = ARTIFACT_URL.format('<build>', build_artifact)
-
-    env = taskdesc['worker'].setdefault('env', {})
-    env.update({'GECKO_INSTALLER_URL': {'task-reference': installer_url}})
-
-
 def support_vcs_checkout(config, job, taskdesc):
     """Update a job/task with parameters to enable a VCS checkout.
 
     The configuration is intended for tasks using "run-task" and its
     VCS checkout behavior.
     """
     level = config.params['level']
 
--- a/taskcluster/taskgraph/transforms/job/mach.py
+++ b/taskcluster/taskgraph/transforms/job/mach.py
@@ -15,22 +15,16 @@ from taskgraph.transforms.job.run_task i
 from taskgraph.util.schema import Schema
 from voluptuous import Required
 
 mach_schema = Schema({
     Required('using'): 'mach',
 
     # The mach command (omitting `./mach`) to run
     Required('mach'): basestring,
-
-    # Whether the job requires a build artifact or not. If True, the task
-    # will depend on a build task and run-task will download and set up the
-    # installer. Build labels are determined by the `dependent-build-platforms`
-    # config in kind.yml.
-    Required('requires-build', default=False): bool,
 })
 
 
 @run_job_using("docker-worker", "mach", schema=mach_schema)
 @run_job_using("native-engine", "mach", schema=mach_schema)
 def docker_worker_mach(config, job, taskdesc):
     run = job['run']
 
--- a/taskcluster/taskgraph/transforms/job/run_task.py
+++ b/taskcluster/taskgraph/transforms/job/run_task.py
@@ -4,53 +4,41 @@
 """
 Support for running jobs that are invoked via the `run-task` script.
 """
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 from taskgraph.transforms.job import run_job_using
 from taskgraph.util.schema import Schema
-from taskgraph.transforms.job.common import (
-    add_build_dependency,
-    support_vcs_checkout,
-)
+from taskgraph.transforms.job.common import support_vcs_checkout
 from voluptuous import Required, Any
 
 run_task_schema = Schema({
     Required('using'): 'run-task',
 
     # if true, add a cache at ~worker/.cache, which is where things like pip
     # tend to hide their caches.  This cache is never added for level-1 jobs.
     Required('cache-dotcache', default=False): bool,
 
     # if true (the default), perform a checkout in /home/worker/checkouts/gecko
     Required('checkout', default=True): bool,
 
     # The command arguments to pass to the `run-task` script, after the
     # checkout arguments.  If a list, it will be passed directly; otherwise
     # it will be included in a single argument to `bash -cx`.
     Required('command'): Any([basestring], basestring),
-
-    # Whether the job requires a build artifact or not. If True, the task
-    # will depend on a build task and run-task will download and set up the
-    # installer. Build labels are determined by the `dependent-build-platforms`
-    # config in kind.yml.
-    Required('requires-build', default=False): bool,
 })
 
 
 def common_setup(config, job, taskdesc):
     run = job['run']
     if run['checkout']:
         support_vcs_checkout(config, job, taskdesc)
 
-    if run['requires-build']:
-        add_build_dependency(config, job, taskdesc)
-
 
 @run_job_using("docker-worker", "run-task", schema=run_task_schema)
 def docker_worker_run_task(config, job, taskdesc):
     run = job['run']
     worker = taskdesc['worker'] = job['worker']
     common_setup(config, job, taskdesc)
 
     if run.get('cache-dotcache') and int(config.params['level']) > 1:
--- a/taskcluster/taskgraph/transforms/source_test.py
+++ b/taskcluster/taskgraph/transforms/source_test.py
@@ -8,39 +8,48 @@ treeherder configuration and attributes 
 """
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 import copy
 
 from taskgraph.transforms.base import TransformSequence
 from taskgraph.transforms.job import job_description_schema
+from taskgraph.util.attributes import keymatch
 from taskgraph.util.schema import (
     validate_schema,
     resolve_keyed_by,
 )
 from voluptuous import (
     Any,
     Extra,
     Required,
     Schema,
 )
 
+ARTIFACT_URL = 'https://queue.taskcluster.net/v1/task/{}/artifacts/{}'
+
 job_description_schema = {str(k): v for k, v in job_description_schema.schema.iteritems()}
 
 source_test_description_schema = Schema({
     # most fields are passed directly through as job fields, and are not
     # repeated here
     Extra: object,
 
     # The platform on which this task runs.  This will be used to set up attributes
     # (for try selection) and treeherder metadata (for display).  If given as a list,
     # the job will be "split" into multiple tasks, one with each platform.
     Required('platform'): Any(basestring, [basestring]),
 
+    # Whether the job requires a build artifact or not. If True, the task will
+    # depend on a build task and the installer url will be saved to the
+    # GECKO_INSTALLER_URL environment variable. Build labels are determined by the
+    # `dependent-build-platforms` config in kind.yml.
+    Required('require-build', default=False): bool,
+
     # These fields can be keyed by "platform", and are otherwise identical to
     # job descriptions.
     Required('worker-type'): Any(
         job_description_schema['worker-type'],
         {'by-platform': {basestring: job_description_schema['worker-type']}},
     ),
     Required('worker'): Any(
         job_description_schema['worker'],
@@ -78,16 +87,42 @@ def expand_platforms(config, jobs):
 
             if 'name' in pjob:
                 pjob['name'] = '{}-{}'.format(pjob['name'], platform)
             else:
                 pjob['label'] = '{}-{}'.format(pjob['label'], platform)
             yield pjob
 
 
+def add_build_dependency(config, job):
+    """
+    Add build dependency to the job and installer_url to env.
+    """
+    key = job['platform']
+    build_labels = config.config.get('dependent-build-platforms', {})
+    matches = keymatch(build_labels, key)
+    if not matches:
+        raise Exception("No build platform found for '{}'. "
+                        "Define 'dependent-build-platforms' in the kind config.".format(key))
+
+    if len(matches) > 1:
+        raise Exception("More than one build platform found for '{}'.".format(key))
+
+    label = matches[0]['label']
+    target = matches[0]['target-name']
+    deps = job.setdefault('dependencies', {})
+    deps.update({'build': label})
+
+    build_artifact = 'public/build/{}'.format(target)
+    installer_url = ARTIFACT_URL.format('<build>', build_artifact)
+
+    env = job['worker'].setdefault('env', {})
+    env.update({'GECKO_INSTALLER_URL': {'task-reference': installer_url}})
+
+
 @transforms.add
 def handle_platform(config, jobs):
     """
     Handle the 'platform' property, setting up treeherder context as well as
     try-related attributes.
     """
     fields = [
         'worker-type',
@@ -98,10 +133,13 @@ def handle_platform(config, jobs):
         platform = job['platform']
 
         for field in fields:
             resolve_keyed_by(job, field, item_name=job['name'])
 
         if 'treeherder' in job:
             job['treeherder']['platform'] = platform
 
+        if job.pop('require-build'):
+            add_build_dependency(config, job)
+
         del job['platform']
         yield job