Bug 1282187 - create task that publishes nightlies on update server via scriptworker -> balrogscript. r=jlund
authorJustin Wood <Callek@gmail.com>
Thu, 10 Nov 2016 14:42:00 -0500
changeset 437482 95f0d61ced0eef56ce73db673facc1fb053bcbbb
parent 437481 599bc83c7daff03721359515312fca3664d02b47
child 439904 c13bc2963773371c8e4db3661cfb20fd1b6f8d3d
push id35425
push userCallek@gmail.com
push dateFri, 11 Nov 2016 00:15:46 +0000
reviewersjlund
bugs1282187
milestone52.0a1
Bug 1282187 - create task that publishes nightlies on update server via scriptworker -> balrogscript. r=jlund MozReview-Commit-ID: 3Cz8tFbD1gQ
taskcluster/ci/balrog/kind.yml
taskcluster/taskgraph/task/balrog.py
taskcluster/taskgraph/transforms/balrog.py
taskcluster/taskgraph/transforms/task.py
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/balrog/kind.yml
@@ -0,0 +1,12 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+implementation: taskgraph.task.balrog:BalrogTask
+
+transforms:
+   - taskgraph.transforms.balrog:transforms
+   - taskgraph.transforms.task:transforms
+
+kind-dependencies:
+  - beetmover
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/task/balrog.py
@@ -0,0 +1,28 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from . import transform
+
+
+class BalrogTask(transform.TransformTask):
+    """
+    A task implementing a balrog submission job.  These depend on beetmover jobs
+    and submits the update to balrog as available after the files are moved into place
+    """
+
+    @classmethod
+    def get_inputs(cls, kind, path, config, params, loaded_tasks):
+        if config.get('kind-dependencies', []) != ["beetmover"]:
+            raise Exception("Balrog kinds must depend on beetmover kinds")
+        for task in loaded_tasks:
+            if not task.attributes.get('nightly'):
+                continue
+            if task.kind != "beetmover":
+                continue
+            beetmover_task = {}
+            beetmover_task['dependent-task'] = task
+
+            yield beetmover_task
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/balrog.py
@@ -0,0 +1,97 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Transform the beetmover task into an actual task description.
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from taskgraph.transforms.base import (
+    validate_schema,
+    TransformSequence
+)
+from taskgraph.transforms.task import task_description_schema
+from voluptuous import Schema, Any, Required, Optional
+
+
+# Voluptuous uses marker objects as dictionary *keys*, but they are not
+# comparable, so we cast all of the keys back to regular strings
+task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
+
+transforms = TransformSequence()
+
+# shortcut for a string where task references are allowed
+taskref_or_string = Any(
+    basestring,
+    {Required('task-reference'): basestring})
+
+balrog_description_schema = Schema({
+    # the dependent task (object) for this balrog job, used to inform balrogworker.
+    Required('dependent-task'): object,
+
+    # unique label to describe this balrog task, defaults to balrog-{dep.label}
+    Optional('label'): basestring,
+
+    # treeherder is allowed here to override any defaults we use for beetmover.  See
+    # taskcluster/taskgraph/transforms/task.py for the schema details, and the
+    # below transforms for defaults of various values.
+    Optional('treeherder'): task_description_schema['treeherder'],
+})
+
+
+@transforms.add
+def validate(config, jobs):
+    for job in jobs:
+        label = job.get('dependent-task', object).__dict__.get('label', '?no-label?')
+        yield validate_schema(
+            balrog_description_schema, job,
+            "In balrog ({!r} kind) task for {!r}:".format(config.kind, label))
+
+
+@transforms.add
+def skip_unsigned_beets(config, jobs):
+    for job in jobs:
+        if 'signing' not in job['dependent-task'].label:
+            # Skip making a balrog task for this
+            continue
+        yield job
+
+
+@transforms.add
+def make_task_description(config, jobs):
+    for job in jobs:
+        dep_job = job['dependent-task']
+
+        treeherder = job.get('treeherder', {})
+        treeherder.setdefault('symbol', 'tc-Up(N)')
+        dep_th_platform = dep_job.task.get('extra', {}).get(
+            'treeherder', {}).get('machine', {}).get('platform', '')
+        treeherder.setdefault('platform',
+                              "{}/opt".format(dep_th_platform))
+        treeherder.setdefault('tier', 2)
+        treeherder.setdefault('kind', 'build')
+
+        label = job.get('label', "balrog-{}".format(dep_job.label))
+
+        task = {
+            'label': label,
+            'description': "{} Balrog".format(
+                dep_job.task["metadata"]["description"]),
+            # do we have to define worker type somewhere?
+            'worker-type': 'scriptworker-prov-v1/balrogworker-v1',
+            'worker': {'implementation': 'balrog',
+                       'taskid': {"task-reference":
+                                  '<beetmover>'}},
+            'scopes': [],
+            'dependencies': {'beetmover': dep_job.label},
+            'attributes': {
+                'nightly': dep_job.attributes.get('nightly', False),
+                'build_platform': dep_job.attributes.get('build_platform'),
+                'build_type': dep_job.attributes.get('build_type'),
+            },
+            'run-on-projects': dep_job.attributes.get('run_on_projects'),
+            'treeherder': treeherder,
+        }
+
+        yield task
--- a/taskcluster/taskgraph/transforms/task.py
+++ b/taskcluster/taskgraph/transforms/task.py
@@ -18,16 +18,18 @@ from taskgraph.transforms.base import (
     validate_schema,
     TransformSequence
 )
 from voluptuous import Schema, Any, Required, Optional, Extra
 
 from .gecko_v2_whitelist import JOB_NAME_WHITELIST, JOB_NAME_WHITELIST_ERROR
 
 
+ARTIFACT_URL = 'https://queue.taskcluster.net/v1/task/{}/artifacts/{}'
+
 # shortcut for a string where task references are allowed
 taskref_or_string = Any(
     basestring,
     {Required('task-reference'): basestring})
 
 # A task description is a general description of a TaskCluster task
 task_description_schema = Schema({
     # the label for this task
@@ -304,16 +306,21 @@ task_description_schema = Schema({
         # taskid of task with artifacts to beetmove
         Required('taskid_to_beetmove'): taskref_or_string,
 
         # taskid of task with artifacts to beetmove
         Required('taskid_of_manifest'): taskref_or_string,
 
         # beetmover template key
         Required('update_manifest'): bool,
+    }, {
+        Required('implementation'): 'balrog',
+
+        # taskid of the signed beetmoved task
+        Required('taskid', default="invalid"): taskref_or_string,
     }),
 
     # The "when" section contains descriptions of the circumstances
     # under which this task can be "optimized", that is, left out of the
     # task graph because it is unnecessary.
     Optional('when'): Any({
         # This task only needs to be run if a file matching one of the given
         # patterns has changed in the push.  The patterns use the mozpack
@@ -334,16 +341,17 @@ GROUP_NAMES = {
     'tc-R': 'Reftests executed by TaskCluster',
     'tc-R-e10s': 'Reftests executed by TaskCluster with e10s',
     'tc-VP': 'VideoPuppeteer tests executed by TaskCluster',
     'tc-W': 'Web platform tests executed by TaskCluster',
     'tc-W-e10s': 'Web platform tests executed by TaskCluster with e10s',
     'tc-X': 'Xpcshell tests executed by TaskCluster',
     'tc-X-e10s': 'Xpcshell tests executed by TaskCluster with e10s',
     'tc-L10n': 'Localised Repacks executed by Taskcluster',
+    'tc-Up': 'Balrog submission of updates, executed by Taskcluster',
     'Aries': 'Aries Device Image',
     'Nexus 5-L': 'Nexus 5-L Device Image',
     'Cc': 'Toolchain builds',
     'SM-tc': 'Spidermonkey builds',
 }
 UNKNOWN_GROUP_NAME = "Treeherder group {} has no name; add it to " + __file__
 
 BUILDBOT_ROUTE_TEMPLATES = [
@@ -508,16 +516,29 @@ def build_beetmover_payload(config, task
         'maxRunTime': worker['max-run-time'],
         'upload_date': config.params['build_date'],
         'taskid_to_beetmove': worker['taskid_to_beetmove'],
         'taskid_of_manifest': worker['taskid_of_manifest'],
         'update_manifest': worker['update_manifest']
     }
 
 
+@payload_builder('balrog')
+def build_balrog_payload(config, task, task_def):
+    worker = task['worker']
+
+    artifact_url = ARTIFACT_URL.format(worker['taskid'], "public")
+
+    task_def['payload'] = {
+        'parent_task_artifacts_url': artifact_url,
+        # signing cert is unused, but required by balrogworker (Bug 1282187 c#7)
+        'signing_cert': "dep",
+    }
+
+
 @payload_builder('macosx-engine')
 def build_macosx_engine_payload(config, task, task_def):
     worker = task['worker']
     artifacts = map(lambda artifact: {
         'name': artifact['name'],
         'path': artifact['path'],
         'type': artifact['type'],
         'expires': task_def['expires'],