Bug 1282187 - create task that publishes nightlies on update server via scriptworker -> balrogscript. r=jlund
MozReview-Commit-ID: 3Cz8tFbD1gQ
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/balrog/kind.yml
@@ -0,0 +1,12 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+implementation: taskgraph.task.balrog:BalrogTask
+
+transforms:
+ - taskgraph.transforms.balrog:transforms
+ - taskgraph.transforms.task:transforms
+
+kind-dependencies:
+ - beetmover
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/task/balrog.py
@@ -0,0 +1,28 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from . import transform
+
+
+class BalrogTask(transform.TransformTask):
+ """
+ A task implementing a balrog submission job. These depend on beetmover jobs
+ and submits the update to balrog as available after the files are moved into place
+ """
+
+ @classmethod
+ def get_inputs(cls, kind, path, config, params, loaded_tasks):
+ if config.get('kind-dependencies', []) != ["beetmover"]:
+ raise Exception("Balrog kinds must depend on beetmover kinds")
+ for task in loaded_tasks:
+ if not task.attributes.get('nightly'):
+ continue
+ if task.kind != "beetmover":
+ continue
+ beetmover_task = {}
+ beetmover_task['dependent-task'] = task
+
+ yield beetmover_task
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/balrog.py
@@ -0,0 +1,97 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Transform the beetmover task into an actual task description.
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from taskgraph.transforms.base import (
+ validate_schema,
+ TransformSequence
+)
+from taskgraph.transforms.task import task_description_schema
+from voluptuous import Schema, Any, Required, Optional
+
+
+# Voluptuous uses marker objects as dictionary *keys*, but they are not
+# comparable, so we cast all of the keys back to regular strings
+task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
+
+transforms = TransformSequence()
+
+# shortcut for a string where task references are allowed
+taskref_or_string = Any(
+ basestring,
+ {Required('task-reference'): basestring})
+
+balrog_description_schema = Schema({
+ # the dependent task (object) for this balrog job, used to inform balrogworker.
+ Required('dependent-task'): object,
+
+ # unique label to describe this balrog task, defaults to balrog-{dep.label}
+ Optional('label'): basestring,
+
+ # treeherder is allowed here to override any defaults we use for beetmover. See
+ # taskcluster/taskgraph/transforms/task.py for the schema details, and the
+ # below transforms for defaults of various values.
+ Optional('treeherder'): task_description_schema['treeherder'],
+})
+
+
+@transforms.add
+def validate(config, jobs):
+ for job in jobs:
+ label = job.get('dependent-task', object).__dict__.get('label', '?no-label?')
+ yield validate_schema(
+ balrog_description_schema, job,
+ "In balrog ({!r} kind) task for {!r}:".format(config.kind, label))
+
+
+@transforms.add
+def skip_unsigned_beets(config, jobs):
+ for job in jobs:
+ if 'signing' not in job['dependent-task'].label:
+ # Skip making a balrog task for this
+ continue
+ yield job
+
+
+@transforms.add
+def make_task_description(config, jobs):
+ for job in jobs:
+ dep_job = job['dependent-task']
+
+ treeherder = job.get('treeherder', {})
+ treeherder.setdefault('symbol', 'tc-Up(N)')
+ dep_th_platform = dep_job.task.get('extra', {}).get(
+ 'treeherder', {}).get('machine', {}).get('platform', '')
+ treeherder.setdefault('platform',
+ "{}/opt".format(dep_th_platform))
+ treeherder.setdefault('tier', 2)
+ treeherder.setdefault('kind', 'build')
+
+ label = job.get('label', "balrog-{}".format(dep_job.label))
+
+ task = {
+ 'label': label,
+ 'description': "{} Balrog".format(
+ dep_job.task["metadata"]["description"]),
+ # do we have to define worker type somewhere?
+ 'worker-type': 'scriptworker-prov-v1/balrogworker-v1',
+ 'worker': {'implementation': 'balrog',
+ 'taskid': {"task-reference":
+ '<beetmover>'}},
+ 'scopes': [],
+ 'dependencies': {'beetmover': dep_job.label},
+ 'attributes': {
+ 'nightly': dep_job.attributes.get('nightly', False),
+ 'build_platform': dep_job.attributes.get('build_platform'),
+ 'build_type': dep_job.attributes.get('build_type'),
+ },
+ 'run-on-projects': dep_job.attributes.get('run_on_projects'),
+ 'treeherder': treeherder,
+ }
+
+ yield task
--- a/taskcluster/taskgraph/transforms/task.py
+++ b/taskcluster/taskgraph/transforms/task.py
@@ -18,16 +18,18 @@ from taskgraph.transforms.base import (
validate_schema,
TransformSequence
)
from voluptuous import Schema, Any, Required, Optional, Extra
from .gecko_v2_whitelist import JOB_NAME_WHITELIST, JOB_NAME_WHITELIST_ERROR
+ARTIFACT_URL = 'https://queue.taskcluster.net/v1/task/{}/artifacts/{}'
+
# shortcut for a string where task references are allowed
taskref_or_string = Any(
basestring,
{Required('task-reference'): basestring})
# A task description is a general description of a TaskCluster task
task_description_schema = Schema({
# the label for this task
@@ -304,16 +306,21 @@ task_description_schema = Schema({
# taskid of task with artifacts to beetmove
Required('taskid_to_beetmove'): taskref_or_string,
# taskid of task with artifacts to beetmove
Required('taskid_of_manifest'): taskref_or_string,
# beetmover template key
Required('update_manifest'): bool,
+ }, {
+ Required('implementation'): 'balrog',
+
+ # taskid of the signed beetmoved task
+ Required('taskid', default="invalid"): taskref_or_string,
}),
# The "when" section contains descriptions of the circumstances
# under which this task can be "optimized", that is, left out of the
# task graph because it is unnecessary.
Optional('when'): Any({
# This task only needs to be run if a file matching one of the given
# patterns has changed in the push. The patterns use the mozpack
@@ -334,16 +341,17 @@ GROUP_NAMES = {
'tc-R': 'Reftests executed by TaskCluster',
'tc-R-e10s': 'Reftests executed by TaskCluster with e10s',
'tc-VP': 'VideoPuppeteer tests executed by TaskCluster',
'tc-W': 'Web platform tests executed by TaskCluster',
'tc-W-e10s': 'Web platform tests executed by TaskCluster with e10s',
'tc-X': 'Xpcshell tests executed by TaskCluster',
'tc-X-e10s': 'Xpcshell tests executed by TaskCluster with e10s',
'tc-L10n': 'Localised Repacks executed by Taskcluster',
+ 'tc-Up': 'Balrog submission of updates, executed by Taskcluster',
'Aries': 'Aries Device Image',
'Nexus 5-L': 'Nexus 5-L Device Image',
'Cc': 'Toolchain builds',
'SM-tc': 'Spidermonkey builds',
}
UNKNOWN_GROUP_NAME = "Treeherder group {} has no name; add it to " + __file__
BUILDBOT_ROUTE_TEMPLATES = [
@@ -508,16 +516,29 @@ def build_beetmover_payload(config, task
'maxRunTime': worker['max-run-time'],
'upload_date': config.params['build_date'],
'taskid_to_beetmove': worker['taskid_to_beetmove'],
'taskid_of_manifest': worker['taskid_of_manifest'],
'update_manifest': worker['update_manifest']
}
+@payload_builder('balrog')
+def build_balrog_payload(config, task, task_def):
+ worker = task['worker']
+
+ artifact_url = ARTIFACT_URL.format(worker['taskid'], "public")
+
+ task_def['payload'] = {
+ 'parent_task_artifacts_url': artifact_url,
+ # signing cert is unused, but required by balrogworker (Bug 1282187 c#7)
+ 'signing_cert': "dep",
+ }
+
+
@payload_builder('macosx-engine')
def build_macosx_engine_payload(config, task, task_def):
worker = task['worker']
artifacts = map(lambda artifact: {
'name': artifact['name'],
'path': artifact['path'],
'type': artifact['type'],
'expires': task_def['expires'],