Bug 1330668 - Schedule balrog submission tasks. r=dustin
MozReview-Commit-ID: ASBDESDps8G
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/balrog/kind.yml
@@ -0,0 +1,13 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+implementation: taskgraph.task.balrog:BalrogTask
+
+transforms:
+ - taskgraph.transforms.balrog:transforms
+ - taskgraph.transforms.task:transforms
+
+kind-dependencies:
+ - beetmover
+ - beetmover-l10n
--- a/taskcluster/docs/attributes.rst
+++ b/taskcluster/docs/attributes.rst
@@ -138,9 +138,9 @@ chunk_locales
=============
For the ``l10n`` and ``nightly-l10n`` kinds, this attribute contains an array of
the individual locales this chunk is responsible for processing.
locale
======
For jobs that operate on only one locale, we set the attribute ``locale`` to the
specific locale involved. Currently this is only in l10n versions of the
-``beetmover`` kinds.
+``beetmover`` and ``balrog`` kinds.
--- a/taskcluster/docs/kinds.rst
+++ b/taskcluster/docs/kinds.rst
@@ -155,16 +155,29 @@ Docker repository
The task definition used to create the image-building tasks is given in
``image.yml`` in the kind directory, and is interpreted as a :doc:`YAML
Template <yaml-templates>`.
android-stuff
--------------
+balrog
+------
+
+Balrog is the Mozilla Update Server. Jobs of this kind are submitting information
+which assists in telling Firefox that an update is available for the related job.
+
+balrog-l10n
+-----------
+
+Balrog is the Mozilla Update Server. Jobs of this kind are submitting information
+which assists in telling Firefox that an update is available for the localized
+job involved.
+
beetmover
---------
Beetmover, takes specific artifacts, "Beets", and pushes them to a location outside
of Taskcluster's task artifacts, (archive.mozilla.org as one place) and in the
process determines the final location and a "pretty" name (versioned product name)
beetmover-l10n
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/task/balrog.py
@@ -0,0 +1,28 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from . import transform
+
+
+class BalrogTask(transform.TransformTask):
+ """
+ A task implementing a balrog submission job. These depend on beetmover jobs
+ and submits the update to balrog as available after the files are moved into place
+ """
+
+ @classmethod
+ def get_inputs(cls, kind, path, config, params, loaded_tasks):
+ if config.get('kind-dependencies', []) != ["beetmover", "beetmover-l10n"]:
+ raise Exception("Balrog kinds must depend on beetmover kinds")
+ for task in loaded_tasks:
+ if not task.attributes.get('nightly'):
+ continue
+ if task.kind not in config.get('kind-dependencies', []):
+ continue
+ beetmover_task = {}
+ beetmover_task['dependent-task'] = task
+
+ yield beetmover_task
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/balrog.py
@@ -0,0 +1,113 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Transform the beetmover task into an actual task description.
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from taskgraph.transforms.base import (
+ validate_schema,
+ TransformSequence
+)
+from taskgraph.transforms.task import task_description_schema
+from voluptuous import Schema, Any, Required, Optional
+
+
+# Voluptuous uses marker objects as dictionary *keys*, but they are not
+# comparable, so we cast all of the keys back to regular strings
+task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
+
+transforms = TransformSequence()
+
+# shortcut for a string where task references are allowed
+taskref_or_string = Any(
+ basestring,
+ {Required('task-reference'): basestring})
+
+balrog_description_schema = Schema({
+ # the dependent task (object) for this balrog job, used to inform balrogworker.
+ Required('dependent-task'): object,
+
+ # unique label to describe this balrog task, defaults to balrog-{dep.label}
+ Optional('label'): basestring,
+
+ # treeherder is allowed here to override any defaults we use for beetmover. See
+ # taskcluster/taskgraph/transforms/task.py for the schema details, and the
+ # below transforms for defaults of various values.
+ Optional('treeherder'): task_description_schema['treeherder'],
+})
+
+
+@transforms.add
+def validate(config, jobs):
+ for job in jobs:
+ label = job.get('dependent-task', object).__dict__.get('label', '?no-label?')
+ yield validate_schema(
+ balrog_description_schema, job,
+ "In balrog ({!r} kind) task for {!r}:".format(config.kind, label))
+
+
+@transforms.add
+def skip_unsigned_beets(config, jobs):
+ for job in jobs:
+ if 'signing' not in job['dependent-task'].label:
+ # Skip making a balrog task for this
+ continue
+ yield job
+
+
+@transforms.add
+def make_task_description(config, jobs):
+ for job in jobs:
+ dep_job = job['dependent-task']
+
+ treeherder = job.get('treeherder', {})
+ treeherder.setdefault('symbol', 'tc-Up(N)')
+ dep_th_platform = dep_job.task.get('extra', {}).get(
+ 'treeherder', {}).get('machine', {}).get('platform', '')
+ treeherder.setdefault('platform',
+ "{}/opt".format(dep_th_platform))
+ treeherder.setdefault('tier', 2)
+ treeherder.setdefault('kind', 'build')
+
+ attributes = {
+ 'nightly': dep_job.attributes.get('nightly', False),
+ 'build_platform': dep_job.attributes.get('build_platform'),
+ 'build_type': dep_job.attributes.get('build_type'),
+ }
+
+ if dep_job.attributes.get('locale'):
+ treeherder['symbol'] = 'tc-Up({})'.format(dep_job.attributes.get('locale'))
+ attributes['locale'] = dep_job.attributes.get('locale')
+
+ label = job.get('label', "balrog-{}".format(dep_job.label))
+
+ upstream_artifacts = [{
+ "taskId": {"task-reference": "<beetmover>"},
+ "taskType": "beetmover",
+ "paths": [
+ "public/manifest.json"
+ ],
+ }]
+
+ task = {
+ 'label': label,
+ 'description': "{} Balrog".format(
+ dep_job.task["metadata"]["description"]),
+ # do we have to define worker type somewhere?
+ 'worker-type': 'scriptworker-prov-v1/balrogworker-v1',
+ 'worker': {
+ 'implementation': 'balrog',
+ 'upstream-artifacts': upstream_artifacts,
+ },
+ # bump this to nightly / release when applicable+permitted
+ 'scopes': ["project:releng:balrog:nightly"],
+ 'dependencies': {'beetmover': dep_job.label},
+ 'attributes': attributes,
+ 'run-on-projects': dep_job.attributes.get('run_on_projects'),
+ 'treeherder': treeherder,
+ }
+
+ yield task
--- a/taskcluster/taskgraph/transforms/task.py
+++ b/taskcluster/taskgraph/transforms/task.py
@@ -307,16 +307,30 @@ task_description_schema = Schema({
Required('taskType'): basestring,
# Paths to the artifacts to sign
Required('paths'): [basestring],
# locale is used to map upload path and allow for duplicate simple names
Required('locale'): basestring,
}],
+ }, {
+ Required('implementation'): 'balrog',
+
+ # list of artifact URLs for the artifacts that should be beetmoved
+ Required('upstream-artifacts'): [{
+ # taskId of the task with the artifact
+ Required('taskId'): taskref_or_string,
+
+ # type of signing task (for CoT)
+ Required('taskType'): basestring,
+
+ # Paths to the artifacts to sign
+ Required('paths'): [basestring],
+ }],
}),
# The "when" section contains descriptions of the circumstances
# under which this task can be "optimized", that is, left out of the
# task graph because it is unnecessary.
Optional('when'): Any({
# This task only needs to be run if a file matching one of the given
# patterns has changed in the push. The patterns use the mozpack
@@ -341,16 +355,17 @@ GROUP_NAMES = {
'tc-T-e10s': 'Talos performance tests executed by TaskCluster with e10s',
'tc-VP': 'VideoPuppeteer tests executed by TaskCluster',
'tc-W': 'Web platform tests executed by TaskCluster',
'tc-W-e10s': 'Web platform tests executed by TaskCluster with e10s',
'tc-X': 'Xpcshell tests executed by TaskCluster',
'tc-X-e10s': 'Xpcshell tests executed by TaskCluster with e10s',
'tc-L10n': 'Localised Repacks executed by Taskcluster',
'tc-BM-L10n': 'Beetmover for locales executed by Taskcluster',
+ 'tc-Up': 'Balrog submission of updates, executed by Taskcluster',
'Aries': 'Aries Device Image',
'Nexus 5-L': 'Nexus 5-L Device Image',
'Cc': 'Toolchain builds',
'SM-tc': 'Spidermonkey builds',
}
UNKNOWN_GROUP_NAME = "Treeherder group {} has no name; add it to " + __file__
V2_ROUTE_TEMPLATES = [
@@ -521,16 +536,27 @@ def build_beetmover_payload(config, task
'upload_date': config.params['build_date'],
'update_manifest': worker['update_manifest'],
'upstreamArtifacts': worker['upstream-artifacts']
}
if worker.get('locale'):
task_def['payload']['locale'] = worker['locale']
+@payload_builder('balrog')
+def build_balrog_payload(config, task, task_def):
+ worker = task['worker']
+
+ task_def['payload'] = {
+ # signing cert is unused, but required by balrogworker (Bug 1282187 c#7)
+ 'signing_cert': "dep",
+ 'upstreamArtifacts': worker['upstream-artifacts']
+ }
+
+
@payload_builder('macosx-engine')
def build_macosx_engine_payload(config, task, task_def):
worker = task['worker']
artifacts = map(lambda artifact: {
'name': artifact['name'],
'path': artifact['path'],
'type': artifact['type'],
'expires': task_def['expires'],