Bug 1319189 - Create tasks that move desktop l10n artifacts to the s3 final locations. r=kmoir
MozReview-Commit-ID: FXYtJwmjBmd
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/beetmover-l10n/kind.yml
@@ -0,0 +1,14 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+implementation: taskgraph.task.beetmover:BeetmoverTask
+
+transforms:
+ - taskgraph.transforms.beetmover_l10n:transforms
+ - taskgraph.transforms.beetmover:transforms
+ - taskgraph.transforms.task:transforms
+
+kind-dependencies:
+ - nightly-l10n
+ - nightly-l10n-signing
--- a/taskcluster/taskgraph/task/beetmover.py
+++ b/taskcluster/taskgraph/task/beetmover.py
@@ -14,19 +14,20 @@ class BeetmoverTask(transform.TransformT
We use a dictionary to create the input to the transforms.
It will have added to it keys `build-label`, the label for the build task,
and `build-platform` / `build-type`, its platform and type.
"""
@classmethod
def get_inputs(cls, kind, path, config, params, loaded_tasks):
- if config.get('kind-dependencies', []) != ["build", "build-signing"]:
+ if config.get('kind-dependencies', []) != ["build", "build-signing"] and \
+ config.get('kind-dependencies', []) != ["nightly-l10n", "nightly-l10n-signing"]:
raise Exception("Beetmover kinds must depend on builds or signing builds")
for task in loaded_tasks:
if not task.attributes.get('nightly'):
continue
- if task.kind != "build-signing" and task.kind != "build":
+ if task.kind not in config.get('kind-dependencies'):
continue
beetmover_task = {}
beetmover_task['dependent-task'] = task
yield beetmover_task
--- a/taskcluster/taskgraph/transforms/beetmover.py
+++ b/taskcluster/taskgraph/transforms/beetmover.py
@@ -36,16 +36,19 @@ beetmover_description_schema = Schema({
# unique label to describe this beetmover task, defaults to {dep.label}-beetmover
Optional('label'): basestring,
# treeherder is allowed here to override any defaults we use for beetmover. See
# taskcluster/taskgraph/transforms/task.py for the schema details, and the
# below transforms for defaults of various values.
Optional('treeherder'): task_description_schema['treeherder'],
+
+ # locale is passed only for l10n beetmoving
+ Optional('locale'): basestring,
})
@transforms.add
def validate(config, jobs):
for job in jobs:
label = job.get('dependent-task', object).__dict__.get('label', '?no-label?')
yield validate_schema(
@@ -75,33 +78,41 @@ def make_task_description(config, jobs):
dependent_kind = str(job['dependent-task'].kind)
# both artifacts are the build artifacts if not the signing task
taskid_of_manifest = "<" + str(dependent_kind) + ">"
taskid_to_beetmove = taskid_of_manifest
update_manifest = False
dependencies = {job['dependent-task'].kind: dep_job.label}
# taskid_of_manifest always refers to the unsigned task
if "signing" in dependent_kind:
- taskid_of_manifest = "<build>"
+ if len(dep_job.dependencies) > 1:
+ raise NotImplementedError(
+ "can't beetmove a signing task with multiple dependencies")
+ dep_name = dep_job.dependencies.keys()[0]
+ taskid_of_manifest = "<" + str(dep_name) + ">"
update_manifest = True
signing_dependencies = dep_job.dependencies
dependencies.update(signing_dependencies)
+ worker = {'implementation': 'beetmover',
+ 'taskid_to_beetmove': {"task-reference":
+ taskid_to_beetmove},
+ 'taskid_of_manifest': {"task-reference":
+ taskid_of_manifest},
+ 'update_manifest': update_manifest}
+ if job.get('locale'):
+ worker['locale'] = job['locale']
+
task = {
'label': label,
'description': "{} Beetmover".format(
dep_job.task["metadata"]["description"]),
# do we have to define worker type somewhere?
'worker-type': 'scriptworker-prov-v1/beetmoverworker-v1',
- 'worker': {'implementation': 'beetmover',
- 'taskid_to_beetmove': {"task-reference":
- taskid_to_beetmove},
- 'taskid_of_manifest': {"task-reference":
- taskid_of_manifest},
- 'update_manifest': update_manifest},
+ 'worker': worker,
'scopes': [],
'dependencies': dependencies,
'attributes': {
'nightly': dep_job.attributes.get('nightly', False),
'build_platform': dep_job.attributes.get('build_platform'),
'build_type': dep_job.attributes.get('build_type'),
},
'run-on-projects': dep_job.attributes.get('run_on_projects'),
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/beetmover_l10n.py
@@ -0,0 +1,48 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Transform the signing task into an actual task description.
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from taskgraph.transforms.base import TransformSequence
+from taskgraph.util.treeherder import join_symbol
+
+transforms = TransformSequence()
+
+
+@transforms.add
+def ignore_android_moving(config, jobs):
+ for job in jobs:
+ if 'android' in job['dependent-task'].label:
+ # skip android until we general balrog_props.json for it
+ continue
+ yield job
+
+
+@transforms.add
+def make_signing_description(config, jobs):
+ for job in jobs:
+ dep_job = job['dependent-task']
+ for locale in dep_job.attributes.get('chunk_locales', []):
+
+ label = job.get('label',
+ "beetmover-{}-{}".format(locale, dep_job.label))
+
+ # add the locale code
+ symbol = locale
+ group = 'tc-BM-L10n'
+
+ treeherder = {
+ 'symbol': join_symbol(group, symbol),
+ }
+
+ beet_description = {
+ 'dependent-task': dep_job,
+ 'treeherder': treeherder,
+ 'label': label,
+ 'locale': locale,
+ }
+ yield beet_description
--- a/taskcluster/taskgraph/transforms/signing.py
+++ b/taskcluster/taskgraph/transforms/signing.py
@@ -88,29 +88,34 @@ def make_task_description(config, jobs):
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform', "{}/opt".format(dep_th_platform))
treeherder.setdefault('tier', 2)
treeherder.setdefault('kind', 'build')
label = job.get('label', "{}-signing".format(dep_job))
+ attributes = {
+ 'nightly': dep_job.attributes.get('nightly', False),
+ 'build_platform': dep_job.attributes.get('build_platform'),
+ 'build_type': dep_job.attributes.get('build_type'),
+ }
+ if dep_job.attributes.get('chunk_locales'):
+ # Used for l10n attribute passthrough
+ attributes['chunk_locales'] = dep_job.attributes.get('chunk_locales')
+
task = {
'label': label,
'description': "{} Signing".format(
dep_job.task["metadata"]["description"]),
'worker-type': "scriptworker-prov-v1/signing-linux-v1",
'worker': {'implementation': 'scriptworker-signing',
'upstream-artifacts': job['upstream-artifacts'],
'max-run-time': 3600},
'scopes': ["project:releng:signing:cert:nightly-signing"] + \
signing_format_scopes,
'dependencies': {job['depname']: dep_job.label},
- 'attributes': {
- 'nightly': dep_job.attributes.get('nightly', False),
- 'build_platform': dep_job.attributes.get('build_platform'),
- 'build_type': dep_job.attributes.get('build_type'),
- },
+ 'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'treeherder': treeherder,
}
yield task
--- a/taskcluster/taskgraph/transforms/task.py
+++ b/taskcluster/taskgraph/transforms/task.py
@@ -304,16 +304,19 @@ task_description_schema = Schema({
# taskid of task with artifacts to beetmove
Required('taskid_to_beetmove'): taskref_or_string,
# taskid of task with artifacts to beetmove
Required('taskid_of_manifest'): taskref_or_string,
# beetmover template key
Required('update_manifest'): bool,
+
+ # locale key, if this is a locale beetmover job
+ Optional('locale'): basestring,
}, {
Required('implementation'): 'balrog',
# taskid of the signed beetmoved task
Required('task_artifact_url'): taskref_or_string,
}),
# The "when" section contains descriptions of the circumstances
@@ -339,16 +342,17 @@ GROUP_NAMES = {
'tc-R': 'Reftests executed by TaskCluster',
'tc-R-e10s': 'Reftests executed by TaskCluster with e10s',
'tc-VP': 'VideoPuppeteer tests executed by TaskCluster',
'tc-W': 'Web platform tests executed by TaskCluster',
'tc-W-e10s': 'Web platform tests executed by TaskCluster with e10s',
'tc-X': 'Xpcshell tests executed by TaskCluster',
'tc-X-e10s': 'Xpcshell tests executed by TaskCluster with e10s',
'tc-L10n': 'Localised Repacks executed by Taskcluster',
+ 'tc-BM-L10n': 'Beetmover for locales executed by Taskcluster',
'tc-Up': 'Balrog submission of updates, executed by Taskcluster',
'Aries': 'Aries Device Image',
'Nexus 5-L': 'Nexus 5-L Device Image',
'Cc': 'Toolchain builds',
'SM-tc': 'Spidermonkey builds',
}
UNKNOWN_GROUP_NAME = "Treeherder group {} has no name; add it to " + __file__
@@ -510,18 +514,20 @@ def build_scriptworker_signing_payload(c
def build_beetmover_payload(config, task, task_def):
worker = task['worker']
task_def['payload'] = {
'maxRunTime': worker['max-run-time'],
'upload_date': config.params['build_date'],
'taskid_to_beetmove': worker['taskid_to_beetmove'],
'taskid_of_manifest': worker['taskid_of_manifest'],
- 'update_manifest': worker['update_manifest']
+ 'update_manifest': worker['update_manifest'],
}
+ if worker.get('locale'):
+ task_def['payload']['locale'] = worker['locale']
@payload_builder('balrog')
def build_balrog_payload(config, task, task_def):
worker = task['worker']
task_def['payload'] = {
'parent_task_artifacts_url': worker['task_artifact_url'],