Bug 1441353 - part 2: Add beetmover job to publish signed langpacks r=Callek
MozReview-Commit-ID: HXjcclf3OeX
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/release-beetmover-signed-langpacks/kind.yml
@@ -0,0 +1,28 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+loader: taskgraph.loader.single_dep:loader
+
+transforms:
+ - taskgraph.transforms.name_sanity:transforms
+ - taskgraph.transforms.release_beetmover_signed_addons:transforms
+ - taskgraph.transforms.release_notifications:transforms
+ - taskgraph.transforms.task:transforms
+
+kind-dependencies:
+ - release-sign-and-push-langpacks
+
+only-for-attributes:
+ - nightly
+
+job-template:
+ description: Beetmover submission for platform-independent langpacks {locales} in {platform} directory
+ worker-type:
+ by-project:
+ mozilla-beta: scriptworker-prov-v1/beetmoverworker-v1
+ mozilla-release: scriptworker-prov-v1/beetmoverworker-v1
+ default: scriptworker-prov-v1/beetmoverworker-dev
+ run-on-projects: []
+ shipping-phase: promote
+ shipping-product: firefox
--- a/taskcluster/ci/release-generate-checksums/kind.yml
+++ b/taskcluster/ci/release-generate-checksums/kind.yml
@@ -2,16 +2,17 @@
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
loader: taskgraph.loader.transform:loader
kind-dependencies:
- beetmover-source
- post-beetmover-checksums-dummy
+ - release-beetmover-signed-langpacks
transforms:
- taskgraph.transforms.build:transforms
- taskgraph.transforms.release_deps:transforms
- taskgraph.transforms.release_generate_checksums:transforms
- taskgraph.transforms.release_notifications:transforms
- taskgraph.transforms.job:transforms
- taskgraph.transforms.task:transforms
--- a/taskcluster/ci/release-snap-repackage/kind.yml
+++ b/taskcluster/ci/release-snap-repackage/kind.yml
@@ -7,16 +7,17 @@ loader: taskgraph.loader.transform:loade
transforms:
- taskgraph.transforms.release_deps:transforms
- taskgraph.transforms.release_snap_repackage:transforms
- taskgraph.transforms.release_notifications:transforms
- taskgraph.transforms.task:transforms
kind-dependencies:
- post-beetmover-dummy
+ - release-beetmover-signed-langpacks
job-defaults:
description: Generates snap image
run-on-projects: [] # to make sure this never runs as part of CI
shipping-phase: promote
scopes: []
treeherder:
platform: linux64/opt
--- a/taskcluster/docs/kinds.rst
+++ b/taskcluster/docs/kinds.rst
@@ -312,16 +312,20 @@ Verifies the contents and package of rel
release-secondary-balrog-publishing
---------------------
Schedule an RC release to go live in Balrog. Usually this will happen on the beta channel, to a smaller audience, before the RC goes live on the release channel.
release-sign-and-push-langpacks
-------------------------------
Sign a langpack XPI and publishes it onto addons.mozilla.org.
+release-beetmover-signed-langpacks
+----------------------------------
+Publishes signed langpacks to archive.mozilla.org
+
release-update-verify
---------------------
Verifies the contents and package of release update MARs.
release-secondary-update-verify
---------------------
Verifies the contents and package of release update MARs.
--- a/taskcluster/taskgraph/transforms/beetmover_repackage.py
+++ b/taskcluster/taskgraph/transforms/beetmover_repackage.py
@@ -50,44 +50,53 @@ logger = logging.getLogger(__name__)
"target.awsy.tests.zip",
"target.test_packages.json",
"target.txt",
"target.web-platform.tests.tar.gz",
"target.xpcshell.tests.zip",
"target_info.txt",
"target.jsshell.zip",
"mozharness.zip",
- "target.langpack.xpi",
-]
-
-# Until bug 1331141 is fixed, if you are adding any new artifacts here that
-# need to be transfered to S3, please be aware you also need to follow-up
-# with a beetmover patch in https://github.com/mozilla-releng/beetmoverscript/.
-# See example in bug 1348286
-_DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_L10N = [
- "target.langpack.xpi",
]
# Until bug 1331141 is fixed, if you are adding any new artifacts here that
# need to be transfered to S3, please be aware you also need to follow-up
# with a beetmover patch in https://github.com/mozilla-releng/beetmoverscript/.
# See example in bug 1348286
UPSTREAM_ARTIFACT_UNSIGNED_PATHS = {
- r'^(linux(|64)|macosx64)(|-devedition)-nightly$':
+ r'^(linux(|64)|macosx64)-nightly$':
_DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US + [
'host/bin/mar',
'host/bin/mbsdiff',
],
- r'^win(32|64)(|-devedition)-nightly$':
+ r'^(linux(|64)|macosx64)-devedition-nightly$':
+ _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US + [
+ 'host/bin/mar',
+ 'host/bin/mbsdiff',
+ # TODO Bug 1453033: Sign devedition langpacks
+ 'target.langpack.xpi',
+ ],
+ r'^win(32|64)-nightly$':
_DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US + [
- "host/bin/mar.exe",
- "host/bin/mbsdiff.exe",
+ 'host/bin/mar.exe',
+ 'host/bin/mbsdiff.exe',
],
- r'^(linux(|64)|macosx64|win(32|64))(|-devedition)-nightly-l10n$':
- _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_L10N,
+ r'^win(32|64)-devedition-nightly$':
+ _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US + [
+ 'host/bin/mar.exe',
+ 'host/bin/mbsdiff.exe',
+ # TODO Bug 1453033: Sign devedition langpacks
+ 'target.langpack.xpi',
+ ],
+ r'^(linux(|64)|macosx64|win(32|64))-nightly-l10n$': [],
+ r'^(linux(|64)|macosx64|win(32|64))-devedition-nightly-l10n$':
+ [
+ # TODO Bug 1453033: Sign devedition langpacks
+ 'target.langpack.xpi',
+ ],
}
# Until bug 1331141 is fixed, if you are adding any new artifacts here that
# need to be transfered to S3, please be aware you also need to follow-up
# with a beetmover patch in https://github.com/mozilla-releng/beetmoverscript/.
# See example in bug 1348286
UPSTREAM_ARTIFACT_SIGNED_PATHS = {
r'^linux(|64)(|-devedition)-nightly(|-l10n)$': ['target.tar.bz2', 'target.tar.bz2.asc'],
@@ -282,22 +291,23 @@ def generate_upstream_artifacts(build_ta
for ref, tasktype, mapping in zip(task_refs, tasktypes, mapping):
plarform_was_previously_matched_by_regex = None
for platform_regex, paths in mapping.iteritems():
if platform_regex.match(platform) is not None:
_check_platform_matched_only_one_regex(
tasktype, platform, plarform_was_previously_matched_by_regex, platform_regex
)
- upstream_artifacts.append({
- "taskId": {"task-reference": ref},
- "taskType": tasktype,
- "paths": ["{}/{}".format(artifact_prefix, path) for path in paths],
- "locale": locale or "en-US",
- })
+ if paths:
+ upstream_artifacts.append({
+ "taskId": {"task-reference": ref},
+ "taskType": tasktype,
+ "paths": ["{}/{}".format(artifact_prefix, path) for path in paths],
+ "locale": locale or "en-US",
+ })
plarform_was_previously_matched_by_regex = platform_regex
return upstream_artifacts
def generate_partials_upstream_artifacts(artifacts, platform, locale=None):
if not locale or locale == 'en-US':
artifact_prefix = 'public/build'
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/release_beetmover_signed_addons.py
@@ -0,0 +1,208 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Transform the beetmover task into an actual task description.
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from taskgraph.transforms.base import TransformSequence
+from taskgraph.transforms.beetmover import craft_release_properties
+from taskgraph.util.attributes import copy_attributes_from_dependent_job
+from taskgraph.util.schema import validate_schema, Schema, optionally_keyed_by, resolve_keyed_by
+from taskgraph.util.scriptworker import (get_beetmover_bucket_scope,
+ get_beetmover_action_scope)
+from taskgraph.transforms.task import task_description_schema
+from taskgraph.transforms.release_sign_and_push_langpacks import get_upstream_task_ref
+from voluptuous import Required, Optional
+
+import logging
+import copy
+
+logger = logging.getLogger(__name__)
+
+
+task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
+
+
+transforms = TransformSequence()
+
+
+beetmover_description_schema = Schema({
+ # the dependent task (object) for this beetmover job, used to inform beetmover.
+ Required('dependent-task'): object,
+
+ # depname is used in taskref's to identify the taskID of the unsigned things
+ Required('depname', default='build'): basestring,
+
+ # unique label to describe this beetmover task, defaults to {dep.label}-beetmover
+ Optional('label'): basestring,
+
+ # treeherder is allowed here to override any defaults we use for beetmover. See
+ # taskcluster/taskgraph/transforms/task.py for the schema details, and the
+ # below transforms for defaults of various values.
+ Optional('treeherder'): task_description_schema['treeherder'],
+
+ Required('description'): basestring,
+ Required('worker-type'): optionally_keyed_by('project', basestring),
+ Required('run-on-projects'): [],
+
+ # locale is passed only for l10n beetmoving
+ Optional('locale'): basestring,
+ Optional('shipping-phase'): task_description_schema['shipping-phase'],
+ Optional('shipping-product'): task_description_schema['shipping-product'],
+})
+
+
+@transforms.add
+def set_label(config, jobs):
+ for job in jobs:
+ job['label'] = job['dependent-task'].label.replace(
+ 'sign-and-push-langpacks', 'beetmover-signed-langpacks'
+ )
+
+ yield job
+
+
+@transforms.add
+def validate(config, jobs):
+ for job in jobs:
+ validate_schema(
+ beetmover_description_schema, job,
+ "In beetmover ({!r} kind) task for {!r}:".format(config.kind, job['label'])
+ )
+ yield job
+
+
+@transforms.add
+def resolve_keys(config, jobs):
+ for job in jobs:
+ resolve_keyed_by(
+ job, 'worker-type', item_name=job['label'], project=config.params['project']
+ )
+ yield job
+
+
+@transforms.add
+def make_task_description(config, jobs):
+ for job in jobs:
+ dep_job = job['dependent-task']
+ attributes = dep_job.attributes
+
+ treeherder = job.get('treeherder', {})
+ treeherder.setdefault('symbol', 'langpack(BM{})'.format(attributes.get('l10n_chunk', '')))
+ dep_th_platform = dep_job.task.get('extra', {}).get(
+ 'treeherder', {}).get('machine', {}).get('platform', '')
+ treeherder.setdefault('platform',
+ "{}/opt".format(dep_th_platform))
+ treeherder.setdefault('tier', 1)
+ treeherder.setdefault('kind', 'build')
+
+ job['attributes'] = copy_attributes_from_dependent_job(dep_job)
+ job['attributes']['chunk_locales'] = dep_job.attributes['chunk_locales']
+
+ job['description'] = job['description'].format(
+ locales='/'.join(job['attributes']['chunk_locales']),
+ platform=job['attributes']['build_platform']
+ )
+
+ job['scopes'] = [
+ get_beetmover_bucket_scope(config),
+ get_beetmover_action_scope(config),
+ ]
+
+ job['dependencies'] = {
+ str(dep_job.kind): dep_job.label
+ }
+
+ job['run-on-projects'] = dep_job.attributes['run_on_projects']
+ job['treeherder'] = treeherder
+ job['shipping-phase'] = dep_job.attributes['shipping_phase']
+ job['shipping-product'] = dep_job.attributes['shipping_product']
+
+ yield job
+
+
+@transforms.add
+def make_task_worker(config, jobs):
+ for job in jobs:
+ signing_task_ref = get_upstream_task_ref(
+ job, expected_kinds=('release-sign-and-push-langpacks',)
+ )
+
+ job['worker'] = {
+ 'implementation': 'beetmover',
+ 'release-properties': craft_release_properties(config, job),
+ 'upstream-artifacts': generate_upstream_artifacts(
+ signing_task_ref, job['attributes']['chunk_locales']
+ ),
+ }
+
+ yield job
+
+
+def generate_upstream_artifacts(upstream_task_ref, locales):
+ return [{
+ 'taskId': {'task-reference': upstream_task_ref},
+ 'taskType': 'scriptworker',
+ 'locale': locale,
+ 'paths': [
+ # addonscript uploads en-US XPI in the en-US folder
+ 'public/build/{}/target.langpack.xpi'.format(locale)
+ ],
+ } for locale in locales]
+
+
+@transforms.add
+def strip_unused_data(config, jobs):
+ for job in jobs:
+ del job['dependent-task']
+
+ yield job
+
+
+@transforms.add
+def yield_all_platform_jobs(config, jobs):
+ # Even though langpacks are now platform independent, we keep beetmoving them at old
+ # platform-specific locations. That's why this transform exist
+ for job in jobs:
+ if 'ja-JP-mac' in job['label']:
+ # This locale must not be copied on any other platform than macos
+ yield job
+ else:
+ for platform in ('linux', 'linux64', 'macosx64', 'win32', 'win64'):
+ platform_job = copy.deepcopy(job)
+ if 'ja' in platform_job['attributes']['chunk_locales'] and platform == 'macosx64':
+ platform_job = _strip_ja_data_from_linux_job(platform_job)
+
+ platform_job = _change_platform_data(platform_job, platform)
+
+ yield platform_job
+
+
+def _strip_ja_data_from_linux_job(platform_job):
+ # Let's take "ja" out the description. This locale is in a substring like "aa/bb/cc/dd", where
+ # "ja" could be any of "aa", "bb", "cc", "dd"
+ platform_job['description'] = platform_job['description'].replace('ja/', '')
+ platform_job['description'] = platform_job['description'].replace('/ja', '')
+
+ platform_job['worker']['upstream-artifacts'] = [
+ artifact
+ for artifact in platform_job['worker']['upstream-artifacts']
+ if artifact['locale'] != 'ja'
+ ]
+
+ return platform_job
+
+
+def _change_platform_data(platform_job, platform):
+ platform_job['attributes']['build_platform'] = platform
+ platform_job['label'] = platform_job['label'].replace('linux64', platform)
+ platform_job['description'] = platform_job['description'].replace('linux64', platform)
+ platform_job['treeherder']['platform'] = platform_job['treeherder']['platform'].replace(
+ 'linux64', platform
+ )
+ platform_job['worker']['release-properties']['platform'] = platform
+
+ return platform_job
--- a/taskcluster/taskgraph/transforms/task.py
+++ b/taskcluster/taskgraph/transforms/task.py
@@ -451,17 +451,17 @@ task_description_schema = Schema({
Required('formats'): [basestring],
}],
}, {
Required('implementation'): 'binary-transparency',
}, {
Required('implementation'): 'beetmover',
# the maximum time to run, in seconds
- Required('max-run-time'): int,
+ Required('max-run-time', default=600): int,
# locale key, if this is a locale beetmover job
Optional('locale'): basestring,
Required('release-properties'): {
'app-name': basestring,
'app-version': basestring,
'branch': basestring,