--- a/taskcluster/ci/balrog/kind.yml
+++ b/taskcluster/ci/balrog/kind.yml
@@ -6,11 +6,12 @@ loader: taskgraph.loader.single_dep:load
transforms:
- taskgraph.transforms.balrog:transforms
- taskgraph.transforms.task:transforms
kind-dependencies:
- beetmover
- beetmover-l10n
+ - beetmover-repackage
only-for-attributes:
- nightly
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/beetmover-repackage/kind.yml
@@ -0,0 +1,15 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+loader: taskgraph.loader.single_dep:loader
+
+transforms:
+ - taskgraph.transforms.beetmover_repackage:transforms
+ - taskgraph.transforms.task:transforms
+
+kind-dependencies:
+ - repackage
+
+only-for-build-platforms:
+ - macosx64-nightly/opt
--- a/taskcluster/ci/beetmover/kind.yml
+++ b/taskcluster/ci/beetmover/kind.yml
@@ -8,8 +8,11 @@ transforms:
- taskgraph.transforms.beetmover:transforms
- taskgraph.transforms.task:transforms
kind-dependencies:
- build-signing
only-for-attributes:
- nightly
+
+not-for-build-platforms:
+ - macosx64-nightly/opt
--- a/taskcluster/ci/checksums-signing/kind.yml
+++ b/taskcluster/ci/checksums-signing/kind.yml
@@ -6,11 +6,12 @@ loader: taskgraph.loader.single_dep:load
transforms:
- taskgraph.transforms.checksums_signing:transforms
- taskgraph.transforms.task:transforms
kind-dependencies:
- beetmover
- beetmover-l10n
+ - beetmover-repackage
only-for-attributes:
- nightly
--- a/taskcluster/docs/kinds.rst
+++ b/taskcluster/docs/kinds.rst
@@ -179,16 +179,24 @@ beetmover-l10n
--------------
Beetmover L10n, takes specific artifacts, "Beets", and pushes them to a location outside
of Taskcluster's task artifacts, (archive.mozilla.org as one place) and in the
process determines the final location and a "pretty" name (versioned product name)
This separate kind uses logic specific to localized artifacts, such as including
the language in the final artifact names.
+beetmover-repackage
+---------
+
+Beetmover-repackage is beetmover but for tasks that need an intermediate step
+between signing and packaging, such as OSX. For more details see the definitions
+of the Beetmover kind above and the repackage kind below.
+
+
checksums-signing
----------------
Checksums-signing take as input the checksums file generated by beetmover tasks
and sign it via the signing scriptworkers. Returns the same file signed and
additional detached signature.
beetmover-checksums
-------------------
--- a/taskcluster/taskgraph/loader/single_dep.py
+++ b/taskcluster/taskgraph/loader/single_dep.py
@@ -7,40 +7,46 @@ from __future__ import absolute_import,
import copy
def loader(kind, path, config, params, loaded_tasks):
"""
Load tasks based on the jobs dependant kinds.
The `only-for-build-platforms` kind configuration, if specified, will limit
- the build platforms for which a job will be created.
+ the build platforms for which a job will be created. Alternatively there is
+ 'not-for-build-platforms' kind configuration which will be consulted only after
+ 'only-for-build-platforms' is checked (if present), and omit any jobs where the
+ build platform matches.
Optional `only-for-attributes` kind configuration, if specified, will limit
the jobs chosen to ones which have the specified attribute, with the specified
value.
Optional `job-template` kind configuration value, if specified, will be used to
pass configuration down to the specified transforms used.
"""
only_platforms = config.get('only-for-build-platforms')
+ not_platforms = config.get('not-for-build-platforms')
only_attributes = config.get('only-for-attributes')
job_template = config.get('job-template')
for task in loaded_tasks:
if task.kind not in config.get('kind-dependencies', []):
continue
- if only_platforms:
+ if only_platforms or not_platforms:
build_platform = task.attributes.get('build_platform')
build_type = task.attributes.get('build_type')
if not build_platform or not build_type:
continue
platform = "{}/{}".format(build_platform, build_type)
- if platform not in only_platforms:
+ if only_platforms and platform not in only_platforms:
+ continue
+ elif not_platforms and platform in not_platforms:
continue
if only_attributes:
config_attrs = set(only_attributes)
if config_attrs - set(task.attributes):
# make sure all attributes exist
continue
--- a/taskcluster/taskgraph/transforms/balrog.py
+++ b/taskcluster/taskgraph/transforms/balrog.py
@@ -47,17 +47,18 @@ def validate(config, jobs):
yield validate_schema(
balrog_description_schema, job,
"In balrog ({!r} kind) task for {!r}:".format(config.kind, label))
@transforms.add
def skip_unsigned_beets(config, jobs):
for job in jobs:
- if 'signing' not in job['dependent-task'].label:
+ if ('signing' not in job['dependent-task'].label and
+ 'beetmover-repackage' not in job['dependent-task'].label):
# Skip making a balrog task for this
continue
yield job
@transforms.add
def make_task_description(config, jobs):
for job in jobs:
--- a/taskcluster/taskgraph/transforms/beetmover.py
+++ b/taskcluster/taskgraph/transforms/beetmover.py
@@ -100,50 +100,56 @@ from voluptuous import Any, Required, Op
]
UPSTREAM_ARTIFACT_UNSIGNED_PATHS = {
'linux64-nightly': _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US,
'linux-nightly': _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US,
'android-x86-nightly': _MOBILE_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US,
'android-api-15-nightly': _MOBILE_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US,
- 'macosx64-nightly': [],
+ 'macosx64-nightly': _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US,
'linux64-nightly-l10n': _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_L10N,
'linux-nightly-l10n': _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_L10N,
'android-x86-nightly-multi': _MOBILE_UPSTREAM_ARTIFACTS_UNSIGNED_MULTI,
'android-api-15-nightly-l10n': ["balrog_props.json"],
'android-api-15-nightly-multi': _MOBILE_UPSTREAM_ARTIFACTS_UNSIGNED_MULTI,
- 'macosx64-nightly-l10n': [],
+ 'macosx64-nightly-l10n': _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_L10N,
}
UPSTREAM_ARTIFACT_SIGNED_PATHS = {
'linux64-nightly': _DESKTOP_UPSTREAM_ARTIFACTS_SIGNED_EN_US + [
"target.tar.bz2",
"target.tar.bz2.asc",
],
'linux-nightly': _DESKTOP_UPSTREAM_ARTIFACTS_SIGNED_EN_US + [
"target.tar.bz2",
"target.tar.bz2.asc",
],
'android-x86-nightly': ["en-US/target.apk"],
'android-api-15-nightly': ["en-US/target.apk"],
- 'macosx64-nightly': [],
+ 'macosx64-nightly': _DESKTOP_UPSTREAM_ARTIFACTS_SIGNED_EN_US + [
+ "target.dmg",
+ "target.dmg.asc",
+ ],
'linux64-nightly-l10n': _DESKTOP_UPSTREAM_ARTIFACTS_SIGNED_L10N + [
"target.tar.bz2",
"target.tar.bz2.asc",
],
'linux-nightly-l10n': _DESKTOP_UPSTREAM_ARTIFACTS_SIGNED_L10N + [
"target.tar.bz2",
"target.tar.bz2.asc",
],
'android-x86-nightly-multi': ["target.apk"],
'android-api-15-nightly-l10n': ["target.apk"],
'android-api-15-nightly-multi': ["target.apk"],
- 'macosx64-nightly-l10n': [],
+ 'macosx64-nightly-l10n': _DESKTOP_UPSTREAM_ARTIFACTS_SIGNED_L10N + [
+ "target.dmg",
+ "target.dmg.asc",
+ ],
}
# Voluptuous uses marker objects as dictionary *keys*, but they are not
# comparable, so we cast all of the keys back to regular strings
task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
transforms = TransformSequence()
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/beetmover_repackage.py
@@ -0,0 +1,244 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Transform the beetmover task into an actual task description.
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from taskgraph.transforms.base import TransformSequence
+from taskgraph.util.schema import validate_schema, Schema
+from taskgraph.util.scriptworker import (get_beetmover_bucket_scope,
+ get_beetmover_action_scope)
+from taskgraph.transforms.task import task_description_schema
+from voluptuous import Any, Required, Optional
+
+import logging
+logger = logging.getLogger(__name__)
+
+# For developers: if you are adding any new artifacts here that need to be
+# transfered to S3, please be aware you also need to follow-up with patch in
+# the actual beetmoverscript logic that lies under
+# https://github.com/mozilla-releng/beetmoverscript/. See example in bug
+# 1348286
+_DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US = [
+ "balrog_props.json",
+ "target.common.tests.zip",
+ "target.cppunittest.tests.zip",
+ "target.crashreporter-symbols.zip",
+ "target.json",
+ "target.mochitest.tests.zip",
+ "target.mozinfo.json",
+ "target.reftest.tests.zip",
+ "target.talos.tests.zip",
+ "target.awsy.tests.zip",
+ "target.test_packages.json",
+ "target.txt",
+ "target.web-platform.tests.zip",
+ "target.xpcshell.tests.zip",
+ "target_info.txt",
+ "target.jsshell.zip",
+ "mozharness.zip",
+ "target.langpack.xpi",
+ "host/bin/mar",
+ "host/bin/mbsdiff",
+]
+_DESKTOP_UPSTREAM_ARTIFACTS_SIGNED_EN_US = [
+ "update/target.complete.mar",
+]
+_DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_L10N = [
+ "target.langpack.xpi",
+ "balrog_props.json",
+]
+_DESKTOP_UPSTREAM_ARTIFACTS_SIGNED_L10N = [
+ "target.complete.mar",
+]
+
+UPSTREAM_ARTIFACT_UNSIGNED_PATHS = {
+ 'macosx64-nightly': _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US,
+ 'macosx64-nightly-l10n': _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_L10N,
+}
+UPSTREAM_ARTIFACT_SIGNED_PATHS = {
+ 'macosx64-nightly': _DESKTOP_UPSTREAM_ARTIFACTS_SIGNED_EN_US,
+ 'macosx64-nightly-l10n': _DESKTOP_UPSTREAM_ARTIFACTS_SIGNED_L10N,
+}
+UPSTREAM_ARTIFACT_REPACKAGE_PATHS = {
+ 'macosx64-nightly': ["target.dmg"],
+ 'macosx64-nightly-l10n': ["target.dmg"],
+}
+
+# Voluptuous uses marker objects as dictionary *keys*, but they are not
+# comparable, so we cast all of the keys back to regular strings
+task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
+
+transforms = TransformSequence()
+
+# shortcut for a string where task references are allowed
+taskref_or_string = Any(
+ basestring,
+ {Required('task-reference'): basestring})
+
+beetmover_description_schema = Schema({
+ # the dependent task (object) for this beetmover job, used to inform beetmover.
+ Required('dependent-task'): object,
+
+ # depname is used in taskref's to identify the taskID of the unsigned things
+ Required('depname', default='build'): basestring,
+
+ # unique label to describe this beetmover task, defaults to {dep.label}-beetmover
+ Optional('label'): basestring,
+
+ # treeherder is allowed here to override any defaults we use for beetmover. See
+ # taskcluster/taskgraph/transforms/task.py for the schema details, and the
+ # below transforms for defaults of various values.
+ Optional('treeherder'): task_description_schema['treeherder'],
+
+ # locale is passed only for l10n beetmoving
+ Optional('locale'): basestring,
+})
+
+
+@transforms.add
+def validate(config, jobs):
+ for job in jobs:
+ label = job.get('dependent-task', object).__dict__.get('label', '?no-label?')
+ yield validate_schema(
+ beetmover_description_schema, job,
+ "In beetmover ({!r} kind) task for {!r}:".format(config.kind, label))
+
+
+@transforms.add
+def make_task_description(config, jobs):
+ for job in jobs:
+ dep_job = job['dependent-task']
+
+ treeherder = job.get('treeherder', {})
+ treeherder.setdefault('symbol', 'tc(BM-R)')
+ dep_th_platform = dep_job.task.get('extra', {}).get(
+ 'treeherder', {}).get('machine', {}).get('platform', '')
+ treeherder.setdefault('platform',
+ "{}/opt".format(dep_th_platform))
+ treeherder.setdefault('tier', 1)
+ treeherder.setdefault('kind', 'build')
+ label = job.get('label', "beetmover-{}".format(dep_job.label))
+ dependent_kind = str(dep_job.kind)
+ dependencies = {dependent_kind: dep_job.label}
+
+ # macosx nightly builds depend on repackage which use in tree docker
+ # images and thus have two dependencies
+ # change the signing_dependencies to be use the ones in
+ docker_dependencies = {"docker-image":
+ dep_job.dependencies["docker-image"]
+ }
+ dependencies.update(docker_dependencies)
+ signing_dependencies = {"build-signing":
+ dep_job.dependencies["build-signing"]
+ }
+ dependencies.update(signing_dependencies)
+ repackage_dependencies = {"repackage":
+ dep_job.label
+ }
+ dependencies.update(repackage_dependencies)
+ build_dependencies = {"build":
+ dep_job.dependencies["build-signing"].replace("signing-","build-")
+ }
+ dependencies.update(build_dependencies)
+
+ attributes = {
+ 'nightly': dep_job.attributes.get('nightly', False),
+ 'build_platform': dep_job.attributes.get('build_platform'),
+ 'build_type': dep_job.attributes.get('build_type'),
+ }
+ if job.get('locale'):
+ attributes['locale'] = job['locale']
+
+ bucket_scope = get_beetmover_bucket_scope(config)
+ action_scope = get_beetmover_action_scope(config)
+
+ task = {
+ 'label': label,
+ 'description': "{} Beetmover".format(
+ dep_job.task["metadata"]["description"]),
+ 'worker-type': 'scriptworker-prov-v1/beetmoverworker-v1',
+ 'scopes': [bucket_scope, action_scope],
+ 'dependencies': dependencies,
+ 'attributes': attributes,
+ 'run-on-projects': dep_job.attributes.get('run_on_projects'),
+ 'treeherder': treeherder,
+ }
+
+ yield task
+
+
+def generate_upstream_artifacts(signing_task_ref, build_task_ref,
+ repackage_task_ref, platform,
+ locale=None):
+
+ signing_mapping = UPSTREAM_ARTIFACT_SIGNED_PATHS
+ build_mapping = UPSTREAM_ARTIFACT_UNSIGNED_PATHS
+ repackage_mapping = UPSTREAM_ARTIFACT_REPACKAGE_PATHS
+
+ artifact_prefix = 'public/build'
+ if locale:
+ artifact_prefix = 'public/build/{}'.format(locale)
+ platform = "{}-l10n".format(platform)
+
+ upstream_artifacts = [{
+ "taskId": {"task-reference": build_task_ref},
+ "taskType": "build",
+ "paths": ["{}/{}".format(artifact_prefix, p)
+ for p in build_mapping[platform]],
+ "locale": locale or "en-US",
+ }, {
+ "taskId": {"task-reference": signing_task_ref},
+ "taskType": "signing",
+ "paths": ["{}/{}".format(artifact_prefix, p)
+ for p in signing_mapping[platform]],
+ "locale": locale or "en-US",
+ }, {
+ "taskId": {"task-reference": repackage_task_ref},
+ "taskType": "repackage",
+ "paths": ["{}/{}".format(artifact_prefix, p)
+ for p in repackage_mapping[platform]],
+ "locale": locale or "en-US",
+ }]
+
+ return upstream_artifacts
+
+
+@transforms.add
+def make_task_worker(config, jobs):
+ for job in jobs:
+ valid_beetmover_job = (len(job["dependencies"]) == 4 and
+ any(['repackage' in j for j in job['dependencies']]))
+ if not valid_beetmover_job:
+ raise NotImplementedError("Beetmover_repackage must have four dependencies.")
+
+ locale = job["attributes"].get("locale")
+ platform = job["attributes"]["build_platform"]
+ build_task = None
+ repackage_task = None
+ signing_task = None
+ for dependency in job["dependencies"].keys():
+ if 'repackage' in dependency:
+ repackage_task = dependency
+ elif 'signing' in dependency:
+ signing_task = dependency
+ else:
+ build_task = "build"
+
+ signing_task_ref = "<" + str(signing_task) + ">"
+ build_task_ref = "<" + str(build_task) + ">"
+ repackage_task_ref = "<" + str(repackage_task) + ">"
+ upstream_artifacts = generate_upstream_artifacts(
+ signing_task_ref, build_task_ref, repackage_task_ref, platform, locale
+ )
+
+ worker = {'implementation': 'beetmover',
+ 'upstream-artifacts': upstream_artifacts}
+ if locale:
+ worker["locale"] = locale
+ job["worker"] = worker
+
+ yield job
--- a/taskcluster/taskgraph/util/scriptworker.py
+++ b/taskcluster/taskgraph/util/scriptworker.py
@@ -79,16 +79,17 @@ BEETMOVER_RELEASE_TARGET_TASKS = set([
"""Map beetmover tasks aliases to sets of target task methods.
This is a list of list-pairs, for ordering.
"""
BEETMOVER_SCOPE_ALIAS_TO_TARGET_TASK = [[
'all-nightly-tasks', set([
'nightly_fennec',
'nightly_linux',
+ 'nightly_macosx',
'mozilla_beta_tasks',
'mozilla_release_tasks',
])
], [
'all-release-tasks', BEETMOVER_RELEASE_TARGET_TASKS
]]
"""Map the beetmover scope aliases to the actual scopes.