new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/push-apk-breakpoint/kind.yml
@@ -0,0 +1,32 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+loader: taskgraph.loader.push_apk:loader
+
+transforms:
+ - taskgraph.transforms.push_apk_breakpoint:transforms
+ - taskgraph.transforms.task:transforms
+
+kind-dependencies:
+ - build-signing
+
+jobs:
+ android-push-apk-breakpoint/opt:
+ description: PushApk breakpoint. Decides whether APK should be published onto Google Play Store
+ attributes:
+ build_platform: android-nightly
+ nightly: true
+ worker-type: # see transforms
+ worker:
+ implementation: push-apk-breakpoint
+ treeherder:
+ symbol: pub(Br)
+ platform: Android/opt
+ tier: 2
+ kind: other
+ run-on-projects:
+ - mozilla-aurora
+ - mozilla-beta
+ - mozilla-release
+ deadline-after: 5 days
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/push-apk/kind.yml
@@ -0,0 +1,38 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+loader: taskgraph.loader.push_apk:loader
+
+transforms:
+ - taskgraph.transforms.push_apk:transforms
+ - taskgraph.transforms.task:transforms
+
+kind-dependencies:
+ - build-signing
+ - push-apk-breakpoint
+
+jobs:
+ push-apk/opt:
+ description: Publishes APK onto Google Play Store
+ attributes:
+ build_platform: android-nightly
+ nightly: true
+ worker-type: scriptworker-prov-v1/pushapk-v1
+ worker:
+ upstream-artifacts: # see transforms
+ google-play-track: # see transforms
+ implementation: push-apk
+ # TODO unhardcode that line
+ dry-run: true
+ scopes: # see transforms
+ treeherder:
+ symbol: pub(gp)
+ platform: Android/opt
+ tier: 2
+ kind: other
+ run-on-projects:
+ - mozilla-aurora
+ - mozilla-beta
+ - mozilla-release
+ deadline-after: 5 days
--- a/taskcluster/docs/kinds.rst
+++ b/taskcluster/docs/kinds.rst
@@ -190,8 +190,20 @@ Checksums-signing take as input the chec
and sign it via the signing scriptworkers. Returns the same file signed and
additional detached signature.
beetmover-checksums
-------------------
Beetmover, takes specific artifact checksums and pushes it to a location outside
of Taskcluster's task artifacts (archive.mozilla.org as one place) and in the
process determines the final location and "pretty" names it (version product name)
+
+push-apk-breakpoint
+-------------------
+Decides whether or not APKs should be published onto Google Play Store. Jobs of this
+kind depend on all the signed multi-locales (aka "multi") APKs for a given release,
+in order to make the decision.
+
+push-apk
+--------
+PushApk publishes Android packages onto Google Play Store. Jobs of this kind take
+all the signed multi-locales (aka "multi") APKs for a given release and upload them
+all at once. They also depend on the breakpoint.
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/loader/push_apk.py
@@ -0,0 +1,32 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from .transform import loader as base_loader
+
+
+def loader(kind, path, config, params, loaded_tasks):
+ """
+ Generate inputs implementing PushApk jobs. These depend on signed multi-locales nightly builds.
+ """
+ jobs = base_loader(kind, path, config, params, loaded_tasks)
+
+ for job in jobs:
+ job['dependent-tasks'] = get_dependent_loaded_tasks(config, loaded_tasks)
+ yield job
+
+
+def get_dependent_loaded_tasks(config, loaded_tasks):
+ nightly_tasks = (
+ task for task in loaded_tasks if task.attributes.get('nightly')
+ )
+ tasks_with_matching_kind = (
+ task for task in nightly_tasks if task.kind in config.get('kind-dependencies')
+ )
+ android_tasks = [
+ task for task in tasks_with_matching_kind if 'android' in task.label
+ ]
+
+ return android_tasks
--- a/taskcluster/taskgraph/target_tasks.py
+++ b/taskcluster/taskgraph/target_tasks.py
@@ -202,17 +202,17 @@ def target_tasks_code_coverage(full_task
@_target_task('nightly_fennec')
def target_tasks_nightly_fennec(full_task_graph, parameters):
"""Select the set of tasks required for a nightly build of fennec. The
nightly build process involves a pipeline of builds, signing,
and, eventually, uploading the tasks to balrog."""
def filter(task):
platform = task.attributes.get('build_platform')
- if platform in ('android-api-15-nightly', 'android-x86-nightly'):
+ if platform in ('android-api-15-nightly', 'android-x86-nightly', 'android-nightly'):
if not task.attributes.get('nightly', False):
return False
return filter_for_project(task, parameters)
return [l for l, t in full_task_graph.tasks.iteritems() if filter(t)]
@_target_task('nightly_linux')
def target_tasks_nightly_linux(full_task_graph, parameters):
@@ -242,16 +242,17 @@ def target_tasks_mozilla_beta(full_task_
return False
if platform in ('linux64', 'linux'):
if task.attributes['build_type'] == 'opt':
return False
# skip l10n, beetmover, balrog
if task.kind in [
'balrog', 'beetmover', 'beetmover-checksums', 'beetmover-l10n',
'checksums-signing', 'nightly-l10n', 'nightly-l10n-signing',
+ 'push-apk', 'push-apk-breakpoint',
]:
return False
return True
return [l for l, t in full_task_graph.tasks.iteritems() if filter(t)]
@_target_task('mozilla_release_tasks')
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/push_apk.py
@@ -0,0 +1,67 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Transform the push-apk kind into an actual task description.
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import functools
+
+from taskgraph.transforms.base import TransformSequence
+from taskgraph.util.schema import Schema
+from taskgraph.util.scriptworker import get_push_apk_scope, get_push_apk_track
+from taskgraph.util.push_apk import fill_labels_tranform, validate_jobs_schema_transform_partial, \
+ validate_dependent_tasks_transform, delete_non_required_fields_transform, generate_dependencies
+from voluptuous import Required
+
+
+transforms = TransformSequence()
+
+push_apk_description_schema = Schema({
+ # the dependent task (object) for this beetmover job, used to inform beetmover.
+ Required('dependent-tasks'): object,
+ Required('name'): basestring,
+ Required('label'): basestring,
+ Required('description'): basestring,
+ Required('attributes'): object,
+ Required('treeherder'): object,
+ Required('run-on-projects'): list,
+ Required('worker-type'): basestring,
+ Required('worker'): object,
+ Required('scopes'): None,
+ Required('deadline-after'): basestring,
+})
+
+validate_jobs_schema_transform = functools.partial(
+ validate_jobs_schema_transform_partial,
+ push_apk_description_schema,
+ 'PushApk'
+)
+
+transforms.add(fill_labels_tranform)
+transforms.add(validate_jobs_schema_transform)
+transforms.add(validate_dependent_tasks_transform)
+
+
+@transforms.add
+def make_task_description(config, jobs):
+ for job in jobs:
+ job['dependencies'] = generate_dependencies(job['dependent-tasks'])
+ job['worker']['upstream-artifacts'] = generate_upstream_artifacts(job['dependencies'])
+ job['worker']['google-play-track'] = get_push_apk_track(config)
+ job['scopes'] = [get_push_apk_scope(config)]
+
+ yield job
+
+
+transforms.add(delete_non_required_fields_transform)
+
+
+def generate_upstream_artifacts(dependencies):
+ return [{
+ 'taskId': {'task-reference': '<{}>'.format(task_kind)},
+ 'taskType': 'signing',
+ 'paths': ['public/build/target.apk'],
+ } for task_kind in dependencies.keys() if 'breakpoint' not in task_kind]
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/push_apk_breakpoint.py
@@ -0,0 +1,68 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Transform the push-apk-breakpoint kind into an actual task description.
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import functools
+
+from taskgraph.transforms.base import TransformSequence
+from taskgraph.util.schema import Schema
+from taskgraph.util.scriptworker import get_push_apk_breakpoint_worker_type
+from taskgraph.util.push_apk import fill_labels_tranform, validate_jobs_schema_transform_partial, \
+ validate_dependent_tasks_transform, delete_non_required_fields_transform, generate_dependencies
+from voluptuous import Required
+
+
+transforms = TransformSequence()
+
+push_apk_breakpoint_description_schema = Schema({
+ # the dependent task (object) for this beetmover job, used to inform beetmover.
+ Required('dependent-tasks'): object,
+ Required('name'): basestring,
+ Required('label'): basestring,
+ Required('description'): basestring,
+ Required('attributes'): object,
+ Required('worker-type'): None,
+ Required('worker'): object,
+ Required('treeherder'): object,
+ Required('run-on-projects'): list,
+ Required('deadline-after'): basestring,
+})
+
+validate_jobs_schema_transform = functools.partial(
+ validate_jobs_schema_transform_partial,
+ push_apk_breakpoint_description_schema,
+ 'PushApkBreakpoint'
+)
+
+transforms.add(fill_labels_tranform)
+transforms.add(validate_jobs_schema_transform)
+transforms.add(validate_dependent_tasks_transform)
+
+
+@transforms.add
+def make_task_description(config, jobs):
+ for job in jobs:
+ job['dependencies'] = generate_dependencies(job['dependent-tasks'])
+
+ worker_type = get_push_apk_breakpoint_worker_type(config)
+ job['worker-type'] = worker_type
+
+ job['worker']['payload'] = {} if 'human' in worker_type else {
+ 'image': 'ubuntu:16.10',
+ 'command': [
+ '/bin/bash',
+ '-c',
+ 'echo "Dummy task while while bug 1351664 is implemented"'
+ ],
+ 'maxRunTime': 600,
+ }
+
+ yield job
+
+
+transforms.add(delete_non_required_fields_transform)
--- a/taskcluster/taskgraph/transforms/task.py
+++ b/taskcluster/taskgraph/transforms/task.py
@@ -340,16 +340,38 @@ task_description_schema = Schema({
Required('taskId'): taskref_or_string,
# type of signing task (for CoT)
Required('taskType'): basestring,
# Paths to the artifacts to sign
Required('paths'): [basestring],
}],
+ }, {
+ Required('implementation'): 'push-apk-breakpoint',
+ Required('payload'): object,
+
+ }, {
+ Required('implementation'): 'push-apk',
+
+ # list of artifact URLs for the artifacts that should be beetmoved
+ Required('upstream-artifacts'): [{
+ # taskId of the task with the artifact
+ Required('taskId'): taskref_or_string,
+
+ # type of signing task (for CoT)
+ Required('taskType'): basestring,
+
+ # Paths to the artifacts to sign
+ Required('paths'): [basestring],
+ }],
+
+ # "Invalid" is a noop for try and other non-supported branches
+ Required('google-play-track'): Any('production', 'beta', 'alpha', 'invalid'),
+ Required('dry-run', default=True): bool,
}),
})
GROUP_NAMES = {
'py': 'Python unit tests',
'tc': 'Executed by TaskCluster',
'tc-e10s': 'Executed by TaskCluster with e10s',
'tc-Fxfn-l': 'Firefox functional tests (local) executed by TaskCluster',
@@ -377,16 +399,17 @@ GROUP_NAMES = {
'Aries': 'Aries Device Image',
'Nexus 5-L': 'Nexus 5-L Device Image',
'I': 'Docker Image Builds',
'TL': 'Toolchain builds for Linux 64-bits',
'TM': 'Toolchain builds for OSX',
'TW32': 'Toolchain builds for Windows 32-bits',
'TW64': 'Toolchain builds for Windows 64-bits',
'SM-tc': 'Spidermonkey builds',
+ 'pub': 'APK publishing',
}
UNKNOWN_GROUP_NAME = "Treeherder group {} has no name; add it to " + __file__
V2_ROUTE_TEMPLATES = [
"index.gecko.v2.{project}.latest.{product}.{job-name}",
"index.gecko.v2.{project}.pushdate.{build_date_long}.{product}.{job-name}",
"index.gecko.v2.{project}.revision.{head_rev}.{product}.{job-name}",
]
@@ -593,16 +616,32 @@ def build_beetmover_payload(config, task
def build_balrog_payload(config, task, task_def):
worker = task['worker']
task_def['payload'] = {
'upstreamArtifacts': worker['upstream-artifacts']
}
+@payload_builder('push-apk')
+def build_push_apk_payload(config, task, task_def):
+ worker = task['worker']
+
+ task_def['payload'] = {
+ 'dry_run': worker['dry-run'],
+ 'upstreamArtifacts': worker['upstream-artifacts'],
+ 'google_play_track': worker['google-play-track'],
+ }
+
+
+@payload_builder('push-apk-breakpoint')
+def build_push_apk_breakpoint_payload(config, task, task_def):
+ task_def['payload'] = task['worker']['payload']
+
+
@payload_builder('native-engine')
def build_macosx_engine_payload(config, task, task_def):
worker = task['worker']
artifacts = map(lambda artifact: {
'name': artifact['name'],
'path': artifact['path'],
'type': artifact['type'],
'expires': task_def['expires'],
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/util/push_apk.py
@@ -0,0 +1,73 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Common functions for both push-apk and push-apk-breakpoint.
+"""
+
+import re
+
+from taskgraph.util.schema import validate_schema
+
+REQUIRED_ARCHITECTURES = ('android-x86', 'android-api-15')
+PLATFORM_REGEX = re.compile(r'signing-android-(\S+)-nightly')
+
+
+def fill_labels_tranform(_, jobs):
+ for job in jobs:
+ job['label'] = job['name']
+
+ yield job
+
+
+def validate_jobs_schema_transform_partial(description_schema, transform_type, config, jobs):
+ for job in jobs:
+ label = job.get('label', '?no-label?')
+ yield validate_schema(
+ description_schema, job,
+ "In {} ({!r} kind) task for {!r}:".format(transform_type, config.kind, label)
+ )
+
+
+def validate_dependent_tasks_transform(_, jobs):
+ for job in jobs:
+ check_every_architecture_is_present_in_dependent_tasks(job['dependent-tasks'])
+ yield job
+
+
+def check_every_architecture_is_present_in_dependent_tasks(dependent_tasks):
+ dependencies_labels = [task.label for task in dependent_tasks]
+
+ is_this_required_architecture_present = {
+ architecture: any(architecture in label for label in dependencies_labels)
+ for architecture in REQUIRED_ARCHITECTURES
+ }
+ are_all_required_achitectures_present = all(is_this_required_architecture_present.values())
+
+ if not are_all_required_achitectures_present:
+ raise Exception('''One or many required architectures are missing.
+
+Required architectures: {}.
+Given dependencies: {}.
+'''.format(REQUIRED_ARCHITECTURES, dependent_tasks)
+ )
+
+
+def delete_non_required_fields_transform(_, jobs):
+ for job in jobs:
+ del job['name']
+ del job['dependent-tasks']
+
+ yield job
+
+
+def generate_dependencies(dependent_tasks):
+ # Because we depend on several tasks that have the same kind, we introduce the platform
+ dependencies = {}
+ for task in dependent_tasks:
+ platform_match = PLATFORM_REGEX.match(task.label)
+ # platform_match is None when the breakpoint task is given
+ task_kind = task.kind if platform_match is None else \
+ '{}-{}'.format(task.kind, platform_match.group(1))
+ dependencies[task_kind] = task.label
+ return dependencies
--- a/taskcluster/taskgraph/util/scriptworker.py
+++ b/taskcluster/taskgraph/util/scriptworker.py
@@ -183,16 +183,54 @@ BALROG_CHANNEL_SCOPES = {
'project:releng:balrog:channel:release-cdntest'
'project:releng:balrog:channel:esr',
'project:releng:balrog:channel:esr-localtest',
'project:releng:balrog:channel:esr-cdntest'
]
}
+PUSH_APK_SCOPE_ALIAS_TO_PROJECT = [[
+ 'aurora', set([
+ 'mozilla-aurora',
+ ])
+], [
+ 'beta', set([
+ 'mozilla-beta',
+ ])
+], [
+ 'release', set([
+ 'mozilla-release',
+ ])
+]]
+
+
+PUSH_APK_SCOPES = {
+ 'aurora': 'project:releng:googleplay:aurora',
+ 'beta': 'project:releng:googleplay:beta',
+ 'release': 'project:releng:googleplay:release',
+ 'default': 'project:releng:googleplay:invalid',
+}
+
+# See https://github.com/mozilla-releng/pushapkscript#aurora-beta-release-vs-alpha-beta-production
+PUSH_APK_GOOGLE_PLAY_TRACT = {
+ 'aurora': 'beta',
+ 'beta': 'production',
+ 'release': 'production',
+ 'default': 'invalid',
+}
+
+PUSH_APK_BREAKPOINT_WORKER_TYPE = {
+ 'aurora': 'aws-provisioner-v1/taskcluster-generic',
+ 'beta': 'null-provisioner/human-breakpoint',
+ 'release': 'null-provisioner/human-breakpoint',
+ 'default': 'invalid/invalid',
+}
+
+
# scope functions {{{1
def get_scope_from_project(alias_to_project_map, alias_to_scope_map, config):
"""Determine the restricted scope from `config.params['project']`.
Args:
alias_to_project_map (list of lists): each list pair contains the
alias and the set of projects that match. This is ordered.
alias_to_scope_map (dict): the alias alias to scope
@@ -281,16 +319,34 @@ get_balrog_server_scope = functools.part
)
get_balrog_channel_scopes = functools.partial(
get_scope_from_project,
BALROG_SCOPE_ALIAS_TO_PROJECT,
BALROG_CHANNEL_SCOPES
)
+get_push_apk_scope = functools.partial(
+ get_scope_from_project,
+ PUSH_APK_SCOPE_ALIAS_TO_PROJECT,
+ PUSH_APK_SCOPES
+)
+
+get_push_apk_track = functools.partial(
+ get_scope_from_project,
+ PUSH_APK_SCOPE_ALIAS_TO_PROJECT,
+ PUSH_APK_GOOGLE_PLAY_TRACT
+)
+
+get_push_apk_breakpoint_worker_type = functools.partial(
+ get_scope_from_project,
+ PUSH_APK_SCOPE_ALIAS_TO_PROJECT,
+ PUSH_APK_BREAKPOINT_WORKER_TYPE
+)
+
# release_config {{{1
def get_release_config(config):
"""Get the build number and version for a release task.
Currently only applies to beetmover tasks.
Args: