Bug 1385401 - Fetch Google Play listings outside of the push-apk task r=aki
MozReview-Commit-ID: HVFVqjliqor
--- a/taskcluster/ci/docker-image/kind.yml
+++ b/taskcluster/ci/docker-image/kind.yml
@@ -30,16 +30,18 @@ jobs:
lint:
symbol: I(lnt)
android-build:
symbol: I(agb)
index-task:
symbol: I(idx)
funsize-update-generator:
symbol: I(pg)
+ google-play-strings:
+ symbol: I(gps)
funsize-balrog-submitter:
symbol: I(fbs)
beet-mover:
symbol: I(bm)
update-verify:
symbol: I(uv)
diffoscope:
symbol: I(diff)
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/google-play-strings/kind.yml
@@ -0,0 +1,54 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+loader: taskgraph.loader.transform:loader
+
+transforms:
+ - taskgraph.transforms.google_play_strings:transforms
+ - taskgraph.transforms.task:transforms
+
+jobs:
+ google-play-strings:
+ description: Download strings to display on Google Play from https://l10n.mozilla-community.org/stores_l10n/
+ attributes:
+ build_type: google_play_strings
+ build_platform: android-nightly
+ nightly: true
+ shipping-phase: promote
+ shipping-product: fennec
+ worker-type: aws-provisioner-v1/gecko-{level}-b-android
+ worker:
+ implementation: docker-worker
+ os: linux
+ docker-image: {in-tree: google-play-strings}
+ chain-of-trust: true
+ max-run-time: 600
+ artifacts:
+ - name: 'public/google_play_strings.json'
+ # XXX The folder depends on the one defined in the Dockerfile
+ path: /builds/worker/google_play_strings.json
+ type: 'file'
+ env:
+ # TODO Use the branch name instead of the android package name
+ PACKAGE_NAME:
+ by-project:
+ mozilla-central: org.mozilla.fennec_aurora
+ mozilla-beta: org.mozilla.firefox_beta
+ mozilla-release: org.mozilla.firefox_beta
+ default: org.mozilla.fennec_aurora # Fetches strings for mozilla-central
+ # XXX The folder depends on the one defined in the Dockerfile
+ GOOGLE_PLAY_STRING_FILE: /builds/worker/google_play_strings.json
+ command:
+ - bash
+ - -cx
+ - >
+ python3 ./mozapkpublisher/get_l10n_strings.py
+ --package-name "${PACKAGE_NAME}"
+ --output-file "${GOOGLE_PLAY_STRING_FILE}"
+ treeherder:
+ symbol: pub(gps)
+ platform: Android/opt
+ tier: 2
+ kind: other
+ run-on-projects: ['maple', 'mozilla-central', 'mozilla-beta', 'mozilla-release']
--- a/taskcluster/ci/push-apk/kind.yml
+++ b/taskcluster/ci/push-apk/kind.yml
@@ -1,43 +1,43 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
loader: taskgraph.loader.push_apk:loader
transforms:
- - taskgraph.transforms.push_apk:transforms
- - taskgraph.transforms.task:transforms
+ - taskgraph.transforms.push_apk:transforms
+ - taskgraph.transforms.task:transforms
kind-dependencies:
- - build-signing
- - push-apk-breakpoint
+ - build-signing
+ - google-play-strings
+ - push-apk-breakpoint
jobs:
- push-apk/opt:
- description: Publishes APK onto Google Play Store
- attributes:
- build_platform: android-nightly
- nightly: true
- shipping-phase: ship
- shipping-product: fennec
- worker-type:
- by-project:
- mozilla-central: scriptworker-prov-v1/pushapk-v1
- mozilla-beta: scriptworker-prov-v1/pushapk-v1
- mozilla-release: scriptworker-prov-v1/pushapk-v1
- default: scriptworker-prov-v1/dep-pushapk
- worker:
- upstream-artifacts: # see transforms
- google-play-track: # see transforms
- implementation: push-apk
- commit: # see transforms
- scopes: # see transforms
- treeherder:
- symbol: pub(gp)
- platform: Android/opt
- tier: 2
- kind: other
- run-on-projects: ['mozilla-central', 'mozilla-beta', 'mozilla-release']
- deadline-after: 5 days
- extra:
- product: fennec
+ push-apk/opt:
+ description: Publishes APK onto Google Play Store
+ attributes:
+ build_platform: android-nightly
+ nightly: true
+ shipping-phase: ship
+ shipping-product: fennec
+ worker-type:
+ by-project:
+ mozilla-central: scriptworker-prov-v1/pushapk-v1
+ mozilla-beta: scriptworker-prov-v1/pushapk-v1
+ mozilla-release: scriptworker-prov-v1/pushapk-v1
+ default: scriptworker-prov-v1/dep-pushapk
+ worker:
+ upstream-artifacts: # see transforms
+ google-play-track: # see transforms
+ implementation: push-apk
+ commit: # see transforms
+ requires: all-resolved
+ scopes: # see transforms
+ treeherder:
+ symbol: pub(gp)
+ platform: Android/opt
+ tier: 2
+ kind: other
+ run-on-projects: ['mozilla-central', 'mozilla-beta', 'mozilla-release', 'maple']
+ deadline-after: 5 days
new file mode 100644
--- /dev/null
+++ b/taskcluster/docker/google-play-strings/Dockerfile
@@ -0,0 +1,19 @@
+FROM ubuntu:16.04
+MAINTAINER Johan Lorenzo <jlorenzo+tc@mozilla.com>
+
+RUN mkdir /builds
+RUN groupadd -g 500 worker
+RUN useradd -u 500 -g 500 -d /builds/worker -s /bin/bash -m worker
+
+RUN apt-get update
+RUN apt-get install --yes git python3-setuptools build-essential libssl-dev libffi-dev python3-dev
+
+WORKDIR /builds/worker/
+RUN git clone https://github.com/mozilla-releng/mozapkpublisher
+WORKDIR /builds/worker/mozapkpublisher
+RUN python3 setup.py develop
+
+RUN chown -R worker:worker /builds/worker
+
+# Set a default command useful for debugging
+CMD ["/bin/bash", "--login"]
--- a/taskcluster/docs/kinds.rst
+++ b/taskcluster/docs/kinds.rst
@@ -212,16 +212,21 @@ and sign it via the signing scriptworker
additional detached signature.
beetmover-checksums
-------------------
Beetmover, takes specific artifact checksums and pushes it to a location outside
of Taskcluster's task artifacts (archive.mozilla.org as one place) and in the
process determines the final location and "pretty" names it (version product name)
+google-play-strings
+-------------------
+Download strings to display on Google Play from https://l10n.mozilla-community.org/stores_l10n/.
+Artifact is then used by push-apk.
+
push-apk-breakpoint
-------------------
Decides whether or not APKs should be published onto Google Play Store. Jobs of this
kind depend on all the signed multi-locales (aka "multi") APKs for a given release,
in order to make the decision.
push-apk
--------
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/google_play_strings.py
@@ -0,0 +1,64 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Transform the push-apk kind into an actual task description.
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import functools
+
+from taskgraph.transforms.base import TransformSequence
+from taskgraph.transforms.task import task_description_schema
+from taskgraph.util.schema import resolve_keyed_by, Schema
+from taskgraph.util.push_apk import fill_labels_tranform, validate_jobs_schema_transform_partial
+
+from voluptuous import Required
+
+
+transforms = TransformSequence()
+
+# Voluptuous uses marker objects as dictionary *keys*, but they are not
+# comparable, so we cast all of the keys back to regular strings
+task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
+
+google_play_description_schema = Schema({
+ Required('name'): basestring,
+ Required('label'): task_description_schema['label'],
+ Required('description'): task_description_schema['description'],
+ Required('job-from'): task_description_schema['job-from'],
+ Required('attributes'): task_description_schema['attributes'],
+ Required('treeherder'): task_description_schema['treeherder'],
+ Required('run-on-projects'): task_description_schema['run-on-projects'],
+ Required('shipping-phase'): task_description_schema['shipping-phase'],
+ Required('shipping-product'): task_description_schema['shipping-product'],
+ Required('worker-type'): task_description_schema['worker-type'],
+ Required('worker'): object,
+})
+
+validate_jobs_schema_transform = functools.partial(
+ validate_jobs_schema_transform_partial,
+ google_play_description_schema,
+ 'GooglePlayStrings'
+)
+
+transforms.add(fill_labels_tranform)
+transforms.add(validate_jobs_schema_transform)
+
+
+@transforms.add
+def set_worker_data(config, jobs):
+ for job in jobs:
+ worker = job['worker']
+
+ env = worker.setdefault('env', {})
+ resolve_keyed_by(
+ env, 'PACKAGE_NAME', item_name=job['name'],
+ project=config.params['project']
+ )
+
+ cot = job.setdefault('extra', {}).setdefault('chainOfTrust', {})
+ cot.setdefault('inputs', {})['docker-image'] = {'task-reference': '<docker-image>'}
+
+ yield job
--- a/taskcluster/taskgraph/transforms/push_apk.py
+++ b/taskcluster/taskgraph/transforms/push_apk.py
@@ -23,32 +23,32 @@ from voluptuous import Optional, Require
transforms = TransformSequence()
# Voluptuous uses marker objects as dictionary *keys*, but they are not
# comparable, so we cast all of the keys back to regular strings
task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
push_apk_description_schema = Schema({
- # the dependent task (object) for this beetmover job, used to inform beetmover.
Required('dependent-tasks'): object,
Required('name'): basestring,
- Required('label'): basestring,
- Required('description'): basestring,
- Required('job-from'): basestring,
- Required('attributes'): object,
- Required('treeherder'): object,
- Required('run-on-projects'): list,
+ Required('label'): task_description_schema['label'],
+ Required('description'): task_description_schema['description'],
+ Required('job-from'): task_description_schema['job-from'],
+ Required('attributes'): task_description_schema['attributes'],
+ Required('treeherder'): task_description_schema['treeherder'],
+ Required('run-on-projects'): task_description_schema['run-on-projects'],
Required('worker-type'): optionally_keyed_by('project', basestring),
Required('worker'): object,
Required('scopes'): None,
+ Required('requires'): task_description_schema['requires'],
Required('deadline-after'): basestring,
Required('shipping-phase'): task_description_schema['shipping-phase'],
Required('shipping-product'): task_description_schema['shipping-product'],
- Optional('extra'): object,
+ Optional('extra'): task_description_schema['extra'],
})
validate_jobs_schema_transform = functools.partial(
validate_jobs_schema_transform_partial,
push_apk_description_schema,
'PushApk'
)
@@ -78,13 +78,26 @@ def make_task_description(config, jobs):
yield job
transforms.add(delete_non_required_fields_transform)
def generate_upstream_artifacts(dependencies):
- return [{
+ apks = [{
'taskId': {'task-reference': '<{}>'.format(task_kind)},
'taskType': 'signing',
'paths': ['public/build/target.apk'],
- } for task_kind in dependencies.keys() if 'breakpoint' not in task_kind]
+ } for task_kind in dependencies.keys()
+ if task_kind not in ('push-apk-breakpoint', 'google-play-strings')
+ ]
+
+ google_play_strings = [{
+ 'taskId': {'task-reference': '<{}>'.format(task_kind)},
+ 'taskType': 'build',
+ 'paths': ['public/google_play_strings.json'],
+ 'optional': True,
+ } for task_kind in dependencies.keys()
+ if 'google-play-strings' in task_kind
+ ]
+
+ return apks + google_play_strings
--- a/taskcluster/taskgraph/transforms/task.py
+++ b/taskcluster/taskgraph/transforms/task.py
@@ -88,16 +88,18 @@ task_description_schema = Schema({
# relative path (from config.path) to the file task was defined in
Optional('job-from'): basestring,
# dependencies of this task, keyed by name; these are passed through
# verbatim and subject to the interpretation of the Task's get_dependencies
# method.
Optional('dependencies'): {basestring: object},
+ Optional('requires'): Any('all-completed', 'all-resolved'),
+
# expiration and deadline times, relative to task creation, with units
# (e.g., "14 days"). Defaults are set based on the project.
Optional('expires-after'): basestring,
Optional('deadline-after'): basestring,
# custom routes for this task; the default treeherder routes will be added
# automatically
Optional('routes'): [basestring],
@@ -566,16 +568,19 @@ task_description_schema = Schema({
# taskId of the task with the artifact
Required('taskId'): taskref_or_string,
# type of signing task (for CoT)
Required('taskType'): basestring,
# Paths to the artifacts to sign
Required('paths'): [basestring],
+
+ # Artifact is optional to run the task
+ Optional('optional', default=False): bool,
}],
# "Invalid" is a noop for try and other non-supported branches
Required('google-play-track'): Any('production', 'beta', 'alpha', 'rollout', 'invalid'),
Required('commit'): bool,
Optional('rollout-percentage'): int,
}),
})
@@ -1466,16 +1471,19 @@ def build_task(config, tasks):
config.params['head_rev'],
config.path),
},
'extra': extra,
'tags': tags,
'priority': task['priority'],
}
+ if task.get('requires', None):
+ task_def['requires'] = task['requires']
+
if task_th:
# link back to treeherder in description
th_push_link = 'https://treeherder.mozilla.org/#/jobs?repo={}&revision={}'.format(
config.params['project'], treeherder_rev)
task_def['metadata']['description'] += ' ([Treeherder push]({}))'.format(
th_push_link)
# add the payload and adjust anything else as required (e.g., scopes)