Bug 1447263 - Snap: Split build and upload task r=rail
MozReview-Commit-ID: DGLQZLTjJDu
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/release-snap-push/kind.yml
@@ -0,0 +1,41 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+loader: taskgraph.loader.transform:loader
+
+transforms:
+ - taskgraph.transforms.release_deps:transforms
+ - taskgraph.transforms.release_snap_push:transforms
+ - taskgraph.transforms.release_notifications:transforms
+ - taskgraph.transforms.task:transforms
+
+kind-dependencies:
+ - release-snap-repackage
+
+job-defaults:
+ description: Pushes (Ubuntu) Snaps onto Snap Store
+ run-on-projects: [] # to make sure this never runs as part of CI
+ shipping-phase: push
+ scopes:
+ by-project:
+ mozilla-beta: ["project:releng:snapcraft:firefox:beta"]
+ mozilla-release: ["project:releng:snapcraft:firefox:candidate"]
+ default: ["project:releng:snapcraft:firefox:mock"]
+ treeherder:
+ platform: linux64/opt
+ kind: build
+ tier: 2
+ worker-type:
+ by-project:
+ mozilla-beta: scriptworker-prov-v1/pushsnap-v1
+ mozilla-release: scriptworker-prov-v1/pushsnap-v1
+ default: scriptworker-prov-v1/dep-pushsnap
+ worker:
+ implementation: push-snap
+
+jobs:
+ firefox:
+ shipping-product: firefox
+ treeherder:
+ symbol: Snap(push)
rename from taskcluster/ci/release-snap/kind.yml
rename to taskcluster/ci/release-snap-repackage/kind.yml
--- a/taskcluster/ci/release-snap/kind.yml
+++ b/taskcluster/ci/release-snap-repackage/kind.yml
@@ -1,32 +1,28 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
loader: taskgraph.loader.transform:loader
transforms:
- taskgraph.transforms.release_deps:transforms
- - taskgraph.transforms.release_snap:transforms
+ - taskgraph.transforms.release_snap_repackage:transforms
- taskgraph.transforms.release_notifications:transforms
- taskgraph.transforms.task:transforms
kind-dependencies:
- post-beetmover-dummy
job-defaults:
description: Generates snap image
run-on-projects: [] # to make sure this never runs as part of CI
shipping-phase: promote
- scopes:
- by-project:
- mozilla-beta: ["secrets:get:project/releng/snapcraft/firefox/edge"]
- mozilla-release: ["secrets:get:project/releng/snapcraft/firefox/candidate"]
- default: []
+ scopes: []
treeherder:
platform: linux64/opt
kind: build
tier: 2
worker-type: aws-provisioner-v1/gecko-{level}-b-linux
worker:
implementation: docker-worker
os: linux
@@ -43,35 +39,26 @@ job-defaults:
curl -o scripts.tar.bz2 {config_params[head_repository]}/archive/{config_params[head_rev]}.tar.bz2/taskcluster/docker/firefox-snap/ &&
mkdir scripts &&
tar xvfj scripts.tar.bz2 -C scripts --strip-components 4 &&
cd scripts &&
./runme.sh
env:
VERSION: "{release_config[version]}"
BUILD_NUMBER: "{release_config[build_number]}"
- PUSH_TO_CHANNEL:
- by-project:
- # The snap stores exposes 4 channels: edge, beta, candidate, release.
- # Let's use edge and candidate for QE to test. Then, on the Snap Store UI,
- # we can promote builds to beta and release, respectively.
- # For more info: https://docs.snapcraft.io/reference/channels
- mozilla-beta: edge
- mozilla-release: candidate
- default: ""
CANDIDATES_DIR:
by-project:
maple:
https://net-mozaws-stage-delivery-firefox.s3.amazonaws.com/pub/{task[shipping-product]}/candidates
default:
https://net-mozaws-prod-delivery-firefox.s3.amazonaws.com/pub/{task[shipping-product]}/candidates
LC_ALL: C.UTF-8
LANG: C.UTF-8
L10N_CHANGESETS: "{config_params[head_repository]}/raw-file/{config_params[head_rev]}/browser/locales/l10n-changesets.json"
- taskcluster-proxy: true
+ chain-of-trust: true
jobs:
firefox:
shipping-product: firefox
attributes:
build_platform: linux64-nightly
treeherder:
- symbol: Snap(BF)
+ symbol: Snap(r)
deleted file mode 100644
--- a/taskcluster/docker/firefox-snap/fetch_macaroons.sh
+++ /dev/null
@@ -1,14 +0,0 @@
-#!/bin/bash
-
-set -ex
-
-url="$1"
-
-CONFIG="$HOME/.config/snapcraft/snapcraft.cfg"
-
-mkdir -p "$( dirname "$CONFIG" )"
-curl -s "$url" | \
- python -c 'import json, sys; a = json.load(sys.stdin); print a["secret"]["content"]' | \
- base64 -d > \
- "$CONFIG"
-chmod 600 "$CONFIG"
--- a/taskcluster/docker/firefox-snap/runme.sh
+++ b/taskcluster/docker/firefox-snap/runme.sh
@@ -10,17 +10,17 @@ test "$L10N_CHANGESETS"
# Optional env variables
: WORKSPACE "${WORKSPACE:=/home/worker/workspace}"
: ARTIFACTS_DIR "${ARTIFACTS_DIR:=/home/worker/artifacts}"
: PUSH_TO_CHANNEL ""
SCRIPT_DIRECTORY="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-TARGET="firefox-${VERSION}.snap"
+TARGET="target.snap"
TARGET_FULL_PATH="$ARTIFACTS_DIR/$TARGET"
SOURCE_DEST="${WORKSPACE}/source"
mkdir -p "$ARTIFACTS_DIR"
rm -rf "$SOURCE_DEST" && mkdir -p "$SOURCE_DEST"
CURL="curl --location --retry 10 --retry-delay 10"
@@ -84,19 +84,8 @@ hash=$(sha512sum "$TARGET.checksums" | a
cat << EOF > signing_manifest.json
[{"file_to_sign": "$TARGET.checksums", "hash": "$hash"}]
EOF
# For posterity
find . -ls
cat "$TARGET.checksums"
cat signing_manifest.json
-
-
-# Upload snaps to Ubuntu Snap Store
-# TODO: Make this part an independent task
-if [[ "$PUSH_TO_CHANNEL" =~ (^(edge|candidate)$) ]]; then
- echo "Uploading to Ubuntu Store on channel $PUSH_TO_CHANNEL"
- bash "$SCRIPT_DIRECTORY/fetch_macaroons.sh" "http://taskcluster/secrets/v1/secret/project/releng/snapcraft/firefox/$PUSH_TO_CHANNEL"
- snapcraft push --release "$PUSH_TO_CHANNEL" "$TARGET_FULL_PATH"
-else
- echo "No upload done: PUSH_TO_CHANNEL value \"$PUSH_TO_CHANNEL\" doesn't match a known channel."
-fi
--- a/taskcluster/docs/kinds.rst
+++ b/taskcluster/docs/kinds.rst
@@ -244,20 +244,24 @@ release-secondary-balrog-scheduling
----------------------
Schedule an RC release to go live in Balrog.
release-binary-transparency
---------------------------
Binary transparency creates a publicly verifiable log of binary shas for downstream
release auditing. https://wiki.mozilla.org/Security/Binary_Transparency
-release-snap
-------------
+release-snap-repackage
+----------------------
Generate an installer using Ubuntu's Snap format.
+release-snap-push
+----------------------
+Pushes Snap repackage on Snap store.
+
release-notify-push
----------------------
Notify when a release has been pushed to CDNs.
release-notify-ship
----------------------
Notify when a release has been shipped.
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/release_snap_push.py
@@ -0,0 +1,73 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Transform the release-snap-push kind into an actual task description.
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from taskgraph.transforms.base import TransformSequence
+from taskgraph.transforms.task import task_description_schema
+from taskgraph.util.schema import optionally_keyed_by, resolve_keyed_by, Schema, validate_schema
+
+from voluptuous import Optional, Required
+
+
+transforms = TransformSequence()
+
+# Voluptuous uses marker objects as dictionary *keys*, but they are not
+# comparable, so we cast all of the keys back to regular strings
+task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
+
+
+push_snap_description_schema = Schema({
+ Required('name'): basestring,
+ Required('job-from'): task_description_schema['job-from'],
+ Required('dependencies'): task_description_schema['dependencies'],
+ Required('description'): task_description_schema['description'],
+ Required('treeherder'): task_description_schema['treeherder'],
+ Required('run-on-projects'): task_description_schema['run-on-projects'],
+ Required('worker-type'): optionally_keyed_by('project', basestring),
+ Required('worker'): object,
+ Required('scopes'): optionally_keyed_by('project', [basestring]),
+ Required('shipping-phase'): task_description_schema['shipping-phase'],
+ Required('shipping-product'): task_description_schema['shipping-product'],
+ Optional('extra'): task_description_schema['extra'],
+})
+
+
+@transforms.add
+def validate_jobs_schema_transform(config, jobs):
+ for job in jobs:
+ label = job.get('label', '?no-label?')
+ validate_schema(
+ push_snap_description_schema, job,
+ "In release_snap_push ({!r} kind) task for {!r}:".format(config.kind, label)
+ )
+ yield job
+
+
+@transforms.add
+def make_task_description(config, jobs):
+ for job in jobs:
+ if len(job['dependencies']) != 1:
+ raise Exception('Exactly 1 dependency is required')
+
+ job['worker']['upstream-artifacts'] = generate_upstream_artifacts(job['dependencies'])
+
+ resolve_keyed_by(job, 'scopes', item_name=job['name'], project=config.params['project'])
+ resolve_keyed_by(
+ job, 'worker-type', item_name=job['name'], project=config.params['project']
+ )
+
+ yield job
+
+
+def generate_upstream_artifacts(dependencies):
+ return [{
+ 'taskId': {'task-reference': '<{}>'.format(task_kind)},
+ # TODO bug 1417960
+ 'taskType': 'build',
+ 'paths': ['public/build/target.snap'],
+ } for task_kind in dependencies.keys()]
rename from taskcluster/taskgraph/transforms/release_snap.py
rename to taskcluster/taskgraph/transforms/release_snap_repackage.py
--- a/taskcluster/taskgraph/transforms/release_snap.py
+++ b/taskcluster/taskgraph/transforms/release_snap_repackage.py
@@ -28,11 +28,9 @@ def format(config, tasks):
command = task.get('worker', {}).get('command', [])
task['worker']['command'] = [x.format(**format_params) for x in command]
env = task.get('worker', {}).get('env', {})
for k in env.keys():
resolve_keyed_by(env, k, 'snap envs', project=config.params['project'])
task['worker']['env'][k] = env[k].format(**format_params)
- resolve_keyed_by(task, 'scopes', 'snap scopes', project=config.params['project'])
-
yield task
--- a/taskcluster/taskgraph/transforms/task.py
+++ b/taskcluster/taskgraph/transforms/task.py
@@ -529,37 +529,35 @@ task_description_schema = Schema({
Extra: object,
}, {
Required('implementation'): 'always-optimized',
Extra: object,
}, {
Required('implementation'): 'push-apk',
-
- # list of artifact URLs for the artifacts that should be beetmoved
Required('upstream-artifacts'): [{
- # taskId of the task with the artifact
Required('taskId'): taskref_or_string,
-
- # type of signing task (for CoT)
Required('taskType'): basestring,
-
- # Paths to the artifacts to sign
Required('paths'): [basestring],
-
- # Artifact is optional to run the task
Optional('optional', default=False): bool,
}],
# "Invalid" is a noop for try and other non-supported branches
Required('google-play-track'): Any('production', 'beta', 'alpha', 'rollout', 'invalid'),
Required('commit'): bool,
Optional('rollout-percentage'): Any(int, None),
}, {
+ Required('implementation'): 'push-snap',
+ Required('upstream-artifacts'): [{
+ Required('taskId'): taskref_or_string,
+ Required('taskType'): basestring,
+ Required('paths'): [basestring],
+ }],
+ }, {
Required('implementation'): 'shipit',
Required('release-name'): basestring,
}, {
Required('implementation'): 'treescript',
Required('tag'): bool,
Required('bump'): bool,
Optional('bump-files'): [basestring],
Required('force-dry-run', default=True): bool,
@@ -1110,16 +1108,25 @@ def build_push_apk_payload(config, task,
'upstreamArtifacts': worker['upstream-artifacts'],
'google_play_track': worker['google-play-track'],
}
if worker.get('rollout-percentage', None):
task_def['payload']['rollout_percentage'] = worker['rollout-percentage']
+@payload_builder('push-snap')
+def build_push_snap_payload(config, task, task_def):
+ worker = task['worker']
+
+ task_def['payload'] = {
+ 'upstreamArtifacts': worker['upstream-artifacts'],
+ }
+
+
@payload_builder('shipit')
def build_ship_it_payload(config, task, task_def):
worker = task['worker']
task_def['payload'] = {
'release_name': worker['release-name']
}