Bug 1447263 - Snap: Split build and upload task r=rail draft
authorJohan Lorenzo <jlorenzo@mozilla.com>
Tue, 20 Mar 2018 17:19:50 +0100
changeset 775177 dcd16e9e7e28f1817de12ff469c1456305a2dafc
parent 775176 b7fa9d95150ec24e3c8d11106153f76be4507356
push id104638
push userbmo:jlorenzo@mozilla.com
push dateFri, 30 Mar 2018 13:27:33 +0000
reviewersrail
bugs1447263
milestone61.0a1
Bug 1447263 - Snap: Split build and upload task r=rail MozReview-Commit-ID: DGLQZLTjJDu
taskcluster/ci/release-snap-push/kind.yml
taskcluster/ci/release-snap-repackage/kind.yml
taskcluster/ci/release-snap/kind.yml
taskcluster/docker/firefox-snap/fetch_macaroons.sh
taskcluster/docker/firefox-snap/runme.sh
taskcluster/docs/kinds.rst
taskcluster/taskgraph/transforms/release_snap.py
taskcluster/taskgraph/transforms/release_snap_push.py
taskcluster/taskgraph/transforms/release_snap_repackage.py
taskcluster/taskgraph/transforms/task.py
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/release-snap-push/kind.yml
@@ -0,0 +1,41 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+loader: taskgraph.loader.transform:loader
+
+transforms:
+   - taskgraph.transforms.release_deps:transforms
+   - taskgraph.transforms.release_snap_push:transforms
+   - taskgraph.transforms.release_notifications:transforms
+   - taskgraph.transforms.task:transforms
+
+kind-dependencies:
+   - release-snap-repackage
+
+job-defaults:
+   description: Pushes (Ubuntu) Snaps onto Snap Store
+   run-on-projects: []  # to make sure this never runs as part of CI
+   shipping-phase: push
+   scopes:
+      by-project:
+         mozilla-beta: ["project:releng:snapcraft:firefox:beta"]
+         mozilla-release: ["project:releng:snapcraft:firefox:candidate"]
+         default: ["project:releng:snapcraft:firefox:mock"]
+   treeherder:
+      platform: linux64/opt
+      kind: build
+      tier: 2
+   worker-type:
+      by-project:
+         mozilla-beta: scriptworker-prov-v1/pushsnap-v1
+         mozilla-release: scriptworker-prov-v1/pushsnap-v1
+         default: scriptworker-prov-v1/dep-pushsnap
+   worker:
+      implementation: push-snap
+
+jobs:
+   firefox:
+      shipping-product: firefox
+      treeherder:
+         symbol: Snap(push)
rename from taskcluster/ci/release-snap/kind.yml
rename to taskcluster/ci/release-snap-repackage/kind.yml
--- a/taskcluster/ci/release-snap/kind.yml
+++ b/taskcluster/ci/release-snap-repackage/kind.yml
@@ -1,32 +1,28 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 loader: taskgraph.loader.transform:loader
 
 transforms:
    - taskgraph.transforms.release_deps:transforms
-   - taskgraph.transforms.release_snap:transforms
+   - taskgraph.transforms.release_snap_repackage:transforms
    - taskgraph.transforms.release_notifications:transforms
    - taskgraph.transforms.task:transforms
 
 kind-dependencies:
    - post-beetmover-dummy
 
 job-defaults:
    description: Generates snap image
    run-on-projects: []  # to make sure this never runs as part of CI
    shipping-phase: promote
-   scopes:
-      by-project:
-         mozilla-beta: ["secrets:get:project/releng/snapcraft/firefox/edge"]
-         mozilla-release: ["secrets:get:project/releng/snapcraft/firefox/candidate"]
-         default: []
+   scopes: []
    treeherder:
       platform: linux64/opt
       kind: build
       tier: 2
    worker-type: aws-provisioner-v1/gecko-{level}-b-linux
    worker:
       implementation: docker-worker
       os: linux
@@ -43,35 +39,26 @@ job-defaults:
            curl -o scripts.tar.bz2 {config_params[head_repository]}/archive/{config_params[head_rev]}.tar.bz2/taskcluster/docker/firefox-snap/ &&
            mkdir scripts &&
            tar xvfj scripts.tar.bz2 -C scripts --strip-components 4 &&
            cd scripts &&
            ./runme.sh
       env:
          VERSION: "{release_config[version]}"
          BUILD_NUMBER: "{release_config[build_number]}"
-         PUSH_TO_CHANNEL:
-            by-project:
-               # The snap stores exposes 4 channels: edge, beta, candidate, release.
-               # Let's use edge and candidate for QE to test. Then, on the Snap Store UI,
-               # we can promote builds to beta and release, respectively.
-               # For more info: https://docs.snapcraft.io/reference/channels
-               mozilla-beta: edge
-               mozilla-release: candidate
-               default: ""
          CANDIDATES_DIR:
             by-project:
                maple:
                   https://net-mozaws-stage-delivery-firefox.s3.amazonaws.com/pub/{task[shipping-product]}/candidates
                default:
                   https://net-mozaws-prod-delivery-firefox.s3.amazonaws.com/pub/{task[shipping-product]}/candidates
          LC_ALL: C.UTF-8
          LANG: C.UTF-8
          L10N_CHANGESETS: "{config_params[head_repository]}/raw-file/{config_params[head_rev]}/browser/locales/l10n-changesets.json"
-      taskcluster-proxy: true
+      chain-of-trust: true
 
 jobs:
    firefox:
       shipping-product: firefox
       attributes:
          build_platform: linux64-nightly
       treeherder:
-         symbol: Snap(BF)
+         symbol: Snap(r)
deleted file mode 100644
--- a/taskcluster/docker/firefox-snap/fetch_macaroons.sh
+++ /dev/null
@@ -1,14 +0,0 @@
-#!/bin/bash
-
-set -ex
-
-url="$1"
-
-CONFIG="$HOME/.config/snapcraft/snapcraft.cfg"
-
-mkdir -p "$( dirname "$CONFIG" )"
-curl -s "$url" | \
-    python -c 'import json, sys; a = json.load(sys.stdin); print a["secret"]["content"]' | \
-    base64 -d > \
-    "$CONFIG"
-chmod 600 "$CONFIG"
--- a/taskcluster/docker/firefox-snap/runme.sh
+++ b/taskcluster/docker/firefox-snap/runme.sh
@@ -10,17 +10,17 @@ test "$L10N_CHANGESETS"
 
 # Optional env variables
 : WORKSPACE                     "${WORKSPACE:=/home/worker/workspace}"
 : ARTIFACTS_DIR                 "${ARTIFACTS_DIR:=/home/worker/artifacts}"
 : PUSH_TO_CHANNEL               ""
 
 SCRIPT_DIRECTORY="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 
-TARGET="firefox-${VERSION}.snap"
+TARGET="target.snap"
 TARGET_FULL_PATH="$ARTIFACTS_DIR/$TARGET"
 SOURCE_DEST="${WORKSPACE}/source"
 
 mkdir -p "$ARTIFACTS_DIR"
 rm -rf "$SOURCE_DEST" && mkdir -p "$SOURCE_DEST"
 
 CURL="curl --location --retry 10 --retry-delay 10"
 
@@ -84,19 +84,8 @@ hash=$(sha512sum "$TARGET.checksums" | a
 cat << EOF > signing_manifest.json
 [{"file_to_sign": "$TARGET.checksums", "hash": "$hash"}]
 EOF
 
 # For posterity
 find . -ls
 cat "$TARGET.checksums"
 cat signing_manifest.json
-
-
-# Upload snaps to Ubuntu Snap Store
-# TODO: Make this part an independent task
-if [[ "$PUSH_TO_CHANNEL" =~ (^(edge|candidate)$)  ]]; then
-  echo "Uploading to Ubuntu Store on channel $PUSH_TO_CHANNEL"
-  bash "$SCRIPT_DIRECTORY/fetch_macaroons.sh" "http://taskcluster/secrets/v1/secret/project/releng/snapcraft/firefox/$PUSH_TO_CHANNEL"
-  snapcraft push --release "$PUSH_TO_CHANNEL" "$TARGET_FULL_PATH"
-else
-  echo "No upload done: PUSH_TO_CHANNEL value \"$PUSH_TO_CHANNEL\" doesn't match a known channel."
-fi
--- a/taskcluster/docs/kinds.rst
+++ b/taskcluster/docs/kinds.rst
@@ -244,20 +244,24 @@ release-secondary-balrog-scheduling
 ----------------------
 Schedule an RC release to go live in Balrog.
 
 release-binary-transparency
 ---------------------------
 Binary transparency creates a publicly verifiable log of binary shas for downstream
 release auditing. https://wiki.mozilla.org/Security/Binary_Transparency
 
-release-snap
-------------
+release-snap-repackage
+----------------------
 Generate an installer using Ubuntu's Snap format.
 
+release-snap-push
+----------------------
+Pushes Snap repackage on Snap store.
+
 release-notify-push
 ----------------------
 Notify when a release has been pushed to CDNs.
 
 release-notify-ship
 ----------------------
 Notify when a release has been shipped.
 
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/release_snap_push.py
@@ -0,0 +1,73 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Transform the release-snap-push kind into an actual task description.
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from taskgraph.transforms.base import TransformSequence
+from taskgraph.transforms.task import task_description_schema
+from taskgraph.util.schema import optionally_keyed_by, resolve_keyed_by, Schema, validate_schema
+
+from voluptuous import Optional, Required
+
+
+transforms = TransformSequence()
+
+# Voluptuous uses marker objects as dictionary *keys*, but they are not
+# comparable, so we cast all of the keys back to regular strings
+task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
+
+
+push_snap_description_schema = Schema({
+    Required('name'): basestring,
+    Required('job-from'): task_description_schema['job-from'],
+    Required('dependencies'): task_description_schema['dependencies'],
+    Required('description'): task_description_schema['description'],
+    Required('treeherder'): task_description_schema['treeherder'],
+    Required('run-on-projects'): task_description_schema['run-on-projects'],
+    Required('worker-type'): optionally_keyed_by('project', basestring),
+    Required('worker'): object,
+    Required('scopes'): optionally_keyed_by('project', [basestring]),
+    Required('shipping-phase'): task_description_schema['shipping-phase'],
+    Required('shipping-product'): task_description_schema['shipping-product'],
+    Optional('extra'): task_description_schema['extra'],
+})
+
+
+@transforms.add
+def validate_jobs_schema_transform(config, jobs):
+    for job in jobs:
+        label = job.get('label', '?no-label?')
+        validate_schema(
+            push_snap_description_schema, job,
+            "In release_snap_push ({!r} kind) task for {!r}:".format(config.kind, label)
+        )
+        yield job
+
+
+@transforms.add
+def make_task_description(config, jobs):
+    for job in jobs:
+        if len(job['dependencies']) != 1:
+            raise Exception('Exactly 1 dependency is required')
+
+        job['worker']['upstream-artifacts'] = generate_upstream_artifacts(job['dependencies'])
+
+        resolve_keyed_by(job, 'scopes', item_name=job['name'], project=config.params['project'])
+        resolve_keyed_by(
+            job, 'worker-type', item_name=job['name'], project=config.params['project']
+        )
+
+        yield job
+
+
+def generate_upstream_artifacts(dependencies):
+    return [{
+        'taskId': {'task-reference': '<{}>'.format(task_kind)},
+        # TODO bug 1417960
+        'taskType': 'build',
+        'paths': ['public/build/target.snap'],
+    } for task_kind in dependencies.keys()]
rename from taskcluster/taskgraph/transforms/release_snap.py
rename to taskcluster/taskgraph/transforms/release_snap_repackage.py
--- a/taskcluster/taskgraph/transforms/release_snap.py
+++ b/taskcluster/taskgraph/transforms/release_snap_repackage.py
@@ -28,11 +28,9 @@ def format(config, tasks):
         command = task.get('worker', {}).get('command', [])
         task['worker']['command'] = [x.format(**format_params) for x in command]
 
         env = task.get('worker', {}).get('env', {})
         for k in env.keys():
             resolve_keyed_by(env, k, 'snap envs', project=config.params['project'])
             task['worker']['env'][k] = env[k].format(**format_params)
 
-        resolve_keyed_by(task, 'scopes', 'snap scopes', project=config.params['project'])
-
         yield task
--- a/taskcluster/taskgraph/transforms/task.py
+++ b/taskcluster/taskgraph/transforms/task.py
@@ -529,37 +529,35 @@ task_description_schema = Schema({
         Extra: object,
 
     }, {
         Required('implementation'): 'always-optimized',
         Extra: object,
 
     }, {
         Required('implementation'): 'push-apk',
-
-        # list of artifact URLs for the artifacts that should be beetmoved
         Required('upstream-artifacts'): [{
-            # taskId of the task with the artifact
             Required('taskId'): taskref_or_string,
-
-            # type of signing task (for CoT)
             Required('taskType'): basestring,
-
-            # Paths to the artifacts to sign
             Required('paths'): [basestring],
-
-            # Artifact is optional to run the task
             Optional('optional', default=False): bool,
         }],
 
         # "Invalid" is a noop for try and other non-supported branches
         Required('google-play-track'): Any('production', 'beta', 'alpha', 'rollout', 'invalid'),
         Required('commit'): bool,
         Optional('rollout-percentage'): Any(int, None),
     }, {
+        Required('implementation'): 'push-snap',
+        Required('upstream-artifacts'): [{
+            Required('taskId'): taskref_or_string,
+            Required('taskType'): basestring,
+            Required('paths'): [basestring],
+        }],
+    }, {
         Required('implementation'): 'shipit',
         Required('release-name'): basestring,
     }, {
         Required('implementation'): 'treescript',
         Required('tag'): bool,
         Required('bump'): bool,
         Optional('bump-files'): [basestring],
         Required('force-dry-run', default=True): bool,
@@ -1110,16 +1108,25 @@ def build_push_apk_payload(config, task,
         'upstreamArtifacts':  worker['upstream-artifacts'],
         'google_play_track': worker['google-play-track'],
     }
 
     if worker.get('rollout-percentage', None):
         task_def['payload']['rollout_percentage'] = worker['rollout-percentage']
 
 
+@payload_builder('push-snap')
+def build_push_snap_payload(config, task, task_def):
+    worker = task['worker']
+
+    task_def['payload'] = {
+        'upstreamArtifacts':  worker['upstream-artifacts'],
+    }
+
+
 @payload_builder('shipit')
 def build_ship_it_payload(config, task, task_def):
     worker = task['worker']
 
     task_def['payload'] = {
         'release_name': worker['release-name']
     }