--- a/taskcluster/ci/balrog/kind.yml
+++ b/taskcluster/ci/balrog/kind.yml
@@ -1,17 +1,17 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
loader: taskgraph.loader.single_dep:loader
transforms:
- taskgraph.transforms.name_sanity:transforms
- - taskgraph.transforms.balrog:transforms
+ - taskgraph.transforms.balrog_submit:transforms
- taskgraph.transforms.release_notifications:transforms
- taskgraph.transforms.task:transforms
kind-dependencies:
- beetmover
- beetmover-l10n
- beetmover-repackage
deleted file mode 100644
--- a/taskcluster/ci/release-balrog-publishing/kind.yml
+++ /dev/null
@@ -1,49 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-loader: taskgraph.loader.transform:loader
-
-transforms:
- - taskgraph.transforms.release_deps:transforms
- - taskgraph.transforms.release_balrog_publishing:transforms
- - taskgraph.transforms.job:transforms
- - taskgraph.transforms.release_notifications:transforms
- - taskgraph.transforms.task:transforms
-
-kind-dependencies:
- - release-bouncer-check
-
-job-defaults:
- description: Schedule publishing in balrog
- worker-type: buildbot-bridge/buildbot-bridge
- run-on-projects: []
- shipping-phase: ship
- run:
- using: buildbot
- release-promotion: true
-
-jobs:
- firefox:
- name: release-firefox_schedule_publishing_in_balrog
- shipping-product: firefox
- run:
- product: firefox
- buildername: release-{branch}-firefox_schedule_publishing_in_balrog
- channels:
- by-project:
- birch: release
- maple: beta
- mozilla-beta: beta
- mozilla-release: release
- mozilla-esr52: esr
- mozilla-esr59: esr
- default: unknown
-
- devedition:
- name: release-devedition_schedule_publishing_in_balrog
- shipping-product: devedition
- run:
- product: devedition
- buildername: release-{branch}-devedition_schedule_publishing_in_balrog
- channels: aurora
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/release-balrog-scheduling/kind.yml
@@ -0,0 +1,65 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+loader: taskgraph.loader.transform:loader
+
+transforms:
+ - taskgraph.transforms.release_deps:transforms
+ - taskgraph.transforms.worker_type:transforms
+ - taskgraph.transforms.release_notifications:transforms
+ - taskgraph.transforms.task:transforms
+
+kind-dependencies:
+ - release-bouncer-check
+ - release-update-verify
+
+job-defaults:
+ worker-type:
+ by-project:
+ maple: scriptworker-prov-v1/balrog-dev
+ birch: scriptworker-prov-v1/balrog-dev
+ mozilla-beta: scriptworker-prov-v1/balrogworker-v1
+ mozilla-release: scriptworker-prov-v1/balrogworker-v1
+ default: invalid/invalid
+ run-on-projects: []
+ shipping-phase: ship
+ worker:
+ implementation: balrog
+ balrog-action: schedule
+
+jobs:
+ firefox:
+ description: Schedule Firefox publishing in balrog
+ name: release-firefox_schedule_publishing_in_balrog
+ shipping-product: firefox
+ worker:
+ product: firefox
+ publish-rules:
+ by-project:
+ maple: [32]
+ birch: [145]
+ mozilla-beta: [32]
+ mozilla-release: [145]
+ default: []
+ treeherder:
+ platform: linux64/opt
+ symbol: Rel(BSFx)
+ tier: 1
+ kind: build
+ devedition:
+ description: Schedule Devedition publishing in balrog
+ name: release-devedition_schedule_publishing_in_balrog
+ shipping-product: devedition
+ worker:
+ product: devedition
+ publish-rules:
+ by-project:
+ maple: [10]
+ mozilla-beta: [10]
+ default: []
+ treeherder:
+ platform: linux64/opt
+ symbol: Rel(BSDev)
+ tier: 1
+ kind: build
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/release-balrog-submit-toplevel/kind.yml
@@ -0,0 +1,111 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+loader: taskgraph.loader.transform:loader
+
+kind-dependencies:
+ # For the $platform_info.txt files.
+ - post-beetmover-dummy
+
+transforms:
+ - taskgraph.transforms.release_deps:transforms
+ - taskgraph.transforms.worker_type:transforms
+ - taskgraph.transforms.release_notifications:transforms
+ - taskgraph.transforms.task:transforms
+
+job-defaults:
+ run-on-projects: [] # to make sure this never runs in CI
+ shipping-phase: promote
+ worker-type:
+ by-project:
+ maple: scriptworker-prov-v1/balrog-dev
+ birch: scriptworker-prov-v1/balrog-dev
+ mozilla-beta: scriptworker-prov-v1/balrogworker-v1
+ mozilla-release: scriptworker-prov-v1/balrogworker-v1
+ default: invalid/invalid
+ worker:
+ implementation: balrog
+ balrog-action: submit-toplevel
+ require-mirrors: true
+
+jobs:
+ firefox:
+ name: submit-toplevel-firefox-release-to-balrog
+ description: Submit toplevel Firefox release to balrog
+ shipping-product: firefox
+ worker:
+ product: firefox
+ archive-domain:
+ by-project:
+ mozilla-beta: archive.mozilla.org
+ mozilla-release: archive.mozilla.org
+ default: ftp.stage.mozaws.net
+ download-domain:
+ by-project:
+ mozilla-beta: download.mozilla.org
+ mozilla-release: download.mozilla.org
+ default: download.mozilla.org
+ channel-names:
+ by-project:
+ maple: ["beta", "beta-localtest", "beta-cdntest"]
+ birch: ["release", "release-localtest", "release-cdntest"]
+ mozilla-beta: ["beta", "beta-localtest", "beta-cdntest"]
+ mozilla-release: ["release", "release-localtest", "release-cdntest"]
+ default: []
+ publish-rules:
+ by-project:
+ maple: [32]
+ birch: [145]
+ mozilla-beta: [32]
+ mozilla-release: [145]
+ default: []
+ rules-to-update:
+ by-project:
+ maple: ["firefox-beta-cdntest", "firefox-beta-localtest"]
+ birch: ["firefox-release-cdntest", "firefox-release-localtest"]
+ mozilla-beta: ["firefox-beta-cdntest", "firefox-beta-localtest"]
+ mozilla-release: ["firefox-release-cdntest", "firefox-release-localtest"]
+ default: []
+ platforms: ["linux", "linux64", "macosx64", "win32", "win64"]
+ treeherder:
+ platform: linux64/opt
+ symbol: Rel(BPFx)
+ tier: 1
+ kind: build
+
+ devedition:
+ name: submit-toplevel-devedition-release-to-balrog
+ description: submit toplevel Devedition release to balrog
+ shipping-product: devedition
+ worker:
+ product: devedition
+ archive-domain:
+ by-project:
+ mozilla-beta: archive.mozilla.org
+ default: ftp.stage.mozaws.net
+ download-domain:
+ by-project:
+ mozilla-beta: download.mozilla.org
+ default: download.mozilla.org
+ channel-names:
+ by-project:
+ maple: ["aurora", "aurora-localtest", "aurora-cdntest"]
+ mozilla-beta: ["aurora", "aurora-localtest", "aurora-cdntest"]
+ default: []
+ publish-rules:
+ by-project:
+ maple: [10]
+ mozilla-beta: [10]
+ default: []
+ rules-to-update:
+ by-project:
+ maple: ["devedition-cdntest", "devedition-localtest"]
+ mozilla-beta: ["devedition-cdntest", "devedition-localtest"]
+ default: []
+ platforms: ["linux", "linux64", "macosx64", "win32", "win64"]
+ treeherder:
+ platform: linux64/opt
+ symbol: Rel(BPDev)
+ tier: 1
+ kind: build
--- a/taskcluster/ci/release-mark-as-shipped/kind.yml
+++ b/taskcluster/ci/release-mark-as-shipped/kind.yml
@@ -7,17 +7,17 @@ loader: taskgraph.loader.transform:loade
transforms:
- taskgraph.transforms.release_deps:transforms
- taskgraph.transforms.release_mark_as_shipped:transforms
- taskgraph.transforms.release_notifications:transforms
- taskgraph.transforms.task:transforms
kind-dependencies:
- push-apk
- - release-balrog-publishing
+ - release-balrog-scheduling
- release-bouncer-aliases
- release-version-bump
job-defaults:
description: mark release as shipped in Ship-It
worker-type:
by-project:
mozilla-central: scriptworker-prov-v1/shipit-v1
--- a/taskcluster/ci/release-notify-ship/kind.yml
+++ b/taskcluster/ci/release-notify-ship/kind.yml
@@ -6,17 +6,17 @@ loader: taskgraph.loader.transform:loade
transforms:
- taskgraph.transforms.release_deps:transforms
- taskgraph.transforms.release_notifications:transforms
- taskgraph.transforms.task:transforms
kind-dependencies:
- push-apk
- - release-balrog-publishing
+ - release-balrog-scheduling
- release-bouncer-aliases
- release-version-bump
- release-mark-as-shipped
job-defaults:
name: notify-release-drivers-ship
description: Sends email to release-drivers telling release was shipped.
run-on-projects: []
deleted file mode 100644
--- a/taskcluster/ci/release-secondary-balrog-publishing/kind.yml
+++ /dev/null
@@ -1,39 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-loader: taskgraph.loader.transform:loader
-
-transforms:
- - taskgraph.transforms.release_deps:transforms
- - taskgraph.transforms.release_balrog_publishing:transforms
- - taskgraph.transforms.job:transforms
- - taskgraph.transforms.release_notifications:transforms
- - taskgraph.transforms.task:transforms
-
-kind-dependencies:
- - post-balrog-dummy
- - post-beetmover-dummy
- - release-updates-builder
-
-job-defaults:
- description: Schedule publishing in balrog
- worker-type: buildbot-bridge/buildbot-bridge
- run-on-projects: []
- shipping-phase: ship
- run:
- using: buildbot
- release-promotion: true
-
-jobs:
- firefox-secondary-rc:
- name: release-firefox_schedule_publishing_in_balrog
- shipping-product: firefox
- run:
- product: firefox
- buildername: release-{branch}-firefox_schedule_publishing_in_balrog
- channels:
- by-project:
- birch: beta
- mozilla-release: beta
- default: unknown
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/release-secondary-balrog-scheduling/kind.yml
@@ -0,0 +1,59 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+loader: taskgraph.loader.transform:loader
+
+transforms:
+ - taskgraph.transforms.release_deps:transforms
+ - taskgraph.transforms.worker_type:transforms
+ - taskgraph.transforms.release_notifications:transforms
+ - taskgraph.transforms.task:transforms
+
+kind-dependencies:
+ - post-balrog-dummy
+ - post-beetmover-dummy
+ - release-secondary-balrog-submit-toplevel
+ - release-secondary-update-verify
+
+job-defaults:
+ worker-type:
+ by-project:
+ maple: scriptworker-prov-v1/balrog-dev
+ birch: scriptworker-prov-v1/balrog-dev
+ mozilla-beta: scriptworker-prov-v1/balrogworker-v1
+ mozilla-release: scriptworker-prov-v1/balrogworker-v1
+ default: invalid/invalid
+ run-on-projects: []
+ shipping-phase: ship
+ worker:
+ implementation: balrog
+ balrog-action: schedule
+
+jobs:
+ firefox-secondary-rc:
+ description: Schedule Beta Firefox publishing in balrog
+ name: release-firefox_schedule_publishing_in_balrog
+ shipping-product: firefox
+ worker:
+ product: firefox
+ channel-names:
+ by-project:
+ birch: ["beta", "beta-localtest", "beta-cdntest"]
+ mozilla-release: ["beta", "beta-localtest", "beta-cdntest"]
+ default: []
+ publish-rules:
+ by-project:
+ birch: [32]
+ mozilla-release: [32]
+ default: []
+ rules-to-update:
+ by-project:
+ birch: ["firefox-beta-cdntest", "firefox-beta-localtest"]
+ mozilla-release: ["firefox-beta-cdntest", "firefox-beta-localtest"]
+ default: []
+ treeherder:
+ platform: linux64/opt
+ symbol: Rel(BSFxRC)
+ tier: 1
+ kind: build
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/release-secondary-balrog-submit-toplevel/kind.yml
@@ -0,0 +1,67 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+loader: taskgraph.loader.transform:loader
+
+kind-dependencies:
+ # For the $platform_info.txt files.
+ - post-beetmover-dummy
+
+transforms:
+ - taskgraph.transforms.release_deps:transforms
+ - taskgraph.transforms.worker_type:transforms
+ - taskgraph.transforms.release_notifications:transforms
+ - taskgraph.transforms.task:transforms
+
+job-defaults:
+ run-on-projects: [] # to make sure this never runs in CI
+ shipping-phase: promote
+ worker-type:
+ by-project:
+ maple: scriptworker-prov-v1/balrog-dev
+ birch: scriptworker-prov-v1/balrog-dev
+ mozilla-beta: scriptworker-prov-v1/balrogworker-v1
+ mozilla-release: scriptworker-prov-v1/balrogworker-v1
+ default: invalid/invalid
+ worker:
+ implementation: balrog
+ balrog-action: submit-toplevel
+ require-mirrors: true
+ platforms: ["linux", "linux64", "macosx64", "win32", "win64"]
+
+jobs:
+ firefox:
+ name: submit-toplevel-rc-firefox-release-to-balrog
+ description: submit toplevel RC Firefox release to balrog
+ shipping-product: firefox
+ worker:
+ product: firefox
+ archive-domain:
+ by-project:
+ mozilla-release: archive.mozilla.org
+ default: ftp.stage.mozaws.net
+ download-domain:
+ by-project:
+ mozilla-release: download.mozilla.org
+ default: download.mozilla.org
+ channel-names:
+ by-project:
+ birch: ["beta", "beta-localtest", "beta-cdntest"]
+ mozilla-release: ["beta", "beta-localtest", "beta-cdntest"]
+ default: []
+ publish-rules:
+ by-project:
+ birch: [32]
+ mozilla-release: [32]
+ default: []
+ rules-to-update:
+ by-project:
+ birch: ["firefox-beta-cdntest", "firefox-beta-localtest"]
+ mozilla-release: ["firefox-beta-cdntest", "firefox-beta-localtest"]
+ default: []
+ treeherder:
+ platform: linux64/opt
+ symbol: Rel(BPFxRC)
+ tier: 1
+ kind: build
--- a/taskcluster/ci/release-secondary-final-verify/kind.yml
+++ b/taskcluster/ci/release-secondary-final-verify/kind.yml
@@ -2,17 +2,17 @@
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
loader: taskgraph.loader.transform:loader
kind-dependencies:
- post-balrog-dummy
- post-beetmover-dummy
- - release-updates-builder
+ - release-secondary-balrog-submit-toplevel
transforms:
- taskgraph.transforms.release_deps:transforms
- taskgraph.transforms.final_verify:transforms
- taskgraph.transforms.release_notifications:transforms
- taskgraph.transforms.task:transforms
job-defaults:
--- a/taskcluster/ci/release-secondary-notify-ship/kind.yml
+++ b/taskcluster/ci/release-secondary-notify-ship/kind.yml
@@ -5,17 +5,17 @@
loader: taskgraph.loader.transform:loader
transforms:
- taskgraph.transforms.release_deps:transforms
- taskgraph.transforms.release_notifications:transforms
- taskgraph.transforms.task:transforms
kind-dependencies:
- - release-secondary-balrog-publishing
+ - release-secondary-balrog-scheduling
- push-apk
job-defaults:
name: notify-release-drivers-ship
description: Sends email to release-drivers telling release was shipped.
run-on-projects: []
shipping-phase: ship
worker-type: aws-provisioner-v1/gecko-{level}-b-linux
--- a/taskcluster/ci/release-secondary-update-verify/kind.yml
+++ b/taskcluster/ci/release-secondary-update-verify/kind.yml
@@ -2,17 +2,17 @@
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
loader: taskgraph.loader.transform:loader
kind-dependencies:
- post-balrog-dummy
- post-beetmover-dummy
- - release-updates-builder
+ - release-secondary-balrog-submit-toplevel
- release-secondary-update-verify-config
transforms:
- taskgraph.transforms.release_deps:transforms
- taskgraph.transforms.update_verify:transforms
- taskgraph.transforms.release_notifications:transforms
- taskgraph.transforms.task:transforms
--- a/taskcluster/ci/release-update-verify/kind.yml
+++ b/taskcluster/ci/release-update-verify/kind.yml
@@ -2,17 +2,17 @@
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
loader: taskgraph.loader.transform:loader
kind-dependencies:
- post-balrog-dummy
- post-beetmover-dummy
- - release-updates-builder
+ - release-balrog-submit-toplevel
- release-update-verify-config
transforms:
- taskgraph.transforms.release_deps:transforms
- taskgraph.transforms.update_verify:transforms
- taskgraph.transforms.release_notifications:transforms
- taskgraph.transforms.task:transforms
deleted file mode 100644
--- a/taskcluster/ci/release-updates-builder/kind.yml
+++ /dev/null
@@ -1,99 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-loader: taskgraph.loader.transform:loader
-
-kind-dependencies:
- # For the $platform_info.txt files.
- - post-beetmover-dummy
-
-transforms:
- - taskgraph.transforms.release_deps:transforms
- - taskgraph.transforms.release_updates:transforms
- - taskgraph.transforms.job:transforms
- - taskgraph.transforms.release_notifications:transforms
- - taskgraph.transforms.task:transforms
-
-job-defaults:
- name: updates
- run-on-projects: [] # to make sure this never runs in CI
- shipping-phase: promote
- worker-type: buildbot-bridge/buildbot-bridge
- run:
- using: buildbot
- release-promotion: true
- worker:
- properties:
- platforms: "linux,linux64,macosx64,win32,win64"
- updater_platform: "linux64"
-
-jobs:
- firefox:
- description: updates builder
- shipping-product: firefox
- scopes:
- - project:releng:buildbot-bridge:builder-name:release-{project}_firefox_updates
- run:
- product: firefox
- buildername: release-{branch}-firefox_updates
- worker:
- properties:
- generate_bz2_blob:
- by-project:
- mozilla-release: true
- default: false
- product: firefox
- balrog_api_root:
- by-project:
- mozilla-beta: https://aus4-admin.mozilla.org/api
- mozilla-release: https://aus4-admin.mozilla.org/api
- mozilla-esr52: https://aus4-admin.mozilla.org/api
- default: https://balrog-admin.stage.mozaws.net/api
- channels:
- by-project:
- jamun: "beta"
- maple: "beta"
- mozilla-beta: "beta"
- # beta gets removed for non RCs by the
- # release_updates transform
- mozilla-release: "beta,release"
- mozilla-esr52: "esr"
- default: "default"
- repo_path:
- by-project:
- birch: "projects/birch"
- jamun: "projects/jamun"
- maple: "projects/maple"
- mozilla-beta: "releases/mozilla-beta"
- mozilla-release: "releases/mozilla-release"
- mozilla-esr52: "releases/mozilla-esr52"
- default: "default"
-
- devedition:
- description: updates builder
- shipping-product: devedition
- scopes:
- - project:releng:buildbot-bridge:builder-name:release-{project}_devedition_updates
- run:
- product: devedition
- buildername: release-{branch}-devedition_updates
- worker:
- properties:
- product: devedition
- balrog_api_root:
- by-project:
- mozilla-beta: https://aus4-admin.mozilla.org/api
- default: https://balrog-admin.stage.mozaws.net/api
- channels:
- by-project:
- jamun: "aurora"
- maple: "aurora"
- mozilla-beta: "aurora"
- default: "default"
- repo_path:
- by-project:
- jamun: "projects/jamun"
- maple: "projects/maple"
- mozilla-beta: "releases/mozilla-beta"
- default: "default"
--- a/taskcluster/docs/kinds.rst
+++ b/taskcluster/docs/kinds.rst
@@ -229,20 +229,32 @@ kind depend on all the signed multi-loca
in order to make the decision.
push-apk
--------
PushApk publishes Android packages onto Google Play Store. Jobs of this kind take
all the signed multi-locales (aka "multi") APKs for a given release and upload them
all at once. They also depend on the breakpoint.
-release-balrog-publishing
+release-balrog-submit-toplevel
+----------------------
+Push a top-level release blob to Balrog.
+
+release-secondary-balrog-submit-toplevel
+----------------------
+Push a top-level RC release blob to Balrog.
+
+release-balrog-scheduling
----------------------
Schedule a release to go live in Balrog.
+release-secondary-balrog-scheduling
+----------------------
+Schedule an RC release to go live in Balrog.
+
release-binary-transparency
---------------------------
Binary transparency creates a publicly verifiable log of binary shas for downstream
release auditing. https://wiki.mozilla.org/Security/Binary_Transparency
release-snap
------------
Generate an installer using Ubuntu's Snap format.
--- a/taskcluster/taskgraph/target_tasks.py
+++ b/taskcluster/taskgraph/target_tasks.py
@@ -325,22 +325,19 @@ def target_tasks_promote_firefox(full_ta
# Allow for {beta,release}_tasks; these will get optimized out to point
# to the previous graph using ``previous_graph_ids`` and
# ``previous_graph_kinds``.
# At some point this should filter by shipping_phase == 'build' and
# shipping_product matches.
if task.label in beta_release_tasks:
return True
- # 'secondary' update/final verify tasks only run for
- # RCs
+ # 'secondary' balrog/update verify/final verify tasks only run for RCs
if parameters.get('release_type') != 'rc':
- if task.kind in ('release-secondary-update-verify',
- 'release-secondary-update-verify-config',
- 'release-secondary-final-verify'):
+ if 'secondary' in task.kind:
return False
if task.attributes.get('shipping_product') == 'firefox' and \
task.attributes.get('shipping_phase') == 'promote':
return True
return [l for l, t in full_task_graph.tasks.iteritems() if filter(t)]
@@ -384,20 +381,17 @@ def target_tasks_ship_firefox(full_task_
def filter(task):
# Include promotion tasks; these will be optimized out
if task.label in filtered_for_candidates:
return True
if task.attributes.get('shipping_product') != 'firefox' or \
task.attributes.get('shipping_phase') != 'ship':
return False
- if task.kind in (
- 'release-secondary-balrog-publishing',
- 'release-secondary-notify-ship',
- ):
+ if 'secondary' in task.kind:
return is_rc
else:
return not is_rc
return [l for l, t in full_task_graph.tasks.iteritems() if filter(t)]
@_target_task('promote_devedition')
rename from taskcluster/taskgraph/transforms/balrog.py
rename to taskcluster/taskgraph/transforms/balrog_submit.py
--- a/taskcluster/taskgraph/transforms/balrog.py
+++ b/taskcluster/taskgraph/transforms/balrog_submit.py
@@ -5,19 +5,16 @@
Transform the beetmover task into an actual task description.
"""
from __future__ import absolute_import, print_function, unicode_literals
from taskgraph.transforms.base import TransformSequence
from taskgraph.util.attributes import copy_attributes_from_dependent_job
from taskgraph.util.schema import validate_schema, Schema
-from taskgraph.util.scriptworker import (get_balrog_server_scope,
- get_balrog_channel_scopes,
- get_phase)
from taskgraph.transforms.task import task_description_schema
from voluptuous import Any, Required, Optional
# Voluptuous uses marker objects as dictionary *keys*, but they are not
# comparable, so we cast all of the keys back to regular strings
task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
@@ -92,30 +89,26 @@ def make_task_description(config, jobs):
upstream_artifacts = [{
"taskId": {"task-reference": "<beetmover>"},
"taskType": "beetmover",
"paths": [
"public/manifest.json"
],
}]
- server_scope = get_balrog_server_scope(config)
- channel_scopes = get_balrog_channel_scopes(config)
- phase = get_phase(config)
-
task = {
'label': label,
'description': description,
'worker-type': 'scriptworker-prov-v1/balrogworker-v1',
'worker': {
'implementation': 'balrog',
'upstream-artifacts': upstream_artifacts,
+ 'balrog-action': 'submit-locale',
},
- 'scopes': [server_scope] + channel_scopes,
'dependencies': {'beetmover': dep_job.label},
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'treeherder': treeherder,
- 'shipping-phase': job.get('shipping-phase', phase),
+ 'shipping-phase': job.get('shipping-phase', 'promote'),
'shipping-product': job.get('shipping-product'),
}
yield task
--- a/taskcluster/taskgraph/transforms/release_balrog_publishing.py
+++ b/taskcluster/taskgraph/transforms/release_balrog_publishing.py
@@ -1,21 +1,18 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""
-Add from parameters.yml into Balrog publishing tasks.
+Resolve worker-type by project
"""
-
from __future__ import absolute_import, print_function, unicode_literals
-
+from taskgraph.util.schema import resolve_keyed_by
from taskgraph.transforms.base import TransformSequence
transforms = TransformSequence()
@transforms.add
-def add_release_eta(config, jobs):
+def resolve_worker_type(config, jobs):
for job in jobs:
- if config.params['release_eta']:
- job['run']['release-eta'] = config.params['release_eta']
-
+ resolve_keyed_by(job, 'worker-type', job['description'], **config.params)
yield job
deleted file mode 100644
--- a/taskcluster/taskgraph/transforms/release_updates.py
+++ /dev/null
@@ -1,44 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-"""
-Transform the update generation task into an actual task description.
-"""
-
-from __future__ import absolute_import, print_function, unicode_literals
-
-from copy import deepcopy
-
-from taskgraph.transforms.base import TransformSequence
-from taskgraph.util.schema import resolve_keyed_by
-from taskgraph.util.scriptworker import get_release_config
-
-transforms = TransformSequence()
-
-
-@transforms.add
-def add_command(config, tasks):
- for task in tasks:
- release_config = get_release_config(config)
-
- real_task = deepcopy(task)
- real_task.setdefault("worker", {}).setdefault("properties", {})
-
- real_task["worker"]["properties"]["version"] = release_config["version"]
- real_task["worker"]["properties"]["appVersion"] = release_config["appVersion"]
- real_task["worker"]["properties"]["build_number"] = release_config["build_number"]
- real_task["worker"]["properties"]["partial_versions"] = release_config.get(
- "partial_versions", ""
- )
-
- for thing in ("generate_bz2_blob", "balrog_api_root", "channels", "repo_path"):
- thing = "worker.properties.{}".format(thing)
- resolve_keyed_by(real_task, thing, thing, **config.params)
-
- # Non-RC builds from mozilla-release shouldn't use the beta channel.
- if config.params.get('project') == 'mozilla-release':
- if config.params.get('release_type') != "rc":
- real_task["worker"]["properties"]["channels"] = \
- real_task["worker"]["properties"]["channels"].replace("beta,", "")
-
- yield real_task
--- a/taskcluster/taskgraph/transforms/task.py
+++ b/taskcluster/taskgraph/transforms/task.py
@@ -18,17 +18,22 @@ from copy import deepcopy
from mozbuild.util import memoize
from mozbuild import schedules
from taskgraph.util.attributes import TRUNK_PROJECTS
from taskgraph.util.hash import hash_path
from taskgraph.util.treeherder import split_symbol
from taskgraph.transforms.base import TransformSequence
from taskgraph.util.schema import validate_schema, Schema, optionally_keyed_by, resolve_keyed_by
-from taskgraph.util.scriptworker import get_release_config
+from taskgraph.util.scriptworker import (
+ BALROG_ACTIONS,
+ get_balrog_action_scope,
+ get_balrog_server_scope,
+ get_release_config,
+)
from voluptuous import Any, Required, Optional, Extra
from taskgraph import GECKO, MAX_DEPENDENCIES
from ..util import docker as dockerutil
RUN_TASK = os.path.join(GECKO, 'taskcluster', 'docker', 'recipes', 'run-task')
@memoize
@@ -530,19 +535,28 @@ task_description_schema = Schema({
}, {
Required('implementation'): 'beetmover-cdns',
# the maximum time to run, in seconds
Required('max-run-time'): int,
Required('product'): basestring,
}, {
Required('implementation'): 'balrog',
+ Required('balrog-action'): Any(*BALROG_ACTIONS),
+ Optional('product'): basestring,
+ Optional('platforms'): [basestring],
+ Optional('channel-names'): optionally_keyed_by('project', [basestring]),
+ Optional('require-mirrors'): bool,
+ Optional('publish-rules'): optionally_keyed_by('project', [int]),
+ Optional('rules-to-update'): optionally_keyed_by('project', [basestring]),
+ Optional('archive-domain'): optionally_keyed_by('project', basestring),
+ Optional('download-domain'): optionally_keyed_by('project', basestring),
# list of artifact URLs for the artifacts that should be beetmoved
- Required('upstream-artifacts'): [{
+ Optional('upstream-artifacts'): [{
# taskId of the task with the artifact
Required('taskId'): taskref_or_string,
# type of signing task (for CoT)
Required('taskType'): basestring,
# Paths to the artifacts to sign
Required('paths'): [basestring],
@@ -1039,20 +1053,55 @@ def build_beetmover_cdns_payload(config,
'version': release_config['version'],
'build_number': release_config['build_number'],
}
@payload_builder('balrog')
def build_balrog_payload(config, task, task_def):
worker = task['worker']
+ release_config = get_release_config(config)
- task_def['payload'] = {
- 'upstreamArtifacts': worker['upstream-artifacts']
- }
+ server_scope = get_balrog_server_scope(config)
+ action_scope = get_balrog_action_scope(config, action=worker['balrog-action'])
+ task_def['scopes'] = [server_scope, action_scope]
+
+ if worker['balrog-action'] == 'submit-locale':
+ task_def['payload'] = {
+ 'upstreamArtifacts': worker['upstream-artifacts']
+ }
+ else:
+ for prop in ('archive-domain', 'channel-names', 'download-domain',
+ 'publish-rules', 'rules-to-update'):
+ if prop in worker:
+ resolve_keyed_by(
+ worker, prop, task['description'],
+ **config.params
+ )
+ task_def['payload'] = {
+ 'build_number': release_config['build_number'],
+ 'product': worker['product'],
+ 'version': release_config['version'],
+ }
+ if worker['balrog-action'] == 'submit-toplevel':
+ task_def['payload'].update({
+ 'app_version': release_config['appVersion'],
+ 'archive_domain': worker['archive-domain'],
+ 'channel_names': worker['channel-names'],
+ 'download_domain': worker['download-domain'],
+ 'partial_versions': release_config.get('partial_versions', ""),
+ 'platforms': worker['platforms'],
+ 'rules_to_update': worker['rules-to-update'],
+ 'require_mirrors': worker['require-mirrors'],
+ })
+ else: # schedule / ship
+ task_def['payload'].update({
+ 'publish_rules': worker['publish-rules'],
+ 'release_eta': config.params.get('release_eta') or '',
+ })
@payload_builder('push-apk')
def build_push_apk_payload(config, task, task_def):
worker = task['worker']
task_def['payload'] = {
'commit': worker['commit'],
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/worker_type.py
@@ -0,0 +1,18 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Resolve worker-type by project
+"""
+from __future__ import absolute_import, print_function, unicode_literals
+from taskgraph.util.schema import resolve_keyed_by
+from taskgraph.transforms.base import TransformSequence
+
+transforms = TransformSequence()
+
+
+@transforms.add
+def resolve_worker_type(config, jobs):
+ for job in jobs:
+ resolve_keyed_by(job, 'worker-type', job['description'], **config.params)
+ yield job
--- a/taskcluster/taskgraph/util/scriptworker.py
+++ b/taskcluster/taskgraph/util/scriptworker.py
@@ -150,16 +150,19 @@ BEETMOVER_ACTION_SCOPES = {
"""Map the beetmover tasks aliases to phases.
"""
PHASES = {
'all-candidates-tasks': 'promote',
'all-push-tasks': 'push',
'default': None,
}
+"""Known balrog actions."""
+BALROG_ACTIONS = ('submit-locale', 'submit-toplevel', 'schedule')
+
"""Map balrog scope aliases to sets of projects.
This is a list of list-pairs, for ordering.
"""
BALROG_SCOPE_ALIAS_TO_PROJECT = [[
'nightly', set([
'mozilla-central',
])
@@ -183,58 +186,16 @@ BALROG_SERVER_SCOPES = {
'nightly': 'balrog:server:nightly',
'aurora': 'balrog:server:aurora',
'beta': 'balrog:server:beta',
'release': 'balrog:server:release',
'esr': 'balrog:server:esr',
'default': 'balrog:server:dep',
}
-"""Map the balrog scope aliases to the actual channel scopes.
-"""
-BALROG_CHANNEL_SCOPES = {
- 'nightly': [
- 'balrog:channel:nightly',
- 'balrog:channel:nightly-old-id',
- 'balrog:channel:aurora',
- ],
- 'aurora': [
- 'balrog:channel:aurora',
- ],
- 'beta': [
- 'balrog:channel:beta',
- 'balrog:channel:beta-localtest',
- 'balrog:channel:beta-cdntest',
- ],
- 'release': [
- 'balrog:channel:release',
- 'balrog:channel:release-localtest',
- 'balrog:channel:release-cdntest',
- ],
- 'esr': [
- 'balrog:channel:esr',
- 'balrog:channel:esr-localtest',
- 'balrog:channel:esr-cdntest',
- ],
- 'default': [
- 'balrog:channel:nightly',
- 'balrog:channel:nightly-old-id',
- 'balrog:channel:aurora',
- 'balrog:channel:beta',
- 'balrog:channel:beta-localtest',
- 'balrog:channel:beta-cdntest',
- 'balrog:channel:release',
- 'balrog:channel:release-localtest',
- 'balrog:channel:release-cdntest',
- 'balrog:channel:esr',
- 'balrog:channel:esr-localtest',
- 'balrog:channel:esr-cdntest',
- ],
-}
-
PUSH_APK_SCOPE_ALIAS_TO_PROJECT = [[
'central', set([
'mozilla-central',
])
], [
'beta', set([
'mozilla-beta',
@@ -383,16 +344,22 @@ def get_phase_from_target_method(config,
string: the phase to use.
"""
for alias, tasks in alias_to_tasks_map:
if config.params['target_tasks_method'] in tasks and alias in alias_to_phase_map:
return alias_to_phase_map[alias]
return alias_to_phase_map['default']
+@with_scope_prefix
+def get_balrog_action_scope(config, action='submit'):
+ assert action in BALROG_ACTIONS
+ return "balrog:action:{}".format(action)
+
+
get_signing_cert_scope = functools.partial(
get_scope_from_project,
alias_to_project_map=SIGNING_SCOPE_ALIAS_TO_PROJECT,
alias_to_scope_map=SIGNING_CERT_SCOPES,
)
get_devedition_signing_cert_scope = functools.partial(
get_scope_from_project,
@@ -420,22 +387,16 @@ get_phase = functools.partial(
)
get_balrog_server_scope = functools.partial(
get_scope_from_project,
alias_to_project_map=BALROG_SCOPE_ALIAS_TO_PROJECT,
alias_to_scope_map=BALROG_SERVER_SCOPES,
)
-get_balrog_channel_scopes = functools.partial(
- get_scope_from_project,
- alias_to_project_map=BALROG_SCOPE_ALIAS_TO_PROJECT,
- alias_to_scope_map=BALROG_CHANNEL_SCOPES,
-)
-
get_push_apk_scope = functools.partial(
get_scope_from_project,
alias_to_project_map=PUSH_APK_SCOPE_ALIAS_TO_PROJECT,
alias_to_scope_map=PUSH_APK_SCOPES,
)
# release_config {{{1
@@ -451,19 +412,20 @@ def get_release_config(config):
dict: containing both `build_number` and `version`. This can be used to
update `task.payload`.
"""
release_config = {}
partial_updates = os.environ.get("PARTIAL_UPDATES", "")
if partial_updates != "" and config.kind in ('release-bouncer-sub',
'release-bouncer-check',
- 'release-updates-builder',
'release-update-verify-config',
'release-secondary-update-verify-config',
+ 'release-balrog-submit-toplevel',
+ 'release-secondary-balrog-submit-toplevel',
):
partial_updates = json.loads(partial_updates)
release_config['partial_versions'] = ', '.join([
'{}build{}'.format(v, info['buildNumber'])
for v, info in partial_updates.items()
])
if release_config['partial_versions'] == "{}":
del release_config['partial_versions']
--- a/taskcluster/taskgraph/util/workertypes.py
+++ b/taskcluster/taskgraph/util/workertypes.py
@@ -36,18 +36,18 @@ WORKER_TYPES = {
'aws-provisioner-v1/gecko-t-win7-32': ('generic-worker', 'windows'),
'aws-provisioner-v1/gecko-t-win7-32-gpu': ('generic-worker', 'windows'),
'releng-hardware/gecko-t-win7-32-hw': ('generic-worker', 'windows'),
'aws-provisioner-v1/taskcluster-generic': ('docker-worker', 'linux'),
'buildbot-bridge/buildbot-bridge': ('buildbot-bridge', None),
'invalid/invalid': ('invalid', None),
'invalid/always-optimized': ('always-optimized', None),
'null-provisioner/human-breakpoint': ('push-apk-breakpoint', None),
- 'null-provisioner/human-breakpoint': ('push-apk-breakpoint', None),
'releng-hardware/gecko-t-linux-talos': ('native-engine', 'linux'),
+ 'scriptworker-prov-v1/balrog-dev': ('balrog', None),
'scriptworker-prov-v1/balrogworker-v1': ('balrog', None),
'scriptworker-prov-v1/beetmoverworker-v1': ('beetmover', None),
'scriptworker-prov-v1/pushapk-v1': ('push-apk', None),
"scriptworker-prov-v1/signing-linux-v1": ('scriptworker-signing', None),
"scriptworker-prov-v1/shipit": ('shipit', None),
"scriptworker-prov-v1/shipit-dev": ('shipit', None),
'releng-hardware/gecko-t-osx-1010': ('generic-worker', 'macosx'),
}