Bug 1433459 - part 1: Move bouncer submission tasks to scriptworker r=mtabara,rail
MozReview-Commit-ID: 6SKhjf1ywoH
--- a/taskcluster/ci/release-bouncer-sub/kind.yml
+++ b/taskcluster/ci/release-bouncer-sub/kind.yml
@@ -1,41 +1,58 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
loader: taskgraph.loader.transform:loader
transforms:
- - taskgraph.transforms.job:transforms
+ - taskgraph.transforms.bouncer_submission:transforms
- taskgraph.transforms.release_notifications:transforms
- taskgraph.transforms.task:transforms
job-defaults:
description: release bouncer submission job
- worker-type: buildbot-bridge/buildbot-bridge
+ worker-type:
+ by-project:
+ mozilla-central: scriptworker-prov-v1/bouncer-v1
+ mozilla-beta: scriptworker-prov-v1/bouncer-v1
+ mozilla-release: scriptworker-prov-v1/bouncer-v1
+ default: scriptworker-prov-v1/bouncer-dev
+ worker:
+ implementation: bouncer-submission
+ scopes:
+ by-project:
+ mozilla-beta:
+ - project:releng:bouncer:action:submission
+ - project:releng:bouncer:server:production
+ mozilla-release:
+ - project:releng:bouncer:action:submission
+ - project:releng:bouncer:server:production
+ default:
+ - project:releng:bouncer:action:submission
+ - project:releng:bouncer:server:staging
run-on-projects: []
shipping-phase: promote
- run:
- using: buildbot
- release-promotion: true
+ shipping-product: firefox
+ locales-file: browser/locales/l10n-changesets.json
jobs:
+ devedition:
+ bouncer-platforms: ['linux', 'linux64', 'osx', 'win', 'win64']
+ bouncer-products: ['complete-mar', 'installer', 'installer-ssl', 'partial-mar', 'stub-installer']
+ shipping-product: devedition
+
fennec:
- name: fennec_release_bouncer_sub
+ bouncer-platforms: ['android', 'android-x86']
+ bouncer-products: ['apk']
shipping-product: fennec
- run:
- product: fennec
- buildername: release-{branch}-fennec_bncr_sub
+ locales-file: mobile/locales/l10n-changesets.json
firefox:
- name: firefox_release_bouncer_sub
+ bouncer-platforms: ['linux', 'linux64', 'osx', 'win', 'win64']
+ bouncer-products: ['complete-mar', 'installer', 'installer-ssl', 'partial-mar', 'stub-installer']
shipping-product: firefox
- run:
- product: firefox
- buildername: release-{branch}_firefox_bncr_sub
- devedition:
- name: devedition_release_bouncer_sub
- shipping-product: devedition
- run:
- product: devedition
- buildername: release-{branch}_devedition_bncr_sub
+ firefox-rc:
+ bouncer-platforms: ['linux', 'linux64', 'osx', 'win', 'win64']
+ bouncer-products: ['partial-mar-candidates']
+ shipping-product: firefox
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/bouncer_submission.py
@@ -0,0 +1,253 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Add from parameters.yml into bouncer submission tasks.
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import logging
+
+from taskgraph.transforms.base import TransformSequence
+from taskgraph.transforms.l10n import parse_locales_file
+from taskgraph.util.schema import resolve_keyed_by
+from taskgraph.util.scriptworker import get_release_config
+
+logger = logging.getLogger(__name__)
+
+
+FTP_PLATFORMS_PER_BOUNCER_PLATFORM = {
+ 'android': 'android-api-16',
+ 'android-x86': 'android-x86',
+ 'linux': 'linux-i686',
+ 'linux64': 'linux-x86_64',
+ 'osx': 'mac',
+ 'win': 'win32',
+ 'win64': 'win64',
+}
+
+# :lang is interpolated by bouncer at runtime
+CANDIDATES_PATH_TEMPLATE = '/{product}/candidates/{version}-candidates/build{build_number}/\
+{update_folder}{ftp_platform}/:lang/{file}'
+RELEASES_PATH_TEMPLATE = '/{product}/releases/{version}/{update_folder}{ftp_platform}/:lang/{file}'
+
+
+CONFIG_PER_BOUNCER_PRODUCT = {
+ 'apk': {
+ 'path_template': RELEASES_PATH_TEMPLATE,
+ 'file_names': {
+ 'android': 'fennec-{version}.:lang.android-arm.apk',
+ 'android-x86': 'fennec-{version}.:lang.android-i386.apk',
+ },
+ },
+ 'complete-mar': {
+ 'path_template': RELEASES_PATH_TEMPLATE,
+ 'file_names': {
+ 'default': 'firefox-{version}.complete.mar',
+ },
+ },
+ 'installer': {
+ 'path_template': RELEASES_PATH_TEMPLATE,
+ 'file_names': {
+ 'linux': 'firefox-{version}.tar.bz2',
+ 'linux64': 'firefox-{version}.tar.bz2',
+ 'osx': 'Firefox%20{version}.dmg',
+ 'win': 'Firefox%20Setup%20{version}.exe',
+ 'win64': 'Firefox%20Setup%20{version}.exe',
+ },
+ },
+ 'partial-mar': {
+ 'path_template': RELEASES_PATH_TEMPLATE,
+ 'file_names': {
+ 'default': 'firefox-{previous_version}-{version}.partial.mar',
+ },
+ },
+ 'partial-mar-candidates': {
+ 'path_template': CANDIDATES_PATH_TEMPLATE,
+ 'file_names': {
+ 'default': 'firefox-{previous_version}-{version}.partial.mar',
+ },
+ },
+ 'stub-installer': {
+ 'path_template': RELEASES_PATH_TEMPLATE,
+ 'file_names': {
+ 'win': 'Firefox%20Installer.exe',
+ 'win64': 'Firefox%20Installer.exe',
+ },
+ },
+}
+CONFIG_PER_BOUNCER_PRODUCT['installer-ssl'] = CONFIG_PER_BOUNCER_PRODUCT['installer']
+
+transforms = TransformSequence()
+
+
+@transforms.add
+def make_task_worker(config, jobs):
+ for job in jobs:
+ resolve_keyed_by(
+ job, 'worker-type', item_name=job['name'], project=config.params['project']
+ )
+ resolve_keyed_by(
+ job, 'scopes', item_name=job['name'], project=config.params['project']
+ )
+
+ # No need to filter out ja-JP-mac, we need to upload both
+ all_locales = list(sorted(parse_locales_file(job['locales-file']).keys()))
+ job['worker']['locales'] = all_locales
+ job['worker']['entries'] = craft_bouncer_entries(config, job)
+
+ del job['locales-file']
+ del job['bouncer-platforms']
+ del job['bouncer-products']
+
+ if job['worker']['entries']:
+ # XXX Because rc jobs are defined within the same kind, we need to delete the
+ # firefox-rc job at this stage, if we're not building an RC. Otherwise, even if
+ # target_tasks.py filters out the rc job, it gets resurected by any kind that depends
+ # on the release-bouncer-sub one (release-notify-promote as of time of this writing).
+ if config.params['release_type'] == 'rc' or job['name'] != 'firefox-rc':
+ yield job
+ else:
+ logger.warn('No bouncer entries defined in bouncer submission task for "{}". \
+Job deleted.'.format(job['name']))
+
+
+def craft_bouncer_entries(config, job):
+ release_config = get_release_config(config)
+
+ product = job['shipping-product']
+ bouncer_platforms = job['bouncer-platforms']
+
+ current_version = release_config['version']
+ current_build_number = release_config['build_number']
+
+ bouncer_products = job['bouncer-products']
+ previous_versions_string = release_config.get('partial_versions', None)
+ if previous_versions_string:
+ previous_versions = previous_versions_string.split(', ')
+ else:
+ logger.warn('No partials defined! Bouncer submission task won\'t send any \
+partial-related entry for "{}"'.format(job['name']))
+ bouncer_products = [
+ bouncer_product
+ for bouncer_product in bouncer_products
+ if 'partial' not in bouncer_product
+ ]
+ previous_versions = [None]
+
+ project = config.params['project']
+
+ return {
+ craft_bouncer_product_name(
+ product, bouncer_product, current_version, current_build_number, previous_version
+ ): {
+ 'options': {
+ 'add_locales': craft_add_locales(product),
+ 'check_uptake': craft_check_uptake(bouncer_product),
+ 'ssl_only': craft_ssl_only(bouncer_product, project),
+ },
+ 'paths_per_bouncer_platform': craft_paths_per_bouncer_platform(
+ product, bouncer_product, bouncer_platforms, current_version,
+ current_build_number, previous_version
+ ),
+ }
+ for bouncer_product in bouncer_products
+ for previous_version in previous_versions
+ }
+
+
+def craft_paths_per_bouncer_platform(product, bouncer_product, bouncer_platforms, current_version,
+ current_build_number, previous_version=None):
+ paths_per_bouncer_platform = {}
+ for bouncer_platform in bouncer_platforms:
+ ftp_platform = FTP_PLATFORMS_PER_BOUNCER_PLATFORM[bouncer_platform]
+
+ file_names_per_platform = CONFIG_PER_BOUNCER_PRODUCT[bouncer_product]['file_names']
+ file_name_template = file_names_per_platform.get(
+ bouncer_platform, file_names_per_platform.get('default', None)
+ )
+ if not file_name_template:
+ # Some bouncer product like stub-installer are only meant to be on Windows.
+ # Thus no default value is defined there
+ continue
+
+ file_name = file_name_template.format(
+ version=current_version, previous_version=strip_build_data(previous_version)
+ )
+
+ path_template = CONFIG_PER_BOUNCER_PRODUCT[bouncer_product]['path_template']
+ file_relative_location = path_template.format(
+ product=product.lower(),
+ version=current_version,
+ build_number=current_build_number,
+ update_folder='updates/' if '-mar' in bouncer_product else '',
+ ftp_platform=ftp_platform,
+ file=file_name,
+ )
+
+ paths_per_bouncer_platform[bouncer_platform] = file_relative_location
+
+ return paths_per_bouncer_platform
+
+
+def craft_bouncer_product_name(product, bouncer_product, current_version,
+ current_build_number=None, previous_version=None):
+ if '-ssl' in bouncer_product:
+ postfix = '-SSL'
+ elif 'stub-' in bouncer_product:
+ postfix = '-stub'
+ elif 'complete-' in bouncer_product:
+ postfix = '-Complete'
+ elif 'partial-' in bouncer_product:
+ if not previous_version:
+ raise Exception('Partial is being processed, but no previous version defined.')
+
+ if '-candidates' in bouncer_product:
+ if not current_build_number:
+ raise Exception('Partial in candidates directory is being processed, \
+but no current build number defined.')
+
+ postfix = 'build{build_number}-Partial-{previous_version_with_build_number}'.format(
+ build_number=current_build_number,
+ previous_version_with_build_number=previous_version,
+ )
+ else:
+ postfix = '-Partial-{previous_version}'.format(
+ previous_version=strip_build_data(previous_version)
+ )
+
+ elif 'sha1-' in bouncer_product:
+ postfix = '-sha1'
+ else:
+ postfix = ''
+
+ return '{product}-{version}{postfix}'.format(
+ product=product.capitalize(), version=current_version, postfix=postfix
+ )
+
+
+def craft_check_uptake(bouncer_product):
+ return bouncer_product != 'complete-mar-candidates'
+
+
+def craft_ssl_only(bouncer_product, project):
+ # XXX ESR is the only channel where we force serve the installer over SSL
+ if '-esr' in project and bouncer_product == 'installer':
+ return True
+
+ return bouncer_product not in (
+ 'complete-mar',
+ 'installer',
+ 'partial-mar',
+ 'partial-mar-candidates',
+ )
+
+
+def craft_add_locales(product):
+ # Do not add locales on Fennec in order to let "multi" work
+ return product != 'fennec'
+
+
+def strip_build_data(version):
+ return version.split('build')[0] if version and 'build' in version else version
--- a/taskcluster/taskgraph/transforms/l10n.py
+++ b/taskcluster/taskgraph/transforms/l10n.py
@@ -158,29 +158,29 @@ l10n_description_schema = Schema({
# Shipping product and phase
Optional('shipping-product'): task_description_schema['shipping-product'],
Optional('shipping-phase'): task_description_schema['shipping-phase'],
})
transforms = TransformSequence()
-def _parse_locales_file(locales_file, platform):
+def parse_locales_file(locales_file, platform=None):
""" Parse the passed locales file for a list of locales.
"""
locales = []
with open(locales_file, mode='r') as f:
if locales_file.endswith('json'):
all_locales = json.load(f)
# XXX Only single locales are fetched
locales = {
locale: data['revision']
for locale, data in all_locales.items()
- if platform in data['platforms']
+ if platform is None or platform in data['platforms']
}
else:
all_locales = f.read().split()
# 'default' is the hg revision at the top of hg repo, in this context
locales = {locale: 'default' for locale in all_locales}
return locales
@@ -294,18 +294,18 @@ def handle_keyed_by(config, jobs):
resolve_keyed_by(item=job, field=field, item_name=job['name'])
yield job
@transforms.add
def all_locales_attribute(config, jobs):
for job in jobs:
locales_platform = job['attributes']['build_platform'].replace("-nightly", "")
- locales_with_changesets = _parse_locales_file(job["locales-file"],
- platform=locales_platform)
+ locales_with_changesets = parse_locales_file(job["locales-file"],
+ platform=locales_platform)
locales_with_changesets = _remove_locales(locales_with_changesets,
to_remove=job['ignore-locales'])
locales = sorted(locales_with_changesets.keys())
attributes = job.setdefault('attributes', {})
attributes["all_locales"] = locales
attributes["all_locales_with_changesets"] = locales_with_changesets
if job.get('shipping-product'):
--- a/taskcluster/taskgraph/transforms/task.py
+++ b/taskcluster/taskgraph/transforms/task.py
@@ -554,19 +554,22 @@ task_description_schema = Schema({
# type of signing task (for CoT)
Required('taskType'): basestring,
# Paths to the artifacts to sign
Required('paths'): [basestring],
}],
}, {
+ Required('implementation'): 'bouncer-submission',
+ Required('locales'): [basestring],
+ Required('entries'): object,
+ }, {
Required('implementation'): 'push-apk-breakpoint',
Required('payload'): object,
-
}, {
Required('implementation'): 'invalid',
# an invalid task is one which should never actually be created; this is used in
# release automation on branches where the task just doesn't make sense
Extra: object,
}, {
Required('implementation'): 'always-optimized',
@@ -1100,16 +1103,26 @@ def build_balrog_payload(config, task, t
})
else: # schedule / ship
task_def['payload'].update({
'publish_rules': worker['publish-rules'],
'release_eta': config.params.get('release_eta') or '',
})
+@payload_builder('bouncer-submission')
+def build_bouncer_submission_payload(config, task, task_def):
+ worker = task['worker']
+
+ task_def['payload'] = {
+ 'locales': worker['locales'],
+ 'submission_entries': worker['entries']
+ }
+
+
@payload_builder('push-apk')
def build_push_apk_payload(config, task, task_def):
worker = task['worker']
task_def['payload'] = {
'commit': worker['commit'],
'upstreamArtifacts': worker['upstream-artifacts'],
'google_play_track': worker['google-play-track'],