Bug 1433459 - part 1: Move bouncer submission tasks to scriptworker r=mtabara,rail draft
authorJohan Lorenzo <jlorenzo@mozilla.com>
Mon, 26 Feb 2018 15:14:46 +0100
changeset 761848 3b38bac41d14585764bf208b98f0ceb822c4815b
parent 761787 e33efdb3e1517d521deb949de3fcd6d9946ea440
child 761849 e769804178a82b7fd62124b04042d9e2e6ccdebe
push id101019
push userbmo:jlorenzo@mozilla.com
push dateThu, 01 Mar 2018 15:24:04 +0000
reviewersmtabara, rail
bugs1433459
milestone60.0a1
Bug 1433459 - part 1: Move bouncer submission tasks to scriptworker r=mtabara,rail MozReview-Commit-ID: 6SKhjf1ywoH
taskcluster/ci/release-bouncer-sub/kind.yml
taskcluster/taskgraph/transforms/bouncer_submission.py
taskcluster/taskgraph/transforms/l10n.py
taskcluster/taskgraph/transforms/task.py
--- a/taskcluster/ci/release-bouncer-sub/kind.yml
+++ b/taskcluster/ci/release-bouncer-sub/kind.yml
@@ -1,41 +1,58 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 loader: taskgraph.loader.transform:loader
 
 transforms:
-   - taskgraph.transforms.job:transforms
+   - taskgraph.transforms.bouncer_submission:transforms
    - taskgraph.transforms.release_notifications:transforms
    - taskgraph.transforms.task:transforms
 
 job-defaults:
    description: release bouncer submission job
-   worker-type: buildbot-bridge/buildbot-bridge
+   worker-type:
+      by-project:
+         mozilla-central: scriptworker-prov-v1/bouncer-v1
+         mozilla-beta: scriptworker-prov-v1/bouncer-v1
+         mozilla-release: scriptworker-prov-v1/bouncer-v1
+         default: scriptworker-prov-v1/bouncer-dev
+   worker:
+      implementation: bouncer-submission
+   scopes:
+      by-project:
+         mozilla-beta:
+            - project:releng:bouncer:action:submission
+            - project:releng:bouncer:server:production
+         mozilla-release:
+            - project:releng:bouncer:action:submission
+            - project:releng:bouncer:server:production
+         default:
+            - project:releng:bouncer:action:submission
+            - project:releng:bouncer:server:staging
    run-on-projects: []
    shipping-phase: promote
-   run:
-      using: buildbot
-      release-promotion: true
+   shipping-product: firefox
+   locales-file: browser/locales/l10n-changesets.json
 
 jobs:
+   devedition:
+      bouncer-platforms: ['linux', 'linux64', 'osx', 'win', 'win64']
+      bouncer-products: ['complete-mar', 'installer', 'installer-ssl', 'partial-mar', 'stub-installer']
+      shipping-product: devedition
+
    fennec:
-      name: fennec_release_bouncer_sub
+      bouncer-platforms: ['android', 'android-x86']
+      bouncer-products: ['apk']
       shipping-product: fennec
-      run:
-         product: fennec
-         buildername: release-{branch}-fennec_bncr_sub
+      locales-file: mobile/locales/l10n-changesets.json
 
    firefox:
-      name: firefox_release_bouncer_sub
+      bouncer-platforms: ['linux', 'linux64', 'osx', 'win', 'win64']
+      bouncer-products: ['complete-mar', 'installer', 'installer-ssl', 'partial-mar', 'stub-installer']
       shipping-product: firefox
-      run:
-         product: firefox
-         buildername: release-{branch}_firefox_bncr_sub
 
-   devedition:
-      name: devedition_release_bouncer_sub
-      shipping-product: devedition
-      run:
-         product: devedition
-         buildername: release-{branch}_devedition_bncr_sub
+   firefox-rc:
+      bouncer-platforms: ['linux', 'linux64', 'osx', 'win', 'win64']
+      bouncer-products: ['partial-mar-candidates']
+      shipping-product: firefox
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/bouncer_submission.py
@@ -0,0 +1,253 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Add from parameters.yml into bouncer submission tasks.
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import logging
+
+from taskgraph.transforms.base import TransformSequence
+from taskgraph.transforms.l10n import parse_locales_file
+from taskgraph.util.schema import resolve_keyed_by
+from taskgraph.util.scriptworker import get_release_config
+
+logger = logging.getLogger(__name__)
+
+
+FTP_PLATFORMS_PER_BOUNCER_PLATFORM = {
+    'android': 'android-api-16',
+    'android-x86': 'android-x86',
+    'linux': 'linux-i686',
+    'linux64': 'linux-x86_64',
+    'osx': 'mac',
+    'win': 'win32',
+    'win64': 'win64',
+}
+
+# :lang is interpolated by bouncer at runtime
+CANDIDATES_PATH_TEMPLATE = '/{product}/candidates/{version}-candidates/build{build_number}/\
+{update_folder}{ftp_platform}/:lang/{file}'
+RELEASES_PATH_TEMPLATE = '/{product}/releases/{version}/{update_folder}{ftp_platform}/:lang/{file}'
+
+
+CONFIG_PER_BOUNCER_PRODUCT = {
+    'apk': {
+        'path_template': RELEASES_PATH_TEMPLATE,
+        'file_names': {
+            'android': 'fennec-{version}.:lang.android-arm.apk',
+            'android-x86': 'fennec-{version}.:lang.android-i386.apk',
+        },
+    },
+    'complete-mar': {
+        'path_template': RELEASES_PATH_TEMPLATE,
+        'file_names': {
+            'default': 'firefox-{version}.complete.mar',
+        },
+    },
+    'installer': {
+        'path_template': RELEASES_PATH_TEMPLATE,
+        'file_names': {
+            'linux': 'firefox-{version}.tar.bz2',
+            'linux64': 'firefox-{version}.tar.bz2',
+            'osx': 'Firefox%20{version}.dmg',
+            'win': 'Firefox%20Setup%20{version}.exe',
+            'win64': 'Firefox%20Setup%20{version}.exe',
+        },
+    },
+    'partial-mar': {
+        'path_template': RELEASES_PATH_TEMPLATE,
+        'file_names': {
+            'default': 'firefox-{previous_version}-{version}.partial.mar',
+        },
+    },
+    'partial-mar-candidates': {
+        'path_template': CANDIDATES_PATH_TEMPLATE,
+        'file_names': {
+            'default': 'firefox-{previous_version}-{version}.partial.mar',
+        },
+    },
+    'stub-installer': {
+        'path_template': RELEASES_PATH_TEMPLATE,
+        'file_names': {
+            'win': 'Firefox%20Installer.exe',
+            'win64': 'Firefox%20Installer.exe',
+        },
+    },
+}
+CONFIG_PER_BOUNCER_PRODUCT['installer-ssl'] = CONFIG_PER_BOUNCER_PRODUCT['installer']
+
+transforms = TransformSequence()
+
+
+@transforms.add
+def make_task_worker(config, jobs):
+    for job in jobs:
+        resolve_keyed_by(
+            job, 'worker-type', item_name=job['name'], project=config.params['project']
+        )
+        resolve_keyed_by(
+            job, 'scopes', item_name=job['name'], project=config.params['project']
+        )
+
+        # No need to filter out ja-JP-mac, we need to upload both
+        all_locales = list(sorted(parse_locales_file(job['locales-file']).keys()))
+        job['worker']['locales'] = all_locales
+        job['worker']['entries'] = craft_bouncer_entries(config, job)
+
+        del job['locales-file']
+        del job['bouncer-platforms']
+        del job['bouncer-products']
+
+        if job['worker']['entries']:
+            # XXX Because rc jobs are defined within the same kind, we need to delete the
+            # firefox-rc job at this stage, if we're not building an RC. Otherwise, even if
+            # target_tasks.py filters out the rc job, it gets resurected by any kind that depends
+            # on the release-bouncer-sub one (release-notify-promote as of time of this writing).
+            if config.params['release_type'] == 'rc' or job['name'] != 'firefox-rc':
+                yield job
+        else:
+            logger.warn('No bouncer entries defined in bouncer submission task for "{}". \
+Job deleted.'.format(job['name']))
+
+
+def craft_bouncer_entries(config, job):
+    release_config = get_release_config(config)
+
+    product = job['shipping-product']
+    bouncer_platforms = job['bouncer-platforms']
+
+    current_version = release_config['version']
+    current_build_number = release_config['build_number']
+
+    bouncer_products = job['bouncer-products']
+    previous_versions_string = release_config.get('partial_versions', None)
+    if previous_versions_string:
+        previous_versions = previous_versions_string.split(', ')
+    else:
+        logger.warn('No partials defined! Bouncer submission task won\'t send any \
+partial-related entry for "{}"'.format(job['name']))
+        bouncer_products = [
+            bouncer_product
+            for bouncer_product in bouncer_products
+            if 'partial' not in bouncer_product
+        ]
+        previous_versions = [None]
+
+    project = config.params['project']
+
+    return {
+        craft_bouncer_product_name(
+            product, bouncer_product, current_version, current_build_number, previous_version
+        ): {
+            'options': {
+                'add_locales': craft_add_locales(product),
+                'check_uptake': craft_check_uptake(bouncer_product),
+                'ssl_only': craft_ssl_only(bouncer_product, project),
+            },
+            'paths_per_bouncer_platform': craft_paths_per_bouncer_platform(
+                product, bouncer_product, bouncer_platforms, current_version,
+                current_build_number, previous_version
+            ),
+        }
+        for bouncer_product in bouncer_products
+        for previous_version in previous_versions
+    }
+
+
+def craft_paths_per_bouncer_platform(product, bouncer_product, bouncer_platforms, current_version,
+                                     current_build_number, previous_version=None):
+    paths_per_bouncer_platform = {}
+    for bouncer_platform in bouncer_platforms:
+        ftp_platform = FTP_PLATFORMS_PER_BOUNCER_PLATFORM[bouncer_platform]
+
+        file_names_per_platform = CONFIG_PER_BOUNCER_PRODUCT[bouncer_product]['file_names']
+        file_name_template = file_names_per_platform.get(
+            bouncer_platform, file_names_per_platform.get('default', None)
+        )
+        if not file_name_template:
+            # Some bouncer product like stub-installer are only meant to be on Windows.
+            # Thus no default value is defined there
+            continue
+
+        file_name = file_name_template.format(
+            version=current_version, previous_version=strip_build_data(previous_version)
+        )
+
+        path_template = CONFIG_PER_BOUNCER_PRODUCT[bouncer_product]['path_template']
+        file_relative_location = path_template.format(
+            product=product.lower(),
+            version=current_version,
+            build_number=current_build_number,
+            update_folder='updates/' if '-mar' in bouncer_product else '',
+            ftp_platform=ftp_platform,
+            file=file_name,
+        )
+
+        paths_per_bouncer_platform[bouncer_platform] = file_relative_location
+
+    return paths_per_bouncer_platform
+
+
+def craft_bouncer_product_name(product, bouncer_product, current_version,
+                               current_build_number=None, previous_version=None):
+    if '-ssl' in bouncer_product:
+        postfix = '-SSL'
+    elif 'stub-' in bouncer_product:
+        postfix = '-stub'
+    elif 'complete-' in bouncer_product:
+        postfix = '-Complete'
+    elif 'partial-' in bouncer_product:
+        if not previous_version:
+            raise Exception('Partial is being processed, but no previous version defined.')
+
+        if '-candidates' in bouncer_product:
+            if not current_build_number:
+                raise Exception('Partial in candidates directory is being processed, \
+but no current build number defined.')
+
+            postfix = 'build{build_number}-Partial-{previous_version_with_build_number}'.format(
+                build_number=current_build_number,
+                previous_version_with_build_number=previous_version,
+            )
+        else:
+            postfix = '-Partial-{previous_version}'.format(
+                previous_version=strip_build_data(previous_version)
+            )
+
+    elif 'sha1-' in bouncer_product:
+        postfix = '-sha1'
+    else:
+        postfix = ''
+
+    return '{product}-{version}{postfix}'.format(
+        product=product.capitalize(), version=current_version, postfix=postfix
+    )
+
+
+def craft_check_uptake(bouncer_product):
+    return bouncer_product != 'complete-mar-candidates'
+
+
+def craft_ssl_only(bouncer_product, project):
+    # XXX ESR is the only channel where we force serve the installer over SSL
+    if '-esr' in project and bouncer_product == 'installer':
+        return True
+
+    return bouncer_product not in (
+        'complete-mar',
+        'installer',
+        'partial-mar',
+        'partial-mar-candidates',
+    )
+
+
+def craft_add_locales(product):
+    # Do not add locales on Fennec in order to let "multi" work
+    return product != 'fennec'
+
+
+def strip_build_data(version):
+    return version.split('build')[0] if version and 'build' in version else version
--- a/taskcluster/taskgraph/transforms/l10n.py
+++ b/taskcluster/taskgraph/transforms/l10n.py
@@ -158,29 +158,29 @@ l10n_description_schema = Schema({
     # Shipping product and phase
     Optional('shipping-product'): task_description_schema['shipping-product'],
     Optional('shipping-phase'): task_description_schema['shipping-phase'],
 })
 
 transforms = TransformSequence()
 
 
-def _parse_locales_file(locales_file, platform):
+def parse_locales_file(locales_file, platform=None):
     """ Parse the passed locales file for a list of locales.
     """
     locales = []
 
     with open(locales_file, mode='r') as f:
         if locales_file.endswith('json'):
             all_locales = json.load(f)
             # XXX Only single locales are fetched
             locales = {
                 locale: data['revision']
                 for locale, data in all_locales.items()
-                if platform in data['platforms']
+                if platform is None or platform in data['platforms']
             }
         else:
             all_locales = f.read().split()
             # 'default' is the hg revision at the top of hg repo, in this context
             locales = {locale: 'default' for locale in all_locales}
     return locales
 
 
@@ -294,18 +294,18 @@ def handle_keyed_by(config, jobs):
             resolve_keyed_by(item=job, field=field, item_name=job['name'])
         yield job
 
 
 @transforms.add
 def all_locales_attribute(config, jobs):
     for job in jobs:
         locales_platform = job['attributes']['build_platform'].replace("-nightly", "")
-        locales_with_changesets = _parse_locales_file(job["locales-file"],
-                                                      platform=locales_platform)
+        locales_with_changesets = parse_locales_file(job["locales-file"],
+                                                     platform=locales_platform)
         locales_with_changesets = _remove_locales(locales_with_changesets,
                                                   to_remove=job['ignore-locales'])
 
         locales = sorted(locales_with_changesets.keys())
         attributes = job.setdefault('attributes', {})
         attributes["all_locales"] = locales
         attributes["all_locales_with_changesets"] = locales_with_changesets
         if job.get('shipping-product'):
--- a/taskcluster/taskgraph/transforms/task.py
+++ b/taskcluster/taskgraph/transforms/task.py
@@ -554,19 +554,22 @@ task_description_schema = Schema({
 
             # type of signing task (for CoT)
             Required('taskType'): basestring,
 
             # Paths to the artifacts to sign
             Required('paths'): [basestring],
         }],
     }, {
+        Required('implementation'): 'bouncer-submission',
+        Required('locales'): [basestring],
+        Required('entries'): object,
+    }, {
         Required('implementation'): 'push-apk-breakpoint',
         Required('payload'): object,
-
     }, {
         Required('implementation'): 'invalid',
         # an invalid task is one which should never actually be created; this is used in
         # release automation on branches where the task just doesn't make sense
         Extra: object,
 
     }, {
         Required('implementation'): 'always-optimized',
@@ -1100,16 +1103,26 @@ def build_balrog_payload(config, task, t
             })
         else:  # schedule / ship
             task_def['payload'].update({
                 'publish_rules': worker['publish-rules'],
                 'release_eta': config.params.get('release_eta') or '',
             })
 
 
+@payload_builder('bouncer-submission')
+def build_bouncer_submission_payload(config, task, task_def):
+    worker = task['worker']
+
+    task_def['payload'] = {
+        'locales':  worker['locales'],
+        'submission_entries': worker['entries']
+    }
+
+
 @payload_builder('push-apk')
 def build_push_apk_payload(config, task, task_def):
     worker = task['worker']
 
     task_def['payload'] = {
         'commit': worker['commit'],
         'upstreamArtifacts':  worker['upstream-artifacts'],
         'google_play_track': worker['google-play-track'],