bug 1423081 - add release partials support. r=callek draft
authorAki Sasaki <asasaki@mozilla.com>
Tue, 05 Dec 2017 19:08:06 -0800
changeset 711348 ee235a72b333f12c2ccb262713893ad5a3036305
parent 711347 7884508f508a65ab9ff9c05b64596b864aead881
child 711349 450dee0302ebddd4e1a171f4b25f5d4d93578f29
push id93059
push userasasaki@mozilla.com
push dateWed, 13 Dec 2017 21:20:02 +0000
reviewerscallek
bugs1423081
milestone59.0a1
bug 1423081 - add release partials support. r=callek MozReview-Commit-ID: GCFevQVeIyy
taskcluster/ci/partials-signing/kind.yml
taskcluster/ci/partials/kind.yml
taskcluster/taskgraph/transforms/partials.py
taskcluster/taskgraph/util/partials.py
--- a/taskcluster/ci/partials-signing/kind.yml
+++ b/taskcluster/ci/partials-signing/kind.yml
@@ -6,8 +6,11 @@ loader: taskgraph.loader.single_dep:load
 
 transforms:
   - taskgraph.transforms.name_sanity:transforms
   - taskgraph.transforms.partials_signing:transforms
   - taskgraph.transforms.task:transforms
 
 kind-dependencies:
   - partials
+
+job-template:
+  shipping-phase: promote
--- a/taskcluster/ci/partials/kind.yml
+++ b/taskcluster/ci/partials/kind.yml
@@ -1,23 +1,31 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 loader: taskgraph.loader.single_dep:loader
 
 transforms:
-  - taskgraph.transforms.name_sanity:transforms
-  - taskgraph.transforms.partials:transforms
-  - taskgraph.transforms.task:transforms
+   - taskgraph.transforms.name_sanity:transforms
+   - taskgraph.transforms.partials:transforms
+   - taskgraph.transforms.task:transforms
 
 kind-dependencies:
-  - repackage-signing
+   - repackage-signing
 
 only-for-attributes:
-  - nightly
+   - nightly
 
 only-for-build-platforms:
-  - macosx64-nightly/opt
-  - win32-nightly/opt
-  - win64-nightly/opt
-  - linux-nightly/opt
-  - linux64-nightly/opt
+   - macosx64-nightly/opt
+   - macosx64-devedition-nightly/opt
+   - win32-nightly/opt
+   - win32-devedition-nightly/opt
+   - win64-nightly/opt
+   - win64-devedition-nightly/opt
+   - linux-nightly/opt
+   - linux-devedition-nightly/opt
+   - linux64-nightly/opt
+   - linux64-devedition-nightly/opt
+
+job-template:
+   shipping-phase: promote
--- a/taskcluster/taskgraph/transforms/partials.py
+++ b/taskcluster/taskgraph/transforms/partials.py
@@ -87,43 +87,63 @@ def make_task_description(config, jobs):
 
         extra = {'funsize': {'partials': list()}}
         update_number = 1
         artifact_path = "{}{}".format(
             get_taskcluster_artifact_prefix(signing_task_ref, locale=locale),
             'target.complete.mar'
         )
         for build in sorted(builds):
-            extra['funsize']['partials'].append({
+            partial_info = {
                 'locale': build_locale,
                 'from_mar': builds[build]['mar_url'],
                 'to_mar': {'task-reference': artifact_path},
                 'platform': get_balrog_platform_name(dep_th_platform),
                 'branch': config.params['project'],
                 'update_number': update_number,
                 'dest_mar': build,
-            })
+            }
+            if 'product' in builds[build]:
+                partial_info['product'] = builds[build]['product']
+            if 'previousVersion' in builds[build]:
+                partial_info['previousVersion'] = builds[build]['previousVersion']
+            if 'previousBuildNumber' in builds[build]:
+                partial_info['previousBuildNumber'] = builds[build]['previousBuildNumber']
+            extra['funsize']['partials'].append(partial_info)
             update_number += 1
 
         cot = extra.setdefault('chainOfTrust', {})
         cot.setdefault('inputs', {})['docker-image'] = {"task-reference": "<docker-image>"}
 
+        mar_channel_id = None
+        if config.params['project'] == 'mozilla-beta':
+            if 'devedition' in label:
+                mar_channel_id = 'firefox-mozilla-aurora'
+            else:
+                mar_channel_id = 'firefox-mozilla-beta'
+        elif config.params['project'] == 'mozilla-release':
+            mar_channel_id = 'firefox-mozilla-release'
+        elif 'esr' in config.params['project']:
+            mar_channel_id = 'firefox-mozilla-esr'
+
         worker = {
             'artifacts': _generate_task_output_files(builds.keys(), locale),
             'implementation': 'docker-worker',
             'docker-image': {'in-tree': 'funsize-update-generator'},
             'os': 'linux',
             'max-run-time': 3600,
             'chain-of-trust': True,
             'taskcluster-proxy': True,
             'env': {
                 'SHA1_SIGNING_CERT': 'nightly_sha1',
-                'SHA384_SIGNING_CERT': 'nightly_sha384'
+                'SHA384_SIGNING_CERT': 'nightly_sha384',
             }
         }
+        if mar_channel_id:
+            worker['env']['ACCEPTED_MAR_CHANNEL_IDS'] = mar_channel_id
 
         level = config.params['level']
 
         task = {
             'label': label,
             'description': "{} Partials".format(
                 dep_job.task["metadata"]["description"]),
             'worker-type': 'aws-provisioner-v1/gecko-%s-b-linux' % level,
--- a/taskcluster/taskgraph/util/partials.py
+++ b/taskcluster/taskgraph/util/partials.py
@@ -40,16 +40,30 @@ BALROG_PLATFORM_MAP = {
         "WINNT_x86-msvc-x64"
     ],
     "win64": [
         "WINNT_x86_64-msvc",
         "WINNT_x86_64-msvc-x64"
     ]
 }
 
+FTP_PLATFORM_MAP = {
+    "Darwin_x86-gcc3": "mac",
+    "Darwin_x86-gcc3-u-i386-x86_64": "mac",
+    "Darwin_x86_64-gcc3": "mac",
+    "Darwin_x86_64-gcc3-u-i386-x86_64": "mac",
+    "Linux_x86-gcc3": "linux-i686",
+    "Linux_x86_64-gcc3": "linux-x86_64",
+    "WINNT_x86-msvc": "win32",
+    "WINNT_x86-msvc-x64": "win32",
+    "WINNT_x86-msvc-x86": "win32",
+    "WINNT_x86_64-msvc": "win64",
+    "WINNT_x86_64-msvc-x64": "win64",
+}
+
 
 def get_balrog_platform_name(platform):
     """Convert build platform names into balrog platform names"""
     if '-nightly' in platform:
         platform = platform.replace('-nightly', '')
     if '-devedition' in platform:
         platform = platform.replace('-devedition', '')
     return PLATFORM_RENAMES.get(platform, platform)
@@ -71,18 +85,26 @@ def get_builds(release_history, platform
 
 def get_partials_artifacts(release_history, platform, locale):
     platform = _sanitize_platform(platform)
     return release_history.get(platform, {}).get(locale, {}).keys()
 
 
 def get_partials_artifact_map(release_history, platform, locale):
     platform = _sanitize_platform(platform)
-    return {k: release_history[platform][locale][k]['buildid']
-            for k in release_history.get(platform, {}).get(locale, {})}
+
+    artifact_map = {}
+    for k in release_history.get(platform, {}).get(locale, {}):
+        details = release_history[platform][locale][k]
+        attributes = ('buildid',
+                      'previousBuildNumber',
+                      'previousVersion')
+        artifact_map[k] = {attr: details[attr] for attr in attributes
+                           if attr in details}
+    return artifact_map
 
 
 def _retry_on_http_errors(url, verify, params, errors):
     if params:
         params_str = "&".join("=".join([k, str(v)])
                               for k, v in params.iteritems())
     else:
         params_str = ''
@@ -128,17 +150,35 @@ def get_sorted_releases(product, branch)
 def get_release_builds(release):
     url = "{}/releases/{}".format(BALROG_API_ROOT, release)
     req = _retry_on_http_errors(
         url=url, verify=True, params=None,
         errors=[500])
     return req.json()
 
 
-def populate_release_history(product, branch, maxbuilds=4, maxsearch=10):
+def find_localtest(fileUrls):
+    for channel in fileUrls:
+        if "-localtest" in channel:
+            return channel
+
+
+def populate_release_history(product, branch, maxbuilds=4, maxsearch=10,
+                             partial_updates=None):
+    # Assuming we are using release branches when we know the list of previous
+    # releases in advance
+    if partial_updates:
+        return _populate_release_history(
+            product, branch, partial_updates=partial_updates)
+    else:
+        return _populate_nightly_history(
+            product, branch, maxbuilds=maxbuilds, maxsearch=maxsearch)
+
+
+def _populate_nightly_history(product, branch, maxbuilds=4, maxsearch=10):
     """Find relevant releases in Balrog
     Not all releases have all platforms and locales, due
     to Taskcluster migration.
 
         Args:
             product (str): capitalized product name, AKA appName, e.g. Firefox
             branch (str): branch name (mozilla-central)
             maxbuilds (int): Maximum number of historical releases to populate
@@ -189,8 +229,43 @@ def populate_release_history(product, br
                 buildid = history['platforms'][platform]['locales'][locale]['buildID']
                 url = history['platforms'][platform]['locales'][locale]['completes'][0]['fileUrl']
                 nextkey = len(builds[platform][locale]) + 1
                 builds[platform][locale][partial_mar_tmpl.format(nextkey)] = {
                     'buildid': buildid,
                     'mar_url': url,
                 }
     return builds
+
+
+def _populate_release_history(product, branch, partial_updates):
+    builds = dict()
+    for version, release in partial_updates.iteritems():
+        prev_release_blob = '{product}-{version}-build{build_number}'.format(
+            product=product, version=version, build_number=release['buildNumber']
+        )
+        partial_mar_key = 'target-{version}.partial.mar'.format(version=version)
+        history = get_release_builds(prev_release_blob)
+        # use one of the localtest channels to avoid relying on bouncer
+        localtest = find_localtest(history['fileUrls'])
+        url_pattern = history['fileUrls'][localtest]['completes']['*']
+
+        for platform in history['platforms']:
+            if 'alias' in history['platforms'][platform]:
+                continue
+            if platform not in builds:
+                builds[platform] = dict()
+            for locale in history['platforms'][platform]['locales']:
+                if locale not in builds[platform]:
+                    builds[platform][locale] = dict()
+                buildid = history['platforms'][platform]['locales'][locale]['buildID']
+                url = url_pattern.replace(
+                    '%OS_FTP%', FTP_PLATFORM_MAP[platform]).replace(
+                    '%LOCALE%', locale
+                )
+                builds[platform][locale][partial_mar_key] = {
+                    'buildid': buildid,
+                    'mar_url': url,
+                    'previousVersion': version,
+                    'previousBuildNumber': release['buildNumber'],
+                    'product': product,
+                }
+    return builds