bug 1423081 - add release partials support. r=callek
MozReview-Commit-ID: GCFevQVeIyy
--- a/taskcluster/ci/partials-signing/kind.yml
+++ b/taskcluster/ci/partials-signing/kind.yml
@@ -6,8 +6,11 @@ loader: taskgraph.loader.single_dep:load
transforms:
- taskgraph.transforms.name_sanity:transforms
- taskgraph.transforms.partials_signing:transforms
- taskgraph.transforms.task:transforms
kind-dependencies:
- partials
+
+job-template:
+ shipping-phase: promote
--- a/taskcluster/ci/partials/kind.yml
+++ b/taskcluster/ci/partials/kind.yml
@@ -1,23 +1,31 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
loader: taskgraph.loader.single_dep:loader
transforms:
- - taskgraph.transforms.name_sanity:transforms
- - taskgraph.transforms.partials:transforms
- - taskgraph.transforms.task:transforms
+ - taskgraph.transforms.name_sanity:transforms
+ - taskgraph.transforms.partials:transforms
+ - taskgraph.transforms.task:transforms
kind-dependencies:
- - repackage-signing
+ - repackage-signing
only-for-attributes:
- - nightly
+ - nightly
only-for-build-platforms:
- - macosx64-nightly/opt
- - win32-nightly/opt
- - win64-nightly/opt
- - linux-nightly/opt
- - linux64-nightly/opt
+ - macosx64-nightly/opt
+ - macosx64-devedition-nightly/opt
+ - win32-nightly/opt
+ - win32-devedition-nightly/opt
+ - win64-nightly/opt
+ - win64-devedition-nightly/opt
+ - linux-nightly/opt
+ - linux-devedition-nightly/opt
+ - linux64-nightly/opt
+ - linux64-devedition-nightly/opt
+
+job-template:
+ shipping-phase: promote
--- a/taskcluster/taskgraph/transforms/partials.py
+++ b/taskcluster/taskgraph/transforms/partials.py
@@ -87,43 +87,63 @@ def make_task_description(config, jobs):
extra = {'funsize': {'partials': list()}}
update_number = 1
artifact_path = "{}{}".format(
get_taskcluster_artifact_prefix(signing_task_ref, locale=locale),
'target.complete.mar'
)
for build in sorted(builds):
- extra['funsize']['partials'].append({
+ partial_info = {
'locale': build_locale,
'from_mar': builds[build]['mar_url'],
'to_mar': {'task-reference': artifact_path},
'platform': get_balrog_platform_name(dep_th_platform),
'branch': config.params['project'],
'update_number': update_number,
'dest_mar': build,
- })
+ }
+ if 'product' in builds[build]:
+ partial_info['product'] = builds[build]['product']
+ if 'previousVersion' in builds[build]:
+ partial_info['previousVersion'] = builds[build]['previousVersion']
+ if 'previousBuildNumber' in builds[build]:
+ partial_info['previousBuildNumber'] = builds[build]['previousBuildNumber']
+ extra['funsize']['partials'].append(partial_info)
update_number += 1
cot = extra.setdefault('chainOfTrust', {})
cot.setdefault('inputs', {})['docker-image'] = {"task-reference": "<docker-image>"}
+ mar_channel_id = None
+ if config.params['project'] == 'mozilla-beta':
+ if 'devedition' in label:
+ mar_channel_id = 'firefox-mozilla-aurora'
+ else:
+ mar_channel_id = 'firefox-mozilla-beta'
+ elif config.params['project'] == 'mozilla-release':
+ mar_channel_id = 'firefox-mozilla-release'
+ elif 'esr' in config.params['project']:
+ mar_channel_id = 'firefox-mozilla-esr'
+
worker = {
'artifacts': _generate_task_output_files(builds.keys(), locale),
'implementation': 'docker-worker',
'docker-image': {'in-tree': 'funsize-update-generator'},
'os': 'linux',
'max-run-time': 3600,
'chain-of-trust': True,
'taskcluster-proxy': True,
'env': {
'SHA1_SIGNING_CERT': 'nightly_sha1',
- 'SHA384_SIGNING_CERT': 'nightly_sha384'
+ 'SHA384_SIGNING_CERT': 'nightly_sha384',
}
}
+ if mar_channel_id:
+ worker['env']['ACCEPTED_MAR_CHANNEL_IDS'] = mar_channel_id
level = config.params['level']
task = {
'label': label,
'description': "{} Partials".format(
dep_job.task["metadata"]["description"]),
'worker-type': 'aws-provisioner-v1/gecko-%s-b-linux' % level,
--- a/taskcluster/taskgraph/util/partials.py
+++ b/taskcluster/taskgraph/util/partials.py
@@ -40,16 +40,30 @@ BALROG_PLATFORM_MAP = {
"WINNT_x86-msvc-x64"
],
"win64": [
"WINNT_x86_64-msvc",
"WINNT_x86_64-msvc-x64"
]
}
+FTP_PLATFORM_MAP = {
+ "Darwin_x86-gcc3": "mac",
+ "Darwin_x86-gcc3-u-i386-x86_64": "mac",
+ "Darwin_x86_64-gcc3": "mac",
+ "Darwin_x86_64-gcc3-u-i386-x86_64": "mac",
+ "Linux_x86-gcc3": "linux-i686",
+ "Linux_x86_64-gcc3": "linux-x86_64",
+ "WINNT_x86-msvc": "win32",
+ "WINNT_x86-msvc-x64": "win32",
+ "WINNT_x86-msvc-x86": "win32",
+ "WINNT_x86_64-msvc": "win64",
+ "WINNT_x86_64-msvc-x64": "win64",
+}
+
def get_balrog_platform_name(platform):
"""Convert build platform names into balrog platform names"""
if '-nightly' in platform:
platform = platform.replace('-nightly', '')
if '-devedition' in platform:
platform = platform.replace('-devedition', '')
return PLATFORM_RENAMES.get(platform, platform)
@@ -71,18 +85,26 @@ def get_builds(release_history, platform
def get_partials_artifacts(release_history, platform, locale):
platform = _sanitize_platform(platform)
return release_history.get(platform, {}).get(locale, {}).keys()
def get_partials_artifact_map(release_history, platform, locale):
platform = _sanitize_platform(platform)
- return {k: release_history[platform][locale][k]['buildid']
- for k in release_history.get(platform, {}).get(locale, {})}
+
+ artifact_map = {}
+ for k in release_history.get(platform, {}).get(locale, {}):
+ details = release_history[platform][locale][k]
+ attributes = ('buildid',
+ 'previousBuildNumber',
+ 'previousVersion')
+ artifact_map[k] = {attr: details[attr] for attr in attributes
+ if attr in details}
+ return artifact_map
def _retry_on_http_errors(url, verify, params, errors):
if params:
params_str = "&".join("=".join([k, str(v)])
for k, v in params.iteritems())
else:
params_str = ''
@@ -128,17 +150,35 @@ def get_sorted_releases(product, branch)
def get_release_builds(release):
url = "{}/releases/{}".format(BALROG_API_ROOT, release)
req = _retry_on_http_errors(
url=url, verify=True, params=None,
errors=[500])
return req.json()
-def populate_release_history(product, branch, maxbuilds=4, maxsearch=10):
+def find_localtest(fileUrls):
+ for channel in fileUrls:
+ if "-localtest" in channel:
+ return channel
+
+
+def populate_release_history(product, branch, maxbuilds=4, maxsearch=10,
+ partial_updates=None):
+ # Assuming we are using release branches when we know the list of previous
+ # releases in advance
+ if partial_updates:
+ return _populate_release_history(
+ product, branch, partial_updates=partial_updates)
+ else:
+ return _populate_nightly_history(
+ product, branch, maxbuilds=maxbuilds, maxsearch=maxsearch)
+
+
+def _populate_nightly_history(product, branch, maxbuilds=4, maxsearch=10):
"""Find relevant releases in Balrog
Not all releases have all platforms and locales, due
to Taskcluster migration.
Args:
product (str): capitalized product name, AKA appName, e.g. Firefox
branch (str): branch name (mozilla-central)
maxbuilds (int): Maximum number of historical releases to populate
@@ -189,8 +229,43 @@ def populate_release_history(product, br
buildid = history['platforms'][platform]['locales'][locale]['buildID']
url = history['platforms'][platform]['locales'][locale]['completes'][0]['fileUrl']
nextkey = len(builds[platform][locale]) + 1
builds[platform][locale][partial_mar_tmpl.format(nextkey)] = {
'buildid': buildid,
'mar_url': url,
}
return builds
+
+
+def _populate_release_history(product, branch, partial_updates):
+ builds = dict()
+ for version, release in partial_updates.iteritems():
+ prev_release_blob = '{product}-{version}-build{build_number}'.format(
+ product=product, version=version, build_number=release['buildNumber']
+ )
+ partial_mar_key = 'target-{version}.partial.mar'.format(version=version)
+ history = get_release_builds(prev_release_blob)
+ # use one of the localtest channels to avoid relying on bouncer
+ localtest = find_localtest(history['fileUrls'])
+ url_pattern = history['fileUrls'][localtest]['completes']['*']
+
+ for platform in history['platforms']:
+ if 'alias' in history['platforms'][platform]:
+ continue
+ if platform not in builds:
+ builds[platform] = dict()
+ for locale in history['platforms'][platform]['locales']:
+ if locale not in builds[platform]:
+ builds[platform][locale] = dict()
+ buildid = history['platforms'][platform]['locales'][locale]['buildID']
+ url = url_pattern.replace(
+ '%OS_FTP%', FTP_PLATFORM_MAP[platform]).replace(
+ '%LOCALE%', locale
+ )
+ builds[platform][locale][partial_mar_key] = {
+ 'buildid': buildid,
+ 'mar_url': url,
+ 'previousVersion': version,
+ 'previousBuildNumber': release['buildNumber'],
+ 'product': product,
+ }
+ return builds