bug 1457034 - generate `release_partner_config` in `release_promotion` r=nthomas
If we're not passed `release_partner_config` in the input, let's poll github for it in the action.
MozReview-Commit-ID: 2swx76vhTE5
--- a/taskcluster/ci/config.yml
+++ b/taskcluster/ci/config.yml
@@ -129,8 +129,17 @@ scriptworker:
'scriptworker-prov-v1/balrogworker-v1':
- 'project:releng:balrog:server:nightly'
- 'project:releng:balrog:server:aurora'
- 'project:releng:balrog:server:beta'
- 'project:releng:balrog:server:release'
- 'project:releng:balrog:server:esr'
'scriptworker-prov-v1/balrog-dev':
- 'project:releng:balrog:server:dep'
+
+
+partner:
+ release:
+ release-partner-repack: git@github.com:mozilla-partners/repack-manifests.git
+ release-eme-free-repack: git@github.com:mozilla-partners/mozilla-EME-free-manifest
+ staging:
+ release-partner-repack: git@github.com:mozilla-releng/staging-repack-manifests.git
+ release-eme-free-repack: git@github.com:mozilla-releng/staging-repack-manifests.git
--- a/taskcluster/ci/release-eme-free-repack/kind.yml
+++ b/taskcluster/ci/release-eme-free-repack/kind.yml
@@ -21,23 +21,16 @@ job-defaults:
shipping-product: firefox
shipping-phase: promote
worker-type: aws-provisioner-v1/gecko-{level}-b-linux
worker:
docker-image:
in-tree: "partner-repack"
chain-of-trust: true
max-run-time: 7200
- env:
- REPACK_MANIFESTS_URL:
- by-project:
- mozilla-beta: "git@github.com:mozilla-partners/mozilla-EME-free-manifest"
- mozilla-release: "git@github.com:mozilla-partners/mozilla-EME-free-manifest"
- maple: "git@github.com:mozilla-partners/mozilla-EME-free-manifest"
- default: "git@github.com:mozilla-releng/staging-repack-manifests.git"
run:
using: mozharness
config:
- partner_repacks/release_mozilla-release_desktop.py
script: mozharness/scripts/desktop_partner_repacks.py
job-script: taskcluster/scripts/builder/repackage.sh
need-xvfb: false
tooltool-downloads: false
--- a/taskcluster/ci/release-partner-repack/kind.yml
+++ b/taskcluster/ci/release-partner-repack/kind.yml
@@ -21,23 +21,16 @@ job-defaults:
shipping-product: firefox
shipping-phase: promote
worker-type: aws-provisioner-v1/gecko-{level}-b-linux
worker:
docker-image:
in-tree: "partner-repack"
chain-of-trust: true
max-run-time: 7200
- env:
- REPACK_MANIFESTS_URL:
- by-project:
- mozilla-beta: "git@github.com:mozilla-partners/repack-manifests.git"
- mozilla-release: "git@github.com:mozilla-partners/repack-manifests.git"
- maple: "git@github.com:mozilla-partners/repack-manifests.git"
- default: "git@github.com:mozilla-releng/staging-repack-manifests.git"
run:
using: mozharness
config:
- partner_repacks/release_mozilla-release_desktop.py
script: mozharness/scripts/desktop_partner_repacks.py
job-script: taskcluster/scripts/builder/repackage.sh
need-xvfb: false
tooltool-downloads: false
--- a/taskcluster/taskgraph/actions/release_promotion.py
+++ b/taskcluster/taskgraph/actions/release_promotion.py
@@ -10,17 +10,24 @@ import json
import os
from .registry import register_callback_action
from .util import (find_decision_task, find_existing_tasks_from_previous_kinds,
find_hg_revision_pushlog_id)
from taskgraph.util.taskcluster import get_artifact
from taskgraph.util.partials import populate_release_history
-from taskgraph.util.partners import fix_partner_config
+from taskgraph.util.partners import (
+ EMEFREE_BRANCHES,
+ PARTNER_BRANCHES,
+ fix_partner_config,
+ get_partner_config_by_url,
+ get_partner_url_config,
+ get_token
+)
from taskgraph.taskgraph import TaskGraph
from taskgraph.decision import taskgraph_decision
from taskgraph.parameters import Parameters
from taskgraph.util.attributes import RELEASE_PROMOTION_PROJECTS
RELEASE_PROMOTION_CONFIG = {
'promote_fennec': {
'target_tasks_method': 'promote_fennec',
@@ -96,24 +103,28 @@ PARTIAL_UPDATES_FLAVORS = (
'push_firefox',
'push_firefox_rc',
'push_devedition',
'ship_firefox',
'ship_firefox_rc',
'ship_devedition',
)
-PARTNER_BRANCHES = ('mozilla-beta', 'mozilla-release', 'maple', 'birch', 'jamun')
-EMEFREE_BRANCHES = ('mozilla-beta', 'mozilla-release', 'maple', 'birch', 'jamun')
-
def is_release_promotion_available(parameters):
return parameters['project'] in RELEASE_PROMOTION_PROJECTS
+def get_partner_config(partner_url_config, github_token):
+ partner_config = {}
+ for kind, url in partner_url_config.items():
+ partner_config[kind] = get_partner_config_by_url(url, kind, github_token)
+ return partner_config
+
+
@register_callback_action(
name='release-promotion',
title='Release Promotion',
symbol='${input.release_promotion_flavor}',
description="Promote a release.",
order=10000,
context=[],
available=is_release_promotion_available,
@@ -339,19 +350,29 @@ def release_promotion_action(parameters,
parameters['target_tasks_method'] = target_tasks_method
parameters['build_number'] = int(input['build_number'])
parameters['next_version'] = next_version
parameters['release_history'] = release_history
parameters['release_type'] = promotion_config.get('release_type', '')
parameters['release_eta'] = input.get('release_eta', '')
parameters['release_enable_partners'] = release_enable_partners
parameters['release_partners'] = input.get('release_partners')
- if input.get('release_partner_config'):
- parameters['release_partner_config'] = fix_partner_config(input['release_partner_config'])
parameters['release_enable_emefree'] = release_enable_emefree
+ partner_config = input.get('release_partner_config')
+ if not partner_config and (release_enable_emefree or release_enable_partners):
+ partner_url_config = get_partner_url_config(
+ parameters, graph_config, enable_emefree=release_enable_emefree,
+ enable_partners=release_enable_partners
+ )
+ github_token = get_token(parameters)
+ partner_config = get_partner_config(partner_url_config, github_token)
+
+ if partner_config:
+ parameters['release_partner_config'] = fix_partner_config(partner_config)
+
if input['version']:
parameters['version'] = input['version']
# make parameters read-only
parameters = Parameters(**parameters)
taskgraph_decision({'root': graph_config.root_dir}, parameters=parameters)
--- a/taskcluster/taskgraph/config.py
+++ b/taskcluster/taskgraph/config.py
@@ -41,16 +41,22 @@ graph_config_schema = Schema({
Required('products'): [basestring],
},
Required('scriptworker'): {
# Prefix to add to scopes controlling scriptworkers
Required('scope-prefix'): basestring,
# Mapping of scriptworker types to scopes they accept
Required('worker-types'): {basestring: [basestring]}
},
+ Required('partner'): {
+ # Release config for partner repacks
+ Required('release'): {basestring: basestring},
+ # Staging config for partner repacks
+ Required('staging'): {basestring: basestring},
+ },
})
@attr.s(frozen=True)
class GraphConfig(object):
_config = attr.ib()
root_dir = attr.ib()
--- a/taskcluster/taskgraph/transforms/partner_repack.py
+++ b/taskcluster/taskgraph/transforms/partner_repack.py
@@ -5,28 +5,37 @@
Transform the partner repack task into an actual task description.
"""
from __future__ import absolute_import, print_function, unicode_literals
from taskgraph.transforms.base import TransformSequence
from taskgraph.util.schema import resolve_keyed_by
from taskgraph.util.scriptworker import get_release_config
-from taskgraph.util.partners import check_if_partners_enabled
+from taskgraph.util.partners import (
+ check_if_partners_enabled,
+ get_partner_url_config,
+)
transforms = TransformSequence()
@transforms.add
-def resolve_properties(config, tasks):
+def populate_repack_manifests_url(config, tasks):
for task in tasks:
- for property in ("REPACK_MANIFESTS_URL", ):
- property = "worker.env.{}".format(property)
- resolve_keyed_by(task, property, property, **config.params)
+ partner_url_config = get_partner_url_config(config.params, config.graph_config)
+
+ for k in partner_url_config:
+ if config.kind.startswith(k):
+ task['worker'].setdefault('env', {})['REPACK_MANIFESTS_URL'] = \
+ partner_url_config[k]
+ break
+ else:
+ raise Exception("Can't find partner REPACK_MANIFESTS_URL")
for property in ("limit-locales", ):
property = "extra.{}".format(property)
resolve_keyed_by(task, property, property, **config.params)
if task['worker']['env']['REPACK_MANIFESTS_URL'].startswith('git@'):
task.setdefault('scopes', []).append(
'secrets:get:project/releng/gecko/build/level-{level}/partner-github-ssh'.format(
@@ -57,16 +66,19 @@ def add_command_arguments(config, tasks)
task['run']['options'] = [
'version={}'.format(release_config['version']),
'build-number={}'.format(release_config['build_number']),
'platform={}'.format(task['attributes']['build_platform'].split('-')[0]),
]
if task['extra']['limit-locales']:
for locale in all_locales:
task['run']['options'].append('limit-locale={}'.format(locale))
+ if 'partner' in config.kind and config.params['release_partners']:
+ for partner in config.params['release_partners']:
+ task['run']['options'].append('p={}'.format(partner))
# The upstream taskIds are stored a special environment variable, because we want to use
# task-reference's to resolve dependencies, but the string handling of MOZHARNESS_OPTIONS
# blocks that. It's space-separated string of ids in the end.
task['worker']['env']['UPSTREAM_TASKIDS'] = {
'task-reference': ' '.join(['<{}>'.format(dep) for dep in task['dependencies']])
}
--- a/taskcluster/taskgraph/util/partners.py
+++ b/taskcluster/taskgraph/util/partners.py
@@ -1,24 +1,39 @@
from __future__ import absolute_import, print_function, unicode_literals
from copy import deepcopy
import json
import logging
import os
+from redo import retry
import requests
import xml.etree.ElementTree as ET
# Suppress chatty requests logging
logging.getLogger("requests").setLevel(logging.WARNING)
log = logging.getLogger(__name__)
GITHUB_API_ENDPOINT = "https://api.github.com/graphql"
+PARTNER_BRANCHES = {
+ 'mozilla-beta': 'release',
+ 'mozilla-release': 'release',
+ 'maple': 'release',
+ 'birch': 'release',
+ 'jamun': 'release',
+}
+EMEFREE_BRANCHES = {
+ 'mozilla-beta': 'release',
+ 'mozilla-release': 'release',
+ 'maple': 'release',
+ 'birch': 'release',
+ 'jamun': 'release',
+}
"""
LOGIN_QUERY, MANIFEST_QUERY, and REPACK_CFG_QUERY are all written to the Github v4 API,
which users GraphQL. See https://developer.github.com/v4/
"""
LOGIN_QUERY = """query {
viewer {
@@ -125,27 +140,51 @@ Example response:
TC_PLATFORM_PER_FTP = {
'linux-i686': 'linux-nightly',
'linux-x86_64': 'linux64-nightly',
'mac': 'macosx64-nightly',
'win32': 'win32-nightly',
'win64': 'win64-nightly',
}
-TASKCLUSTER_PROXY_SECRET_ROOT = 'http://taskcluster/secrets/v1/secret/'
+TASKCLUSTER_PROXY_SECRET_ROOT = 'http://taskcluster/secrets/v1/secret'
LOCALES_FILE = os.path.join(
os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))),
'browser', 'locales', 'l10n-changesets.json'
)
# cache data at the module level
partner_configs = {}
+# TODO - grant private repo access to P.A.T.
+# TODO - add level-3 as well, cleanup as of level
+def get_token(params):
+ """ We use a Personal Access Token from Github to lookup partner config. No extra scopes are
+ needed on the token to read public repositories, but need the 'repo' scope to see private
+ repositories. This is not fine grained and also grants r/w access, but is revoked at the repo
+ level.
+ """
+
+ # The 'usual' method - via taskClusterProxy for decision tasks
+ # TODO use {level}? Or allow the token to level 1 and remove level from the path?
+ url = "{secret_root}/project/releng/gecko/build/level-2/partner-github-api".format(
+ secret_root=TASKCLUSTER_PROXY_SECRET_ROOT, **params
+ )
+ try:
+ resp = retry(requests.get, attempts=2, sleeptime=10,
+ args=(url, ),
+ kwargs={'timeout': 60, 'headers': ''})
+ j = resp.json()
+ return j['secret']['key']
+ except (requests.ConnectionError, ValueError, KeyError):
+ raise RuntimeError('Could not get Github API token to lookup partner data')
+
+
def query_api(query, token):
""" Make a query with a Github auth header, returning the json """
headers = {'Authorization': 'bearer %s' % token}
r = requests.post(GITHUB_API_ENDPOINT, json={'query': query}, headers=headers)
r.raise_for_status()
j = r.json()
if 'errors' in j:
@@ -261,19 +300,17 @@ def get_partner_config_by_url(manifest_u
log.info('Looking up data for %s from %s', kind, manifest_url)
check_login(token)
partners = get_partners(manifest_url, token)
partner_configs[kind] = {}
for partner, partner_url in partners.items():
if partner_subset and partner not in partner_subset:
continue
- partner_configs[kind][partner] = get_repack_configs(
- partner_url, token, partner_subset
- )
+ partner_configs[kind][partner] = get_repack_configs(partner_url, token)
return partner_configs[kind]
def check_if_partners_enabled(config, tasks):
if (
config.params['release_enable_partners'] and
config.kind.startswith('release-partner-repack')
) or (
@@ -358,8 +395,22 @@ def get_ftp_platform(platform):
# Ugh
def locales_per_build_platform(build_platform, locales):
if build_platform.startswith('mac'):
exclude = ['ja']
else:
exclude = ['ja-JP-mac']
return [locale for locale in locales if locale not in exclude]
+
+
+def get_partner_url_config(parameters, graph_config, enable_emefree=True, enable_partners=True):
+ partner_url_config = {}
+ project = parameters['project']
+ if enable_emefree:
+ alias = EMEFREE_BRANCHES[project]
+ partner_url_config['release-eme-free-repack'] = \
+ graph_config['partner'][alias]['release-eme-free-repack']
+ if enable_partners:
+ alias = PARTNER_BRANCHES[project]
+ partner_url_config['release-partner-repack'] = \
+ graph_config['partner'][alias]['release-partner-repack']
+ return partner_url_config