bug 1415391 - move release indexes into index_builder('release'). r=dustin
In
bug 1412690, Dustin noted that the scopes and routes don't belong at the worker level.
To deal with the release indexes, we now have a new `index_builder`. We also add the default
release bbb scope in `build_buildbot_bridge_payload`.
We can and should still move the product info to attributes. I left that for another patch.
MozReview-Commit-ID: 4ZqvnY577S7
--- a/taskcluster/ci/beetmover-cdns/kind.yml
+++ b/taskcluster/ci/beetmover-cdns/kind.yml
@@ -19,13 +19,16 @@ job-defaults:
mozilla-beta: scriptworker-prov-v1/beetmoverworker-v1
default: scriptworker-prov-v1/beetmoverworker-dev
run-on-projects: []
jobs:
fennec-push-to-cdns:
name: fennec_push_to_cdns
product: fennec
- run:
- routes:
- - index.releases.v1.{branch}.latest.fennec.latest.beetmover_cdns
- - index.releases.v1.{branch}.{revision}.fennec.{underscore_version}.build{build_number}.beetmover_cdns
+ routes:
+ - index.releases.v1.{branch}.latest.fennec.latest.beetmover_cdns
+ - index.releases.v1.{branch}.{revision}.fennec.{underscore_version}.build{build_number}.beetmover_cdns
treeherder-platform: Android/opt
+ index:
+ type: release
+ product: fennec
+ job-name: android-api-16-opt
--- a/taskcluster/ci/release-bouncer-aliases/kind.yml
+++ b/taskcluster/ci/release-bouncer-aliases/kind.yml
@@ -18,26 +18,30 @@ jobs:
description: Update bouncer aliases job
worker-type: buildbot-bridge/buildbot-bridge
run-on-projects: []
run:
using: buildbot
product: fennec
buildername: release-{branch}-fennec_bouncer_aliases
release-promotion: true
- routes:
- - index.releases.v1.{branch}.latest.fennec.latest.bouncer_submitter
- - index.releases.v1.{branch}.{revision}.fennec.{underscore_version}.build{build_number}.bouncer_submitter
properties:
tuxedo_server_url:
by-project:
mozilla-beta: https://bounceradmin.mozilla.com/api
mozilla-release: https://bounceradmin.mozilla.com/api
maple: https://admin-bouncer.stage.mozaws.net/api/
default: http://localhost/api
+ routes:
+ - index.releases.v1.{branch}.latest.fennec.latest.bouncer_submitter
+ - index.releases.v1.{branch}.{revision}.fennec.{underscore_version}.build{build_number}.bouncer_submitter
+ index:
+ type: release
+ product: fennec
+ job-name: android-api-16-opt
notifications:
completed:
by-project:
maple:
- "release-drivers-staging"
try:
#- "{task[tags][createdForUser]}"
default:
--- a/taskcluster/ci/release-bouncer-sub/kind.yml
+++ b/taskcluster/ci/release-bouncer-sub/kind.yml
@@ -15,19 +15,23 @@ jobs:
description: release bouncer submission job
worker-type: buildbot-bridge/buildbot-bridge
run-on-projects: []
run:
using: buildbot
product: fennec
buildername: release-{branch}-fennec_bncr_sub
release-promotion: true
- routes:
- - index.releases.v1.{branch}.latest.fennec.latest.bouncer_submitter
- - index.releases.v1.{branch}.{revision}.fennec.{underscore_version}.build{build_number}.bouncer_submitter
+ routes:
+ - index.releases.v1.{branch}.latest.fennec.latest.bouncer_submitter
+ - index.releases.v1.{branch}.{revision}.fennec.{underscore_version}.build{build_number}.bouncer_submitter
+ index:
+ type: release
+ product: fennec
+ job-name: android-api-16-opt
notifications:
completed:
by-project:
maple:
- "release-drivers-staging"
try:
#- "{task[tags][createdForUser]}"
default:
--- a/taskcluster/ci/release-mark-as-shipped/kind.yml
+++ b/taskcluster/ci/release-mark-as-shipped/kind.yml
@@ -21,19 +21,23 @@ jobs:
description: mark release as shipped in Ship-It
worker-type: buildbot-bridge/buildbot-bridge
run-on-projects: []
run:
using: buildbot
product: fennec
buildername: release-{branch}-fennec_mark_as_shipped
release-promotion: true
- routes:
- - index.releases.v1.{branch}.latest.fennec.latest.mark_as_shipped
- - index.releases.v1.{branch}.{revision}.fennec.{underscore_version}.build{build_number}.mark_as_shipped
+ routes:
+ - index.releases.v1.{branch}.latest.fennec.latest.mark_as_shipped
+ - index.releases.v1.{branch}.{revision}.fennec.{underscore_version}.build{build_number}.mark_as_shipped
+ index:
+ type: release
+ product: fennec
+ job-name: android-api-16-opt
notifications:
completed:
by-project:
maple:
- "release-drivers-staging"
try:
#- "{task[tags][createdForUser]}"
default:
--- a/taskcluster/ci/release-uptake-monitoring/kind.yml
+++ b/taskcluster/ci/release-uptake-monitoring/kind.yml
@@ -18,28 +18,32 @@ jobs:
description: Uptake monitoring job
worker-type: buildbot-bridge/buildbot-bridge
run-on-projects: []
run:
using: buildbot
product: fennec
buildername: release-{branch}-fennec_uptake_monitoring
release-promotion: true
- routes:
- - index.releases.v1.{branch}.latest.fennec.latest.uptake_monitoring
- - index.releases.v1.{branch}.{revision}.fennec.{underscore_version}.build{build_number}.uptake_monitoring
properties:
# TODO: Calculate "platforms" dynamically
platforms: "android-api-16, android-x86"
tuxedo_server_url:
by-project:
mozilla-beta: https://bounceradmin.mozilla.com/api
mozilla-release: https://bounceradmin.mozilla.com/api
maple: https://admin-bouncer.stage.mozaws.net/api/
default: http://localhost/api
+ routes:
+ - index.releases.v1.{branch}.latest.fennec.latest.uptake_monitoring
+ - index.releases.v1.{branch}.{revision}.fennec.{underscore_version}.build{build_number}.uptake_monitoring
+ index:
+ type: release
+ product: fennec
+ job-name: android-api-16-opt
notifications:
completed:
by-project:
maple:
- "release-drivers-staging"
try:
#- "{task[tags][createdForUser]}"
default:
--- a/taskcluster/ci/release-version-bump/kind.yml
+++ b/taskcluster/ci/release-version-bump/kind.yml
@@ -18,19 +18,23 @@ jobs:
description: Release Promotion version bump
worker-type: buildbot-bridge/buildbot-bridge
run-on-projects: []
run:
using: buildbot
product: fennec
buildername: release-{branch}-fennec_version_bump
release-promotion: true
- routes:
- - index.releases.v1.{branch}.latest.fennec.latest.version_bump
- - index.releases.v1.{branch}.{revision}.fennec.{underscore_version}.build{build_number}.version_bump
+ routes:
+ - index.releases.v1.{branch}.latest.fennec.latest.version_bump
+ - index.releases.v1.{branch}.{revision}.fennec.{underscore_version}.build{build_number}.version_bump
+ index:
+ type: release
+ product: fennec
+ job-name: android-api-16-opt
notifications:
completed:
by-project:
maple:
- "release-drivers-staging"
try:
#- "{task[tags][createdForUser]}"
default:
--- a/taskcluster/taskgraph/transforms/beetmover_cdns.py
+++ b/taskcluster/taskgraph/transforms/beetmover_cdns.py
@@ -35,16 +35,18 @@ beetmover_cdns_description_schema = Sche
Optional('job-from'): task_description_schema['job-from'],
Optional('run'): {basestring: object},
Optional('run-on-projects'): task_description_schema['run-on-projects'],
Required('worker-type'): Any(
job_description_schema['worker-type'],
{'by-project': {basestring: job_description_schema['worker-type']}},
),
Optional('dependencies'): {basestring: taskref_or_string},
+ Optional('index'): {basestring: basestring},
+ Optional('routes'): [basestring],
})
@transforms.add
def validate(config, jobs):
for job in jobs:
label = job['name']
yield validate_schema(
--- a/taskcluster/taskgraph/transforms/job/buildbot.py
+++ b/taskcluster/taskgraph/transforms/job/buildbot.py
@@ -25,52 +25,32 @@ buildbot_run_schema = Schema({
# the buildername to use for buildbot-bridge, will expand {branch} in name from
# the current project.
Required('buildername'): basestring,
# the product to use
Required('product'): Any('firefox', 'mobile', 'fennec', 'devedition', 'thunderbird'),
Optional('release-promotion'): bool,
- Optional('routes'): [basestring],
Optional('properties'): {basestring: optionally_keyed_by('project', basestring)},
})
def bb_release_worker(config, worker, run):
# props
release_props = get_release_config(config, force=True)
repo_path = urlparse(config.params['head_repository']).path.lstrip('/')
revision = config.params['head_rev']
- branch = config.params['project']
- buildername = worker['buildername']
- underscore_version = release_props['version'].replace('.', '_')
release_props.update({
'release_promotion': True,
'repo_path': repo_path,
'revision': revision,
'script_repo_revision': revision,
})
worker['properties'].update(release_props)
- # scopes
- worker['scopes'] = [
- "project:releng:buildbot-bridge:builder-name:{}".format(buildername)
- ]
- # routes
- if run.get('routes'):
- worker['routes'] = []
- repl_dict = {
- 'branch': branch,
- 'build_number': str(release_props['build_number']),
- 'revision': revision,
- 'underscore_version': underscore_version,
- }
- for route in run['routes']:
- route = route.format(**repl_dict)
- worker['routes'].append(route)
def bb_ci_worker(config, worker):
worker['properties'].update({
'who': config.params['owner'],
'upload_to_task_id': slugid.nice(),
})
--- a/taskcluster/taskgraph/transforms/task.py
+++ b/taskcluster/taskgraph/transforms/task.py
@@ -127,27 +127,28 @@ task_description_schema = Schema({
},
# information for indexing this build so its artifacts can be discovered;
# if omitted, the build will not be indexed.
Optional('index'): {
# the name of the product this build produces
'product': Any(
'firefox',
+ 'fennec',
'mobile',
'static-analysis',
'devedition',
'source',
),
# the names to use for this job in the TaskCluster index
'job-name': basestring,
# Type of gecko v2 index to use
- 'type': Any('generic', 'nightly', 'l10n', 'nightly-with-multi-l10n'),
+ 'type': Any('generic', 'nightly', 'l10n', 'nightly-with-multi-l10n', 'release'),
# The rank that the task will receive in the TaskCluster
# index. A newly completed task supercedes the currently
# indexed task iff it has a higher rank. If unspecified,
# 'by-tier' behavior will be used.
'rank': Any(
# Rank is equal the timestamp of the build_date for tier-1
# tasks, and zero for non-tier-1. This sorts tier-{2,3}
@@ -406,18 +407,16 @@ task_description_schema = Schema({
},
Required('properties'): {
'product': basestring,
Optional('build_number'): int,
Optional('release_promotion'): bool,
Optional('tuxedo_server_url'): optionally_keyed_by('project', basestring),
Extra: taskref_or_string, # additional properties are allowed
},
- Optional('scopes'): [basestring],
- Optional('routes'): [basestring],
}, {
Required('implementation'): 'native-engine',
Required('os'): Any('macosx', 'linux'),
# A link for an executable to download
Optional('context'): basestring,
# Tells the worker whether machine should reboot
@@ -660,16 +659,22 @@ def superseder_url(config, task):
size = task['coalesce']['size']
return SUPERSEDER_URL.format(
age=age,
size=size,
key=key
)
+def verify_index_job_name(index):
+ job_name = index['job-name']
+ if job_name not in JOB_NAME_WHITELIST:
+ raise Exception(JOB_NAME_WHITELIST_ERROR.format(job_name))
+
+
@payload_builder('docker-worker')
def build_docker_worker_payload(config, task, task_def):
worker = task['worker']
level = int(config.params['level'])
image = worker['docker-image']
if isinstance(image, dict):
if 'in-tree' in image:
@@ -1003,18 +1008,21 @@ def build_buildbot_bridge_payload(config
task['extra'].pop('treeherder', None)
task['extra'].pop('treeherderEnv', None)
worker = task['worker']
task_def['payload'] = {
'buildername': worker['buildername'],
'sourcestamp': worker['sourcestamp'],
'properties': worker['properties'],
}
- task_def['scopes'].extend(worker.get('scopes', []))
- task_def['routes'].extend(worker.get('routes', []))
+ task_def.setdefault('scopes', [])
+ if worker['properties'].get('release_promotion'):
+ task_def['scopes'].append(
+ "project:releng:buildbot-bridge:builder-name:{}".format(worker['buildername'])
+ )
transforms = TransformSequence()
@transforms.add
def task_name_from_label(config, tasks):
for task in tasks:
@@ -1035,22 +1043,20 @@ def validate(config, tasks):
"In task {!r}:".format(task.get('label', '?no-label?')))
@index_builder('generic')
def add_generic_index_routes(config, task):
index = task.get('index')
routes = task.setdefault('routes', [])
- job_name = index['job-name']
- if job_name not in JOB_NAME_WHITELIST:
- raise Exception(JOB_NAME_WHITELIST_ERROR.format(job_name))
+ verify_index_job_name(index)
subs = config.params.copy()
- subs['job-name'] = job_name
+ subs['job-name'] = index['job-name']
subs['build_date_long'] = time.strftime("%Y.%m.%d.%Y%m%d%H%M%S",
time.gmtime(config.params['build_date']))
subs['product'] = index['product']
project = config.params.get('project')
for tpl in V2_ROUTE_TEMPLATES:
routes.append(tpl.format(**subs))
@@ -1064,55 +1070,74 @@ def add_generic_index_routes(config, tas
return task
@index_builder('nightly')
def add_nightly_index_routes(config, task):
index = task.get('index')
routes = task.setdefault('routes', [])
- job_name = index['job-name']
- if job_name not in JOB_NAME_WHITELIST:
- raise Exception(JOB_NAME_WHITELIST_ERROR.format(job_name))
+ verify_index_job_name(index)
subs = config.params.copy()
- subs['job-name'] = job_name
+ subs['job-name'] = index['job-name']
subs['build_date_long'] = time.strftime("%Y.%m.%d.%Y%m%d%H%M%S",
time.gmtime(config.params['build_date']))
subs['build_date'] = time.strftime("%Y.%m.%d",
time.gmtime(config.params['build_date']))
subs['product'] = index['product']
for tpl in V2_NIGHTLY_TEMPLATES:
routes.append(tpl.format(**subs))
# Also add routes for en-US
task = add_l10n_index_routes(config, task, force_locale="en-US")
return task
+@index_builder('release')
+def add_release_index_routes(config, task):
+ index = task.get('index')
+ routes = []
+ release_config = get_release_config(config, force=True)
+
+ verify_index_job_name(index)
+
+ subs = config.params.copy()
+ subs['build_number'] = str(release_config['build_number'])
+ subs['revision'] = subs['head_rev']
+ subs['underscore_version'] = release_config['version'].replace('.', '_')
+ subs['product'] = index['product']
+ subs['branch'] = subs['project']
+
+ for rt in task.get('routes', []):
+ routes.append(rt.format(**subs))
+
+ task['routes'] = routes
+
+ return task
+
+
@index_builder('nightly-with-multi-l10n')
def add_nightly_multi_index_routes(config, task):
task = add_nightly_index_routes(config, task)
task = add_l10n_index_routes(config, task, force_locale="multi")
return task
@index_builder('l10n')
def add_l10n_index_routes(config, task, force_locale=None):
index = task.get('index')
routes = task.setdefault('routes', [])
- job_name = index['job-name']
- if job_name not in JOB_NAME_WHITELIST:
- raise Exception(JOB_NAME_WHITELIST_ERROR.format(job_name))
+ verify_index_job_name(index)
subs = config.params.copy()
- subs['job-name'] = job_name
+ subs['job-name'] = index['job-name']
subs['build_date_long'] = time.strftime("%Y.%m.%d.%Y%m%d%H%M%S",
time.gmtime(config.params['build_date']))
subs['product'] = index['product']
locales = task['attributes'].get('chunk_locales',
task['attributes'].get('all_locales'))
# Some tasks has only one locale set
if task['attributes'].get('locale'):