--- a/taskcluster/ci/config.yml
+++ b/taskcluster/ci/config.yml
@@ -51,16 +51,152 @@ treeherder:
'Searchfox': 'Searchfox builds'
'SM-tc': 'Spidermonkey builds'
'pub': 'APK publishing'
'p': 'Partial generation'
'ps': 'Partials signing'
'Rel': 'Release promotion'
'Snap': 'Snap image generation'
+index:
+ job-names:
+ # This list contains a whitelist of gecko.v2 index route job names. The intent
+ # of this whitelist is to raise an alarm when new jobs are added. If those jobs
+ # already run in Buildbot, then it's important that the generated index routes
+ # match (and that only one of Buildbot and TaskCluster be tier-1 at any time).
+ # If the jobs are new and never ran in Buildbot, then their job name can be added
+ # here without any further fuss.
+ #
+ # Once all jobs have been ported from Buildbot, this list can be removed.
+ - 'android-aarch64-opt'
+ - 'android-api-16-debug'
+ - 'android-api-16-gradle-opt'
+ - 'android-api-16-old-id-opt'
+ - 'android-api-16-opt'
+ - 'android-checkstyle'
+ - 'android-dependencies'
+ - 'android-findbugs'
+ - 'android-lint'
+ - 'android-test'
+ - 'android-x86-old-id-opt'
+ - 'android-x86-opt'
+ - 'browser-haz-debug'
+ - 'linux-debug'
+ - 'linux-devedition'
+ - 'linux-devedition-opt'
+ - 'linux-devedition-nightly-repackage'
+ - 'linux-devedition-nightly-repackage-signing'
+ - 'linux-nightly-repackage'
+ - 'linux-nightly-repackage-signing'
+ - 'linux-opt'
+ - 'linux-pgo'
+ - 'linux-rusttests-opt'
+ - 'linux-rusttests-debug'
+ - 'linux64-add-on-devel'
+ - 'linux64-artifact-opt'
+ - 'linux64-asan-debug'
+ - 'linux64-asan-opt'
+ - 'linux64-asan-reporter-opt'
+ - 'linux64-base-toolchains-debug'
+ - 'linux64-base-toolchains-opt'
+ - 'linux64-fuzzing-asan-opt'
+ - 'linux64-fuzzing-debug'
+ - 'linux64-ccov-opt'
+ - 'linux64-clang-tidy'
+ - 'linux64-debug'
+ - 'linux64-devedition'
+ - 'linux64-devedition-opt'
+ - 'linux64-devedition-nightly-repackage'
+ - 'linux64-devedition-nightly-repackage-signing'
+ - 'linux64-jsdcov-opt'
+ - 'linux64-nightly-repackage'
+ - 'linux64-nightly-repackage-signing'
+ - 'linux64-noopt-debug'
+ - 'linux64-opt'
+ - 'linux64-pgo'
+ - 'linux64-rusttests-opt'
+ - 'linux64-rusttests-debug'
+ - 'linux64-searchfox-debug'
+ - 'linux64-st-an-debug'
+ - 'linux64-st-an-opt'
+ - 'linux64-valgrind-opt'
+ - 'linux64-dmd-opt'
+ - 'linux64-source-opt'
+ - 'linux64-devedition-source-opt'
+ - 'linux64-fennec-source-opt'
+ - 'macosx64-add-on-devel'
+ - 'macosx64-clang-tidy'
+ - 'macosx64-debug'
+ - 'macosx64-devedition-opt'
+ - 'macosx64-nightly-repackage'
+ - 'macosx64-nightly-repackage-signing'
+ - 'macosx64-noopt-debug'
+ - 'macosx64-opt'
+ - 'macosx64-devedition-nightly-repackage'
+ - 'macosx64-devedition-nightly-repackage-signing'
+ - 'macosx64-st-an-debug'
+ - 'macosx64-st-an-opt'
+ - 'macosx64-searchfox-debug'
+ - 'macosx64-dmd-opt'
+ - 'shell-haz-debug'
+ - 'sm-arm-sim-linux32-debug'
+ - 'sm-arm64-sim-linux64-debug'
+ - 'sm-asan-linux64-opt'
+ - 'sm-compacting-linux64-debug'
+ - 'sm-compacting-win32-debug'
+ - 'sm-fuzzing-linux64'
+ - 'sm-mozjs-sys-linux64-debug'
+ - 'sm-msan-linux64-opt'
+ - 'sm-nonunified-linux64-debug'
+ - 'sm-package-linux64-opt'
+ - 'sm-plain-linux64-opt'
+ - 'sm-plain-win32-opt'
+ - 'sm-plain-linux64-debug'
+ - 'sm-plain-win32-debug'
+ - 'sm-rootanalysis-linux64-debug'
+ - 'sm-rust-bindings-linux64-debug'
+ - 'sm-tsan-linux64-opt'
+ - 'source-bugzilla-info'
+ - 'win32-add-on-devel'
+ - 'win32-clang-tidy'
+ - 'win32-debug'
+ - 'win32-devedition-nightly-repackage'
+ - 'win32-devedition-nightly-repackage-signing'
+ - 'win32-devedition-opt'
+ - 'win32-nightly-repackage'
+ - 'win32-nightly-repackage-signing'
+ - 'win32-noopt-debug'
+ - 'win32-opt'
+ - 'win32-pgo'
+ - 'win32-rusttests-opt'
+ - 'win32-searchfox-debug'
+ - 'win32-st-an-debug'
+ - 'win32-st-an-opt'
+ - 'win32-dmd-opt'
+ - 'win64-ccov-debug'
+ - 'win64-add-on-devel'
+ - 'win64-clang-tidy'
+ - 'win64-debug'
+ - 'win64-devedition-opt'
+ - 'win64-devedition-nightly-repackage'
+ - 'win64-devedition-nightly-repackage-signing'
+ - 'win64-nightly-repackage'
+ - 'win64-nightly-repackage-signing'
+ - 'win64-noopt-debug'
+ - 'win64-opt'
+ - 'win64-pgo'
+ - 'win64-rusttests-opt'
+ - 'win64-st-an-debug'
+ - 'win64-st-an-opt'
+ - 'win64-asan-debug'
+ - 'win64-asan-opt'
+ - 'win64-dmd-opt'
+ - 'win32-mingw32-debug'
+
+
try:
# We have a few platforms for which we want to do some "extra" builds, or at
# least build-ish things. Sort of. Anyway, these other things are implemented
# as different "platforms". These do *not* automatically ride along with "-p
# all"
ridealong-builds:
'android-api-16':
- 'android-api-16-l10n'
--- a/taskcluster/taskgraph/config.py
+++ b/taskcluster/taskgraph/config.py
@@ -1,25 +1,29 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import, print_function, unicode_literals
from .util.schema import validate_schema, Schema
-from voluptuous import Required
+from voluptuous import Required, Optional
graph_config_schema = Schema({
# The trust-domain for this graph.
# (See https://firefox-source-docs.mozilla.org/taskcluster/taskcluster/taskgraph.html#taskgraph-trust-domain) # noqa
Required('trust-domain'): basestring,
Required('treeherder'): {
# Mapping of treeherder group symbols to descriptive names
Required('group-names'): {basestring: basestring}
},
+ Required('index'): {
+ # A whitelist of gecko.v2 index route job names.
+ Optional('job-names'): [basestring],
+ },
Required('try'): {
# We have a few platforms for which we want to do some "extra" builds, or at
# least build-ish things. Sort of. Anyway, these other things are implemented
# as different "platforms". These do *not* automatically ride along with "-p
# all"
Required('ridealong-builds', default={}): {basestring: [basestring]},
},
})
deleted file mode 100644
--- a/taskcluster/taskgraph/transforms/gecko_v2_whitelist.py
+++ /dev/null
@@ -1,150 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-"""
-This file contains a whitelist of gecko.v2 index route job names. The intent
-of this whitelist is to raise an alarm when new jobs are added. If those jobs
-already run in Buildbot, then it's important that the generated index routes
-match (and that only one of Buildbot and TaskCluster be tier-1 at any time).
-If the jobs are new and never ran in Buildbot, then their job name can be added
-here without any further fuss.
-
-Once all jobs have been ported from Buildbot, this file can be removed.
-"""
-
-from __future__ import absolute_import, print_function, unicode_literals
-
-# please keep me in lexical order
-JOB_NAME_WHITELIST = set([
- 'android-aarch64-opt',
- 'android-api-16-debug',
- 'android-api-16-gradle-opt',
- 'android-api-16-old-id-opt',
- 'android-api-16-opt',
- 'android-checkstyle',
- 'android-dependencies',
- 'android-findbugs',
- 'android-lint',
- 'android-test',
- 'android-x86-old-id-opt',
- 'android-x86-opt',
- 'browser-haz-debug',
- 'linux-debug',
- 'linux-devedition',
- 'linux-devedition-opt',
- 'linux-devedition-nightly-repackage',
- 'linux-devedition-nightly-repackage-signing',
- 'linux-nightly-repackage',
- 'linux-nightly-repackage-signing',
- 'linux-opt',
- 'linux-pgo',
- 'linux-rusttests-opt',
- 'linux-rusttests-debug',
- 'linux64-add-on-devel',
- 'linux64-artifact-opt',
- 'linux64-asan-debug',
- 'linux64-asan-opt',
- 'linux64-asan-reporter-opt',
- 'linux64-base-toolchains-debug',
- 'linux64-base-toolchains-opt',
- 'linux64-fuzzing-asan-opt',
- 'linux64-fuzzing-debug',
- 'linux64-ccov-opt',
- 'linux64-clang-tidy',
- 'linux64-debug',
- 'linux64-devedition',
- 'linux64-devedition-opt',
- 'linux64-devedition-nightly-repackage',
- 'linux64-devedition-nightly-repackage-signing',
- 'linux64-jsdcov-opt',
- 'linux64-nightly-repackage',
- 'linux64-nightly-repackage-signing',
- 'linux64-noopt-debug',
- 'linux64-opt',
- 'linux64-pgo',
- 'linux64-rusttests-opt',
- 'linux64-rusttests-debug',
- 'linux64-searchfox-debug',
- 'linux64-st-an-debug',
- 'linux64-st-an-opt',
- 'linux64-valgrind-opt',
- 'linux64-dmd-opt',
- 'linux64-source-opt',
- 'linux64-devedition-source-opt',
- 'linux64-fennec-source-opt',
- 'macosx64-add-on-devel',
- 'macosx64-clang-tidy',
- 'macosx64-debug',
- 'macosx64-devedition-opt',
- 'macosx64-nightly-repackage',
- 'macosx64-nightly-repackage-signing',
- 'macosx64-noopt-debug',
- 'macosx64-opt',
- 'macosx64-devedition-nightly-repackage',
- 'macosx64-devedition-nightly-repackage-signing',
- 'macosx64-st-an-debug',
- 'macosx64-st-an-opt',
- 'macosx64-searchfox-debug',
- 'macosx64-dmd-opt',
- 'shell-haz-debug',
- 'sm-arm-sim-linux32-debug',
- 'sm-arm64-sim-linux64-debug',
- 'sm-asan-linux64-opt',
- 'sm-compacting-linux64-debug',
- 'sm-compacting-win32-debug',
- 'sm-fuzzing-linux64',
- 'sm-mozjs-sys-linux64-debug',
- 'sm-msan-linux64-opt',
- 'sm-nonunified-linux64-debug',
- 'sm-package-linux64-opt',
- 'sm-plain-linux64-opt',
- 'sm-plain-win32-opt',
- 'sm-plain-linux64-debug',
- 'sm-plain-win32-debug',
- 'sm-rootanalysis-linux64-debug',
- 'sm-rust-bindings-linux64-debug',
- 'sm-tsan-linux64-opt',
- 'source-bugzilla-info',
- 'win32-add-on-devel',
- 'win32-clang-tidy',
- 'win32-debug',
- 'win32-devedition-nightly-repackage',
- 'win32-devedition-nightly-repackage-signing',
- 'win32-devedition-opt',
- 'win32-nightly-repackage',
- 'win32-nightly-repackage-signing',
- 'win32-noopt-debug',
- 'win32-opt',
- 'win32-pgo',
- 'win32-rusttests-opt',
- 'win32-searchfox-debug',
- 'win32-st-an-debug',
- 'win32-st-an-opt',
- 'win32-dmd-opt',
- 'win64-ccov-debug',
- 'win64-add-on-devel',
- 'win64-clang-tidy',
- 'win64-debug',
- 'win64-devedition-opt',
- 'win64-devedition-nightly-repackage',
- 'win64-devedition-nightly-repackage-signing',
- 'win64-nightly-repackage',
- 'win64-nightly-repackage-signing',
- 'win64-noopt-debug',
- 'win64-opt',
- 'win64-pgo',
- 'win64-rusttests-opt',
- 'win64-st-an-debug',
- 'win64-st-an-opt',
- 'win64-asan-debug',
- 'win64-asan-opt',
- 'win64-dmd-opt',
- 'win32-mingw32-debug',
-])
-
-JOB_NAME_WHITELIST_ERROR = """\
-The gecko-v2 job name {} is not in the whitelist in gecko_v2_whitelist.py.
-If this job runs on Buildbot, please ensure that the job names match between
-Buildbot and TaskCluster, then add the job name to the whitelist. If this is a
-new job, there is nothing to check -- just add the job to the whitelist.
-"""
--- a/taskcluster/taskgraph/transforms/task.py
+++ b/taskcluster/taskgraph/transforms/task.py
@@ -24,19 +24,16 @@ from taskgraph.util.hash import hash_pat
from taskgraph.util.treeherder import split_symbol
from taskgraph.transforms.base import TransformSequence
from taskgraph.util.schema import validate_schema, Schema, optionally_keyed_by, resolve_keyed_by
from taskgraph.util.scriptworker import get_release_config
from voluptuous import Any, Required, Optional, Extra
from taskgraph import GECKO
from ..util import docker as dockerutil
-from .gecko_v2_whitelist import JOB_NAME_WHITELIST, JOB_NAME_WHITELIST_ERROR
-
-
RUN_TASK = os.path.join(GECKO, 'taskcluster', 'docker', 'recipes', 'run-task')
@memoize
def _run_task_suffix():
"""String to append to cache names under control of run-task."""
return hash_path(RUN_TASK)[0:20]
@@ -718,20 +715,29 @@ def superseder_url(config, task):
size = task['coalesce']['size']
return SUPERSEDER_URL.format(
age=age,
size=size,
key=key
)
-def verify_index_job_name(index):
- job_name = index['job-name']
- if job_name not in JOB_NAME_WHITELIST:
- raise Exception(JOB_NAME_WHITELIST_ERROR.format(job_name))
+JOB_NAME_WHITELIST_ERROR = """\
+The gecko-v2 job name {job_name} is not in the whitelist in `taskcluster/ci/config.yml`.
+If this job runs on Buildbot, please ensure that the job names match between
+Buildbot and TaskCluster, then add the job name to the whitelist. If this is a
+new job, there is nothing to check -- just add the job to the whitelist.
+"""
+
+
+def verify_index(config, index):
+ if 'job-names' in config.graph_config['index']:
+ job_name = index['job-name']
+ if job_name not in config.graph_config['index']['job-names']:
+ raise Exception(JOB_NAME_WHITELIST_ERROR.format(job_name=job_name))
@payload_builder('docker-worker')
def build_docker_worker_payload(config, task, task_def):
worker = task['worker']
level = int(config.params['level'])
image = worker['docker-image']
@@ -1133,17 +1139,17 @@ def validate(config, tasks):
"In task {!r}:".format(task.get('label', '?no-label?')))
@index_builder('generic')
def add_generic_index_routes(config, task):
index = task.get('index')
routes = task.setdefault('routes', [])
- verify_index_job_name(index)
+ verify_index(config, index)
subs = config.params.copy()
subs['job-name'] = index['job-name']
subs['build_date_long'] = time.strftime("%Y.%m.%d.%Y%m%d%H%M%S",
time.gmtime(config.params['build_date']))
subs['product'] = index['product']
subs['trust-domain'] = config.graph_config['trust-domain']
@@ -1161,17 +1167,17 @@ def add_generic_index_routes(config, tas
return task
@index_builder('nightly')
def add_nightly_index_routes(config, task):
index = task.get('index')
routes = task.setdefault('routes', [])
- verify_index_job_name(index)
+ verify_index(config, index)
subs = config.params.copy()
subs['job-name'] = index['job-name']
subs['build_date_long'] = time.strftime("%Y.%m.%d.%Y%m%d%H%M%S",
time.gmtime(config.params['build_date']))
subs['build_date'] = time.strftime("%Y.%m.%d",
time.gmtime(config.params['build_date']))
subs['product'] = index['product']
@@ -1220,17 +1226,17 @@ def add_nightly_multi_index_routes(confi
return task
@index_builder('l10n')
def add_l10n_index_routes(config, task, force_locale=None):
index = task.get('index')
routes = task.setdefault('routes', [])
- verify_index_job_name(index)
+ verify_index(config, index)
subs = config.params.copy()
subs['job-name'] = index['job-name']
subs['build_date_long'] = time.strftime("%Y.%m.%d.%Y%m%d%H%M%S",
time.gmtime(config.params['build_date']))
subs['product'] = index['product']
subs['trust-domain'] = config.graph_config['trust-domain']
@@ -1259,17 +1265,17 @@ def add_l10n_index_routes(config, task,
return task
@index_builder('nightly-l10n')
def add_nightly_l10n_index_routes(config, task, force_locale=None):
index = task.get('index')
routes = task.setdefault('routes', [])
- verify_index_job_name(index)
+ verify_index(config, index)
subs = config.params.copy()
subs['job-name'] = index['job-name']
subs['build_date_long'] = time.strftime("%Y.%m.%d.%Y%m%d%H%M%S",
time.gmtime(config.params['build_date']))
subs['product'] = index['product']
subs['trust-domain'] = config.graph_config['trust-domain']
