Bug 1301495 - Taskcluster l10n indexing should match mozharness' l10n indexing. r=dustin
Adds l10n and nightly indexing, matching (better) what Buildbot is currently doing with these types of tasks
(This patch is against `date`, will be grafted on review for real landing, using autoland)
MozReview-Commit-ID: K0BYwaCm6xL
--- a/taskcluster/ci/build/android.yml
+++ b/taskcluster/ci/build/android.yml
@@ -48,17 +48,18 @@ android-x86/opt:
tooltool-downloads: internal
android-x86-nightly/opt:
description: "Android 4.2 x86 Nightly"
attributes:
nightly: true
index:
product: mobile
- job-name: android-x86-nightly-opt
+ job-name: android-x86-opt
+ type: nightly
treeherder:
platform: android-4-2-x86/opt
symbol: tc(N)
tier: 2
worker-type: aws-provisioner-v1/gecko-{level}-b-android
worker:
implementation: docker-worker
max-run-time: 7200
@@ -102,17 +103,18 @@ android-api-15/opt:
tooltool-downloads: internal
android-api-15-nightly/opt:
description: "Android 4.0 API15+ Nightly"
attributes:
nightly: true
index:
product: mobile
- job-name: android-api-15-nightly-opt
+ job-name: android-api-15-opt
+ type: nightly
treeherder:
platform: android-4-0-armv7-api15/opt
symbol: tc(N)
tier: 2
worker-type: aws-provisioner-v1/gecko-{level}-b-android
worker:
implementation: docker-worker
max-run-time: 7200
--- a/taskcluster/ci/build/linux.yml
+++ b/taskcluster/ci/build/linux.yml
@@ -150,17 +150,18 @@ linux/pgo:
need-xvfb: true
linux-nightly/opt:
description: "Linux32 Nightly"
attributes:
nightly: true
index:
product: firefox
- job-name: linux32-nightly-opt
+ job-name: linux-opt
+ type: nightly
treeherder:
platform: linux32/opt
symbol: tc(N)
tier: 2
worker-type: aws-provisioner-v1/gecko-{level}-b-linux
worker:
implementation: docker-worker
max-run-time: 36000
@@ -227,17 +228,18 @@ linux64-asan/debug:
need-xvfb: true
linux64-nightly/opt:
description: "Linux64 Nightly"
attributes:
nightly: true
index:
product: firefox
- job-name: linux64-nightly-opt
+ job-name: linux64-opt
+ type: nightly
treeherder:
platform: linux64/opt
symbol: tc(N)
tier: 2
worker-type: aws-provisioner-v1/gecko-{level}-b-linux
worker:
implementation: docker-worker
max-run-time: 36000
--- a/taskcluster/ci/build/macosx.yml
+++ b/taskcluster/ci/build/macosx.yml
@@ -39,8 +39,10 @@ macosx64/opt:
using: mozharness
actions: [get-secrets build generate-build-stats update]
config:
- builds/releng_base_mac_64_cross_builds.py
- balrog/production.py
script: "mozharness/scripts/fx_desktop_build.py"
secrets: true
tooltool-downloads: internal
+
+ type: nightly
\ No newline at end of file
--- a/taskcluster/ci/l10n/kind.yml
+++ b/taskcluster/ci/l10n/kind.yml
@@ -32,26 +32,27 @@ job-template:
by-build-platform:
default: 36000
android-api-15-l10n: 18000
tooltool:
by-build-platform:
default: public
android-api-15-l10n: internal
index:
+ type: l10n
product:
by-build-platform:
default: firefox
android-api-15-l10n: mobile
job-name:
by-build-platform:
- linux-l10n: linux32-l10n-opt
- linux64-l10n: linux64-l10n-opt
- macosx64: macosx64-l10n-opt
- android-api-15-l10n: android-l10n-opt
+ linux-l10n: linux-opt
+ linux64-l10n: linux64-opt
+ macosx64: macosx64-opt
+ android-api-15-l10n: android-api-15-opt
worker-type:
by-build-platform:
default: aws-provisioner-v1/gecko-{level}-b-linux
android: aws-provisioner-v1/gecko-{level}-b-android
treeherder:
symbol: tc(L10n)
tier:
by-build-platform:
--- a/taskcluster/ci/nightly-l10n/kind.yml
+++ b/taskcluster/ci/nightly-l10n/kind.yml
@@ -31,27 +31,28 @@ job-template:
run-time:
by-build-platform:
default: 36000
android-api-15-nightly: 18000
tooltool:
by-build-platform:
default: public
android-api-15-nightly: internal
-# index:
-# product:
-# by-build-platform:
-# default: firefox
-# android-api-15-nightly: mobile
-# job-name:
-# by-build-platform:
-# linux-nightly: linux32-nightly-l10n-opt
-# linux64-nightly: linux64-nightly-l10n-opt
-# macosx64-nightly: macosx64-nightly-l10n-opt
-# android-api-15-nightly: android-nightly-l10n-opt
+ index:
+ type: l10n
+ product:
+ by-build-platform:
+ default: firefox
+ android-api-15-nightly: mobile
+ job-name:
+ by-build-platform:
+ linux-nightly: linux-opt
+ linux64-nightly: linux64-opt
+ macosx64-nightly: macosx64-opt
+ android-api-15-nightly: android-api-15-opt
worker-type:
by-build-platform:
default: aws-provisioner-v1/gecko-{level}-b-linux
android-api-15-nightly: aws-provisioner-v1/gecko-{level}-b-android
treeherder:
symbol: tc-L10n(N)
tier:
by-build-platform:
--- a/taskcluster/taskgraph/transforms/l10n.py
+++ b/taskcluster/taskgraph/transforms/l10n.py
@@ -63,16 +63,19 @@ l10n_description_schema = Schema({
},
# Items for the taskcluster index
Optional('index'): {
# Product to identify as in the taskcluster index
Required('product'): _by_platform(basestring),
# Job name to identify as in the taskcluster index
Required('job-name'): _by_platform(basestring),
+
+ # Type of index
+ Optional('type'): basestring,
},
# Description of the localized task
Required('description'): _by_platform(basestring),
# task object of the dependent task
Required('dependent-task'): object,
# worker-type to utilize
@@ -350,16 +353,17 @@ def make_job_description(config, jobs):
},
'run-on-projects': [],
}
if job.get('index'):
job_description['index'] = {
'product': job['index']['product'],
'job-name': job['index']['job-name'],
+ 'type': job['index'].get('type', 'generic'),
}
if job.get('dependencies'):
job_description['dependencies'] = job['dependencies']
if job.get('env'):
job_description['worker']['env'] = job['env']
if job.get('when', {}).get('files-changed'):
job_description.setdefault('when', {})
--- a/taskcluster/taskgraph/transforms/task.py
+++ b/taskcluster/taskgraph/transforms/task.py
@@ -87,16 +87,19 @@ task_description_schema = Schema({
# if omitted, the build will not be indexed.
Optional('index'): {
# the name of the product this build produces
'product': Any('firefox', 'mobile', 'static-analysis'),
# the names to use for this job in the TaskCluster index
'job-name': basestring,
+ # Type of gecko v2 index to use
+ 'type': Any('generic', 'nightly', 'l10n'),
+
# The rank that the task will receive in the TaskCluster
# index. A newly completed task supercedes the currently
# indexed task iff it has a higher rank. If unspecified,
# 'by-tier' behavior will be used.
'rank': Any(
# Rank is equal the timestamp of the build_date for tier-1
# tasks, and zero for non-tier-1. This sorts tier-{2,3}
# builds below tier-1 in the index.
@@ -326,16 +329,29 @@ GROUP_NAMES = {
UNKNOWN_GROUP_NAME = "Treeherder group {} has no name; add it to " + __file__
V2_ROUTE_TEMPLATES = [
"index.gecko.v2.{project}.latest.{product}.{job-name}",
"index.gecko.v2.{project}.pushdate.{build_date_long}.{product}.{job-name}",
"index.gecko.v2.{project}.revision.{head_rev}.{product}.{job-name}",
]
+V2_NIGHTLY_TEMPLATES = [
+ "index.gecko.v2.{project}.nightly.latest.{product}.{job-name}",
+ "index.gecko.v2.{project}.nightly.{build_date}.revision.{head_rev}.{product}.{job-name}",
+ "index.gecko.v2.{project}.nightly.{build_date}.latest.{product}.{job-name}",
+ "index.gecko.v2.{project}.nightly.revision.{head_rev}.{product}.{job-name}",
+]
+
+V2_L10N_TEMPLATES = [
+ "index.gecko.v2.{project}.revision.{head_rev}.{product}-l10n.{job-name}.{locale}",
+ "index.gecko.v2.{project}.pushdate.{build_date_long}.{product}-l10n.{job-name}.{locale}",
+ "index.gecko.v2.{project}.latest.{product}-l10n.{job-name}.{locale}",
+]
+
# the roots of the treeherder routes, keyed by treeherder environment
TREEHERDER_ROUTE_ROOTS = {
'production': 'tc-treeherder',
'staging': 'tc-treeherder-stage',
}
COALESCE_KEY = 'builds.{project}.{name}'
@@ -344,16 +360,26 @@ payload_builders = {}
def payload_builder(name):
def wrap(func):
payload_builders[name] = func
return func
return wrap
+# define a collection of index builders, depending on the type implementation
+index_builders = {}
+
+
+def index_builder(name):
+ def wrap(func):
+ index_builders[name] = func
+ return func
+ return wrap
+
@payload_builder('docker-worker')
def build_docker_worker_payload(config, task, task_def):
worker = task['worker']
image = worker['docker-image']
if isinstance(image, dict):
docker_image_task = 'build-docker-image-' + image['in-tree']
@@ -523,38 +549,104 @@ transforms = TransformSequence()
@transforms.add
def validate(config, tasks):
for task in tasks:
yield validate_schema(
task_description_schema, task,
"In task {!r}:".format(task.get('label', '?no-label?')))
+@index_builder('generic')
+def add_generic_index_routes(config, task):
+ index = task.get('index')
+ routes = task.setdefault('routes', [])
+
+ job_name = index['job-name']
+ if job_name not in JOB_NAME_WHITELIST:
+ raise Exception(JOB_NAME_WHITELIST_ERROR.format(job_name))
+
+ subs = config.params.copy()
+ subs['job-name'] = job_name
+ subs['build_date_long'] = time.strftime("%Y.%m.%d.%Y%m%d%H%M%S",
+ time.gmtime(config.params['build_date']))
+ subs['product'] = index['product']
+
+ for tpl in V2_ROUTE_TEMPLATES:
+ routes.append(tpl.format(**subs))
+
+ return task
+
+
+@index_builder('nightly')
+def add_nightly_index_routes(config, task):
+ index = task.get('index')
+ routes = task.setdefault('routes', [])
+
+ job_name = index['job-name']
+ if job_name not in JOB_NAME_WHITELIST:
+ raise Exception(JOB_NAME_WHITELIST_ERROR.format(job_name))
+
+ subs = config.params.copy()
+ subs['job-name'] = job_name
+ subs['build_date_long'] = time.strftime("%Y.%m.%d.%Y%m%d%H%M%S",
+ time.gmtime(config.params['build_date']))
+ subs['build_date'] = time.strftime("%Y.%m.%d",
+ time.gmtime(config.params['build_date']))
+ subs['product'] = index['product']
+
+ for tpl in V2_NIGHTLY_TEMPLATES:
+ routes.append(tpl.format(**subs))
+
+ return task
+
+
+@index_builder('l10n')
+def add_l10n_index_routes(config, task):
+ index = task.get('index')
+ routes = task.setdefault('routes', [])
+
+ job_name = index['job-name']
+ if job_name not in JOB_NAME_WHITELIST:
+ raise Exception(JOB_NAME_WHITELIST_ERROR.format(job_name))
+
+ subs = config.params.copy()
+ subs['job-name'] = job_name
+ subs['build_date_long'] = time.strftime("%Y.%m.%d.%Y%m%d%H%M%S",
+ time.gmtime(config.params['build_date']))
+ subs['product'] = index['product']
+
+ locales = task['attributes'].get('chunk_locales',
+ task['attributes'].get('all_locales'))
+
+ if not locales:
+ raise Exception("Error: Unable to use l10n index for tasks without locales")
+
+ # If there are too many locales, we can't write a route for all of them
+ # See Bug 1323792
+ if len(locales) > 18: # 18 * 3 = 54, max routes = 64
+ return task
+
+ for locale in locales:
+ for tpl in V2_L10N_TEMPLATES:
+ routes.append(tpl.format(locale=locale, **subs))
+
+ return task
+
+
@transforms.add
def add_index_routes(config, tasks):
for task in tasks:
index = task.get('index')
- routes = task.setdefault('routes', [])
if not index:
yield task
continue
- job_name = index['job-name']
- if job_name not in JOB_NAME_WHITELIST:
- raise Exception(JOB_NAME_WHITELIST_ERROR.format(job_name))
-
- subs = config.params.copy()
- subs['job-name'] = job_name
- subs['build_date_long'] = time.strftime("%Y.%m.%d.%Y%m%d%H%M%S",
- time.gmtime(config.params['build_date']))
- subs['product'] = index['product']
-
- for tpl in V2_ROUTE_TEMPLATES:
- routes.append(tpl.format(**subs))
+ index_type = index.get('type', 'generic')
+ task = index_builders[index_type](config, task)
# The default behavior is to rank tasks according to their tier
extra_index = task.setdefault('extra', {}).setdefault('index', {})
rank = index.get('rank', 'by-tier')
if rank == 'by-tier':
# rank is zero for non-tier-1 tasks and based on pushid for others;
# this sorts tier-{2,3} builds below tier-1 in the index
@@ -658,24 +750,33 @@ def build_task(config, tasks):
# Check that the v2 route templates match those used by Mozharness. This can
# go away once Mozharness builds are no longer performed in Buildbot, and the
# Mozharness code referencing routes.json is deleted.
def check_v2_routes():
with open("testing/mozharness/configs/routes.json", "rb") as f:
routes_json = json.load(f)
- # we only deal with the 'routes' key here
- routes = routes_json['routes']
+ for key in ('routes', 'nightly', 'l10n'):
+ if key == 'routes':
+ tc_template = V2_ROUTE_TEMPLATES
+ elif key == 'nightly':
+ tc_template = V2_NIGHTLY_TEMPLATES
+ elif key == 'l10n':
+ tc_template = V2_L10N_TEMPLATES
+
+ routes = routes_json[key]
- # we use different variables than mozharness
- for mh, tg in [
- ('{index}', 'index'),
- ('{build_product}', '{product}'),
- ('{build_name}-{build_type}', '{job-name}'),
- ('{year}.{month}.{day}.{pushdate}', '{build_date_long}')]:
- routes = [r.replace(mh, tg) for r in routes]
+ # we use different variables than mozharness
+ for mh, tg in [
+ ('{index}', 'index'),
+ ('{build_product}', '{product}'),
+ ('{build_name}-{build_type}', '{job-name}'),
+ ('{year}.{month}.{day}.{pushdate}', '{build_date_long}'),
+ ('{year}.{month}.{day}', '{build_date}')]:
+ routes = [r.replace(mh, tg) for r in routes]
- if sorted(routes) != sorted(V2_ROUTE_TEMPLATES):
- raise Exception("V2_ROUTE_TEMPLATES does not match Mozharness's routes.json: "
- "%s vs %s" % (V2_ROUTE_TEMPLATES, routes))
+ if sorted(routes) != sorted(tc_template):
+ raise Exception("V2 TEMPLATES do not match Mozharness's routes.json: "
+ "(tc):%s vs (mh):%s" % (tc_template, routes))
+
check_v2_routes()