Bug 1381613 - Populate a repository-less index route for "trunk" repos; r=dustin
There are a few places where we walk commit ancestry looking for things
attached to a specific revision. Because the repository name is attached
to the index path and because a revision can exist in multiple
repositories, we often have to perform N index lookups to find a result
for a specific revision. This is inefficient.
To facilitate faster index lookups by revision, we introduce a new route
that doesn't contain the repository name. In theory, we should be able
to do this globally - for all repos. However, the configuration of
tasks can vary significantly by repo. So e.g. a linux64 build on
"central" is sufficiently different from a linux64 build on "beta" or
"release." For that reason, this commit takes the conservative
approach and only defines a shared route for repositories with a similar
configuration: the "trunk" repositories.
MozReview-Commit-ID: 8rIgUbzW4eL
--- a/taskcluster/taskgraph/transforms/task.py
+++ b/taskcluster/taskgraph/transforms/task.py
@@ -10,16 +10,17 @@ complexities of worker implementations,
from __future__ import absolute_import, print_function, unicode_literals
import json
import os
import time
from copy import deepcopy
+from taskgraph.util.attributes import TRUNK_PROJECTS
from taskgraph.util.treeherder import split_symbol
from taskgraph.transforms.base import TransformSequence
from taskgraph.util.schema import validate_schema, Schema
from taskgraph.util.scriptworker import get_release_config
from voluptuous import Any, Required, Optional, Extra
from taskgraph import GECKO
from .gecko_v2_whitelist import JOB_NAME_WHITELIST, JOB_NAME_WHITELIST_ERROR
@@ -471,16 +472,22 @@ UNKNOWN_GROUP_NAME = "Treeherder group {
V2_ROUTE_TEMPLATES = [
"index.gecko.v2.{project}.latest.{product}.{job-name}",
"index.gecko.v2.{project}.pushdate.{build_date_long}.{product}.{job-name}",
"index.gecko.v2.{project}.pushlog-id.{pushlog_id}.{product}.{job-name}",
"index.gecko.v2.{project}.revision.{head_rev}.{product}.{job-name}",
]
+# {central, inbound, autoland} write to a "trunk" index prefix. This facilitates
+# walking of tasks with similar configurations.
+V2_TRUNK_ROUTE_TEMPLATES = [
+ "index.gecko.v2.trunk.revision.{head_rev}.{product}.{job-name}",
+]
+
V2_NIGHTLY_TEMPLATES = [
"index.gecko.v2.{project}.nightly.latest.{product}.{job-name}",
"index.gecko.v2.{project}.nightly.{build_date}.revision.{head_rev}.{product}.{job-name}",
"index.gecko.v2.{project}.nightly.{build_date}.latest.{product}.{job-name}",
"index.gecko.v2.{project}.nightly.revision.{head_rev}.{product}.{job-name}",
]
V2_L10N_TEMPLATES = [
@@ -815,19 +822,27 @@ def add_generic_index_routes(config, tas
raise Exception(JOB_NAME_WHITELIST_ERROR.format(job_name))
subs = config.params.copy()
subs['job-name'] = job_name
subs['build_date_long'] = time.strftime("%Y.%m.%d.%Y%m%d%H%M%S",
time.gmtime(config.params['build_date']))
subs['product'] = index['product']
+ project = config.params.get('project')
+
for tpl in V2_ROUTE_TEMPLATES:
routes.append(tpl.format(**subs))
+ # Additionally alias all tasks for "trunk" repos into a common
+ # namespace.
+ if project and project in TRUNK_PROJECTS:
+ for tpl in V2_TRUNK_ROUTE_TEMPLATES:
+ routes.append(tpl.format(**subs))
+
return task
@index_builder('nightly')
def add_nightly_index_routes(config, task):
index = task.get('index')
routes = task.setdefault('routes', [])
