Bug 1410513: Treeherder group names from config file; r?dustin
MozReview-Commit-ID: CuEQyn03Anh
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/config.yml
@@ -0,0 +1,54 @@
+treeherder:
+ group-names:
+ 'cram': 'Cram tests'
+ 'mocha': 'Mocha unit tests'
+ 'py': 'Python unit tests'
+ 'tc': 'Executed by TaskCluster'
+ 'tc-A': 'Android Gradle tests executed by TaskCluster'
+ 'tc-e10s': 'Executed by TaskCluster with e10s'
+ 'tc-Fxfn-l': 'Firefox functional tests (local) executed by TaskCluster'
+ 'tc-Fxfn-l-e10s': 'Firefox functional tests (local) executed by TaskCluster with e10s'
+ 'tc-Fxfn-r': 'Firefox functional tests (remote) executed by TaskCluster'
+ 'tc-Fxfn-r-e10s': 'Firefox functional tests (remote) executed by TaskCluster with e10s'
+ 'tc-M': 'Mochitests executed by TaskCluster'
+ 'tc-M-e10s': 'Mochitests executed by TaskCluster with e10s'
+ 'tc-M-V': 'Mochitests on Valgrind executed by TaskCluster'
+ 'tc-R': 'Reftests executed by TaskCluster'
+ 'tc-R-e10s': 'Reftests executed by TaskCluster with e10s'
+ 'tc-T': 'Talos performance tests executed by TaskCluster'
+ 'tc-Tsd': 'Talos performance tests executed by TaskCluster with Stylo disabled'
+ 'tc-Tss': 'Talos performance tests executed by TaskCluster with Stylo sequential'
+ 'tc-T-e10s': 'Talos performance tests executed by TaskCluster with e10s'
+ 'tc-Tsd-e10s': 'Talos performance tests executed by TaskCluster with e10s, Stylo disabled'
+ 'tc-Tss-e10s': 'Talos performance tests executed by TaskCluster with e10s, Stylo sequential'
+ 'tc-tt-c': 'Telemetry client marionette tests'
+ 'tc-tt-c-e10s': 'Telemetry client marionette tests with e10s'
+ 'tc-SY-e10s': 'Are we slim yet tests by TaskCluster with e10s'
+ 'tc-SYsd-e10s': 'Are we slim yet tests by TaskCluster with e10s, Stylo disabled'
+ 'tc-SYss-e10s': 'Are we slim yet tests by TaskCluster with e10s, Stylo sequential'
+ 'tc-VP': 'VideoPuppeteer tests executed by TaskCluster'
+ 'tc-W': 'Web platform tests executed by TaskCluster'
+ 'tc-W-e10s': 'Web platform tests executed by TaskCluster with e10s'
+ 'tc-X': 'Xpcshell tests executed by TaskCluster'
+ 'tc-X-e10s': 'Xpcshell tests executed by TaskCluster with e10s'
+ 'tc-L10n': 'Localised Repacks executed by Taskcluster'
+ 'tc-L10n-Rpk': 'Localized Repackaged Repacks executed by Taskcluster'
+ 'tc-BM-L10n': 'Beetmover for locales executed by Taskcluster'
+ 'tc-BMR-L10n': 'Beetmover repackages for locales executed by Taskcluster'
+ 'c-Up': 'Balrog submission of complete updates'
+ 'tc-cs': 'Checksum signing executed by Taskcluster'
+ 'tc-rs': 'Repackage signing executed by Taskcluster'
+ 'tc-BMcs': 'Beetmover checksums, executed by Taskcluster'
+ 'Aries': 'Aries Device Image'
+ 'Nexus 5-L': 'Nexus 5-L Device Image'
+ 'I': 'Docker Image Builds'
+ 'TL': 'Toolchain builds for Linux 64-bits'
+ 'TM': 'Toolchain builds for OSX'
+ 'TMW': 'Toolchain builds for Windows MinGW'
+ 'TW32': 'Toolchain builds for Windows 32-bits'
+ 'TW64': 'Toolchain builds for Windows 64-bits'
+ 'SM-tc': 'Spidermonkey builds'
+ 'pub': 'APK publishing'
+ 'p': 'Partial generation'
+ 'ps': 'Partials signing'
+ 'Rel': 'Release promotion'
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/config.py
@@ -0,0 +1,19 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from .util.schema import validate_schema, Schema
+from voluptuous import Required
+
+graph_config_schema = Schema({
+ Required('treeherder'): {
+ # Mapping of treeherder group symbols to descriptive names
+ Required('group-names'): {basestring: basestring}
+ }
+})
+
+
+def validate_graph_config(config):
+ return validate_schema(graph_config_schema, config, "Invalid graph configuration:")
--- a/taskcluster/taskgraph/generator.py
+++ b/taskcluster/taskgraph/generator.py
@@ -15,26 +15,28 @@ from .task import Task
from .optimize import optimize_task_graph
from .morph import morph
from .util.python_path import find_object
from .transforms.base import TransformSequence, TransformConfig
from .util.verify import (
verify_docs,
verifications,
)
+from .config import validate_graph_config
logger = logging.getLogger(__name__)
class Kind(object):
- def __init__(self, name, path, config):
+ def __init__(self, name, path, config, graph_config):
self.name = name
self.path = path
self.config = config
+ self.graph_config = graph_config
def _get_loader(self):
try:
loader = self.config['loader']
except KeyError:
raise KeyError("{!r} does not define `loader`".format(self.path))
return find_object(loader)
@@ -50,17 +52,17 @@ class Kind(object):
transforms = TransformSequence()
for xform_path in config['transforms']:
transform = find_object(xform_path)
transforms.add(transform)
# perform the transformations on the loaded inputs
trans_config = TransformConfig(self.name, self.path, config, parameters,
- kind_dependencies_tasks)
+ kind_dependencies_tasks, self.graph_config)
tasks = [Task(self.name,
label=task_dict['label'],
attributes=task_dict['attributes'],
task=task_dict['task'],
optimization=task_dict.get('optimization'),
dependencies=task_dict.get('dependencies'))
for task_dict in transforms(trans_config, inputs)]
return tasks
@@ -174,38 +176,52 @@ class TaskGraphGenerator(object):
The optimized task graph, with any subsequent morphs applied. This graph
will have the same meaning as the optimized task graph, but be in a form
more palatable to TaskCluster.
@type: TaskGraph
"""
return self._run_until('morphed_task_graph')
- def _load_kinds(self):
+ def _load_kinds(self, graph_config):
for path in os.listdir(self.root_dir):
path = os.path.join(self.root_dir, path)
if not os.path.isdir(path):
continue
kind_name = os.path.basename(path)
kind_yml = os.path.join(path, 'kind.yml')
if not os.path.exists(kind_yml):
continue
logger.debug("loading kind `{}` from `{}`".format(kind_name, path))
with open(kind_yml) as f:
config = yaml.load(f)
- yield Kind(kind_name, path, config)
+ yield Kind(kind_name, path, config, graph_config)
+
+ def _load_graph_config(self):
+ config_yml = os.path.join(self.root_dir, "config.yml")
+ if not os.path.exists(config_yml):
+ raise Exception("Couldn't find taskgraph configuration: {}".format(config_yml))
+
+ logger.debug("loading config from `{}`".format(config_yml))
+ with open(config_yml) as f:
+ config = yaml.load(f)
+
+ return validate_graph_config(config)
def _run(self):
+ logger.info("Loading graph configuration.")
+ graph_config = self._load_graph_config()
+
logger.info("Loading kinds")
# put the kinds into a graph and sort topologically so that kinds are loaded
# in post-order
- kinds = {kind.name: kind for kind in self._load_kinds()}
+ kinds = {kind.name: kind for kind in self._load_kinds(graph_config)}
self.verify_kinds(kinds)
edges = set()
for kind in kinds.itervalues():
for dep in kind.config.get('kind-dependencies', []):
edges.add((kind.name, dep, 'kind-dependency'))
kind_graph = Graph(set(kinds), edges)
--- a/taskcluster/taskgraph/test/test_generator.py
+++ b/taskcluster/taskgraph/test/test_generator.py
@@ -52,16 +52,19 @@ class WithFakeKind(TaskGraphGenerator):
for kind_name, cfg in self.parameters['_kinds']:
config = {
'transforms': [],
}
if cfg:
config.update(cfg)
yield FakeKind(kind_name, '/fake', config)
+ def _load_graph_config(self):
+ return {}
+
class FakeParameters(dict):
strict = True
class FakeOptimization(OptimizationStrategy):
def __init__(self, mode, *args, **kwargs):
super(FakeOptimization, self).__init__(*args, **kwargs)
--- a/taskcluster/taskgraph/transforms/base.py
+++ b/taskcluster/taskgraph/transforms/base.py
@@ -5,33 +5,36 @@
from __future__ import absolute_import, print_function, unicode_literals
class TransformConfig(object):
"""A container for configuration affecting transforms. The `config`
argument to transforms is an instance of this class, possibly with
additional kind-specific attributes beyond those set here."""
def __init__(self, kind, path, config, params,
- kind_dependencies_tasks=None):
+ kind_dependencies_tasks=None, graph_config=None):
# the name of the current kind
self.kind = kind
# the path to the kind configuration directory
self.path = path
# the parsed contents of kind.yml
self.config = config
# the parameters for this task-graph generation run
self.params = params
# a list of all the tasks associated with the kind dependencies of the
# current kind
self.kind_dependencies_tasks = kind_dependencies_tasks
+ # Global configuration of the taskgraph
+ self.graph_config = graph_config or {}
+
class TransformSequence(object):
"""
Container for a sequence of transforms. Each transform is represented as a
callable taking (config, items) and returning a generator which will yield
transformed items. The resulting sequence has the same interface.
This is convenient to use in a file full of transforms, as it provides a
--- a/taskcluster/taskgraph/transforms/task.py
+++ b/taskcluster/taskgraph/transforms/task.py
@@ -536,70 +536,16 @@ task_description_schema = Schema({
Required('dry-run', default=True): bool,
Optional('rollout-percentage'): int,
}),
})
TC_TREEHERDER_SCHEMA_URL = 'https://github.com/taskcluster/taskcluster-treeherder/' \
'blob/master/schemas/task-treeherder-config.yml'
-GROUP_NAMES = {
- 'cram': 'Cram tests',
- 'mocha': 'Mocha unit tests',
- 'py': 'Python unit tests',
- 'tc': 'Executed by TaskCluster',
- 'tc-A': 'Android Gradle tests executed by TaskCluster',
- 'tc-e10s': 'Executed by TaskCluster with e10s',
- 'tc-Fxfn-l': 'Firefox functional tests (local) executed by TaskCluster',
- 'tc-Fxfn-l-e10s': 'Firefox functional tests (local) executed by TaskCluster with e10s',
- 'tc-Fxfn-r': 'Firefox functional tests (remote) executed by TaskCluster',
- 'tc-Fxfn-r-e10s': 'Firefox functional tests (remote) executed by TaskCluster with e10s',
- 'tc-M': 'Mochitests executed by TaskCluster',
- 'tc-M-e10s': 'Mochitests executed by TaskCluster with e10s',
- 'tc-M-V': 'Mochitests on Valgrind executed by TaskCluster',
- 'tc-R': 'Reftests executed by TaskCluster',
- 'tc-R-e10s': 'Reftests executed by TaskCluster with e10s',
- 'tc-T': 'Talos performance tests executed by TaskCluster',
- 'tc-Tsd': 'Talos performance tests executed by TaskCluster with Stylo disabled',
- 'tc-Tss': 'Talos performance tests executed by TaskCluster with Stylo sequential',
- 'tc-T-e10s': 'Talos performance tests executed by TaskCluster with e10s',
- 'tc-Tsd-e10s': 'Talos performance tests executed by TaskCluster with e10s, Stylo disabled',
- 'tc-Tss-e10s': 'Talos performance tests executed by TaskCluster with e10s, Stylo sequential',
- 'tc-tt-c': 'Telemetry client marionette tests',
- 'tc-tt-c-e10s': 'Telemetry client marionette tests with e10s',
- 'tc-SY-e10s': 'Are we slim yet tests by TaskCluster with e10s',
- 'tc-SYsd-e10s': 'Are we slim yet tests by TaskCluster with e10s, Stylo disabled',
- 'tc-SYss-e10s': 'Are we slim yet tests by TaskCluster with e10s, Stylo sequential',
- 'tc-VP': 'VideoPuppeteer tests executed by TaskCluster',
- 'tc-W': 'Web platform tests executed by TaskCluster',
- 'tc-W-e10s': 'Web platform tests executed by TaskCluster with e10s',
- 'tc-X': 'Xpcshell tests executed by TaskCluster',
- 'tc-X-e10s': 'Xpcshell tests executed by TaskCluster with e10s',
- 'tc-L10n': 'Localised Repacks executed by Taskcluster',
- 'tc-L10n-Rpk': 'Localized Repackaged Repacks executed by Taskcluster',
- 'tc-BM-L10n': 'Beetmover for locales executed by Taskcluster',
- 'tc-BMR-L10n': 'Beetmover repackages for locales executed by Taskcluster',
- 'c-Up': 'Balrog submission of complete updates',
- 'tc-cs': 'Checksum signing executed by Taskcluster',
- 'tc-rs': 'Repackage signing executed by Taskcluster',
- 'tc-BMcs': 'Beetmover checksums, executed by Taskcluster',
- 'Aries': 'Aries Device Image',
- 'Nexus 5-L': 'Nexus 5-L Device Image',
- 'I': 'Docker Image Builds',
- 'TL': 'Toolchain builds for Linux 64-bits',
- 'TM': 'Toolchain builds for OSX',
- 'TMW': 'Toolchain builds for Windows MinGW',
- 'TW32': 'Toolchain builds for Windows 32-bits',
- 'TW64': 'Toolchain builds for Windows 64-bits',
- 'SM-tc': 'Spidermonkey builds',
- 'pub': 'APK publishing',
- 'p': 'Partial generation',
- 'ps': 'Partials signing',
- 'Rel': 'Release promotion',
-}
UNKNOWN_GROUP_NAME = "Treeherder group {} has no name; add it to " + __file__
V2_ROUTE_TEMPLATES = [
"index.gecko.v2.{project}.latest.{product}.{job-name}",
"index.gecko.v2.{project}.pushdate.{build_date_long}.{product}.{job-name}",
"index.gecko.v2.{project}.pushlog-id.{pushlog_id}.{product}.{job-name}",
"index.gecko.v2.{project}.revision.{head_rev}.{product}.{job-name}",
@@ -1239,22 +1185,23 @@ def build_task(config, tasks):
extra['treeherderEnv'] = task_th['environments']
treeherder = extra.setdefault('treeherder', {})
machine_platform, collection = task_th['platform'].split('/', 1)
treeherder['machine'] = {'platform': machine_platform}
treeherder['collection'] = {collection: True}
+ group_names = config.graph_config['treeherder']['group-names']
groupSymbol, symbol = split_symbol(task_th['symbol'])
if groupSymbol != '?':
treeherder['groupSymbol'] = groupSymbol
- if groupSymbol not in GROUP_NAMES:
+ if groupSymbol not in group_names:
raise Exception(UNKNOWN_GROUP_NAME.format(groupSymbol))
- treeherder['groupName'] = GROUP_NAMES[groupSymbol]
+ treeherder['groupName'] = group_names[groupSymbol]
treeherder['symbol'] = symbol
if len(symbol) > 25 or len(groupSymbol) > 25:
raise RuntimeError("Treeherder group and symbol names must not be longer than "
"25 characters: {} (see {})".format(
task_th['symbol'],
TC_TREEHERDER_SCHEMA_URL,
))
treeherder['jobKind'] = task_th['kind']