Bug 1410513: Treeherder group names from config file; r?dustin draft
authorTom Prince <mozilla@hocat.ca>
Thu, 02 Nov 2017 11:37:20 -0600
changeset 693753 feafd4cd37b484f6679ae1d038b909d16f89334f
parent 693752 1127bce0340da41426b89fb04bc63cef51e4c14d
child 693754 b93297ca203c8f911abd76fd6963694a639d528d
push id87900
push userbmo:mozilla@hocat.ca
push dateMon, 06 Nov 2017 19:45:00 +0000
reviewersdustin
bugs1410513
milestone58.0a1
Bug 1410513: Treeherder group names from config file; r?dustin MozReview-Commit-ID: CuEQyn03Anh
taskcluster/ci/config.yml
taskcluster/taskgraph/config.py
taskcluster/taskgraph/generator.py
taskcluster/taskgraph/test/test_generator.py
taskcluster/taskgraph/transforms/base.py
taskcluster/taskgraph/transforms/task.py
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/config.yml
@@ -0,0 +1,54 @@
+treeherder:
+    group-names:
+        'cram': 'Cram tests'
+        'mocha': 'Mocha unit tests'
+        'py': 'Python unit tests'
+        'tc': 'Executed by TaskCluster'
+        'tc-A': 'Android Gradle tests executed by TaskCluster'
+        'tc-e10s': 'Executed by TaskCluster with e10s'
+        'tc-Fxfn-l': 'Firefox functional tests (local) executed by TaskCluster'
+        'tc-Fxfn-l-e10s': 'Firefox functional tests (local) executed by TaskCluster with e10s'
+        'tc-Fxfn-r': 'Firefox functional tests (remote) executed by TaskCluster'
+        'tc-Fxfn-r-e10s': 'Firefox functional tests (remote) executed by TaskCluster with e10s'
+        'tc-M': 'Mochitests executed by TaskCluster'
+        'tc-M-e10s': 'Mochitests executed by TaskCluster with e10s'
+        'tc-M-V': 'Mochitests on Valgrind executed by TaskCluster'
+        'tc-R': 'Reftests executed by TaskCluster'
+        'tc-R-e10s': 'Reftests executed by TaskCluster with e10s'
+        'tc-T': 'Talos performance tests executed by TaskCluster'
+        'tc-Tsd': 'Talos performance tests executed by TaskCluster with Stylo disabled'
+        'tc-Tss': 'Talos performance tests executed by TaskCluster with Stylo sequential'
+        'tc-T-e10s': 'Talos performance tests executed by TaskCluster with e10s'
+        'tc-Tsd-e10s': 'Talos performance tests executed by TaskCluster with e10s, Stylo disabled'
+        'tc-Tss-e10s': 'Talos performance tests executed by TaskCluster with e10s, Stylo sequential'
+        'tc-tt-c': 'Telemetry client marionette tests'
+        'tc-tt-c-e10s': 'Telemetry client marionette tests with e10s'
+        'tc-SY-e10s': 'Are we slim yet tests by TaskCluster with e10s'
+        'tc-SYsd-e10s': 'Are we slim yet tests by TaskCluster with e10s, Stylo disabled'
+        'tc-SYss-e10s': 'Are we slim yet tests by TaskCluster with e10s, Stylo sequential'
+        'tc-VP': 'VideoPuppeteer tests executed by TaskCluster'
+        'tc-W': 'Web platform tests executed by TaskCluster'
+        'tc-W-e10s': 'Web platform tests executed by TaskCluster with e10s'
+        'tc-X': 'Xpcshell tests executed by TaskCluster'
+        'tc-X-e10s': 'Xpcshell tests executed by TaskCluster with e10s'
+        'tc-L10n': 'Localised Repacks executed by Taskcluster'
+        'tc-L10n-Rpk': 'Localized Repackaged Repacks executed by Taskcluster'
+        'tc-BM-L10n': 'Beetmover for locales executed by Taskcluster'
+        'tc-BMR-L10n': 'Beetmover repackages for locales executed by Taskcluster'
+        'c-Up': 'Balrog submission of complete updates'
+        'tc-cs': 'Checksum signing executed by Taskcluster'
+        'tc-rs': 'Repackage signing executed by Taskcluster'
+        'tc-BMcs': 'Beetmover checksums, executed by Taskcluster'
+        'Aries': 'Aries Device Image'
+        'Nexus 5-L': 'Nexus 5-L Device Image'
+        'I': 'Docker Image Builds'
+        'TL': 'Toolchain builds for Linux 64-bits'
+        'TM': 'Toolchain builds for OSX'
+        'TMW': 'Toolchain builds for Windows MinGW'
+        'TW32': 'Toolchain builds for Windows 32-bits'
+        'TW64': 'Toolchain builds for Windows 64-bits'
+        'SM-tc': 'Spidermonkey builds'
+        'pub': 'APK publishing'
+        'p': 'Partial generation'
+        'ps': 'Partials signing'
+        'Rel': 'Release promotion'
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/config.py
@@ -0,0 +1,19 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from .util.schema import validate_schema, Schema
+from voluptuous import Required
+
+graph_config_schema = Schema({
+    Required('treeherder'): {
+        # Mapping of treeherder group symbols to descriptive names
+        Required('group-names'): {basestring: basestring}
+    }
+})
+
+
+def validate_graph_config(config):
+    return validate_schema(graph_config_schema, config, "Invalid graph configuration:")
--- a/taskcluster/taskgraph/generator.py
+++ b/taskcluster/taskgraph/generator.py
@@ -15,26 +15,28 @@ from .task import Task
 from .optimize import optimize_task_graph
 from .morph import morph
 from .util.python_path import find_object
 from .transforms.base import TransformSequence, TransformConfig
 from .util.verify import (
     verify_docs,
     verifications,
 )
+from .config import validate_graph_config
 
 logger = logging.getLogger(__name__)
 
 
 class Kind(object):
 
-    def __init__(self, name, path, config):
+    def __init__(self, name, path, config, graph_config):
         self.name = name
         self.path = path
         self.config = config
+        self.graph_config = graph_config
 
     def _get_loader(self):
         try:
             loader = self.config['loader']
         except KeyError:
             raise KeyError("{!r} does not define `loader`".format(self.path))
         return find_object(loader)
 
@@ -50,17 +52,17 @@ class Kind(object):
 
         transforms = TransformSequence()
         for xform_path in config['transforms']:
             transform = find_object(xform_path)
             transforms.add(transform)
 
         # perform the transformations on the loaded inputs
         trans_config = TransformConfig(self.name, self.path, config, parameters,
-                                       kind_dependencies_tasks)
+                                       kind_dependencies_tasks, self.graph_config)
         tasks = [Task(self.name,
                       label=task_dict['label'],
                       attributes=task_dict['attributes'],
                       task=task_dict['task'],
                       optimization=task_dict.get('optimization'),
                       dependencies=task_dict.get('dependencies'))
                  for task_dict in transforms(trans_config, inputs)]
         return tasks
@@ -174,38 +176,52 @@ class TaskGraphGenerator(object):
         The optimized task graph, with any subsequent morphs applied. This graph
         will have the same meaning as the optimized task graph, but be in a form
         more palatable to TaskCluster.
 
         @type: TaskGraph
         """
         return self._run_until('morphed_task_graph')
 
-    def _load_kinds(self):
+    def _load_kinds(self, graph_config):
         for path in os.listdir(self.root_dir):
             path = os.path.join(self.root_dir, path)
             if not os.path.isdir(path):
                 continue
             kind_name = os.path.basename(path)
 
             kind_yml = os.path.join(path, 'kind.yml')
             if not os.path.exists(kind_yml):
                 continue
 
             logger.debug("loading kind `{}` from `{}`".format(kind_name, path))
             with open(kind_yml) as f:
                 config = yaml.load(f)
 
-            yield Kind(kind_name, path, config)
+            yield Kind(kind_name, path, config, graph_config)
+
+    def _load_graph_config(self):
+        config_yml = os.path.join(self.root_dir, "config.yml")
+        if not os.path.exists(config_yml):
+            raise Exception("Couldn't find taskgraph configuration: {}".format(config_yml))
+
+        logger.debug("loading config from `{}`".format(config_yml))
+        with open(config_yml) as f:
+            config = yaml.load(f)
+
+        return validate_graph_config(config)
 
     def _run(self):
+        logger.info("Loading graph configuration.")
+        graph_config = self._load_graph_config()
+
         logger.info("Loading kinds")
         # put the kinds into a graph and sort topologically so that kinds are loaded
         # in post-order
-        kinds = {kind.name: kind for kind in self._load_kinds()}
+        kinds = {kind.name: kind for kind in self._load_kinds(graph_config)}
         self.verify_kinds(kinds)
 
         edges = set()
         for kind in kinds.itervalues():
             for dep in kind.config.get('kind-dependencies', []):
                 edges.add((kind.name, dep, 'kind-dependency'))
         kind_graph = Graph(set(kinds), edges)
 
--- a/taskcluster/taskgraph/test/test_generator.py
+++ b/taskcluster/taskgraph/test/test_generator.py
@@ -52,16 +52,19 @@ class WithFakeKind(TaskGraphGenerator):
         for kind_name, cfg in self.parameters['_kinds']:
             config = {
                 'transforms': [],
             }
             if cfg:
                 config.update(cfg)
             yield FakeKind(kind_name, '/fake', config)
 
+    def _load_graph_config(self):
+        return {}
+
 
 class FakeParameters(dict):
     strict = True
 
 
 class FakeOptimization(OptimizationStrategy):
     def __init__(self, mode, *args, **kwargs):
         super(FakeOptimization, self).__init__(*args, **kwargs)
--- a/taskcluster/taskgraph/transforms/base.py
+++ b/taskcluster/taskgraph/transforms/base.py
@@ -5,33 +5,36 @@
 from __future__ import absolute_import, print_function, unicode_literals
 
 
 class TransformConfig(object):
     """A container for configuration affecting transforms.  The `config`
     argument to transforms is an instance of this class, possibly with
     additional kind-specific attributes beyond those set here."""
     def __init__(self, kind, path, config, params,
-                 kind_dependencies_tasks=None):
+                 kind_dependencies_tasks=None, graph_config=None):
         # the name of the current kind
         self.kind = kind
 
         # the path to the kind configuration directory
         self.path = path
 
         # the parsed contents of kind.yml
         self.config = config
 
         # the parameters for this task-graph generation run
         self.params = params
 
         # a list of all the tasks associated with the kind dependencies of the
         # current kind
         self.kind_dependencies_tasks = kind_dependencies_tasks
 
+        # Global configuration of the taskgraph
+        self.graph_config = graph_config or {}
+
 
 class TransformSequence(object):
     """
     Container for a sequence of transforms.  Each transform is represented as a
     callable taking (config, items) and returning a generator which will yield
     transformed items.  The resulting sequence has the same interface.
 
     This is convenient to use in a file full of transforms, as it provides a
--- a/taskcluster/taskgraph/transforms/task.py
+++ b/taskcluster/taskgraph/transforms/task.py
@@ -536,70 +536,16 @@ task_description_schema = Schema({
         Required('dry-run', default=True): bool,
         Optional('rollout-percentage'): int,
     }),
 })
 
 TC_TREEHERDER_SCHEMA_URL = 'https://github.com/taskcluster/taskcluster-treeherder/' \
                            'blob/master/schemas/task-treeherder-config.yml'
 
-GROUP_NAMES = {
-    'cram': 'Cram tests',
-    'mocha': 'Mocha unit tests',
-    'py': 'Python unit tests',
-    'tc': 'Executed by TaskCluster',
-    'tc-A': 'Android Gradle tests executed by TaskCluster',
-    'tc-e10s': 'Executed by TaskCluster with e10s',
-    'tc-Fxfn-l': 'Firefox functional tests (local) executed by TaskCluster',
-    'tc-Fxfn-l-e10s': 'Firefox functional tests (local) executed by TaskCluster with e10s',
-    'tc-Fxfn-r': 'Firefox functional tests (remote) executed by TaskCluster',
-    'tc-Fxfn-r-e10s': 'Firefox functional tests (remote) executed by TaskCluster with e10s',
-    'tc-M': 'Mochitests executed by TaskCluster',
-    'tc-M-e10s': 'Mochitests executed by TaskCluster with e10s',
-    'tc-M-V': 'Mochitests on Valgrind executed by TaskCluster',
-    'tc-R': 'Reftests executed by TaskCluster',
-    'tc-R-e10s': 'Reftests executed by TaskCluster with e10s',
-    'tc-T': 'Talos performance tests executed by TaskCluster',
-    'tc-Tsd': 'Talos performance tests executed by TaskCluster with Stylo disabled',
-    'tc-Tss': 'Talos performance tests executed by TaskCluster with Stylo sequential',
-    'tc-T-e10s': 'Talos performance tests executed by TaskCluster with e10s',
-    'tc-Tsd-e10s': 'Talos performance tests executed by TaskCluster with e10s, Stylo disabled',
-    'tc-Tss-e10s': 'Talos performance tests executed by TaskCluster with e10s, Stylo sequential',
-    'tc-tt-c': 'Telemetry client marionette tests',
-    'tc-tt-c-e10s': 'Telemetry client marionette tests with e10s',
-    'tc-SY-e10s': 'Are we slim yet tests by TaskCluster with e10s',
-    'tc-SYsd-e10s': 'Are we slim yet tests by TaskCluster with e10s, Stylo disabled',
-    'tc-SYss-e10s': 'Are we slim yet tests by TaskCluster with e10s, Stylo sequential',
-    'tc-VP': 'VideoPuppeteer tests executed by TaskCluster',
-    'tc-W': 'Web platform tests executed by TaskCluster',
-    'tc-W-e10s': 'Web platform tests executed by TaskCluster with e10s',
-    'tc-X': 'Xpcshell tests executed by TaskCluster',
-    'tc-X-e10s': 'Xpcshell tests executed by TaskCluster with e10s',
-    'tc-L10n': 'Localised Repacks executed by Taskcluster',
-    'tc-L10n-Rpk': 'Localized Repackaged Repacks executed by Taskcluster',
-    'tc-BM-L10n': 'Beetmover for locales executed by Taskcluster',
-    'tc-BMR-L10n': 'Beetmover repackages for locales executed by Taskcluster',
-    'c-Up': 'Balrog submission of complete updates',
-    'tc-cs': 'Checksum signing executed by Taskcluster',
-    'tc-rs': 'Repackage signing executed by Taskcluster',
-    'tc-BMcs': 'Beetmover checksums, executed by Taskcluster',
-    'Aries': 'Aries Device Image',
-    'Nexus 5-L': 'Nexus 5-L Device Image',
-    'I': 'Docker Image Builds',
-    'TL': 'Toolchain builds for Linux 64-bits',
-    'TM': 'Toolchain builds for OSX',
-    'TMW': 'Toolchain builds for Windows MinGW',
-    'TW32': 'Toolchain builds for Windows 32-bits',
-    'TW64': 'Toolchain builds for Windows 64-bits',
-    'SM-tc': 'Spidermonkey builds',
-    'pub': 'APK publishing',
-    'p': 'Partial generation',
-    'ps': 'Partials signing',
-    'Rel': 'Release promotion',
-}
 
 UNKNOWN_GROUP_NAME = "Treeherder group {} has no name; add it to " + __file__
 
 V2_ROUTE_TEMPLATES = [
     "index.gecko.v2.{project}.latest.{product}.{job-name}",
     "index.gecko.v2.{project}.pushdate.{build_date_long}.{product}.{job-name}",
     "index.gecko.v2.{project}.pushlog-id.{pushlog_id}.{product}.{job-name}",
     "index.gecko.v2.{project}.revision.{head_rev}.{product}.{job-name}",
@@ -1239,22 +1185,23 @@ def build_task(config, tasks):
             extra['treeherderEnv'] = task_th['environments']
 
             treeherder = extra.setdefault('treeherder', {})
 
             machine_platform, collection = task_th['platform'].split('/', 1)
             treeherder['machine'] = {'platform': machine_platform}
             treeherder['collection'] = {collection: True}
 
+            group_names = config.graph_config['treeherder']['group-names']
             groupSymbol, symbol = split_symbol(task_th['symbol'])
             if groupSymbol != '?':
                 treeherder['groupSymbol'] = groupSymbol
-                if groupSymbol not in GROUP_NAMES:
+                if groupSymbol not in group_names:
                     raise Exception(UNKNOWN_GROUP_NAME.format(groupSymbol))
-                treeherder['groupName'] = GROUP_NAMES[groupSymbol]
+                treeherder['groupName'] = group_names[groupSymbol]
             treeherder['symbol'] = symbol
             if len(symbol) > 25 or len(groupSymbol) > 25:
                 raise RuntimeError("Treeherder group and symbol names must not be longer than "
                                    "25 characters: {} (see {})".format(
                                        task_th['symbol'],
                                        TC_TREEHERDER_SCHEMA_URL,
                                        ))
             treeherder['jobKind'] = task_th['kind']