Bug 1415199: Move some logic for loading taskgraph kinds from mozbuild.mach_bootstrap to taskgraph.
This fixes the fallout from the mach command not being updated for 69c60c8fc528d8d038e22a2ae62f5ff7b1131231.
MozReview-Commit-ID: 7WZ8wW4gnCY
--- a/python/mozbuild/mozbuild/mach_commands.py
+++ b/python/mozbuild/mozbuild/mach_commands.py
@@ -1779,17 +1779,17 @@ class PackageFrontend(MachCommandBase):
open_manifest,
unpack_file,
)
from requests.adapters import HTTPAdapter
import redo
import requests
import shutil
- from taskgraph.generator import Kind
+ from taskgraph.generator import load_graph_config, Kind
from taskgraph.util.taskcluster import (
get_artifact_url,
list_artifacts,
)
import yaml
self._set_log_level(verbose)
# Normally, we'd use self.log_manager.enable_unstructured(),
@@ -1900,25 +1900,24 @@ class PackageFrontend(MachCommandBase):
'head_rev': '',
'moz_build_date': '',
'build_date': 0,
'pushlog_id': 0,
'owner': '',
}
# TODO: move to the taskcluster package
- def tasks(kind):
- kind_path = mozpath.join(self.topsrcdir, 'taskcluster', 'ci', kind)
- with open(mozpath.join(kind_path, 'kind.yml')) as f:
- config = yaml.load(f)
- tasks = Kind(kind, kind_path, config).load_tasks(params, {})
- return {
- task.task['metadata']['name']: task
- for task in tasks
- }
+ def tasks(kind_name):
+ root_path = mozpath.join(self.topsrcdir, 'taskcluster', 'ci')
+ graph_config = load_graph_config(root_path)
+ tasks = Kind.load(root_path, graph_config, kind_name).load_tasks(params, {})
+ return {
+ task.task['metadata']['name']: task
+ for task in tasks
+ }
toolchains = tasks('toolchain')
aliases = {}
for t in toolchains.values():
alias = t.attributes.get('toolchain-alias')
if alias:
aliases['toolchain-{}'.format(alias)] = \
--- a/taskcluster/taskgraph/generator.py
+++ b/taskcluster/taskgraph/generator.py
@@ -20,16 +20,22 @@ from .util.verify import (
verify_docs,
verifications,
)
from .config import validate_graph_config
logger = logging.getLogger(__name__)
+class KindNotFound(Exception):
+ """
+ Raised when trying to load kind from a directory without a kind.yml.
+ """
+
+
class Kind(object):
def __init__(self, name, path, config, graph_config):
self.name = name
self.path = path
self.config = config
self.graph_config = graph_config
@@ -62,16 +68,41 @@ class Kind(object):
label=task_dict['label'],
attributes=task_dict['attributes'],
task=task_dict['task'],
optimization=task_dict.get('optimization'),
dependencies=task_dict.get('dependencies'))
for task_dict in transforms(trans_config, inputs)]
return tasks
+ @classmethod
+ def load(cls, root_dir, graph_config, kind_name):
+ path = os.path.join(root_dir, kind_name)
+ kind_yml = os.path.join(path, 'kind.yml')
+ if not os.path.exists(kind_yml):
+ raise KindNotFound(kind_yml)
+
+ logger.debug("loading kind `{}` from `{}`".format(kind_name, path))
+ with open(kind_yml) as f:
+ config = yaml.load(f)
+
+ return cls(kind_name, path, config, graph_config)
+
+
+def load_graph_config(root_dir):
+ config_yml = os.path.join(root_dir, "config.yml")
+ if not os.path.exists(config_yml):
+ raise Exception("Couldn't find taskgraph configuration: {}".format(config_yml))
+
+ logger.debug("loading config from `{}`".format(config_yml))
+ with open(config_yml) as f:
+ config = yaml.load(f)
+
+ return validate_graph_config(config)
+
class TaskGraphGenerator(object):
"""
The central controller for taskgraph. This handles all phases of graph
generation. The task is generated from all of the kinds defined in
subdirectories of the generator's root directory.
Access to the results of this generation, as well as intermediate values at
@@ -177,46 +208,25 @@ class TaskGraphGenerator(object):
will have the same meaning as the optimized task graph, but be in a form
more palatable to TaskCluster.
@type: TaskGraph
"""
return self._run_until('morphed_task_graph')
def _load_kinds(self, graph_config):
- for path in os.listdir(self.root_dir):
- path = os.path.join(self.root_dir, path)
- if not os.path.isdir(path):
- continue
- kind_name = os.path.basename(path)
-
- kind_yml = os.path.join(path, 'kind.yml')
- if not os.path.exists(kind_yml):
+ for kind_name in os.listdir(self.root_dir):
+ try:
+ yield Kind.load(self.root_dir, graph_config, kind_name)
+ except KindNotFound:
continue
- logger.debug("loading kind `{}` from `{}`".format(kind_name, path))
- with open(kind_yml) as f:
- config = yaml.load(f)
-
- yield Kind(kind_name, path, config, graph_config)
-
- def _load_graph_config(self):
- config_yml = os.path.join(self.root_dir, "config.yml")
- if not os.path.exists(config_yml):
- raise Exception("Couldn't find taskgraph configuration: {}".format(config_yml))
-
- logger.debug("loading config from `{}`".format(config_yml))
- with open(config_yml) as f:
- config = yaml.load(f)
-
- return validate_graph_config(config)
-
def _run(self):
logger.info("Loading graph configuration.")
- graph_config = self._load_graph_config()
+ graph_config = load_graph_config(self.root_dir)
logger.info("Loading kinds")
# put the kinds into a graph and sort topologically so that kinds are loaded
# in post-order
kinds = {kind.name: kind for kind in self._load_kinds(graph_config)}
self.verify_kinds(kinds)
edges = set()
--- a/taskcluster/taskgraph/test/test_generator.py
+++ b/taskcluster/taskgraph/test/test_generator.py
@@ -7,16 +7,17 @@ from __future__ import absolute_import,
import pytest
import unittest
from mozunit import main
from taskgraph.generator import TaskGraphGenerator, Kind
from taskgraph.optimize import OptimizationStrategy
from taskgraph.util.templates import merge
from taskgraph import (
+ generator,
graph,
optimize as optimize_mod,
target_tasks as target_tasks_mod,
)
def fake_loader(kind, path, config, parameters, loaded_tasks):
for i in range(3):
@@ -52,18 +53,19 @@ class WithFakeKind(TaskGraphGenerator):
for kind_name, cfg in self.parameters['_kinds']:
config = {
'transforms': [],
}
if cfg:
config.update(cfg)
yield FakeKind(kind_name, '/fake', config, graph_config)
- def _load_graph_config(self):
- return {}
+
+def fake_load_graph_config(root_dir):
+ return {}
class FakeParameters(dict):
strict = True
class FakeOptimization(OptimizationStrategy):
def __init__(self, mode, *args, **kwargs):
@@ -103,16 +105,18 @@ class TestGenerator(unittest.TestCase):
parameters = FakeParameters({
'_kinds': kinds,
'target_tasks_method': 'test_method',
'try_mode': None,
})
parameters.update(params)
+ self.patch.setattr(generator, 'load_graph_config', fake_load_graph_config)
+
return WithFakeKind('/root', parameters)
def test_kind_ordering(self):
"When task kinds depend on each other, they are loaded in postorder"
self.tgg = self.maketgg(kinds=[
('_fake3', {'kind-dependencies': ['_fake2', '_fake1']}),
('_fake2', {'kind-dependencies': ['_fake1']}),
('_fake1', {'kind-dependencies': []}),