Bug 1258497: add partial tests for legacy kind; r?gps
MozReview-Commit-ID: I15IRDbM8sV
--- a/taskcluster/mach_commands.py
+++ b/taskcluster/mach_commands.py
@@ -150,17 +150,17 @@ class MachCommands(MachCommandBase):
def taskgraph_decision(self, **options):
# load parameters from env vars, command line, etc.
parameters = self.get_decision_parameters(options)
# create a TaskGraphGenerator instance
import taskgraph.generator
import taskgraph.create
tgg = taskgraph.generator.TaskGraphGenerator(
- root=options['root'],
+ root_dir=options['root'],
log=self.log,
parameters=parameters,
optimization_finder=None) # XXX
# produce some artifacts
def write_artifact(filename, data):
self.log(logging.INFO, 'writing-artifact', {
'filename': filename,
@@ -269,17 +269,17 @@ class MachCommands(MachCommandBase):
def get_taskgraph_generator(self, options, parameters):
import taskgraph.generator
if options['optimize']:
optimization_finder = None # XXX function that searches index
else:
# null optmization finder
optimization_finder = lambda keys: {}
tgg = taskgraph.generator.TaskGraphGenerator(
- root=options['root'],
+ root_dir=options['root'],
log=self.log,
parameters=parameters,
optimization_finder=optimization_finder)
if 'task_set' in parameters:
tgg.set_task_set(parameters['task_set'])
return tgg
def show_taskgraph(self, taskgraph, options):
--- a/taskcluster/taskgraph/generator.py
+++ b/taskcluster/taskgraph/generator.py
@@ -5,31 +5,32 @@ import yaml
import functools
from taskgraph.graph import Graph
from taskgraph.types import Task, TaskGraph
class TaskGraphGenerator(object):
"""
The central controller for taskgraph. This handles all phases of graph
- generation.
+ generation. The task is generated from all of the kinds defined in
+ subdirectories of the generator's root directory.
Access to the results of this generation, at various phases, is available
via properties. This encourages the provision of all generation inputs at
instance construction time. The exception is `target_tasks`, which can be
set at any time until `target_task_set` is accessed; this allows the target
tasks to be determined based on `full_task_graph`.
"""
# Task-graph generation is implemented as a Python generator that yields
# each "phase" of generation. This allows some mach subcommands to short-
# circuit generation of the entire graph by never completing the generator.
- def __init__(self, root, log, parameters, optimization_finder):
- self.root = root
+ def __init__(self, root_dir, log, parameters, optimization_finder):
+ self.root_dir = root_dir
self.log = log
self.parameters = parameters
self.optimization_finder = optimization_finder
# this can be set up until the time the target task set is generated;
# it defaults to parameters['target_tasks']
self._target_tasks = parameters.get('target_tasks')
@@ -88,18 +89,18 @@ class TaskGraphGenerator(object):
have been optimized out are either omitted or replaced with a Task
instance containing only a task_id.
@type: TaskGraph
"""
return self._run_until('optimized_task_graph')
def _load_kinds(self):
- for path in os.listdir(self.root):
- path = os.path.join(self.root, path)
+ for path in os.listdir(self.root_dir):
+ path = os.path.join(self.root_dir, path)
if not os.path.isdir(path):
continue
name = os.path.basename(path)
self.log(logging.DEBUG, 'loading-kind', {
'name': name,
'path': path,
}, "loading kind `{name}` from {path}")
--- a/taskcluster/taskgraph/graph.py
+++ b/taskcluster/taskgraph/graph.py
@@ -2,27 +2,28 @@
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import unicode_literals
import collections
class Graph(object):
"""
- Generic representation of a directed acyclic graph with labeled edges.
- Graph operations are implemented in a functional manner, so the data
- structure is immutable.
+ Generic representation of a directed acyclic graph with labeled edges
+ connecting the nodes. Graph operations are implemented in a functional
+ manner, so the data structure is immutable.
- It permits at most one edge of a given anme between any set of nodes. The
+ It permits at most one edge of a given name between any set of nodes. The
graph is not checked for cycles, and methods may hang or otherwise fail if
given a cyclic graph.
The `nodes` and `edges` attributes may be accessed in a read-only fashion.
The `nodes` attribute is a set of node names, while `edges` is a set of
- `(start, end, label)` tuples.
+ `(left, right, name)` tuples representing an edge named `name` going from
+ node `left` to node `right..
"""
__slots__ = ['nodes', 'edges']
def __init__(self, nodes, edges):
"""
Create a graph. Nodes is a set of node names, while edges is a set of
(start, end) pairs of node names. Both values are used by reference,
@@ -49,18 +50,18 @@ class Graph(object):
assert nodes <= self.nodes
# generate a new graph by expanding along edges until reaching a fixed
# point
new_nodes, new_edges = nodes, set()
nodes, edges = set(), set()
while (new_nodes, new_edges) != (nodes, edges):
nodes, edges = new_nodes, new_edges
- add_edges = set((l, r, n) for (l, r, n) in self.edges if l in nodes)
- add_nodes = set(r for (l, r, n) in add_edges)
+ add_edges = set((left, right, name) for (left, right, name) in self.edges if left in nodes)
+ add_nodes = set(right for (_, right, _) in add_edges)
new_nodes = nodes | add_nodes
new_edges = edges | add_edges
return Graph(new_nodes, new_edges)
def visit_postorder(self):
"""
Generate a sequence of nodes in postorder, such that every node is
visited *after* any nodes it links to.
@@ -85,21 +86,21 @@ class Graph(object):
queue.append(node)
def links_dict(self):
"""
Return a dictionary mapping each node to a set of its downstream
nodes (omitting edge names)
"""
links = collections.defaultdict(lambda: set())
- for l, r, n in self.edges:
- links[l].add(r)
+ for left, right, _ in self.edges:
+ links[left].add(right)
return links
def reverse_links_dict(self):
"""
Return a dictionary mapping each node to a set of its upstream
nodes (omitting edge names)
"""
links = collections.defaultdict(lambda: set())
- for l, r, n in self.edges:
- links[r].add(l)
+ for left, right, _ in self.edges:
+ links[right].add(left)
return links
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/test/test_kind_legacy.py
@@ -0,0 +1,42 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+from __future__ import unicode_literals
+
+import sys
+import unittest
+
+from taskgraph.kind.legacy import LegacyKind, TASKID_PLACEHOLDER
+from taskgraph.types import Task
+
+from mozunit import main
+
+
+class TestLegacyKind(unittest.TestCase):
+ # NOTE: much of LegacyKind is copy-pasted from the old legacy code, which
+ # is emphatically *not* designed for testing, so this test class does not
+ # attempt to test the entire class.
+
+ def setUp(self):
+ def log(level, name, data, message):
+ pass
+ self.kind = LegacyKind('/root', {}, log)
+
+ def test_get_task_definition_artifact_sub(self):
+ "get_task_definition correctly substiatutes artifact URLs"
+ task_def = {
+ 'input_file': TASKID_PLACEHOLDER.format("G5BoWlCBTqOIhn3K3HyvWg"),
+ 'embedded': 'TASK={} FETCH=lazy'.format(
+ TASKID_PLACEHOLDER.format('G5BoWlCBTqOIhn3K3HyvWg')),
+ }
+ task = Task(self.kind, 'label', task=task_def)
+ dep_taskids = {TASKID_PLACEHOLDER.format('G5BoWlCBTqOIhn3K3HyvWg'): 'parent-taskid'}
+ task_def = self.kind.get_task_definition(task, dep_taskids)
+ self.assertEqual(task_def, {
+ 'input_file': 'parent-taskid',
+ 'embedded': 'TASK=parent-taskid FETCH=lazy',
+ })
+
+
+if __name__ == '__main__':
+ main()