Bug 1258497: add partial tests for legacy kind; r?gps draft
authorDustin J. Mitchell <dustin@mozilla.com>
Fri, 06 May 2016 16:21:02 +0000
changeset 364484 90e43e742dbd6ad8e64fd5edec2c3f58557ace75
parent 364483 1f63e71b12e87e4318612f437e22c4ee3b2b444e
child 364485 977bf0586ee122413e8f075c5453e38a758b18dc
child 364496 a0f01abbd288eaf7111ff2989d7c1979a414f6b7
push id17466
push userdmitchell@mozilla.com
push dateFri, 06 May 2016 18:22:06 +0000
reviewersgps
bugs1258497
milestone49.0a1
Bug 1258497: add partial tests for legacy kind; r?gps MozReview-Commit-ID: I15IRDbM8sV
taskcluster/mach_commands.py
taskcluster/taskgraph/generator.py
taskcluster/taskgraph/graph.py
taskcluster/taskgraph/test/test_kind_legacy.py
--- a/taskcluster/mach_commands.py
+++ b/taskcluster/mach_commands.py
@@ -150,17 +150,17 @@ class MachCommands(MachCommandBase):
     def taskgraph_decision(self, **options):
         # load parameters from env vars, command line, etc.
         parameters = self.get_decision_parameters(options)
 
         # create a TaskGraphGenerator instance
         import taskgraph.generator
         import taskgraph.create
         tgg = taskgraph.generator.TaskGraphGenerator(
-            root=options['root'],
+            root_dir=options['root'],
             log=self.log,
             parameters=parameters,
             optimization_finder=None)  # XXX
 
         # produce some artifacts
         def write_artifact(filename, data):
             self.log(logging.INFO, 'writing-artifact', {
                 'filename': filename,
@@ -269,17 +269,17 @@ class MachCommands(MachCommandBase):
     def get_taskgraph_generator(self, options, parameters):
         import taskgraph.generator
         if options['optimize']:
             optimization_finder = None  # XXX function that searches index
         else:
             # null optmization finder
             optimization_finder = lambda keys: {}
         tgg = taskgraph.generator.TaskGraphGenerator(
-            root=options['root'],
+            root_dir=options['root'],
             log=self.log,
             parameters=parameters,
             optimization_finder=optimization_finder)
         if 'task_set' in parameters:
             tgg.set_task_set(parameters['task_set'])
         return tgg
 
     def show_taskgraph(self, taskgraph, options):
--- a/taskcluster/taskgraph/generator.py
+++ b/taskcluster/taskgraph/generator.py
@@ -5,31 +5,32 @@ import yaml
 import functools
 
 from taskgraph.graph import Graph
 from taskgraph.types import Task, TaskGraph
 
 class TaskGraphGenerator(object):
     """
     The central controller for taskgraph.  This handles all phases of graph
-    generation.
+    generation.  The task is generated from all of the kinds defined in
+    subdirectories of the generator's root directory.
 
     Access to the results of this generation, at various phases, is available
     via properties.  This encourages the provision of all generation inputs at
     instance construction time.  The exception is `target_tasks`, which can be
     set at any time until `target_task_set` is accessed; this allows the target
     tasks to be determined based on `full_task_graph`.
     """
 
     # Task-graph generation is implemented as a Python generator that yields
     # each "phase" of generation.  This allows some mach subcommands to short-
     # circuit generation of the entire graph by never completing the generator.
 
-    def __init__(self, root, log, parameters, optimization_finder):
-        self.root = root
+    def __init__(self, root_dir, log, parameters, optimization_finder):
+        self.root_dir = root_dir
         self.log = log
         self.parameters = parameters
         self.optimization_finder = optimization_finder
 
         # this can be set up until the time the target task set is generated;
         # it defaults to parameters['target_tasks']
         self._target_tasks = parameters.get('target_tasks')
 
@@ -88,18 +89,18 @@ class TaskGraphGenerator(object):
         have been optimized out are either omitted or replaced with a Task
         instance containing only a task_id.
 
         @type: TaskGraph
         """
         return self._run_until('optimized_task_graph')
 
     def _load_kinds(self):
-        for path in os.listdir(self.root):
-            path = os.path.join(self.root, path)
+        for path in os.listdir(self.root_dir):
+            path = os.path.join(self.root_dir, path)
             if not os.path.isdir(path):
                 continue
             name = os.path.basename(path)
             self.log(logging.DEBUG, 'loading-kind', {
                 'name': name,
                 'path': path,
             }, "loading kind `{name}` from {path}")
 
--- a/taskcluster/taskgraph/graph.py
+++ b/taskcluster/taskgraph/graph.py
@@ -2,27 +2,28 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 from __future__ import unicode_literals
 
 import collections
 
 class Graph(object):
     """
-    Generic representation of a directed acyclic graph with labeled edges.
-    Graph operations are implemented in a functional manner, so the data
-    structure is immutable.
+    Generic representation of a directed acyclic graph with labeled edges
+    connecting the nodes.  Graph operations are implemented in a functional
+    manner, so the data structure is immutable.
 
-    It permits at most one edge of a given anme between any set of nodes.  The
+    It permits at most one edge of a given name between any set of nodes.  The
     graph is not checked for cycles, and methods may hang or otherwise fail if
     given a cyclic graph.
 
     The `nodes` and `edges` attributes may be accessed in a read-only fashion.
     The `nodes` attribute is a set of node names, while `edges` is a set of
-    `(start, end, label)` tuples.
+    `(left, right, name)` tuples representing an edge named `name` going from
+    node `left` to node `right..
     """
 
     __slots__ = ['nodes', 'edges']
 
     def __init__(self, nodes, edges):
         """
         Create a graph.  Nodes is a set of node names, while edges is a set of
         (start, end) pairs of node names.  Both values are used by reference,
@@ -49,18 +50,18 @@ class Graph(object):
         assert nodes <= self.nodes
 
         # generate a new graph by expanding along edges until reaching a fixed
         # point
         new_nodes, new_edges = nodes, set()
         nodes, edges = set(), set()
         while (new_nodes, new_edges) != (nodes, edges):
             nodes, edges = new_nodes, new_edges
-            add_edges = set((l, r, n) for (l, r, n) in self.edges if l in nodes)
-            add_nodes = set(r for (l, r, n) in add_edges)
+            add_edges = set((left, right, name) for (left, right, name) in self.edges if left in nodes)
+            add_nodes = set(right for (_, right, _) in add_edges)
             new_nodes = nodes | add_nodes
             new_edges = edges | add_edges
         return Graph(new_nodes, new_edges)
 
     def visit_postorder(self):
         """
         Generate a sequence of nodes in postorder, such that every node is
         visited *after* any nodes it links to.
@@ -85,21 +86,21 @@ class Graph(object):
                 queue.append(node)
 
     def links_dict(self):
         """
         Return a dictionary mapping each node to a set of its downstream
         nodes (omitting edge names)
         """
         links = collections.defaultdict(lambda: set())
-        for l, r, n in self.edges:
-            links[l].add(r)
+        for left, right, _ in self.edges:
+            links[left].add(right)
         return links
 
     def reverse_links_dict(self):
         """
         Return a dictionary mapping each node to a set of its upstream
         nodes (omitting edge names)
         """
         links = collections.defaultdict(lambda: set())
-        for l, r, n in self.edges:
-            links[r].add(l)
+        for left, right, _ in self.edges:
+            links[right].add(left)
         return links
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/test/test_kind_legacy.py
@@ -0,0 +1,42 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+from __future__ import unicode_literals
+
+import sys
+import unittest
+
+from taskgraph.kind.legacy import LegacyKind, TASKID_PLACEHOLDER
+from taskgraph.types import Task
+
+from mozunit import main
+
+
+class TestLegacyKind(unittest.TestCase):
+    # NOTE: much of LegacyKind is copy-pasted from the old legacy code, which
+    # is emphatically *not* designed for testing, so this test class does not
+    # attempt to test the entire class.
+
+    def setUp(self):
+        def log(level, name, data, message):
+            pass
+        self.kind = LegacyKind('/root', {}, log)
+
+    def test_get_task_definition_artifact_sub(self):
+        "get_task_definition correctly substiatutes artifact URLs"
+        task_def = {
+            'input_file': TASKID_PLACEHOLDER.format("G5BoWlCBTqOIhn3K3HyvWg"),
+            'embedded': 'TASK={} FETCH=lazy'.format(
+                TASKID_PLACEHOLDER.format('G5BoWlCBTqOIhn3K3HyvWg')),
+        }
+        task = Task(self.kind, 'label', task=task_def)
+        dep_taskids = {TASKID_PLACEHOLDER.format('G5BoWlCBTqOIhn3K3HyvWg'): 'parent-taskid'}
+        task_def = self.kind.get_task_definition(task, dep_taskids)
+        self.assertEqual(task_def, {
+            'input_file': 'parent-taskid',
+            'embedded': 'TASK=parent-taskid FETCH=lazy',
+        })
+
+
+if __name__ == '__main__':
+    main()