Bug 1275409: move templates to taskgraph.util; r?wcosta draft
authorDustin J. Mitchell <dustin@mozilla.com>
Sun, 05 Jun 2016 18:34:22 +0000
changeset 376699 f75814b82cc857ae345a90fbd9e2aad11053e8fd
parent 376698 0ef8fac5b8ee0b33aae690a350ba28ffede63a6a
child 376700 c51efa5b58107106349f6769620daf0286a3df73
push id20643
push userdmitchell@mozilla.com
push dateWed, 08 Jun 2016 13:31:04 +0000
reviewerswcosta
bugs1275409
milestone50.0a1
Bug 1275409: move templates to taskgraph.util; r?wcosta MozReview-Commit-ID: 3vdnm20W4OD
taskcluster/docs/index.rst
taskcluster/docs/yaml-templates.rst
taskcluster/taskgraph/kind/docker_image.py
taskcluster/taskgraph/kind/legacy.py
taskcluster/taskgraph/test/test_util.py
taskcluster/taskgraph/test/test_util_docker.py
taskcluster/taskgraph/test/test_util_templates.py
taskcluster/taskgraph/util.py
taskcluster/taskgraph/util/__init__.py
taskcluster/taskgraph/util/docker.py
taskcluster/taskgraph/util/templates.py
testing/taskcluster/taskcluster_graph/image_builder.py
testing/taskcluster/taskcluster_graph/templates.py
testing/taskcluster/tests/fixtures/child_pass.yml
testing/taskcluster/tests/fixtures/circular.yml
testing/taskcluster/tests/fixtures/circular_ref.yml
testing/taskcluster/tests/fixtures/deep/1.yml
testing/taskcluster/tests/fixtures/deep/2.yml
testing/taskcluster/tests/fixtures/deep/3.yml
testing/taskcluster/tests/fixtures/deep/4.yml
testing/taskcluster/tests/fixtures/extend_child.yml
testing/taskcluster/tests/fixtures/extend_parent.yml
testing/taskcluster/tests/fixtures/inherit.yml
testing/taskcluster/tests/fixtures/inherit_pass.yml
testing/taskcluster/tests/fixtures/simple.yml
testing/taskcluster/tests/fixtures/templates.yml
testing/taskcluster/tests/test_templates.py
--- a/taskcluster/docs/index.rst
+++ b/taskcluster/docs/index.rst
@@ -12,9 +12,10 @@ than you might suppose!  This implementa
  * "Try" pushes, with special means to select a subset of the graph for execution
  * Optimization -- skipping tasks that have already been performed
 
 .. toctree::
 
     taskgraph
     parameters
     attributes
+    yaml-templates
     old
new file mode 100644
--- /dev/null
+++ b/taskcluster/docs/yaml-templates.rst
@@ -0,0 +1,45 @@
+Task Definition YAML Templates
+==============================
+
+Many kinds of tasks are described using YAML files.  These files allow some
+limited forms of inheritance and template substitution as well as the usual
+YAML features, as described below.
+
+Please use these features sparingly.  In many cases, it is better to add a
+feature to the implementation of a task kind rather than add complexity to the
+YAML files.
+
+Inheritance
+-----------
+
+One YAML file can "inherit" from another by including a top-level ``$inherits``
+key.  That key specifies the parent file in ``from``, and optionally a
+collection of variables in ``variables``.  For example:
+
+.. code-block:: yaml
+
+    $inherits:
+      from: 'tasks/builds/base_linux32.yml'
+      variables:
+        build_name: 'linux32'
+        build_type: 'dbg'
+
+Inheritance proceeds as follows: First, the child document has its template
+substitutions performed and is parsed as YAML.  Then, the parent document is
+parsed, with substitutions specified by ``variables`` added to the template
+substitutions.  Finally, the child document is merged with the parent.
+
+To merge two JSON objects (dictionaries), each value is merged individually.
+Lists are merged by concatenating the lists from the parent and child
+documents.  Atomic values (strings, numbers, etc.) are merged by preferring the
+child document's value.
+
+Substitution
+------------
+
+Each document is expanded using the PyStache template engine before it is
+parsed as YAML.  The parameters for this expansion are specific to the task
+kind.
+
+Simple value substitution looks like ``{{variable}}``.  Function calls look
+like ``{{#function}}argument{{/function}}``.
--- a/taskcluster/taskgraph/kind/docker_image.py
+++ b/taskcluster/taskgraph/kind/docker_image.py
@@ -9,20 +9,20 @@ import json
 import os
 import urllib2
 import hashlib
 import tarfile
 import time
 
 from . import base
 from ..types import Task
-from taskgraph.util import docker_image
+from taskgraph.util.docker import docker_image
 import taskcluster_graph.transform.routes as routes_transform
 import taskcluster_graph.transform.treeherder as treeherder_transform
-from taskcluster_graph.templates import Templates
+from taskgraph.util.templates import Templates
 from taskcluster_graph.from_now import (
     json_time_from_now,
     current_json_time,
 )
 
 logger = logging.getLogger(__name__)
 GECKO = os.path.realpath(os.path.join(__file__, '..', '..', '..', '..'))
 ARTIFACT_URL = 'https://queue.taskcluster.net/v1/task/{}/artifacts/{}'
--- a/taskcluster/taskgraph/kind/legacy.py
+++ b/taskcluster/taskgraph/kind/legacy.py
@@ -27,19 +27,19 @@ from taskcluster_graph.mach_util import 
 )
 import taskcluster_graph.transform.routes as routes_transform
 import taskcluster_graph.transform.treeherder as treeherder_transform
 from taskcluster_graph.commit_parser import parse_commit
 from taskcluster_graph.from_now import (
     json_time_from_now,
     current_json_time,
 )
-from taskcluster_graph.templates import Templates
+from taskgraph.util.templates import Templates
 import taskcluster_graph.build_task
-from taskgraph.util import docker_image
+from taskgraph.util.docker import docker_image
 
 # TASKID_PLACEHOLDER is the "internal" form of a taskid; it is substituted with
 # actual taskIds at the very last minute, in get_task_definition
 TASKID_PLACEHOLDER = 'TaskLabel=={}'
 
 DEFINE_TASK = 'queue:define-task:aws-provisioner-v1/{}'
 DEFAULT_TRY = 'try: -b do -p all -u all -t all'
 DEFAULT_JOB_PATH = os.path.join(
rename from taskcluster/taskgraph/test/test_util.py
rename to taskcluster/taskgraph/test/test_util_docker.py
--- a/taskcluster/taskgraph/test/test_util.py
+++ b/taskcluster/taskgraph/test/test_util_docker.py
@@ -1,17 +1,17 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 import unittest
 
-from ..util import docker_image, DOCKER_ROOT
+from ..util.docker import docker_image, DOCKER_ROOT
 from mozunit import main, MockedOpen
 
 
 class TestDockerImage(unittest.TestCase):
 
     def test_docker_image_explicit_registry(self):
         files = {}
         files["{}/myimage/REGISTRY".format(DOCKER_ROOT)] = "cool-images"
rename from testing/taskcluster/tests/test_templates.py
rename to taskcluster/taskgraph/test/test_util_templates.py
--- a/testing/taskcluster/tests/test_templates.py
+++ b/taskcluster/taskgraph/test/test_util_templates.py
@@ -1,23 +1,124 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
 import os
 
 import unittest
 import mozunit
-from taskcluster_graph.templates import (
+import textwrap
+from taskgraph.util.templates import (
     Templates,
     TemplatesException
 )
 
+files = {}
+files['/fixtures/circular.yml'] = textwrap.dedent("""\
+    $inherits:
+      from: 'circular_ref.yml'
+      variables:
+        woot: 'inherit'
+    """)
+
+files['/fixtures/inherit.yml'] = textwrap.dedent("""\
+    $inherits:
+      from: 'templates.yml'
+      variables:
+        woot: 'inherit'
+    """)
+
+files['/fixtures/extend_child.yml'] = textwrap.dedent("""\
+    list: ['1', '2', '3']
+    was_list: ['1']
+    obj:
+      level: 1
+      deeper:
+        woot: 'bar'
+        list: ['baz']
+    """)
+
+files['/fixtures/circular_ref.yml'] = textwrap.dedent("""\
+    $inherits:
+      from: 'circular.yml'
+    """)
+
+files['/fixtures/child_pass.yml'] = textwrap.dedent("""\
+    values:
+      - {{a}}
+      - {{b}}
+      - {{c}}
+    """)
+
+files['/fixtures/inherit_pass.yml'] = textwrap.dedent("""\
+    $inherits:
+      from: 'child_pass.yml'
+      variables:
+        a: 'a'
+        b: 'b'
+        c: 'c'
+    """)
+
+files['/fixtures/deep/2.yml'] = textwrap.dedent("""\
+    $inherits:
+      from: deep/1.yml
+
+    """)
+
+files['/fixtures/deep/3.yml'] = textwrap.dedent("""\
+    $inherits:
+      from: deep/2.yml
+
+    """)
+
+files['/fixtures/deep/4.yml'] = textwrap.dedent("""\
+    $inherits:
+      from: deep/3.yml
+    """)
+
+files['/fixtures/deep/1.yml'] = textwrap.dedent("""\
+    variable: {{value}}
+    """)
+
+files['/fixtures/simple.yml'] = textwrap.dedent("""\
+    is_simple: true
+    """)
+
+files['/fixtures/templates.yml'] = textwrap.dedent("""\
+    content: 'content'
+    variable: '{{woot}}'
+    """)
+
+files['/fixtures/extend_parent.yml'] = textwrap.dedent("""\
+    $inherits:
+      from: 'extend_child.yml'
+
+    list: ['4']
+    was_list:
+      replaced: true
+    obj:
+      level: 2
+      from_parent: true
+      deeper:
+        list: ['bar']
+    """)
+
+
 class TemplatesTest(unittest.TestCase):
 
     def setUp(self):
-        abs_path = os.path.abspath(os.path.dirname(__file__))
-        self.subject = Templates(os.path.join(abs_path, 'fixtures'))
+        self.mocked_open = mozunit.MockedOpen(files)
+        self.mocked_open.__enter__()
+        self.subject = Templates('/fixtures')
 
+    def tearDown(self):
+        self.mocked_open.__exit__(None, None, None)
 
     def test_invalid_path(self):
         with self.assertRaisesRegexp(TemplatesException, 'must be a directory'):
             Templates('/zomg/not/a/dir')
 
     def test_no_templates(self):
         content = self.subject.load('simple.yml', {})
         self.assertEquals(content, {
rename from taskcluster/taskgraph/util.py
rename to taskcluster/taskgraph/util/__init__.py
--- a/taskcluster/taskgraph/util.py
+++ b/taskcluster/taskgraph/util/__init__.py
@@ -1,25 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-from __future__ import absolute_import, print_function, unicode_literals
-
-import os
-
-GECKO = os.path.realpath(os.path.join(__file__, '..', '..', '..'))
-DOCKER_ROOT = os.path.join(GECKO, 'testing', 'docker')
-
-def docker_image(name):
-    ''' Determine the docker image name, including repository and tag, from an
-    in-tree docker file'''
-    try:
-        with open(os.path.join(DOCKER_ROOT, name, 'REGISTRY')) as f:
-            registry = f.read().strip()
-    except IOError:
-        with open(os.path.join(DOCKER_ROOT, 'REGISTRY')) as f:
-            registry = f.read().strip()
-
-    with open(os.path.join(DOCKER_ROOT, name, 'VERSION')) as f:
-        version = f.read().strip()
-
-    return '{}/{}:{}'.format(registry, name, version)
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/util/docker.py
@@ -0,0 +1,26 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import os
+
+GECKO = os.path.realpath(os.path.join(__file__, '..', '..', '..', '..'))
+DOCKER_ROOT = os.path.join(GECKO, 'testing', 'docker')
+
+def docker_image(name):
+    '''Determine the docker image name, including repository and tag, from an
+    in-tree docker file.'''
+    try:
+        with open(os.path.join(DOCKER_ROOT, name, 'REGISTRY')) as f:
+            registry = f.read().strip()
+    except IOError:
+        with open(os.path.join(DOCKER_ROOT, 'REGISTRY')) as f:
+            registry = f.read().strip()
+
+    with open(os.path.join(DOCKER_ROOT, name, 'VERSION')) as f:
+        version = f.read().strip()
+
+    return '{}/{}:{}'.format(registry, name, version)
+
rename from testing/taskcluster/taskcluster_graph/templates.py
rename to taskcluster/taskgraph/util/templates.py
--- a/testing/taskcluster/taskcluster_graph/image_builder.py
+++ b/testing/taskcluster/taskcluster_graph/image_builder.py
@@ -3,17 +3,17 @@ import json
 import os
 import subprocess
 import tarfile
 import urllib2
 
 import taskcluster_graph.transform.routes as routes_transform
 import taskcluster_graph.transform.treeherder as treeherder_transform
 from slugid import nice as slugid
-from taskcluster_graph.templates import Templates
+from taskgraph.util.templates import Templates
 
 TASKCLUSTER_ROOT = os.path.abspath(os.path.join(os.path.dirname(os.path.realpath(__file__)), '..'))
 IMAGE_BUILD_TASK = os.path.join(TASKCLUSTER_ROOT, 'tasks', 'image.yml')
 GECKO = os.path.realpath(os.path.join(TASKCLUSTER_ROOT, '..', '..'))
 DOCKER_ROOT = os.path.join(GECKO, 'testing', 'docker')
 REGISTRY = open(os.path.join(DOCKER_ROOT, 'REGISTRY')).read().strip()
 INDEX_URL = 'https://index.taskcluster.net/v1/task/docker.images.v1.{}.{}.hash.{}'
 ARTIFACT_URL = 'https://queue.taskcluster.net/v1/task/{}/artifacts/{}'
deleted file mode 100644
--- a/testing/taskcluster/tests/fixtures/child_pass.yml
+++ /dev/null
@@ -1,4 +0,0 @@
-values:
-  - {{a}}
-  - {{b}}
-  - {{c}}
deleted file mode 100644
--- a/testing/taskcluster/tests/fixtures/circular.yml
+++ /dev/null
@@ -1,4 +0,0 @@
-$inherits:
-  from: 'circular_ref.yml'
-  variables:
-    woot: 'inherit'
deleted file mode 100644
--- a/testing/taskcluster/tests/fixtures/circular_ref.yml
+++ /dev/null
@@ -1,2 +0,0 @@
-$inherits:
-  from: 'circular.yml'
deleted file mode 100644
--- a/testing/taskcluster/tests/fixtures/deep/1.yml
+++ /dev/null
@@ -1,1 +0,0 @@
-variable: {{value}}
deleted file mode 100644
--- a/testing/taskcluster/tests/fixtures/deep/2.yml
+++ /dev/null
@@ -1,3 +0,0 @@
-$inherits:
-  from: deep/1.yml
-
deleted file mode 100644
--- a/testing/taskcluster/tests/fixtures/deep/3.yml
+++ /dev/null
@@ -1,3 +0,0 @@
-$inherits:
-  from: deep/2.yml
-
deleted file mode 100644
--- a/testing/taskcluster/tests/fixtures/deep/4.yml
+++ /dev/null
@@ -1,2 +0,0 @@
-$inherits:
-  from: deep/3.yml
deleted file mode 100644
--- a/testing/taskcluster/tests/fixtures/extend_child.yml
+++ /dev/null
@@ -1,7 +0,0 @@
-list: ['1', '2', '3']
-was_list: ['1']
-obj:
-  level: 1
-  deeper:
-    woot: 'bar'
-    list: ['baz']
deleted file mode 100644
--- a/testing/taskcluster/tests/fixtures/extend_parent.yml
+++ /dev/null
@@ -1,11 +0,0 @@
-$inherits:
-  from: 'extend_child.yml'
-
-list: ['4']
-was_list:
-  replaced: true
-obj:
-  level: 2
-  from_parent: true
-  deeper:
-    list: ['bar']
deleted file mode 100644
--- a/testing/taskcluster/tests/fixtures/inherit.yml
+++ /dev/null
@@ -1,4 +0,0 @@
-$inherits:
-  from: 'templates.yml'
-  variables:
-    woot: 'inherit'
deleted file mode 100644
--- a/testing/taskcluster/tests/fixtures/inherit_pass.yml
+++ /dev/null
@@ -1,6 +0,0 @@
-$inherits:
-  from: 'child_pass.yml'
-  variables:
-    a: 'a'
-    b: 'b'
-    c: 'c'
deleted file mode 100644
--- a/testing/taskcluster/tests/fixtures/simple.yml
+++ /dev/null
@@ -1,1 +0,0 @@
-is_simple: true
deleted file mode 100644
--- a/testing/taskcluster/tests/fixtures/templates.yml
+++ /dev/null
@@ -1,2 +0,0 @@
-content: 'content'
-variable: '{{woot}}'