Bug 1275409: move templates to taskgraph.util; r?wcosta
MozReview-Commit-ID: 3vdnm20W4OD
--- a/taskcluster/docs/index.rst
+++ b/taskcluster/docs/index.rst
@@ -12,9 +12,10 @@ than you might suppose! This implementa
* "Try" pushes, with special means to select a subset of the graph for execution
* Optimization -- skipping tasks that have already been performed
.. toctree::
taskgraph
parameters
attributes
+ yaml-templates
old
new file mode 100644
--- /dev/null
+++ b/taskcluster/docs/yaml-templates.rst
@@ -0,0 +1,45 @@
+Task Definition YAML Templates
+==============================
+
+Many kinds of tasks are described using YAML files. These files allow some
+limited forms of inheritance and template substitution as well as the usual
+YAML features, as described below.
+
+Please use these features sparingly. In many cases, it is better to add a
+feature to the implementation of a task kind rather than add complexity to the
+YAML files.
+
+Inheritance
+-----------
+
+One YAML file can "inherit" from another by including a top-level ``$inherits``
+key. That key specifies the parent file in ``from``, and optionally a
+collection of variables in ``variables``. For example:
+
+.. code-block:: yaml
+
+ $inherits:
+ from: 'tasks/builds/base_linux32.yml'
+ variables:
+ build_name: 'linux32'
+ build_type: 'dbg'
+
+Inheritance proceeds as follows: First, the child document has its template
+substitutions performed and is parsed as YAML. Then, the parent document is
+parsed, with substitutions specified by ``variables`` added to the template
+substitutions. Finally, the child document is merged with the parent.
+
+To merge two JSON objects (dictionaries), each value is merged individually.
+Lists are merged by concatenating the lists from the parent and child
+documents. Atomic values (strings, numbers, etc.) are merged by preferring the
+child document's value.
+
+Substitution
+------------
+
+Each document is expanded using the PyStache template engine before it is
+parsed as YAML. The parameters for this expansion are specific to the task
+kind.
+
+Simple value substitution looks like ``{{variable}}``. Function calls look
+like ``{{#function}}argument{{/function}}``.
--- a/taskcluster/taskgraph/kind/docker_image.py
+++ b/taskcluster/taskgraph/kind/docker_image.py
@@ -9,20 +9,20 @@ import json
import os
import urllib2
import hashlib
import tarfile
import time
from . import base
from ..types import Task
-from taskgraph.util import docker_image
+from taskgraph.util.docker import docker_image
import taskcluster_graph.transform.routes as routes_transform
import taskcluster_graph.transform.treeherder as treeherder_transform
-from taskcluster_graph.templates import Templates
+from taskgraph.util.templates import Templates
from taskcluster_graph.from_now import (
json_time_from_now,
current_json_time,
)
logger = logging.getLogger(__name__)
GECKO = os.path.realpath(os.path.join(__file__, '..', '..', '..', '..'))
ARTIFACT_URL = 'https://queue.taskcluster.net/v1/task/{}/artifacts/{}'
--- a/taskcluster/taskgraph/kind/legacy.py
+++ b/taskcluster/taskgraph/kind/legacy.py
@@ -27,19 +27,19 @@ from taskcluster_graph.mach_util import
)
import taskcluster_graph.transform.routes as routes_transform
import taskcluster_graph.transform.treeherder as treeherder_transform
from taskcluster_graph.commit_parser import parse_commit
from taskcluster_graph.from_now import (
json_time_from_now,
current_json_time,
)
-from taskcluster_graph.templates import Templates
+from taskgraph.util.templates import Templates
import taskcluster_graph.build_task
-from taskgraph.util import docker_image
+from taskgraph.util.docker import docker_image
# TASKID_PLACEHOLDER is the "internal" form of a taskid; it is substituted with
# actual taskIds at the very last minute, in get_task_definition
TASKID_PLACEHOLDER = 'TaskLabel=={}'
DEFINE_TASK = 'queue:define-task:aws-provisioner-v1/{}'
DEFAULT_TRY = 'try: -b do -p all -u all -t all'
DEFAULT_JOB_PATH = os.path.join(
rename from taskcluster/taskgraph/test/test_util.py
rename to taskcluster/taskgraph/test/test_util_docker.py
--- a/taskcluster/taskgraph/test/test_util.py
+++ b/taskcluster/taskgraph/test/test_util_docker.py
@@ -1,17 +1,17 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import, print_function, unicode_literals
import unittest
-from ..util import docker_image, DOCKER_ROOT
+from ..util.docker import docker_image, DOCKER_ROOT
from mozunit import main, MockedOpen
class TestDockerImage(unittest.TestCase):
def test_docker_image_explicit_registry(self):
files = {}
files["{}/myimage/REGISTRY".format(DOCKER_ROOT)] = "cool-images"
rename from testing/taskcluster/tests/test_templates.py
rename to taskcluster/taskgraph/test/test_util_templates.py
--- a/testing/taskcluster/tests/test_templates.py
+++ b/taskcluster/taskgraph/test/test_util_templates.py
@@ -1,23 +1,124 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
import os
import unittest
import mozunit
-from taskcluster_graph.templates import (
+import textwrap
+from taskgraph.util.templates import (
Templates,
TemplatesException
)
+files = {}
+files['/fixtures/circular.yml'] = textwrap.dedent("""\
+ $inherits:
+ from: 'circular_ref.yml'
+ variables:
+ woot: 'inherit'
+ """)
+
+files['/fixtures/inherit.yml'] = textwrap.dedent("""\
+ $inherits:
+ from: 'templates.yml'
+ variables:
+ woot: 'inherit'
+ """)
+
+files['/fixtures/extend_child.yml'] = textwrap.dedent("""\
+ list: ['1', '2', '3']
+ was_list: ['1']
+ obj:
+ level: 1
+ deeper:
+ woot: 'bar'
+ list: ['baz']
+ """)
+
+files['/fixtures/circular_ref.yml'] = textwrap.dedent("""\
+ $inherits:
+ from: 'circular.yml'
+ """)
+
+files['/fixtures/child_pass.yml'] = textwrap.dedent("""\
+ values:
+ - {{a}}
+ - {{b}}
+ - {{c}}
+ """)
+
+files['/fixtures/inherit_pass.yml'] = textwrap.dedent("""\
+ $inherits:
+ from: 'child_pass.yml'
+ variables:
+ a: 'a'
+ b: 'b'
+ c: 'c'
+ """)
+
+files['/fixtures/deep/2.yml'] = textwrap.dedent("""\
+ $inherits:
+ from: deep/1.yml
+
+ """)
+
+files['/fixtures/deep/3.yml'] = textwrap.dedent("""\
+ $inherits:
+ from: deep/2.yml
+
+ """)
+
+files['/fixtures/deep/4.yml'] = textwrap.dedent("""\
+ $inherits:
+ from: deep/3.yml
+ """)
+
+files['/fixtures/deep/1.yml'] = textwrap.dedent("""\
+ variable: {{value}}
+ """)
+
+files['/fixtures/simple.yml'] = textwrap.dedent("""\
+ is_simple: true
+ """)
+
+files['/fixtures/templates.yml'] = textwrap.dedent("""\
+ content: 'content'
+ variable: '{{woot}}'
+ """)
+
+files['/fixtures/extend_parent.yml'] = textwrap.dedent("""\
+ $inherits:
+ from: 'extend_child.yml'
+
+ list: ['4']
+ was_list:
+ replaced: true
+ obj:
+ level: 2
+ from_parent: true
+ deeper:
+ list: ['bar']
+ """)
+
+
class TemplatesTest(unittest.TestCase):
def setUp(self):
- abs_path = os.path.abspath(os.path.dirname(__file__))
- self.subject = Templates(os.path.join(abs_path, 'fixtures'))
+ self.mocked_open = mozunit.MockedOpen(files)
+ self.mocked_open.__enter__()
+ self.subject = Templates('/fixtures')
+ def tearDown(self):
+ self.mocked_open.__exit__(None, None, None)
def test_invalid_path(self):
with self.assertRaisesRegexp(TemplatesException, 'must be a directory'):
Templates('/zomg/not/a/dir')
def test_no_templates(self):
content = self.subject.load('simple.yml', {})
self.assertEquals(content, {
rename from taskcluster/taskgraph/util.py
rename to taskcluster/taskgraph/util/__init__.py
--- a/taskcluster/taskgraph/util.py
+++ b/taskcluster/taskgraph/util/__init__.py
@@ -1,25 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-from __future__ import absolute_import, print_function, unicode_literals
-
-import os
-
-GECKO = os.path.realpath(os.path.join(__file__, '..', '..', '..'))
-DOCKER_ROOT = os.path.join(GECKO, 'testing', 'docker')
-
-def docker_image(name):
- ''' Determine the docker image name, including repository and tag, from an
- in-tree docker file'''
- try:
- with open(os.path.join(DOCKER_ROOT, name, 'REGISTRY')) as f:
- registry = f.read().strip()
- except IOError:
- with open(os.path.join(DOCKER_ROOT, 'REGISTRY')) as f:
- registry = f.read().strip()
-
- with open(os.path.join(DOCKER_ROOT, name, 'VERSION')) as f:
- version = f.read().strip()
-
- return '{}/{}:{}'.format(registry, name, version)
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/util/docker.py
@@ -0,0 +1,26 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import os
+
+GECKO = os.path.realpath(os.path.join(__file__, '..', '..', '..', '..'))
+DOCKER_ROOT = os.path.join(GECKO, 'testing', 'docker')
+
+def docker_image(name):
+ '''Determine the docker image name, including repository and tag, from an
+ in-tree docker file.'''
+ try:
+ with open(os.path.join(DOCKER_ROOT, name, 'REGISTRY')) as f:
+ registry = f.read().strip()
+ except IOError:
+ with open(os.path.join(DOCKER_ROOT, 'REGISTRY')) as f:
+ registry = f.read().strip()
+
+ with open(os.path.join(DOCKER_ROOT, name, 'VERSION')) as f:
+ version = f.read().strip()
+
+ return '{}/{}:{}'.format(registry, name, version)
+
rename from testing/taskcluster/taskcluster_graph/templates.py
rename to taskcluster/taskgraph/util/templates.py
--- a/testing/taskcluster/taskcluster_graph/image_builder.py
+++ b/testing/taskcluster/taskcluster_graph/image_builder.py
@@ -3,17 +3,17 @@ import json
import os
import subprocess
import tarfile
import urllib2
import taskcluster_graph.transform.routes as routes_transform
import taskcluster_graph.transform.treeherder as treeherder_transform
from slugid import nice as slugid
-from taskcluster_graph.templates import Templates
+from taskgraph.util.templates import Templates
TASKCLUSTER_ROOT = os.path.abspath(os.path.join(os.path.dirname(os.path.realpath(__file__)), '..'))
IMAGE_BUILD_TASK = os.path.join(TASKCLUSTER_ROOT, 'tasks', 'image.yml')
GECKO = os.path.realpath(os.path.join(TASKCLUSTER_ROOT, '..', '..'))
DOCKER_ROOT = os.path.join(GECKO, 'testing', 'docker')
REGISTRY = open(os.path.join(DOCKER_ROOT, 'REGISTRY')).read().strip()
INDEX_URL = 'https://index.taskcluster.net/v1/task/docker.images.v1.{}.{}.hash.{}'
ARTIFACT_URL = 'https://queue.taskcluster.net/v1/task/{}/artifacts/{}'
deleted file mode 100644
--- a/testing/taskcluster/tests/fixtures/child_pass.yml
+++ /dev/null
@@ -1,4 +0,0 @@
-values:
- - {{a}}
- - {{b}}
- - {{c}}
deleted file mode 100644
--- a/testing/taskcluster/tests/fixtures/circular.yml
+++ /dev/null
@@ -1,4 +0,0 @@
-$inherits:
- from: 'circular_ref.yml'
- variables:
- woot: 'inherit'
deleted file mode 100644
--- a/testing/taskcluster/tests/fixtures/circular_ref.yml
+++ /dev/null
@@ -1,2 +0,0 @@
-$inherits:
- from: 'circular.yml'
deleted file mode 100644
--- a/testing/taskcluster/tests/fixtures/deep/1.yml
+++ /dev/null
@@ -1,1 +0,0 @@
-variable: {{value}}
deleted file mode 100644
--- a/testing/taskcluster/tests/fixtures/deep/2.yml
+++ /dev/null
@@ -1,3 +0,0 @@
-$inherits:
- from: deep/1.yml
-
deleted file mode 100644
--- a/testing/taskcluster/tests/fixtures/deep/3.yml
+++ /dev/null
@@ -1,3 +0,0 @@
-$inherits:
- from: deep/2.yml
-
deleted file mode 100644
--- a/testing/taskcluster/tests/fixtures/deep/4.yml
+++ /dev/null
@@ -1,2 +0,0 @@
-$inherits:
- from: deep/3.yml
deleted file mode 100644
--- a/testing/taskcluster/tests/fixtures/extend_child.yml
+++ /dev/null
@@ -1,7 +0,0 @@
-list: ['1', '2', '3']
-was_list: ['1']
-obj:
- level: 1
- deeper:
- woot: 'bar'
- list: ['baz']
deleted file mode 100644
--- a/testing/taskcluster/tests/fixtures/extend_parent.yml
+++ /dev/null
@@ -1,11 +0,0 @@
-$inherits:
- from: 'extend_child.yml'
-
-list: ['4']
-was_list:
- replaced: true
-obj:
- level: 2
- from_parent: true
- deeper:
- list: ['bar']
deleted file mode 100644
--- a/testing/taskcluster/tests/fixtures/inherit.yml
+++ /dev/null
@@ -1,4 +0,0 @@
-$inherits:
- from: 'templates.yml'
- variables:
- woot: 'inherit'
deleted file mode 100644
--- a/testing/taskcluster/tests/fixtures/inherit_pass.yml
+++ /dev/null
@@ -1,6 +0,0 @@
-$inherits:
- from: 'child_pass.yml'
- variables:
- a: 'a'
- b: 'b'
- c: 'c'
deleted file mode 100644
--- a/testing/taskcluster/tests/fixtures/simple.yml
+++ /dev/null
@@ -1,1 +0,0 @@
-is_simple: true
deleted file mode 100644
--- a/testing/taskcluster/tests/fixtures/templates.yml
+++ /dev/null
@@ -1,2 +0,0 @@
-content: 'content'
-variable: '{{woot}}'