Bug 1274611: implement docker image builds as a distinct kind; r?wcosta draft
authorDustin J. Mitchell <dustin@mozilla.com>
Sun, 05 Jun 2016 19:15:53 +0000
changeset 375895 15a2bfe11cb9ca68c9e986584a0417c8b1ebbecd
parent 375505 9c8429f2f93f865dca5fc19784bb150eff3565c3
child 375896 a1aa4dc77d1c63815f32b6b07e1a17d562d63c09
push id20422
push userdmitchell@mozilla.com
push dateMon, 06 Jun 2016 22:28:57 +0000
reviewerswcosta
bugs1274611
milestone49.0a1
Bug 1274611: implement docker image builds as a distinct kind; r?wcosta MozReview-Commit-ID: 81Ad9LcBdx6
taskcluster/ci/docker-image/image.yml
taskcluster/ci/docker-image/kind.yml
taskcluster/docs/attributes.rst
taskcluster/taskgraph/kind/docker_image.py
taskcluster/taskgraph/kind/legacy.py
taskcluster/taskgraph/test/test_kind_docker_image.py
taskcluster/taskgraph/test/test_util.py
taskcluster/taskgraph/util.py
testing/taskcluster/routes.json
testing/taskcluster/taskcluster_graph/image_builder.py
testing/taskcluster/tasks/builds/android_api_15_gradle_dependencies.yml
testing/taskcluster/tasks/builds/b2g_base.yml
testing/taskcluster/tasks/builds/firefox_base.yml
testing/taskcluster/tasks/builds/mobile_base.yml
testing/taskcluster/tasks/harness_test.yml
testing/taskcluster/tasks/post-builds/mulet_simulator.yml
testing/taskcluster/tasks/test.yml
testing/taskcluster/tasks/tests/eslint-gecko.yml
testing/taskcluster/tasks/tests/fx_test_base.yml
testing/taskcluster/tasks/tests/mozharness-gecko.yml
testing/taskcluster/tasks/tests/mozlint-flake8.yml
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/docker-image/image.yml
@@ -0,0 +1,56 @@
+---
+task:
+  created: '{{now}}'
+  deadline: '{{#from_now}}24 hours{{/from_now}}'
+  metadata:
+    name: 'Docker Image Build: {{image_name}}'
+    description: 'Build the docker image {{image_name}} for use by dependent tasks'
+    source: '{{source}}'
+    owner: mozilla-taskcluster-maintenance@mozilla.com
+  tags:
+    createdForUser: {{owner}}
+
+  workerType: taskcluster-images
+  provisionerId: aws-provisioner-v1
+
+  routes:
+      - index.docker.images.v1.{{project}}.{{image_name}}.latest
+      - index.docker.images.v1.{{project}}.{{image_name}}.pushdate.{{year}}.{{month}}-{{day}}-{{pushtime}}
+      - index.docker.images.v1.{{project}}.{{image_name}}.hash.{{context_hash}}
+      - tc-treeherder.{{project}}.{{revision_hash}}
+      - tc-treeherder-stage.{{project}}.{{revision_hash}}
+
+  payload:
+    env:
+      HASH: '{{context_hash}}'
+      PROJECT: '{{project}}'
+      CONTEXT_URL: '{{context_url}}'
+      CONTEXT_PATH: '{{context_path}}'
+      BASE_REPOSITORY: '{{base_repository}}'
+      HEAD_REPOSITORY: '{{head_repository}}'
+      HEAD_REV: '{{head_rev}}'
+      HEAD_REF: '{{head_ref}}'
+    features:
+      dind: true
+    image: '{{#docker_image}}image_builder{{/docker_image}}'
+    command:
+      - /bin/bash
+      - -c
+      - /home/worker/bin/build_image.sh
+    maxRunTime: 3600
+    artifacts:
+      '{{artifact_path}}':
+        type: 'file'
+        path: '/artifacts/image.tar'
+        expires: '{{#from_now}}1 year{{/from_now}}'
+  extra:
+    treeherderEnv:
+      - staging
+      - production
+    treeherder:
+      revision: {{head_rev}}
+      revision_hash: {{revision_hash}}
+      build:
+        platform: 'taskcluster-images'
+      symbol: 'I'
+
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/docker-image/kind.yml
@@ -0,0 +1,19 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+implementation: 'taskgraph.kind.docker_image:DockerImageKind'
+images_path: '../../../testing/docker'
+
+# make a task for each docker-image we might want.  For the moment, since we
+# write artifacts for each, these are whitelisted, but ideally that will change
+# (to use subdirectory clones of the proper directory), at which point we can
+# generate tasks for every docker image in the directory, secure in the
+# knowledge that unnecessary images will be omitted from the target task graph
+images:
+  - desktop-test
+  - desktop-build
+  - builder
+  - tester
+  - lint
+  - android-gradle-build
--- a/taskcluster/docs/attributes.rst
+++ b/taskcluster/docs/attributes.rst
@@ -88,8 +88,12 @@ job
 of a post-build job).  This is valid only for the ``legacy`` kind.
 
 post_build
 ==========
 
 (deprecated) The name of the post-build activity.  This is valid only for the
 ``legacy`` kind.
 
+image_name
+==========
+
+For the ``docker_image`` kind, this attribute contains the docker image name.
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/kind/docker_image.py
@@ -0,0 +1,162 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import logging
+import json
+import os
+import urllib2
+import hashlib
+import tarfile
+import time
+
+from . import base
+from ..types import Task
+from taskgraph.util import docker_image
+import taskcluster_graph.transform.routes as routes_transform
+import taskcluster_graph.transform.treeherder as treeherder_transform
+from taskcluster_graph.templates import Templates
+from taskcluster_graph.from_now import (
+    json_time_from_now,
+    current_json_time,
+)
+
+logger = logging.getLogger(__name__)
+GECKO = os.path.realpath(os.path.join(__file__, '..', '..', '..', '..'))
+ARTIFACT_URL = 'https://queue.taskcluster.net/v1/task/{}/artifacts/{}'
+INDEX_URL = 'https://index.taskcluster.net/v1/task/{}'
+
+
+class DockerImageKind(base.Kind):
+
+    def load_tasks(self, params):
+        # TODO: make this match the pushdate (get it from a parameter rather than vcs)
+        pushdate = time.strftime('%Y%m%d%H%M%S', time.gmtime())
+
+        parameters = {
+            'pushdate': pushdate,
+            'pushtime': pushdate[8:],
+            'year': pushdate[0:4],
+            'month': pushdate[4:6],
+            'day': pushdate[6:8],
+            'project': params['project'],
+            'docker_image': docker_image,
+            'base_repository': params['base_repository'] or params['head_repository'],
+            'head_repository': params['head_repository'],
+            'head_ref': params['head_ref'] or params['head_rev'],
+            'head_rev': params['head_rev'],
+            'owner': params['owner'],
+            'level': params['level'],
+            'from_now': json_time_from_now,
+            'now': current_json_time(),
+            'revision_hash': params['revision_hash'],
+            'source': '{repo}file/{rev}/testing/taskcluster/tasks/image.yml'
+                    .format(repo=params['head_repository'], rev=params['head_rev']),
+        }
+
+        tasks = []
+        templates = Templates(self.path)
+        for image_name in self.config['images']:
+            context_path = os.path.join('testing', 'docker', image_name)
+            context_hash = self.generate_context_hash(context_path)
+
+            image_parameters = dict(parameters)
+            image_parameters['context_hash'] = context_hash
+            image_parameters['context_path'] = context_path
+            image_parameters['artifact_path'] = 'public/image.tar'
+            image_parameters['image_name'] = image_name
+
+            image_artifact_path = "public/decision_task/image_contexts/{}/context.tar.gz".format(image_name)
+            if os.environ.get('TASK_ID'):
+                destination = os.path.join(
+                    os.environ['HOME'],
+                    "artifacts/decision_task/image_contexts/{}/context.tar.gz".format(image_name))
+                image_parameters['context_url'] = ARTIFACT_URL.format(os.environ['TASK_ID'], image_artifact_path)
+                self.create_context_tar(context_path, destination, image_name)
+            else:
+                # skip context generation since this isn't a decision task
+                # TODO: generate context tarballs using subdirectory clones in
+                # the image-building task so we don't have to worry about this.
+                image_parameters['context_url'] = 'file:///tmp/' + image_artifact_path
+
+            image_task = templates.load('image.yml', image_parameters)
+
+            attributes = {
+                'kind': self.name,
+                'image_name': image_name,
+            }
+
+            # As an optimization, if the context hash exists for mozilla-central, that image
+            # task ID will be used.  The reasoning behind this is that eventually everything ends
+            # up on mozilla-central at some point if most tasks use this as a common image
+            # for a given context hash, a worker within Taskcluster does not need to contain
+            # the same image per branch.
+            index_paths = ['docker.images.v1.{}.{}.hash.{}'.format(project, image_name, context_hash)
+                           for project in ['mozilla-central', params['project']]]
+
+            tasks.append(Task(self, 'build-docker-image-' + image_name,
+                              task=image_task['task'], attributes=attributes,
+                              index_paths=index_paths))
+
+        return tasks
+
+    def get_task_dependencies(self, task, taskgraph):
+        return []
+
+    def optimize_task(self, task, taskgraph):
+        for index_path in task.extra['index_paths']:
+            try:
+                url = INDEX_URL.format(index_path)
+                existing_task = json.load(urllib2.urlopen(url))
+                # Only return the task ID if the artifact exists for the indexed
+                # task.  Otherwise, continue on looking at each of the branches.  Method
+                # continues trying other branches in case mozilla-central has an expired
+                # artifact, but 'project' might not. Only return no task ID if all
+                # branches have been tried
+                request = urllib2.Request(ARTIFACT_URL.format(existing_task['taskId'], 'public/image.tar'))
+                request.get_method = lambda: 'HEAD'
+                urllib2.urlopen(request)
+
+                # HEAD success on the artifact is enough
+                return True, existing_task['taskId']
+            except urllib2.HTTPError:
+                pass
+
+        return False, None
+
+    def create_context_tar(self, context_dir, destination, image_name):
+        'Creates a tar file of a particular context directory.'
+        destination = os.path.abspath(destination)
+        if not os.path.exists(os.path.dirname(destination)):
+            os.makedirs(os.path.dirname(destination))
+
+        with tarfile.open(destination, 'w:gz') as tar:
+            tar.add(context_dir, arcname=image_name)
+
+    def generate_context_hash(self, image_path):
+        '''Generates a sha256 hash for context directory used to build an image.
+
+        Contents of the directory are sorted alphabetically, contents of each file is hashed,
+        and then a hash is created for both the file hashes as well as their paths.
+
+        This ensures that hashs are consistent and also change based on if file locations
+        within the context directory change.
+        '''
+        context_hash = hashlib.sha256()
+        files = []
+
+        for dirpath, dirnames, filenames in os.walk(os.path.join(GECKO, image_path)):
+            for filename in filenames:
+                files.append(os.path.join(dirpath, filename))
+
+        for filename in sorted(files):
+            relative_filename = filename.replace(GECKO, '')
+            with open(filename, 'rb') as f:
+                file_hash = hashlib.sha256()
+                data = f.read()
+                file_hash.update(data)
+                context_hash.update(file_hash.hexdigest() + '\t' + relative_filename + '\n')
+
+        return context_hash.hexdigest()
--- a/taskcluster/taskgraph/kind/legacy.py
+++ b/taskcluster/taskgraph/kind/legacy.py
@@ -23,27 +23,23 @@ from taskcluster_graph.mach_util import 
     configure_dependent_task,
     set_interactive_task,
     remove_caches_from_task,
     query_vcs_info
 )
 import taskcluster_graph.transform.routes as routes_transform
 import taskcluster_graph.transform.treeherder as treeherder_transform
 from taskcluster_graph.commit_parser import parse_commit
-from taskcluster_graph.image_builder import (
-    docker_image,
-    normalize_image_details,
-    task_id_for_image
-)
 from taskcluster_graph.from_now import (
     json_time_from_now,
     current_json_time,
 )
 from taskcluster_graph.templates import Templates
 import taskcluster_graph.build_task
+from taskgraph.util import docker_image
 
 # TASKID_PLACEHOLDER is the "internal" form of a taskid; it is substituted with
 # actual taskIds at the very last minute, in get_task_definition
 TASKID_PLACEHOLDER = 'TaskLabel=={}'
 
 DEFINE_TASK = 'queue:define-task:aws-provisioner-v1/{}'
 DEFAULT_TRY = 'try: -b do -p all -u all -t all'
 DEFAULT_JOB_PATH = os.path.join(
@@ -71,16 +67,18 @@ def set_expiration(task, timestamp):
     try:
         artifacts = task_def['payload']['artifacts']
     except KeyError:
         return
 
     for artifact in artifacts.values():
         artifact['expires'] = timestamp
 
+
+
 class LegacyKind(base.Kind):
     """
     This kind generates a full task graph from the old YAML files in
     `testing/taskcluster/tasks`.  The tasks already have dependency links.
 
     The existing task-graph generation generates slugids for tasks during task
     generation, so this kind labels tasks using those slugids, with a prefix of
     "TaskLabel==".  These labels are unfortunately not stable from run to run.
@@ -116,23 +114,21 @@ class LegacyKind(base.Kind):
             logger.debug('{} commits influencing task scheduling:'.format(len(vcs_info.changesets)))
             for c in vcs_info.changesets:
                 logger.debug("{cset} {desc}".format(
                     cset=c['node'][0:12],
                     desc=c['desc'].splitlines()[0].encode('ascii', 'ignore')))
                 changed_files |= set(c['files'])
 
         # Template parameters used when expanding the graph
-        seen_images = {}
         parameters = dict(gaia_info().items() + {
             'index': 'index',
             'project': project,
             'pushlog_id': params.get('pushlog_id', 0),
             'docker_image': docker_image,
-            'task_id_for_image': partial(task_id_for_image, seen_images, project),
             'base_repository': params['base_repository'] or
             params['head_repository'],
             'head_repository': params['head_repository'],
             'head_ref': params['head_ref'] or params['head_rev'],
             'head_rev': params['head_rev'],
             'pushdate': pushdate,
             'pushtime': pushdate[8:],
             'year': pushdate[0:4],
@@ -226,21 +222,16 @@ class LegacyKind(base.Kind):
 
             # Copy build_* attributes to expose them to post-build tasks
             # as well as json routes and tests
             task_extra = build_task['task']['extra']
             build_parameters['build_name'] = task_extra['build_name']
             build_parameters['build_type'] = task_extra['build_type']
             build_parameters['build_product'] = task_extra['build_product']
 
-            normalize_image_details(graph,
-                                    build_task,
-                                    seen_images,
-                                    build_parameters,
-                                    os.environ.get('TASK_ID', None))
             set_interactive_task(build_task, interactive)
 
             # try builds don't use cache
             if project == "try":
                 remove_caches_from_task(build_task)
                 set_expiration(build_task, json_time_from_now(TRY_EXPIRATION))
 
             if params['revision_hash']:
@@ -314,21 +305,16 @@ class LegacyKind(base.Kind):
                 # enabled for build tasks
                 post_parameters = merge_dicts(build_parameters,
                                               post_build.get('additional-parameters', {}))
                 post_task = configure_dependent_task(post_build['task'],
                                                      post_parameters,
                                                      mklabel(),
                                                      templates,
                                                      build_treeherder_config)
-                normalize_image_details(graph,
-                                        post_task,
-                                        seen_images,
-                                        build_parameters,
-                                        os.environ.get('TASK_ID', None))
                 set_interactive_task(post_task, interactive)
                 treeherder_transform.add_treeherder_revision_info(post_task['task'],
                                                                   params['head_rev'],
                                                                   params['revision_hash'])
 
                 if project == "try":
                     set_expiration(post_task, json_time_from_now(TRY_EXPIRATION))
 
@@ -368,21 +354,16 @@ class LegacyKind(base.Kind):
 
                     if chunked:
                         test_parameters['chunk'] = chunk
                     test_task = configure_dependent_task(test['task'],
                                                          test_parameters,
                                                          mklabel(),
                                                          templates,
                                                          build_treeherder_config)
-                    normalize_image_details(graph,
-                                            test_task,
-                                            seen_images,
-                                            build_parameters,
-                                            os.environ.get('TASK_ID', None))
                     set_interactive_task(test_task, interactive)
 
                     if params['revision_hash']:
                         treeherder_transform.add_treeherder_revision_info(test_task['task'],
                                                                           params['head_rev'],
                                                                           params['revision_hash'])
                         routes_transform.decorate_task_treeherder_routes(
                             test_task['task'],
@@ -428,14 +409,20 @@ class LegacyKind(base.Kind):
         # taskId for each task, and we use those as the *labels* for the tasks;
         # taskgraph will later assign them new taskIds.
         return [Task(self, t['taskId'], task=t['task'], attributes=t['attributes'])
                 for t in self.graph['tasks']]
 
     def get_task_dependencies(self, task, taskgraph):
         # fetch dependency information from the cached graph
         taskdict = self.tasks_by_label[task.label]
-        return [(label, label) for label in taskdict.get('requires', [])]
+        deps = [(label, label) for label in taskdict.get('requires', [])]
+
+        # add a dependency on an image task, if needed
+        if 'docker-image' in taskdict:
+            deps.append(('build-docker-image-{docker-image}'.format(**taskdict), 'docker-image'))
+
+        return deps
 
     def optimize_task(self, task, taskgraph):
         # no optimization for the moment
         return False, None
 
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/test/test_kind_docker_image.py
@@ -0,0 +1,54 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import unittest
+import tempfile
+import shutil
+import os
+
+from ..kind import docker_image
+from ..types import Task
+from mozunit import main, MockedOpen
+
+
+class TestDockerImageKind(unittest.TestCase):
+
+    def setUp(self):
+        self.kind = docker_image.DockerImageKind(
+                os.path.join(docker_image.GECKO, 'taskcluster', 'ci', 'docker-image'),
+                {})
+
+    def test_get_task_dependencies(self):
+        # this one's easy!
+        self.assertEqual(self.kind.get_task_dependencies(None, None), [])
+
+    # TODO: optimize_task
+
+    def test_create_context_tar(self):
+        image_dir = os.path.join(docker_image.GECKO, 'testing', 'docker', 'image_builder')
+        tarball = tempfile.mkstemp()[1]
+        self.kind.create_context_tar(image_dir, tarball, 'image_builder')
+        self.failUnless(os.path.exists(tarball))
+        os.unlink(tarball)
+
+    def test_generate_context_hash(self):
+        tmpdir = tempfile.mkdtemp()
+        old_GECKO = docker_image.GECKO
+        docker_image.GECKO = tmpdir
+        try:
+            os.makedirs(os.path.join(tmpdir, 'docker', 'my-image'))
+            with open(os.path.join(tmpdir, 'docker', 'my-image', 'Dockerfile'), "w") as f:
+                f.write("FROM node\nADD a-file\n")
+            with open(os.path.join(tmpdir, 'docker', 'my-image', 'a-file'), "w") as f:
+                f.write("data\n")
+            self.assertEqual(self.kind.generate_context_hash('docker/my-image'),
+                    '781143fcc6cc72c9024b058665265cb6bae3fb8031cad7227dd169ffbfced434')
+        finally:
+            docker_image.GECKO = old_GECKO
+            shutil.rmtree(tmpdir)
+
+if __name__ == '__main__':
+    main()
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/test/test_util.py
@@ -0,0 +1,27 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import unittest
+
+from ..util import docker_image, DOCKER_ROOT
+from mozunit import main, MockedOpen
+
+
+class TestDockerImage(unittest.TestCase):
+
+    def test_docker_image_explicit_registry(self):
+        files = {}
+        files["{}/myimage/REGISTRY".format(DOCKER_ROOT)] = "cool-images"
+        files["{}/myimage/VERSION".format(DOCKER_ROOT)] = "1.2.3"
+        with MockedOpen(files):
+            self.assertEqual(docker_image('myimage'), "cool-images/myimage:1.2.3")
+
+    def test_docker_image_default_registry(self):
+        files = {}
+        files["{}/REGISTRY".format(DOCKER_ROOT)] = "mozilla"
+        files["{}/myimage/VERSION".format(DOCKER_ROOT)] = "1.2.3"
+        with MockedOpen(files):
+            self.assertEqual(docker_image('myimage'), "mozilla/myimage:1.2.3")
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/util.py
@@ -0,0 +1,25 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import os
+
+GECKO = os.path.realpath(os.path.join(__file__, '..', '..', '..'))
+DOCKER_ROOT = os.path.join(GECKO, 'testing', 'docker')
+
+def docker_image(name):
+    ''' Determine the docker image name, including repository and tag, from an
+    in-tree docker file'''
+    try:
+        with open(os.path.join(DOCKER_ROOT, name, 'REGISTRY')) as f:
+            registry = f.read().strip()
+    except IOError:
+        with open(os.path.join(DOCKER_ROOT, 'REGISTRY')) as f:
+            registry = f.read().strip()
+
+    with open(os.path.join(DOCKER_ROOT, name, 'VERSION')) as f:
+        version = f.read().strip()
+
+    return '{}/{}:{}'.format(registry, name, version)
--- a/testing/taskcluster/routes.json
+++ b/testing/taskcluster/routes.json
@@ -9,15 +9,10 @@
         "{index}.gecko.v2.{project}.nightly.{year}.{month}.{day}.latest.{build_product}.{build_name}-{build_type}",
         "{index}.gecko.v2.{project}.nightly.revision.{head_rev}.{build_product}.{build_name}-{build_type}",
         "{index}.gecko.v2.{project}.nightly.latest.{build_product}.{build_name}-{build_type}"
     ],
     "l10n": [
         "{index}.gecko.v2.{project}.revision.{head_rev}.{build_product}-l10n.{build_name}-{build_type}.{locale}",
         "{index}.gecko.v2.{project}.pushdate.{year}.{month}.{day}.{pushdate}.{build_product}-l10n.{build_name}-{build_type}.{locale}",
         "{index}.gecko.v2.{project}.latest.{build_product}-l10n.{build_name}-{build_type}.{locale}"
-    ],
-    "docker_images": [
-        "{index}.docker.images.v1.{project}.{image_name}.latest",
-        "{index}.docker.images.v1.{project}.{image_name}.pushdate.{year}.{month}-{day}-{pushtime}",
-        "{index}.docker.images.v1.{project}.{image_name}.hash.{context_hash}"
     ]
 }
--- a/testing/taskcluster/taskcluster_graph/image_builder.py
+++ b/testing/taskcluster/taskcluster_graph/image_builder.py
@@ -23,28 +23,16 @@ def is_docker_registry_image(registry_pa
     return os.path.isfile(registry_path)
 
 # make a task label; in old decision tasks, this is a regular slugid, but when called
 # from the taskgraph generator's legacy kind, this is monkey-patched to return a label
 # (`TaskLabel==..`)
 def mklabel():
     return slugid()
 
-def docker_image(name):
-    ''' Determine the docker tag/revision from an in tree docker file '''
-    repository_path = os.path.join(DOCKER_ROOT, name, 'REGISTRY')
-    repository = REGISTRY
-
-    version = open(os.path.join(DOCKER_ROOT, name, 'VERSION')).read().strip()
-
-    if os.path.isfile(repository_path):
-        repository = open(repository_path).read().strip()
-
-    return '{}/{}:{}'.format(repository, name, version)
-
 def task_id_for_image(seen_images, project, name, create=True):
     if name in seen_images:
         return seen_images[name]['taskId']
 
     context_path = os.path.join('testing', 'docker', name)
     context_hash = generate_context_hash(context_path)
     task_id = get_task_id_for_namespace(project, name, context_hash)
 
@@ -128,128 +116,16 @@ def generate_context_hash(image_path):
                 data = f.read()
                 if not data:
                     break
                 file_hash.update(data)
             context_hash.update(file_hash.hexdigest() + '\t' + relative_filename + '\n')
 
     return context_hash.hexdigest()
 
-def create_context_tar(context_dir, destination, image_name):
-    ''' Creates a tar file of a particular context directory '''
-    if not os.path.exists(os.path.dirname(destination)):
-        os.makedirs(os.path.dirname(destination))
-
-    with tarfile.open(destination, 'w:gz') as tar:
-        tar.add(context_dir, arcname=image_name)
-
-def image_requires_building(details):
-    ''' Returns true if an image task should be created for a particular image '''
-    if 'path' in details and 'hash' in details:
-        return True
-    else:
-        return False
-
-def create_image_task_parameters(params, name, details):
-    image_parameters = dict(params)
-    image_parameters['context_hash'] = details['hash']
-    image_parameters['context_path'] = details['path']
-    image_parameters['artifact_path'] = 'public/image.tar'
-    image_parameters['image_slugid'] =  details['taskId']
-    image_parameters['image_name'] = name
-
-    return image_parameters
-
-def get_image_details(seen_images, task_id):
-    '''
-    Based on a collection of image details, return the details
-    for an image matching the requested task_id.
-
-    Image details can include a path and hash indicating that the image requires
-    building.
-    '''
-    for name, details in seen_images.items():
-        if details['taskId'] == task_id:
-            return [name, details]
-    return None
-
-def get_json_routes():
-    ''' Returns routes that should be included in the image task. '''
-    routes_file = os.path.join(TASKCLUSTER_ROOT, 'routes.json')
-    with open(routes_file) as f:
-        contents = json.load(f)
-        json_routes = contents['docker_images']
-    return json_routes
-
-def normalize_image_details(graph, task, seen_images, params, decision_task_id):
-    '''
-    This takes a task-image payload and creates an image task to build that
-    image.
-
-    task-image payload is then converted to use a specific task ID of that
-    built image.  All tasks within the graph requiring this same image will have their
-    image details normalized and require the same image build task.
-    '''
-    image = task['task']['payload']['image']
-    if isinstance(image, str) or image.get('type', 'docker-image') == 'docker-image':
-        return
-
-    if 'requires' not in task:
-        task['requires'] = []
-
-    name, details = get_image_details(seen_images, image['taskId'])
-
-    if details.get('required', False) is True or image_requires_building(details) is False:
-        if 'required' in details:
-            task['requires'].append(details['taskId'])
-        return
-
-    image_parameters = create_image_task_parameters(params, name, details)
-
-    if decision_task_id:
-        image_artifact_path = "public/decision_task/image_contexts/{}/context.tar.gz".format(name)
-        destination = "/home/worker/artifacts/decision_task/image_contexts/{}/context.tar.gz".format(name)
-        image_parameters['context_url'] = ARTIFACT_URL.format(decision_task_id, image_artifact_path)
-
-        create_context_tar(image_parameters['context_path'], destination, name)
-
-    templates = Templates(TASKCLUSTER_ROOT)
-    image_task = templates.load(IMAGE_BUILD_TASK, image_parameters)
-    if params['revision_hash']:
-        treeherder_transform.add_treeherder_revision_info(
-            image_task['task'],
-            params['head_rev'],
-            params['revision_hash']
-        )
-        routes_transform.decorate_task_treeherder_routes(
-            image_task['task'],
-            "{}.{}".format(params['project'], params['revision_hash'])
-        )
-        routes_transform.decorate_task_json_routes(image_task['task'],
-                                                   get_json_routes(),
-                                                   image_parameters)
-
-    image_task['attributes'] = {
-        'kind': 'legacy',
-    }
-
-    graph['tasks'].append(image_task);
-    task['requires'].append(details['taskId'])
-
-    define_task = DEFINE_TASK.format(
-        image_task['task']['workerType']
-    )
-
-    graph['scopes'].add(define_task)
-    graph['scopes'] |= set(image_task['task'].get('scopes', []))
-    route_scopes = map(lambda route: 'queue:route:' + route, image_task['task'].get('routes', []))
-    graph['scopes'] |= set(route_scopes)
-
-    details['required'] = True
-
 def docker_load_from_url(url):
     """Get a docker image from a `docker save` tarball at the given URL,
     loading it into the running daemon and returning the image name."""
 
     # because we need to read this file twice (and one read is not all the way
     # through), it is difficult to stream it.  So we downlaod to disk and then
     # read it back.
     filename = 'temp-docker-image.tar'
--- a/testing/taskcluster/tasks/builds/android_api_15_gradle_dependencies.yml
+++ b/testing/taskcluster/tasks/builds/android_api_15_gradle_dependencies.yml
@@ -1,16 +1,17 @@
 # A build-like task to fetch Android Gradle dependencies from jcentral and
 # package them for consumption by tooltool users.  Normally invoked manually.
 
 $inherits:
   from: 'tasks/builds/mobile_base.yml'
   variables:
     build_name: 'android-api-15-gradle-dependencies'
     build_type: 'opt'
+docker-image: android-gradle-build
 task:
   metadata:
       name: '[TC] Android armv7 API 15+ gradle dependencies'
       description: 'Android armv7 API 15+ gradle dependencies'
 
   workerType: android-api-15
 
   routes:
@@ -45,17 +46,18 @@ task:
       MH_BUILD_POOL: taskcluster
       GRADLE_USER_HOME: '/home/worker/workspace/build/src/dotgradle-online'
 
     maxRunTime: 36000
 
     image:
       type: 'task-image'
       path: 'public/image.tar'
-      taskId: '{{#task_id_for_image}}android-gradle-build{{/task_id_for_image}}'
+      taskId:
+        task-reference: "<docker-image>"
 
     command:
       - /bin/bash
       - -c
       - >
         /home/worker/bin/before.sh &&
         /home/worker/bin/build.sh &&
         /home/worker/bin/after.sh &&
--- a/testing/taskcluster/tasks/builds/b2g_base.yml
+++ b/testing/taskcluster/tasks/builds/b2g_base.yml
@@ -1,14 +1,16 @@
 $inherits:
   from: 'tasks/build.yml'
   variables:
     build_product: 'b2g'
+docker-image: builder
 task:
   payload:
     image:
       type: 'task-image'
       path: 'public/image.tar'
-      taskId: '{{#task_id_for_image}}builder{{/task_id_for_image}}'
+      taskId:
+        task-reference: "<docker-image>"
   extra:
     locations:
       test_packages: 'public/build/target.test_packages.json'
       mozharness: 'public/build/mozharness.zip'
--- a/testing/taskcluster/tasks/builds/firefox_base.yml
+++ b/testing/taskcluster/tasks/builds/firefox_base.yml
@@ -1,18 +1,20 @@
 $inherits:
   from: 'tasks/build.yml'
   variables:
     build_product: 'firefox'
+docker-image: desktop-build
 task:
   scopes:
     - "secrets:get:project/releng/gecko/build/level-{{level}}/*"
   extra:
     locations:
       mozharness: 'public/build/mozharness.zip'
       test_packages: 'public/build/target.test_packages.json'
   payload:
     image:
       type: 'task-image'
       path: 'public/image.tar'
-      taskId: '{{#task_id_for_image}}desktop-build{{/task_id_for_image}}'
+      taskId:
+        task-reference: "<docker-image>"
     features:
       taskclusterProxy: true
--- a/testing/taskcluster/tasks/builds/mobile_base.yml
+++ b/testing/taskcluster/tasks/builds/mobile_base.yml
@@ -1,11 +1,13 @@
 $inherits:
   from: 'tasks/build.yml'
   variables:
     build_product: 'mobile'
+docker-image: desktop-build
 task:
   payload:
     image:
       type: 'task-image'
       path: 'public/image.tar'
-      taskId: '{{#task_id_for_image}}desktop-build{{/task_id_for_image}}'
+      taskId:
+        task-reference: "<docker-image>"
 
--- a/testing/taskcluster/tasks/harness_test.yml
+++ b/testing/taskcluster/tasks/harness_test.yml
@@ -1,15 +1,15 @@
 # This is the "base" task which contains the common values all harness
 # unit tests must provide.
 # It's inspired by build.yml, since harness unit tests just depend on
 # checking out the source tree.
 ---
 taskId: {{build_slugid}}
-
+docker-image: desktop-build
 task:
   created: '{{now}}'
   deadline: '{{#from_now}}24 hours{{/from_now}}'
   metadata:
     source: '{{source}}'
     owner: mozilla-taskcluster-maintenance@mozilla.com
   tags:
     createdForUser: {{owner}}
@@ -23,17 +23,18 @@ task:
     - 'docker-worker:cache:level-{{level}}-{{project}}-tc-vcs'
     - 'docker-worker:cache:level-{{level}}-{{project}}-test-workspace'
   payload:
     # 30 minutes should be more than enough for harness unit tests
     maxRunTime: 1800
     image:
       type: 'task-image'
       path: 'public/image.tar'
-      taskId: '{{#task_id_for_image}}desktop-build{{/task_id_for_image}}'
+      taskId:
+        task-reference: "<docker-image>"
     cache:
       # The taskcluster-vcs tooling stores the large clone caches in this
       # directory and will reuse them for new requests this saves about 20s~
       # and is the most generic cache possible.
       level-{{level}}-{{project}}-tc-vcs: '/home/worker/.tc-vcs'
       level-{{level}}-{{project}}-test-workspace: '/home/worker/workspace'
     artifacts:
       'public/logs/':
--- a/testing/taskcluster/tasks/post-builds/mulet_simulator.yml
+++ b/testing/taskcluster/tasks/post-builds/mulet_simulator.yml
@@ -1,12 +1,12 @@
 # This tasks takes a mulet build, pull gaia and craft a xpi file for FxOS simulator addon
 ---
-taskId: {{taskId}}
-
+taskId: {{build_slugid}}
+docker-image: builder
 task:
   created: '{{now}}'
   deadline: '{{#from_now}}24 hours{{/from_now}}'
   metadata:
     source: '{{source}}'
     owner: apoirot@mozilla.com
     name: '[TC] FxOS Simulator'
     description: 'Firefox OS Simulator addon'
@@ -22,17 +22,18 @@ task:
 
   scopes:
     - 'docker-worker:cache:level-{{level}}-{{project}}-tc-vcs'
 
   payload:
     image:
       type: 'task-image'
       path: 'public/image.tar'
-      taskId: '{{#task_id_for_image}}builder{{/task_id_for_image}}'
+      taskId:
+        task-reference: "<docker-image>"
 
     maxRunTime: 600
 
     command:
       - /bin/bash
       - -exc
       - task-reference: >
         tc-vcs checkout ./gecko {{base_repository}} {{head_repository}} {{head_rev}} {{head_ref}} &&
--- a/testing/taskcluster/tasks/test.yml
+++ b/testing/taskcluster/tasks/test.yml
@@ -1,26 +1,32 @@
 # This task is the base for most tests in gecko.
+docker-image: tester
 task:
   created: '{{now}}'
   deadline: '{{#from_now}}24 hours{{/from_now}}'
   metadata:
     source: '{{source}}'
     owner: mozilla-taskcluster-maintenance@mozilla.com
   tags:
     createdForUser: {{owner}}
   workerType: b2gtest
   provisionerId: aws-provisioner-v1
   schedulerId: task-graph-scheduler
 
   scopes:
     - 'docker-worker:feature:allowPtrace'
 
   payload:
-    image: '{{#docker_image}}tester{{/docker_image}}'
+    image:
+      type: 'task-image'
+      path: 'public/image.tar'
+      taskId:
+        task-reference: "<docker-image>"
+
     maxRunTime: 3600
     env:
       MOZILLA_BUILD_URL: {"task-reference": "https://queue.taskcluster.net/v1/task/<{{build_slugid}}>/artifacts/{{build_location}}"}
       MOZHARNESS_URL: {"task-reference": "https://queue.taskcluster.net/v1/task/<{{build_slugid}}>/artifacts/{{mozharness_location}}"}
 
     # All builds share a common artifact directory for ease of uploading.
     artifacts:
       'public/logs/':
--- a/testing/taskcluster/tasks/tests/eslint-gecko.yml
+++ b/testing/taskcluster/tasks/tests/eslint-gecko.yml
@@ -1,26 +1,28 @@
 ---
 $inherits:
     from: 'tasks/lint.yml'
     variables:
       build_product: 'lint'
       build_name: 'eslint-gecko'
       build_type: 'opt'
 
+docker-image: lint
 task:
   metadata:
     name: '[TC] - ESLint'
     description: 'ESLint test'
 
   payload:
     image:
       type: 'task-image'
       path: 'public/image.tar'
-      taskId: '{{#task_id_for_image}}lint{{/task_id_for_image}}'
+      taskId:
+        task-reference: "<docker-image>"
 
     command:
       - bash
       - -cx
       - >
           tc-vcs checkout ./gecko {{base_repository}} {{head_repository}} {{head_rev}} {{head_ref}} &&
           cd gecko/testing/eslint &&
           /build/tooltool.py fetch -m manifest.tt &&
--- a/testing/taskcluster/tasks/tests/fx_test_base.yml
+++ b/testing/taskcluster/tasks/tests/fx_test_base.yml
@@ -1,20 +1,23 @@
 ---
 $inherits:
   from: 'tasks/test.yml'
+docker-image: desktop-test
 task:
   workerType: desktop-test
   scopes:
     - docker-worker:cache:level-{{level}}-{{project}}-test-workspace
   payload:
     image:
       type: 'task-image'
       path: 'public/image.tar'
-      taskId: '{{#task_id_for_image}}desktop-test{{/task_id_for_image}}'
+      taskId:
+        task-reference: "<docker-image>"
+
     env:
         NEED_WINDOW_MANAGER: true
         NEED_PULSEAUDIO: true
         GECKO_HEAD_REPOSITORY: '{{{head_repository}}}'
         GECKO_HEAD_REV: '{{{head_rev}}}'
         MOZ_NODE_PATH: '/usr/local/bin/node'
 
     cache:
--- a/testing/taskcluster/tasks/tests/mozharness-gecko.yml
+++ b/testing/taskcluster/tasks/tests/mozharness-gecko.yml
@@ -1,29 +1,31 @@
 ---
 $inherits:
     from: 'tasks/lint.yml'
     variables:
       build_product: 'lint'
       build_name: 'mozharness-tox'
       build_type: 'opt'
 
+docker-image: desktop-test
 task:
   metadata:
     name: '[TC] - Mozharness Tox'
     description: 'Mozharness integration tests'
 
   scopes:
     - 'docker-worker:cache:level-{{level}}-{{project}}-dotcache'
 
   payload:
     image:
       type: 'task-image'
       path: 'public/image.tar'
-      taskId: '{{#task_id_for_image}}desktop-test{{/task_id_for_image}}'
+      taskId:
+        task-reference: "<docker-image>"
 
     cache:
       level-{{level}}-{{project}}-dotcache: '/home/worker/.cache'
 
     command:
       - bash
       - -cx
       - >
--- a/testing/taskcluster/tasks/tests/mozlint-flake8.yml
+++ b/testing/taskcluster/tasks/tests/mozlint-flake8.yml
@@ -1,25 +1,27 @@
 ---
 $inherits:
   from: 'tasks/lint.yml'
   variables:
     build_product: 'lint'
     build_name: 'flake8-gecko'
     build_type: 'opt'
 
+docker-image: lint
 task:
   metadata:
     name: '[TC] - Flake8'
     description: 'Python flake8 linter'
   payload:
     image:
       type: 'task-image'
       path: 'public/image.tar'
-      taskId: '{{#task_id_for_image}}lint{{/task_id_for_image}}'
+      taskId:
+        task-reference: "<docker-image>"
     command:
       - bash
       - -cx
       - >
           tc-vcs checkout ./gecko {{base_repository}} {{head_repository}} {{head_rev}} {{head_ref}} &&
           cd gecko &&
           ./mach lint -l flake8 -f treeherder
   extra: