Bug 1419638 - Allow to share docker image definitions. r=dustin draft
authorMike Hommey <mh+mozilla@glandium.org>
Sun, 24 Dec 2017 07:58:08 +0900
changeset 715672 72335a93d9a1b16af067eccd6fd5dfd7c5bc5c4d
parent 715671 56cbc4041aed566952dbe81eccc8e62cb95fa428
child 744852 c0d633596adbd187c21c13ba3e8b2ed6f7bb1484
push id94224
push userbmo:mh+mozilla@glandium.org
push dateThu, 04 Jan 2018 11:02:43 +0000
reviewersdustin
bugs1419638
milestone59.0a1
Bug 1419638 - Allow to share docker image definitions. r=dustin Instead of duplicating Dockerfiles between taskcluster/docker/* directories, which can be error prone for very close images, it can be desirable to use the same file. This change allows to set the `definition` keyword on a docker image definition in kind.yml that will make the task use the files from taskcluster/docker/<definition> instead of taskcluster/docker/<image_name>.
taskcluster/taskgraph/docker.py
taskcluster/taskgraph/transforms/docker_image.py
taskcluster/taskgraph/util/docker.py
--- a/taskcluster/taskgraph/docker.py
+++ b/taskcluster/taskgraph/docker.py
@@ -56,32 +56,32 @@ def load_image_by_task_id(task_id, tag=N
 def build_context(name, outputFile, args=None):
     """Build a context.tar for image with specified name.
     """
     if not name:
         raise ValueError('must provide a Docker image name')
     if not outputFile:
         raise ValueError('must provide a outputFile')
 
-    image_dir = os.path.join(docker.IMAGE_DIR, name)
+    image_dir = docker.image_path(name)
     if not os.path.isdir(image_dir):
         raise Exception('image directory does not exist: %s' % image_dir)
 
     docker.create_context_tar(GECKO, image_dir, outputFile, "", args)
 
 
 def build_image(name, args=None):
     """Build a Docker image of specified name.
 
     Output from image building process will be printed to stdout.
     """
     if not name:
         raise ValueError('must provide a Docker image name')
 
-    image_dir = os.path.join(docker.IMAGE_DIR, name)
+    image_dir = docker.image_path(name)
     if not os.path.isdir(image_dir):
         raise Exception('image directory does not exist: %s' % image_dir)
 
     tag = docker.docker_image(name, by_tag=True)
 
     docker_bin = which.which('docker')
 
     # Verify that Docker is working.
--- a/taskcluster/taskgraph/transforms/docker_image.py
+++ b/taskcluster/taskgraph/transforms/docker_image.py
@@ -32,16 +32,20 @@ docker_image_schema = Schema({
     Required('symbol'): basestring,
 
     # relative path (from config.path) to the file the docker image was defined
     # in.
     Optional('job-from'): basestring,
 
     # Arguments to use for the Dockerfile.
     Optional('args'): {basestring: basestring},
+
+    # Name of the docker image definition under taskcluster/docker, when
+    # different from the docker image name.
+    Optional('definition'): basestring,
 })
 
 
 @transforms.add
 def validate(config, tasks):
     for task in tasks:
         yield validate_schema(
             docker_image_schema, task,
@@ -49,18 +53,19 @@ def validate(config, tasks):
 
 
 @transforms.add
 def fill_template(config, tasks):
     for task in tasks:
         image_name = task.pop('name')
         job_symbol = task.pop('symbol')
         args = task.pop('args', {})
+        definition = task.pop('definition', image_name)
 
-        context_path = os.path.join('taskcluster', 'docker', image_name)
+        context_path = os.path.join('taskcluster', 'docker', definition)
         context_hash = generate_context_hash(
             GECKO, context_path, image_name, args)
 
         description = 'Build the docker image {} for use by dependent tasks'.format(
             image_name)
 
         # Adjust the zstandard compression level based on the execution level.
         # We use faster compression for level 1 because we care more about
--- a/taskcluster/taskgraph/util/docker.py
+++ b/taskcluster/taskgraph/util/docker.py
@@ -6,16 +6,17 @@ from __future__ import absolute_import, 
 
 import hashlib
 import os
 import re
 import shutil
 import subprocess
 import tarfile
 import tempfile
+import yaml
 
 from mozbuild.util import memoize
 from mozpack.files import GeneratedFile
 from mozpack.archive import (
     create_tar_gz_from_files,
 )
 from .. import GECKO
 
@@ -185,21 +186,43 @@ def build_from_context(docker_bin, conte
         res = subprocess.call(args, cwd=os.path.join(d, prefix))
         if res:
             raise Exception('error building image')
     finally:
         shutil.rmtree(d)
 
 
 @memoize
+def image_paths():
+    """Return a map of image name to paths containing their Dockerfile.
+    """
+    with open(os.path.join(GECKO, 'taskcluster', 'ci', 'docker-image',
+                           'kind.yml')) as fh:
+        config = yaml.load(fh)
+        return {
+            k: os.path.join(IMAGE_DIR, v.get('definition', k))
+            for k, v in config['jobs'].items()
+        }
+
+
+def image_path(name):
+    paths = image_paths()
+    if name in paths:
+        return paths[name]
+    return os.path.join(IMAGE_DIR, name)
+
+
+@memoize
 def parse_volumes(image):
     """Parse VOLUME entries from a Dockerfile for an image."""
     volumes = set()
 
-    with open(os.path.join(IMAGE_DIR, image, 'Dockerfile'), 'rb') as fh:
+    path = image_path(image)
+
+    with open(os.path.join(path, 'Dockerfile'), 'rb') as fh:
         for line in fh:
             line = line.strip()
             # We assume VOLUME definitions don't use %ARGS.
             if not line.startswith(b'VOLUME '):
                 continue
 
             v = line.split(None, 1)[1]
             if v.startswith(b'['):