Bug 1419638 - Allow to share docker image definitions. r=dustin
Instead of duplicating Dockerfiles between taskcluster/docker/*
directories, which can be error prone for very close images, it can be
desirable to use the same file. This change allows to set the
`definition` keyword on a docker image definition in kind.yml that
will make the task use the files from taskcluster/docker/<definition>
instead of taskcluster/docker/<image_name>.
--- a/taskcluster/taskgraph/docker.py
+++ b/taskcluster/taskgraph/docker.py
@@ -56,32 +56,32 @@ def load_image_by_task_id(task_id, tag=N
def build_context(name, outputFile, args=None):
"""Build a context.tar for image with specified name.
"""
if not name:
raise ValueError('must provide a Docker image name')
if not outputFile:
raise ValueError('must provide a outputFile')
- image_dir = os.path.join(docker.IMAGE_DIR, name)
+ image_dir = docker.image_path(name)
if not os.path.isdir(image_dir):
raise Exception('image directory does not exist: %s' % image_dir)
docker.create_context_tar(GECKO, image_dir, outputFile, "", args)
def build_image(name, args=None):
"""Build a Docker image of specified name.
Output from image building process will be printed to stdout.
"""
if not name:
raise ValueError('must provide a Docker image name')
- image_dir = os.path.join(docker.IMAGE_DIR, name)
+ image_dir = docker.image_path(name)
if not os.path.isdir(image_dir):
raise Exception('image directory does not exist: %s' % image_dir)
tag = docker.docker_image(name, by_tag=True)
docker_bin = which.which('docker')
# Verify that Docker is working.
--- a/taskcluster/taskgraph/transforms/docker_image.py
+++ b/taskcluster/taskgraph/transforms/docker_image.py
@@ -32,16 +32,20 @@ docker_image_schema = Schema({
Required('symbol'): basestring,
# relative path (from config.path) to the file the docker image was defined
# in.
Optional('job-from'): basestring,
# Arguments to use for the Dockerfile.
Optional('args'): {basestring: basestring},
+
+ # Name of the docker image definition under taskcluster/docker, when
+ # different from the docker image name.
+ Optional('definition'): basestring,
})
@transforms.add
def validate(config, tasks):
for task in tasks:
yield validate_schema(
docker_image_schema, task,
@@ -49,18 +53,19 @@ def validate(config, tasks):
@transforms.add
def fill_template(config, tasks):
for task in tasks:
image_name = task.pop('name')
job_symbol = task.pop('symbol')
args = task.pop('args', {})
+ definition = task.pop('definition', image_name)
- context_path = os.path.join('taskcluster', 'docker', image_name)
+ context_path = os.path.join('taskcluster', 'docker', definition)
context_hash = generate_context_hash(
GECKO, context_path, image_name, args)
description = 'Build the docker image {} for use by dependent tasks'.format(
image_name)
# Adjust the zstandard compression level based on the execution level.
# We use faster compression for level 1 because we care more about
--- a/taskcluster/taskgraph/util/docker.py
+++ b/taskcluster/taskgraph/util/docker.py
@@ -6,16 +6,17 @@ from __future__ import absolute_import,
import hashlib
import os
import re
import shutil
import subprocess
import tarfile
import tempfile
+import yaml
from mozbuild.util import memoize
from mozpack.files import GeneratedFile
from mozpack.archive import (
create_tar_gz_from_files,
)
from .. import GECKO
@@ -185,21 +186,43 @@ def build_from_context(docker_bin, conte
res = subprocess.call(args, cwd=os.path.join(d, prefix))
if res:
raise Exception('error building image')
finally:
shutil.rmtree(d)
@memoize
+def image_paths():
+ """Return a map of image name to paths containing their Dockerfile.
+ """
+ with open(os.path.join(GECKO, 'taskcluster', 'ci', 'docker-image',
+ 'kind.yml')) as fh:
+ config = yaml.load(fh)
+ return {
+ k: os.path.join(IMAGE_DIR, v.get('definition', k))
+ for k, v in config['jobs'].items()
+ }
+
+
+def image_path(name):
+ paths = image_paths()
+ if name in paths:
+ return paths[name]
+ return os.path.join(IMAGE_DIR, name)
+
+
+@memoize
def parse_volumes(image):
"""Parse VOLUME entries from a Dockerfile for an image."""
volumes = set()
- with open(os.path.join(IMAGE_DIR, image, 'Dockerfile'), 'rb') as fh:
+ path = image_path(image)
+
+ with open(os.path.join(path, 'Dockerfile'), 'rb') as fh:
for line in fh:
line = line.strip()
# We assume VOLUME definitions don't use %ARGS.
if not line.startswith(b'VOLUME '):
continue
v = line.split(None, 1)[1]
if v.startswith(b'['):