Bug 1290531 - Build Docker images from custom tar contexts; r?dustin
Now that Docker image building is called from Python, we can start to
do advanced stuff with it.
With this commit, we switch from building Docker images directly from
the source directory ("the Docker way") to using our custom Docker image
build contexts.
The main advantage of this is that locally-built Docker images can now
use our custom Dockerfile syntax to include extra files in the build
context!
The code for building a Docker image from a context has been extracted
to its own standalone function. I have nefarious plans for this in the
future, such as the ability to override the FROM syntax to specify
URLs of images. This would allow us to host base images on our own
server, which removes a dependency on Docker Hub and improves
determinism, since images on Docker Hub change all the time.
MozReview-Commit-ID: 5lTdV8yEHkc
--- a/taskcluster/taskgraph/docker.py
+++ b/taskcluster/taskgraph/docker.py
@@ -5,16 +5,17 @@
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import, print_function, unicode_literals
import json
import os
import subprocess
import tarfile
+import tempfile
import urllib2
import which
from taskgraph.util import docker
GECKO = os.path.realpath(os.path.join(__file__, '..', '..', '..'))
IMAGE_DIR = os.path.join(GECKO, 'testing', 'docker')
INDEX_URL = 'https://index.taskcluster.net/v1/task/docker.images.v1.{}.{}.hash.{}'
@@ -83,28 +84,27 @@ def build_image(name):
# Verify that Docker is working.
try:
subprocess.check_output([docker_bin, '--version'])
except subprocess.CalledProcessError:
raise Exception('Docker server is unresponsive. Run `docker ps` and '
'check that Docker is running')
- args = [
- docker_bin,
- 'build',
- # Use --no-cache so we always get the latest package updates.
- '--no-cache',
- '-t', tag,
- name,
- ]
-
- res = subprocess.call(args, cwd=IMAGE_DIR)
- if res:
- raise Exception('error building image')
+ # We obtain a context archive and build from that. Going through the
+ # archive creation is important: it normalizes things like file owners
+ # and mtimes to increase the chances that image generation is
+ # deterministic.
+ fd, context_path = tempfile.mkstemp()
+ os.close(fd)
+ try:
+ docker.create_context_tar(GECKO, image_dir, context_path, name)
+ docker.build_from_context(docker_bin, context_path, name, tag)
+ finally:
+ os.unlink(context_path)
print('Successfully built %s and tagged with %s' % (name, tag))
if tag.endswith(':latest'):
print('*' * 50)
print('WARNING: no VERSION file found in image directory.')
print('Image is not suitable for deploying/pushing.')
print('Create an image suitable for deploying/pushing by creating')
--- a/taskcluster/taskgraph/util/docker.py
+++ b/taskcluster/taskgraph/util/docker.py
@@ -1,16 +1,19 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import, print_function, unicode_literals
import hashlib
import os
+import shutil
+import subprocess
+import tarfile
import tempfile
from mozpack.archive import (
create_tar_gz_from_files,
)
GECKO = os.path.realpath(os.path.join(__file__, '..', '..', '..', '..'))
@@ -115,8 +118,42 @@ def create_context_tar(topsrcdir, contex
h = hashlib.sha256()
with open(out_path, 'rb') as fh:
while True:
data = fh.read(32768)
if not data:
break
h.update(data)
return h.hexdigest()
+
+
+def build_from_context(docker_bin, context_path, prefix, tag=None):
+ """Build a Docker image from a context archive.
+
+ Given the path to a `docker` binary, a image build tar.gz (produced with
+ ``create_context_tar()``, a prefix in that context containing files, and
+ an optional ``tag`` for the produced image, build that Docker image.
+ """
+ d = tempfile.mkdtemp()
+ try:
+ with tarfile.open(context_path, 'r:gz') as tf:
+ tf.extractall(d)
+
+ # If we wanted to do post-processing of the Dockerfile, this is
+ # where we'd do it.
+
+ args = [
+ docker_bin,
+ 'build',
+ # Use --no-cache so we always get the latest package updates.
+ '--no-cache',
+ ]
+
+ if tag:
+ args.extend(['-t', tag])
+
+ args.append('.')
+
+ res = subprocess.call(args, cwd=os.path.join(d, prefix))
+ if res:
+ raise Exception('error building image')
+ finally:
+ shutil.rmtree(d)