Bug 1432390 - Directly call the docker API over its unix socket instead of calling `docker build`. r?dustin
This allows to avoid writing out a tar file to then extract it to feed
it to `docker build`. This is essentially what the image-builder docker
image does, except it uses a temporary file for the tar.
--- a/taskcluster/taskgraph/docker.py
+++ b/taskcluster/taskgraph/docker.py
@@ -3,20 +3,17 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import, print_function, unicode_literals
import json
import os
-import subprocess
import tarfile
-import tempfile
-import which
from io import BytesIO
from taskgraph.util import docker
from taskgraph.util.taskcluster import (
find_task_id,
get_artifact_url,
get_session,
)
@@ -76,36 +73,19 @@ def build_image(name, args=None):
raise ValueError('must provide a Docker image name')
image_dir = docker.image_path(name)
if not os.path.isdir(image_dir):
raise Exception('image directory does not exist: %s' % image_dir)
tag = docker.docker_image(name, by_tag=True)
- docker_bin = which.which('docker')
-
- # Verify that Docker is working.
- try:
- subprocess.check_output([docker_bin, '--version'])
- except subprocess.CalledProcessError:
- raise Exception('Docker server is unresponsive. Run `docker ps` and '
- 'check that Docker is running')
-
- # We obtain a context archive and build from that. Going through the
- # archive creation is important: it normalizes things like file owners
- # and mtimes to increase the chances that image generation is
- # deterministic.
- fd, context_path = tempfile.mkstemp()
- os.close(fd)
- try:
- docker.create_context_tar(GECKO, image_dir, context_path, name, args)
- docker.build_from_context(docker_bin, context_path, name, tag)
- finally:
- os.unlink(context_path)
+ buf = BytesIO()
+ docker.stream_context_tar(GECKO, image_dir, buf, '', args)
+ docker.post_to_docker(buf.getvalue(), '/build', nocache=1, t=tag)
print('Successfully built %s and tagged with %s' % (name, tag))
if tag.endswith(':latest'):
print('*' * 50)
print('WARNING: no VERSION file found in image directory.')
print('Image is not suitable for deploying/pushing.')
print('Create an image suitable for deploying/pushing by creating')
--- a/taskcluster/taskgraph/util/docker.py
+++ b/taskcluster/taskgraph/util/docker.py
@@ -4,21 +4,17 @@
from __future__ import absolute_import, print_function, unicode_literals
import hashlib
import json
import os
import re
import requests_unixsocket
-import shutil
-import subprocess
import sys
-import tarfile
-import tempfile
import urllib
import urlparse
import yaml
from mozbuild.util import memoize
from mozpack.files import GeneratedFile
from mozpack.archive import (
create_tar_gz_from_files,
@@ -256,50 +252,16 @@ def stream_context_tar(topsrcdir, contex
archive_files[os.path.join(prefix, 'Dockerfile')] = \
GeneratedFile(b''.join(content))
writer = HashingWriter(out_file)
create_tar_gz_from_files(writer, archive_files, '%s.tar.gz' % prefix)
return writer.hexdigest()
-def build_from_context(docker_bin, context_path, prefix, tag=None):
- """Build a Docker image from a context archive.
-
- Given the path to a `docker` binary, a image build tar.gz (produced with
- ``create_context_tar()``, a prefix in that context containing files, and
- an optional ``tag`` for the produced image, build that Docker image.
- """
- d = tempfile.mkdtemp()
- try:
- with tarfile.open(context_path, 'r:gz') as tf:
- tf.extractall(d)
-
- # If we wanted to do post-processing of the Dockerfile, this is
- # where we'd do it.
-
- args = [
- docker_bin,
- 'build',
- # Use --no-cache so we always get the latest package updates.
- '--no-cache',
- ]
-
- if tag:
- args.extend(['-t', tag])
-
- args.append('.')
-
- res = subprocess.call(args, cwd=os.path.join(d, prefix))
- if res:
- raise Exception('error building image')
- finally:
- shutil.rmtree(d)
-
-
@memoize
def image_paths():
"""Return a map of image name to paths containing their Dockerfile.
"""
with open(os.path.join(GECKO, 'taskcluster', 'ci', 'docker-image',
'kind.yml')) as fh:
config = yaml.load(fh)
return {