Bug 1432390 - Directly call the docker API over its unix socket instead of calling `docker build`. r?dustin draft
authorMike Hommey <mh+mozilla@glandium.org>
Wed, 24 Jan 2018 15:55:31 +0900
changeset 747406 df22ccb689f6d3857f8f7d18cc38ef3f82a8749c
parent 747405 b9962c6edd18aba26f86acc5e3c3cfc9cdc85bc5
child 747407 2ffde75bc224df7e9679758052bb2e5891a213ad
push id96904
push userbmo:mh+mozilla@glandium.org
push dateFri, 26 Jan 2018 01:20:26 +0000
reviewersdustin
bugs1432390
milestone60.0a1
Bug 1432390 - Directly call the docker API over its unix socket instead of calling `docker build`. r?dustin This allows to avoid writing out a tar file to then extract it to feed it to `docker build`. This is essentially what the image-builder docker image does, except it uses a temporary file for the tar.
taskcluster/taskgraph/docker.py
taskcluster/taskgraph/util/docker.py
--- a/taskcluster/taskgraph/docker.py
+++ b/taskcluster/taskgraph/docker.py
@@ -3,20 +3,17 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 import json
 import os
-import subprocess
 import tarfile
-import tempfile
-import which
 from io import BytesIO
 
 from taskgraph.util import docker
 from taskgraph.util.taskcluster import (
     find_task_id,
     get_artifact_url,
     get_session,
 )
@@ -76,36 +73,19 @@ def build_image(name, args=None):
         raise ValueError('must provide a Docker image name')
 
     image_dir = docker.image_path(name)
     if not os.path.isdir(image_dir):
         raise Exception('image directory does not exist: %s' % image_dir)
 
     tag = docker.docker_image(name, by_tag=True)
 
-    docker_bin = which.which('docker')
-
-    # Verify that Docker is working.
-    try:
-        subprocess.check_output([docker_bin, '--version'])
-    except subprocess.CalledProcessError:
-        raise Exception('Docker server is unresponsive. Run `docker ps` and '
-                        'check that Docker is running')
-
-    # We obtain a context archive and build from that. Going through the
-    # archive creation is important: it normalizes things like file owners
-    # and mtimes to increase the chances that image generation is
-    # deterministic.
-    fd, context_path = tempfile.mkstemp()
-    os.close(fd)
-    try:
-        docker.create_context_tar(GECKO, image_dir, context_path, name, args)
-        docker.build_from_context(docker_bin, context_path, name, tag)
-    finally:
-        os.unlink(context_path)
+    buf = BytesIO()
+    docker.stream_context_tar(GECKO, image_dir, buf, '', args)
+    docker.post_to_docker(buf.getvalue(), '/build', nocache=1, t=tag)
 
     print('Successfully built %s and tagged with %s' % (name, tag))
 
     if tag.endswith(':latest'):
         print('*' * 50)
         print('WARNING: no VERSION file found in image directory.')
         print('Image is not suitable for deploying/pushing.')
         print('Create an image suitable for deploying/pushing by creating')
--- a/taskcluster/taskgraph/util/docker.py
+++ b/taskcluster/taskgraph/util/docker.py
@@ -4,21 +4,17 @@
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 import hashlib
 import json
 import os
 import re
 import requests_unixsocket
-import shutil
-import subprocess
 import sys
-import tarfile
-import tempfile
 import urllib
 import urlparse
 import yaml
 
 from mozbuild.util import memoize
 from mozpack.files import GeneratedFile
 from mozpack.archive import (
     create_tar_gz_from_files,
@@ -256,50 +252,16 @@ def stream_context_tar(topsrcdir, contex
     archive_files[os.path.join(prefix, 'Dockerfile')] = \
         GeneratedFile(b''.join(content))
 
     writer = HashingWriter(out_file)
     create_tar_gz_from_files(writer, archive_files, '%s.tar.gz' % prefix)
     return writer.hexdigest()
 
 
-def build_from_context(docker_bin, context_path, prefix, tag=None):
-    """Build a Docker image from a context archive.
-
-    Given the path to a `docker` binary, a image build tar.gz (produced with
-    ``create_context_tar()``, a prefix in that context containing files, and
-    an optional ``tag`` for the produced image, build that Docker image.
-    """
-    d = tempfile.mkdtemp()
-    try:
-        with tarfile.open(context_path, 'r:gz') as tf:
-            tf.extractall(d)
-
-        # If we wanted to do post-processing of the Dockerfile, this is
-        # where we'd do it.
-
-        args = [
-            docker_bin,
-            'build',
-            # Use --no-cache so we always get the latest package updates.
-            '--no-cache',
-        ]
-
-        if tag:
-            args.extend(['-t', tag])
-
-        args.append('.')
-
-        res = subprocess.call(args, cwd=os.path.join(d, prefix))
-        if res:
-            raise Exception('error building image')
-    finally:
-        shutil.rmtree(d)
-
-
 @memoize
 def image_paths():
     """Return a map of image name to paths containing their Dockerfile.
     """
     with open(os.path.join(GECKO, 'taskcluster', 'ci', 'docker-image',
                            'kind.yml')) as fh:
         config = yaml.load(fh)
         return {