Bug 1329282 - QEMU image for building docker images draft
authorJonas Finnemann Jensen <jopsen@gmail.com>
Wed, 27 Sep 2017 13:27:40 -0700
changeset 671395 033876cfe3e7833f2beb48604b3fd0079225cf13
parent 671394 bcb9b81d50acf46893eda26524d105e8f8fb4d96
child 671396 a9f7cee48133266e0f1a19741e8f004195d5ebf0
push id81953
push userjojensen@mozilla.com
push dateWed, 27 Sep 2017 22:49:28 +0000
bugs1329282
milestone58.0a1
Bug 1329282 - QEMU image for building docker images This adds a QEMU image based on Ubuntu 16.04 for in-tree docker images. Changes to docker-image tasks are not part of this commit. MozReview-Commit-ID: FnSGRfMCCEn
taskcluster/qemu/docker-image-builder/custom-data/build-image.sh
taskcluster/qemu/docker-image-builder/custom-data/config.yml
taskcluster/qemu/docker-image-builder/custom-data/download-and-compress
taskcluster/qemu/docker-image-builder/custom-data/install.sh
taskcluster/qemu/docker-image-builder/custom-data/taskcluster-worker.service
taskcluster/qemu/docker-image-builder/custom-data/worker.seed
taskcluster/qemu/docker-image-builder/custom-data/wrap-install.sh
taskcluster/qemu/docker-image-builder/image.yml
new file mode 100644
--- /dev/null
+++ b/taskcluster/qemu/docker-image-builder/custom-data/build-image.sh
@@ -0,0 +1,47 @@
+#!/bin/bash -vex
+
+# Set bash options to exit immediately if a pipeline exists non-zero, expand
+# print a trace of commands, and make output verbose (print shell input as it's
+# read)
+# See https://www.gnu.org/software/bash/manual/html_node/The-Set-Builtin.html
+set -x -e -v -o pipefail
+
+# Prefix errors with taskcluster error prefix so that they are parsed by Treeherder
+raise_error() {
+  echo
+  echo "[taskcluster-image-build:error] $1"
+  exit 1
+}
+
+# Ensure that the PROJECT is specified so the image can be indexed
+test -n "$PROJECT"    || raise_error "PROJECT must be provided."
+test -n "$HASH"       || raise_error "Context HASH must be provided."
+test -n "$IMAGE_NAME" || raise_error "IMAGE_NAME must be provided."
+
+# Construct a CONTEXT_FILE
+CONTEXT_FILE=/home/worker/context.tar
+
+# Run ./mach taskcluster-build-image with --context-only to build context
+run-task \
+  --vcs-checkout "/home/worker/checkouts/gecko" \
+  -- \
+  /home/worker/checkouts/gecko/mach taskcluster-build-image \
+  --context-only "$CONTEXT_FILE" \
+  "$IMAGE_NAME"
+test -f "$CONTEXT_FILE" || raise_error "Context file wasn't created"
+
+# Create artifact folder (note that this must occur after run-task)
+mkdir -p /home/worker/artifacts
+
+# Build image from tarball
+docker build -t "$IMAGE_NAME:$HASH" - < "$CONTEXT_FILE"
+
+# Get image from docker daemon (try up to 10 times)
+# This interacts directly with the docker remote API, see:
+# https://docs.docker.com/engine/reference/api/docker_remote_api_v1.18/
+#
+# The script will retry up to 10 times.
+/usr/local/bin/download-and-compress \
+    http+unix://%2Fvar%2Frun%2Fdocker.sock/images/${IMAGE_NAME}:${HASH}/get \
+    /home/worker/image.tar.zst.tmp \
+    /home/worker/artifacts/image.tar.zst
new file mode 100644
--- /dev/null
+++ b/taskcluster/qemu/docker-image-builder/custom-data/config.yml
@@ -0,0 +1,19 @@
+entrypoint: ['bash', '-bash', '-c', 'exec "$@"', '--']
+user: worker
+env:
+  # Set variable normally configured at login, by the shells parent process, these
+  # are taken from GNU su manual
+  HOME: /home/worker
+  SHELL: /bin/bash
+  USER: worker
+  LOGNAME: worker
+  HOSTNAME: taskcluster-worker
+  LC_ALL: C
+  # Set terminal emulator
+  TERM: xterm
+  # Set HG_STORE_PATH for run-task
+  HG_STORE_PATH: /home/worker/checkouts/hg-store
+  # Set MOZ_AUTOMATION to make mach and other scripts stop complaining
+  MOZ_AUTOMATION: '1'
+shell: ['bash', '-bash', '-li']
+workdir: /home/worker
new file mode 100644
--- /dev/null
+++ b/taskcluster/qemu/docker-image-builder/custom-data/download-and-compress
@@ -0,0 +1,85 @@
+#!/usr/bin/python2.7
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os
+import sys
+import time
+
+import requests
+import requests_unixsocket
+import zstd
+
+# Allow requests to fetch from UNIX domain sockets.
+requests_unixsocket.monkeypatch()
+
+
+def download_and_compress(url, path, level):
+    r = requests.get(url, stream=True)
+
+    if r.status_code != 200:
+        raise Exception('non-200 response: %d' % r.status_code)
+
+    in_size = 0
+    out_size = 0
+    last_progress = time.time()
+
+    # Use all available CPU cores for multi-threaded compression.
+    cctx = zstd.ZstdCompressor(threads=-1, level=level, write_checksum=True)
+    cobj = cctx.compressobj()
+    with open(path, 'wb') as fh:
+        for raw in r.iter_content(zstd.COMPRESSION_RECOMMENDED_INPUT_SIZE):
+            # Print output periodically, for humans.
+            now = time.time()
+            if now - last_progress > 5.0:
+                print('%d -> %d' % (in_size, out_size))
+                last_progress = now
+
+            in_size += len(raw)
+            chunk = cobj.compress(raw)
+            if not chunk:
+                continue
+
+            out_size += len(chunk)
+            fh.write(chunk)
+
+        chunk = cobj.flush()
+        out_size += len(chunk)
+        fh.write(chunk)
+
+    return in_size, out_size
+
+
+if __name__ == '__main__':
+    url, temp_path, final_path = sys.argv[1:]
+
+    # Default zstd level is 3. We default to 10 because multi-threaded
+    # compression allows us to burn lots of CPU for significant image
+    # size reductions without a major wall time penalty.
+    level = int(os.environ.get('DOCKER_IMAGE_ZSTD_LEVEL', '10'))
+    print('using zstandard compression level %d' % level)
+
+    count = 0
+    while count < 10:
+        count += 1
+
+        try:
+            t_start = time.time()
+            raw_size, compress_size = download_and_compress(url, temp_path,
+                                                            level)
+            elapsed = time.time() - t_start
+            # Move to final path at end so partial image isn't uploaded as
+            # an artifact.
+            os.rename(temp_path, final_path)
+            speed = int(raw_size / elapsed) / 1000000
+            print('compression ratio: %.2f (%d -> %d) @ %d MB/s' % (
+                float(compress_size) / float(raw_size),
+                raw_size, compress_size, speed))
+            sys.exit(0)
+        except Exception as e:
+            print('exception: %s' % e)
+            time.sleep(5)
+
+    print('reached maximum retry attempts; giving up')
+    sys.exit(1)
new file mode 100644
--- /dev/null
+++ b/taskcluster/qemu/docker-image-builder/custom-data/install.sh
@@ -0,0 +1,88 @@
+#!/bin/bash -e
+
+export DEBIAN_FRONTEND=noninteractive
+
+echo ' - Removing password from "worker" user'
+passwd -d worker
+
+echo ' - Allow "worker" user to do sudo without password'
+echo 'worker ALL=(ALL) NOPASSWD: ALL' >> /etc/sudoers
+
+echo ' - Installing taskcluster-worker qemu-guest-tools'
+cp /tmp/custom-data/config.yml /etc/taskcluster-worker-qemu-guest-tools.yml
+cp /tmp/custom-data/taskcluster-worker /usr/local/bin/taskcluster-worker
+chmod +x /usr/local/bin/taskcluster-worker
+cp /tmp/custom-data/taskcluster-worker.service /etc/systemd/system/taskcluster-worker.service
+chmod 644 /etc/systemd/system/taskcluster-worker.service
+systemctl enable taskcluster-worker.service
+
+echo ' - Installing docker'
+apt-get install -y apt-transport-https ca-certificates
+apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D
+echo 'deb https://apt.dockerproject.org/repo ubuntu-xenial main' > /etc/apt/sources.list.d/docker.list
+apt-get update -y
+apt-get install -y docker-engine=17.05.0~ce-0~ubuntu-xenial
+
+echo ' - Grant worker user access to docker'
+usermod -aG docker worker
+
+echo ' - Installing python'
+apt-get install -y python=2.7.11-1
+
+echo ' - Installing mercurial'
+mkdir -p /setup /build
+cp /tmp/custom-data/tooltool.py /setup/tooltool.py
+mkdir -p /usr/local/mercurial
+cp /tmp/custom-data/robustcheckout.py /usr/local/mercurial/robustcheckout.py
+. /tmp/custom-data/common.sh
+. /tmp/custom-data/install-mercurial.sh
+
+echo ' - Configuring mercurial to use uncompressed bundles'
+cat << EOF > /home/worker/.hgrc
+[ui]
+clonebundleprefers = VERSION=packed1
+EOF
+
+echo ' - Installing test script'
+cp /tmp/custom-data/run-task /usr/local/bin/run-task
+chmod +x /usr/local/bin/run-task
+
+echo ' - chown ~/'
+chown -R worker:worker /home/worker/
+
+echo ' - Install dependencies'
+apt-get install -y --no-install-recommends \
+    curl \
+    tar \
+    jq \
+    python \
+    python-requests \
+    python-requests-unixsocket \
+    python-setuptools \
+    build-essential \
+    python-dev \
+    python-pip
+
+echo ' - Install build-image.sh script'
+cp /tmp/custom-data/build-image.sh /usr/local/bin/build-image.sh
+cp /tmp/custom-data/download-and-compress /usr/local/bin/download-and-compress
+chmod +x /usr/local/bin/build-image.sh
+chmod +x /usr/local/bin/download-and-compress
+
+echo ' - Install python-zstandard'
+cd /tmp
+tooltool_fetch <<EOF
+[
+  {
+    "size": 463794,
+    "visibility": "public",
+    "digest": "c6ba906403e5c18b374faf9f676b10f0988b9f4067bd6c52c548d7dee58fac79974babfd5c438aef8da0a5260158116db69b11f2a52a775772d9904b9d86fdbc",
+    "algorithm": "sha512",
+    "filename": "zstandard-0.8.0.tar.gz"
+  }
+]
+EOF
+cd -
+/usr/bin/pip -v install /tmp/zstandard-0.8.0.tar.gz
+
+echo ' - Setup completed'
new file mode 100644
--- /dev/null
+++ b/taskcluster/qemu/docker-image-builder/custom-data/taskcluster-worker.service
@@ -0,0 +1,11 @@
+[Unit]
+Description=QEMU guest tools for taskcluster-worker
+After=docker.service
+Requires=network-online.target
+
+[Service]
+Type=simple
+ExecStart=/usr/local/bin/taskcluster-worker qemu-guest-tools -c /etc/taskcluster-worker-qemu-guest-tools.yml
+
+[Install]
+WantedBy=multi-user.target
new file mode 100644
--- /dev/null
+++ b/taskcluster/qemu/docker-image-builder/custom-data/worker.seed
@@ -0,0 +1,101 @@
+# Defaults from: https://github.com/netson/ubuntu-unattended/blob/master/netson.seed
+# More good docs at: https://www.debian.org/releases/jessie/amd64/apbs04.html.en
+
+# Always install the virtual kernel
+d-i base-installer/kernel/override-image                    string      linux-virtual
+
+# Localization
+d-i debian-installer/language                               string      en_US:en
+d-i debian-installer/country                                string      US
+d-i debian-installer/locale                                 string      en_US
+d-i localechooser/supported-locales                         multiselect en_US.UTF-8
+d-i pkgsel/install-language-support                         boolean     false
+
+# Keyboard selection
+d-i console-setup/ask_detect                                boolean     false
+d-i keyboard-configuration/modelcode                        string      pc105
+d-i keyboard-configuration/layoutcode                       string      us
+d-i keyboard-configuration/variantcode                      string      intl
+d-i keyboard-configuration/xkb-keymap                       select      us(intl)
+d-i debconf/language                                        string      en_US:en
+
+# Network settings
+d-i netcfg/choose_interface                                 select      auto
+d-i netcfg/dhcp_timeout                                     string      30
+d-i netcfg/get_hostname                                     string      worker-vm
+d-i netcfg/get_domain                                       string      worker-vm
+d-i hw-detect/load_firmware                                 boolean     true
+
+# Mirror settings
+d-i mirror/country                                          string      manual
+d-i mirror/http/hostname                                    string      archive.ubuntu.com
+d-i mirror/http/directory                                   string      /ubuntu
+d-i mirror/http/proxy                                       string
+
+# Configure apt
+d-i apt-setup/restricted                                    boolean     true
+d-i apt-setup/universe                                      boolean     true
+d-i apt-setup/backports                                     boolean     true
+d-i apt-setup/services-select                               multiselect security
+d-i apt-setup/security_host                                 string      security.ubuntu.com
+d-i apt-setup/security_path                                 string      /ubuntu
+tasksel tasksel/first                                       multiselect Basic Ubuntu server
+d-i pkgsel/upgrade                                          select      full-upgrade
+d-i pkgsel/update-policy                                    select      none
+d-i pkgsel/updatedb                                         boolean     true
+d-i debconf debconf/frontend                                select      Noninteractive
+popularity-contest popularity-contest/participate           boolean     false
+
+# User configuration...
+d-i passwd/root-login                                       boolean     false
+d-i passwd/make-user                                        boolean     true
+d-i passwd/user-fullname                                    string      worker
+d-i passwd/username                                         string      worker
+d-i passwd/user-password                                    password    worker
+d-i passwd/user-password-again                              password    worker
+d-i passwd/user-uid                                         string
+d-i user-setup/allow-password-weak                          boolean     true
+d-i user-setup/encrypt-home                                 boolean     false
+
+# Clock and time (use UTC, don't sync with NTP, timezone UTC)
+d-i clock-setup/utc                                         boolean     true
+d-i clock-setup/ntp                                         boolean     false
+d-i time/zone                                               string      UTC
+
+# Partitioning
+d-i partman/confirm_write_new_label                         boolean     true
+d-i partman/choose_partition                                select      finish
+d-i partman/confirm_nooverwrite                             boolean     true
+d-i partman/confirm                                         boolean     true
+d-i partman-auto/purge_lvm_from_device                      boolean     true
+d-i partman-lvm/device_remove_lvm                           boolean     true
+d-i partman-lvm/confirm                                     boolean     true
+d-i partman-lvm/confirm_nooverwrite                         boolean     true
+d-i partman-auto-lvm/no_boot                                boolean     true
+d-i partman-md/device_remove_md                             boolean     true
+d-i partman-md/confirm                                      boolean     true
+d-i partman-md/confirm_nooverwrite                          boolean     true
+d-i partman-auto/method                                     string      lvm
+d-i partman-auto-lvm/guided_size                            string      max
+d-i partman-partitioning/confirm_write_new_label            boolean     true
+
+# Package selection
+d-i pkgsel/include                                          string      curl
+
+# No verbose output and no boot splash screen.
+d-i debian-installer/quiet                                  boolean     true
+d-i debian-installer/splash                                 boolean     false
+
+# Bootloader
+d-i grub-installer/timeout                                  string      0
+d-i grub-installer/only_debian                              boolean     true
+d-i grub-installer/with_other_os                            boolean     true
+
+# Command to run before finishing install
+d-i preseed/late_command                                    string      /cdrom/custom-data/wrap-install.sh
+
+# Poweroff the machine after install
+d-i finish-install/reboot_in_progress                       note
+d-i finish-install/keep-consoles                            boolean     false
+d-i cdrom-detect/eject                                      boolean     false
+d-i debian-installer/exit/poweroff                          boolean     true
new file mode 100644
--- /dev/null
+++ b/taskcluster/qemu/docker-image-builder/custom-data/wrap-install.sh
@@ -0,0 +1,6 @@
+#!/bin/sh
+
+cp -r /cdrom/custom-data /target/tmp/custom-data
+chmod +x /target/tmp/custom-data/install.sh
+chroot /target /bin/bash --login /tmp/custom-data/install.sh | /cdrom/custom-data/taskcluster-worker qemu-guest-tools post-log -
+rm -rf /target/tmp/custom-data
new file mode 100644
--- /dev/null
+++ b/taskcluster/qemu/docker-image-builder/image.yml
@@ -0,0 +1,82 @@
+symbol: 'I(docker-image-builder)'
+private: false
+description: |
+  Automatically built Ubuntu image for building in-tree docker images.
+  This image contains
+     * python 2.7
+     * mercurial
+     * docker
+     * run-task (from taskcluster/docker/recipes/run-task)
+disksize: 30 # GB
+machine: # virtual machine definition
+  uuid: 52bab607-10f1-4049-a0f8-ee4725cb715b
+  chipset: pc-i440fx-2.8
+  usb: nec-usb-xhci
+  network: e1000
+  mac: aa:54:1a:30:5c:de
+  storage: virtio-blk-pci
+  graphics: qxl-vga
+  sound: none
+  keyboard: usb-kbd
+  keyboardLayout: en-us
+  mouse: usb-mouse
+  tablet: usb-tablet
+cdromA:
+  # Download an extract ubuntu 16.04 server install media
+  - extract:
+      url: http://releases.ubuntu.com/16.04.2/ubuntu-16.04.2-server-amd64.iso
+      sha256: 737ae7041212c628de5751d15c3016058b0e833fdc32e7420209b76ca3d0a535
+    format: iso
+    target: /
+
+  # Modify the grub boot menu to have a timeout of 1 second
+  - sed: 's/timeout\s\+[0-9]\+/timeout 1/g'
+    target: /isolinux/isolinux.cfg
+
+  # Modify boot options to specify preseed in kernel parameters
+  - sed: '/label install/ilabel autoinstall\nmenu label ^Autoinstall Ubuntu Worker\nkernel /install/vmlinuz\nappend file=/cdrom/custom-data/worker.seed initrd=/install/initrd.gz auto=true priority=high preseed/file=/cdrom/custom-data/worker.seed --'
+    target: /isolinux/txt.cfg
+
+  # Add custom-data which contains preseed, install scripts, etc
+  - copy: ./custom-data
+    target: /custom-data
+  - chmod: +x
+    target: /custom-data/wrap-install.sh
+
+  # Add run-task and install-mercurial.sh as needed by custom-data/install.sh
+  - copy: /taskcluster/docker/recipes/run-task
+    target: /custom-data/run-task
+  - copy: /python/mozbuild/mozbuild/action/tooltool.py
+    target: /custom-data/tooltool.py
+  - copy: /taskcluster/docker/recipes/common.sh
+    target: /custom-data/common.sh
+  - copy: /testing/mozharness/external_tools/robustcheckout.py
+    target: /custom-data/robustcheckout.py
+  - copy: /taskcluster/docker/recipes/install-mercurial.sh
+    target: /custom-data/install-mercurial.sh
+
+  # Download taskcluster-worker and add it to custom-data
+  - copy:
+      url: https://github.com/taskcluster/taskcluster-worker/releases/download/v0.1.9/taskcluster-worker-0.1.9-linux-amd64
+      sha256: dc184f3c741ed4098c05350c531739f176d6e2146d38314311e6b019b5f84727
+    target: /custom-data/taskcluster-worker
+
+  # Make taskcluster-worker binary executable
+  - chmod: +x
+    target: /custom-data/taskcluster-worker
+
+  # Package everything into an ISO
+  - genisoimage:
+     - '-JlDrV'
+     - 'UBUNTU_INSTALLER'
+     - '-input-charset'
+     - 'utf-8'
+     - '-cache-inodes'
+     - '-b'
+     - 'isolinux/isolinux.bin'
+     - '-c'
+     - 'isolinux/boot.cat'
+     - '-no-emul-boot'
+     - '-boot-load-size'
+     - '4'
+     - '-boot-info-table'