Bug 1329282 - QEMU image for building docker images
This adds a QEMU image based on Ubuntu 16.04 for in-tree
docker images. Changes to docker-image tasks are not part
of this commit.
MozReview-Commit-ID: FnSGRfMCCEn
new file mode 100644
--- /dev/null
+++ b/taskcluster/qemu/docker-image-builder/custom-data/build-image.sh
@@ -0,0 +1,47 @@
+#!/bin/bash -vex
+
+# Set bash options to exit immediately if a pipeline exists non-zero, expand
+# print a trace of commands, and make output verbose (print shell input as it's
+# read)
+# See https://www.gnu.org/software/bash/manual/html_node/The-Set-Builtin.html
+set -x -e -v -o pipefail
+
+# Prefix errors with taskcluster error prefix so that they are parsed by Treeherder
+raise_error() {
+ echo
+ echo "[taskcluster-image-build:error] $1"
+ exit 1
+}
+
+# Ensure that the PROJECT is specified so the image can be indexed
+test -n "$PROJECT" || raise_error "PROJECT must be provided."
+test -n "$HASH" || raise_error "Context HASH must be provided."
+test -n "$IMAGE_NAME" || raise_error "IMAGE_NAME must be provided."
+
+# Construct a CONTEXT_FILE
+CONTEXT_FILE=/home/worker/context.tar
+
+# Run ./mach taskcluster-build-image with --context-only to build context
+run-task \
+ --vcs-checkout "/home/worker/checkouts/gecko" \
+ -- \
+ /home/worker/checkouts/gecko/mach taskcluster-build-image \
+ --context-only "$CONTEXT_FILE" \
+ "$IMAGE_NAME"
+test -f "$CONTEXT_FILE" || raise_error "Context file wasn't created"
+
+# Create artifact folder (note that this must occur after run-task)
+mkdir -p /home/worker/artifacts
+
+# Build image from tarball
+docker build -t "$IMAGE_NAME:$HASH" - < "$CONTEXT_FILE"
+
+# Get image from docker daemon (try up to 10 times)
+# This interacts directly with the docker remote API, see:
+# https://docs.docker.com/engine/reference/api/docker_remote_api_v1.18/
+#
+# The script will retry up to 10 times.
+/usr/local/bin/download-and-compress \
+ http+unix://%2Fvar%2Frun%2Fdocker.sock/images/${IMAGE_NAME}:${HASH}/get \
+ /home/worker/image.tar.zst.tmp \
+ /home/worker/artifacts/image.tar.zst
new file mode 100644
--- /dev/null
+++ b/taskcluster/qemu/docker-image-builder/custom-data/config.yml
@@ -0,0 +1,19 @@
+entrypoint: ['bash', '-bash', '-c', 'exec "$@"', '--']
+user: worker
+env:
+ # Set variable normally configured at login, by the shells parent process, these
+ # are taken from GNU su manual
+ HOME: /home/worker
+ SHELL: /bin/bash
+ USER: worker
+ LOGNAME: worker
+ HOSTNAME: taskcluster-worker
+ LC_ALL: C
+ # Set terminal emulator
+ TERM: xterm
+ # Set HG_STORE_PATH for run-task
+ HG_STORE_PATH: /home/worker/checkouts/hg-store
+ # Set MOZ_AUTOMATION to make mach and other scripts stop complaining
+ MOZ_AUTOMATION: '1'
+shell: ['bash', '-bash', '-li']
+workdir: /home/worker
new file mode 100644
--- /dev/null
+++ b/taskcluster/qemu/docker-image-builder/custom-data/download-and-compress
@@ -0,0 +1,85 @@
+#!/usr/bin/python2.7
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os
+import sys
+import time
+
+import requests
+import requests_unixsocket
+import zstd
+
+# Allow requests to fetch from UNIX domain sockets.
+requests_unixsocket.monkeypatch()
+
+
+def download_and_compress(url, path, level):
+ r = requests.get(url, stream=True)
+
+ if r.status_code != 200:
+ raise Exception('non-200 response: %d' % r.status_code)
+
+ in_size = 0
+ out_size = 0
+ last_progress = time.time()
+
+ # Use all available CPU cores for multi-threaded compression.
+ cctx = zstd.ZstdCompressor(threads=-1, level=level, write_checksum=True)
+ cobj = cctx.compressobj()
+ with open(path, 'wb') as fh:
+ for raw in r.iter_content(zstd.COMPRESSION_RECOMMENDED_INPUT_SIZE):
+ # Print output periodically, for humans.
+ now = time.time()
+ if now - last_progress > 5.0:
+ print('%d -> %d' % (in_size, out_size))
+ last_progress = now
+
+ in_size += len(raw)
+ chunk = cobj.compress(raw)
+ if not chunk:
+ continue
+
+ out_size += len(chunk)
+ fh.write(chunk)
+
+ chunk = cobj.flush()
+ out_size += len(chunk)
+ fh.write(chunk)
+
+ return in_size, out_size
+
+
+if __name__ == '__main__':
+ url, temp_path, final_path = sys.argv[1:]
+
+ # Default zstd level is 3. We default to 10 because multi-threaded
+ # compression allows us to burn lots of CPU for significant image
+ # size reductions without a major wall time penalty.
+ level = int(os.environ.get('DOCKER_IMAGE_ZSTD_LEVEL', '10'))
+ print('using zstandard compression level %d' % level)
+
+ count = 0
+ while count < 10:
+ count += 1
+
+ try:
+ t_start = time.time()
+ raw_size, compress_size = download_and_compress(url, temp_path,
+ level)
+ elapsed = time.time() - t_start
+ # Move to final path at end so partial image isn't uploaded as
+ # an artifact.
+ os.rename(temp_path, final_path)
+ speed = int(raw_size / elapsed) / 1000000
+ print('compression ratio: %.2f (%d -> %d) @ %d MB/s' % (
+ float(compress_size) / float(raw_size),
+ raw_size, compress_size, speed))
+ sys.exit(0)
+ except Exception as e:
+ print('exception: %s' % e)
+ time.sleep(5)
+
+ print('reached maximum retry attempts; giving up')
+ sys.exit(1)
new file mode 100644
--- /dev/null
+++ b/taskcluster/qemu/docker-image-builder/custom-data/install.sh
@@ -0,0 +1,88 @@
+#!/bin/bash -e
+
+export DEBIAN_FRONTEND=noninteractive
+
+echo ' - Removing password from "worker" user'
+passwd -d worker
+
+echo ' - Allow "worker" user to do sudo without password'
+echo 'worker ALL=(ALL) NOPASSWD: ALL' >> /etc/sudoers
+
+echo ' - Installing taskcluster-worker qemu-guest-tools'
+cp /tmp/custom-data/config.yml /etc/taskcluster-worker-qemu-guest-tools.yml
+cp /tmp/custom-data/taskcluster-worker /usr/local/bin/taskcluster-worker
+chmod +x /usr/local/bin/taskcluster-worker
+cp /tmp/custom-data/taskcluster-worker.service /etc/systemd/system/taskcluster-worker.service
+chmod 644 /etc/systemd/system/taskcluster-worker.service
+systemctl enable taskcluster-worker.service
+
+echo ' - Installing docker'
+apt-get install -y apt-transport-https ca-certificates
+apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D
+echo 'deb https://apt.dockerproject.org/repo ubuntu-xenial main' > /etc/apt/sources.list.d/docker.list
+apt-get update -y
+apt-get install -y docker-engine=17.05.0~ce-0~ubuntu-xenial
+
+echo ' - Grant worker user access to docker'
+usermod -aG docker worker
+
+echo ' - Installing python'
+apt-get install -y python=2.7.11-1
+
+echo ' - Installing mercurial'
+mkdir -p /setup /build
+cp /tmp/custom-data/tooltool.py /setup/tooltool.py
+mkdir -p /usr/local/mercurial
+cp /tmp/custom-data/robustcheckout.py /usr/local/mercurial/robustcheckout.py
+. /tmp/custom-data/common.sh
+. /tmp/custom-data/install-mercurial.sh
+
+echo ' - Configuring mercurial to use uncompressed bundles'
+cat << EOF > /home/worker/.hgrc
+[ui]
+clonebundleprefers = VERSION=packed1
+EOF
+
+echo ' - Installing test script'
+cp /tmp/custom-data/run-task /usr/local/bin/run-task
+chmod +x /usr/local/bin/run-task
+
+echo ' - chown ~/'
+chown -R worker:worker /home/worker/
+
+echo ' - Install dependencies'
+apt-get install -y --no-install-recommends \
+ curl \
+ tar \
+ jq \
+ python \
+ python-requests \
+ python-requests-unixsocket \
+ python-setuptools \
+ build-essential \
+ python-dev \
+ python-pip
+
+echo ' - Install build-image.sh script'
+cp /tmp/custom-data/build-image.sh /usr/local/bin/build-image.sh
+cp /tmp/custom-data/download-and-compress /usr/local/bin/download-and-compress
+chmod +x /usr/local/bin/build-image.sh
+chmod +x /usr/local/bin/download-and-compress
+
+echo ' - Install python-zstandard'
+cd /tmp
+tooltool_fetch <<EOF
+[
+ {
+ "size": 463794,
+ "visibility": "public",
+ "digest": "c6ba906403e5c18b374faf9f676b10f0988b9f4067bd6c52c548d7dee58fac79974babfd5c438aef8da0a5260158116db69b11f2a52a775772d9904b9d86fdbc",
+ "algorithm": "sha512",
+ "filename": "zstandard-0.8.0.tar.gz"
+ }
+]
+EOF
+cd -
+/usr/bin/pip -v install /tmp/zstandard-0.8.0.tar.gz
+
+echo ' - Setup completed'
new file mode 100644
--- /dev/null
+++ b/taskcluster/qemu/docker-image-builder/custom-data/taskcluster-worker.service
@@ -0,0 +1,11 @@
+[Unit]
+Description=QEMU guest tools for taskcluster-worker
+After=docker.service
+Requires=network-online.target
+
+[Service]
+Type=simple
+ExecStart=/usr/local/bin/taskcluster-worker qemu-guest-tools -c /etc/taskcluster-worker-qemu-guest-tools.yml
+
+[Install]
+WantedBy=multi-user.target
new file mode 100644
--- /dev/null
+++ b/taskcluster/qemu/docker-image-builder/custom-data/worker.seed
@@ -0,0 +1,101 @@
+# Defaults from: https://github.com/netson/ubuntu-unattended/blob/master/netson.seed
+# More good docs at: https://www.debian.org/releases/jessie/amd64/apbs04.html.en
+
+# Always install the virtual kernel
+d-i base-installer/kernel/override-image string linux-virtual
+
+# Localization
+d-i debian-installer/language string en_US:en
+d-i debian-installer/country string US
+d-i debian-installer/locale string en_US
+d-i localechooser/supported-locales multiselect en_US.UTF-8
+d-i pkgsel/install-language-support boolean false
+
+# Keyboard selection
+d-i console-setup/ask_detect boolean false
+d-i keyboard-configuration/modelcode string pc105
+d-i keyboard-configuration/layoutcode string us
+d-i keyboard-configuration/variantcode string intl
+d-i keyboard-configuration/xkb-keymap select us(intl)
+d-i debconf/language string en_US:en
+
+# Network settings
+d-i netcfg/choose_interface select auto
+d-i netcfg/dhcp_timeout string 30
+d-i netcfg/get_hostname string worker-vm
+d-i netcfg/get_domain string worker-vm
+d-i hw-detect/load_firmware boolean true
+
+# Mirror settings
+d-i mirror/country string manual
+d-i mirror/http/hostname string archive.ubuntu.com
+d-i mirror/http/directory string /ubuntu
+d-i mirror/http/proxy string
+
+# Configure apt
+d-i apt-setup/restricted boolean true
+d-i apt-setup/universe boolean true
+d-i apt-setup/backports boolean true
+d-i apt-setup/services-select multiselect security
+d-i apt-setup/security_host string security.ubuntu.com
+d-i apt-setup/security_path string /ubuntu
+tasksel tasksel/first multiselect Basic Ubuntu server
+d-i pkgsel/upgrade select full-upgrade
+d-i pkgsel/update-policy select none
+d-i pkgsel/updatedb boolean true
+d-i debconf debconf/frontend select Noninteractive
+popularity-contest popularity-contest/participate boolean false
+
+# User configuration...
+d-i passwd/root-login boolean false
+d-i passwd/make-user boolean true
+d-i passwd/user-fullname string worker
+d-i passwd/username string worker
+d-i passwd/user-password password worker
+d-i passwd/user-password-again password worker
+d-i passwd/user-uid string
+d-i user-setup/allow-password-weak boolean true
+d-i user-setup/encrypt-home boolean false
+
+# Clock and time (use UTC, don't sync with NTP, timezone UTC)
+d-i clock-setup/utc boolean true
+d-i clock-setup/ntp boolean false
+d-i time/zone string UTC
+
+# Partitioning
+d-i partman/confirm_write_new_label boolean true
+d-i partman/choose_partition select finish
+d-i partman/confirm_nooverwrite boolean true
+d-i partman/confirm boolean true
+d-i partman-auto/purge_lvm_from_device boolean true
+d-i partman-lvm/device_remove_lvm boolean true
+d-i partman-lvm/confirm boolean true
+d-i partman-lvm/confirm_nooverwrite boolean true
+d-i partman-auto-lvm/no_boot boolean true
+d-i partman-md/device_remove_md boolean true
+d-i partman-md/confirm boolean true
+d-i partman-md/confirm_nooverwrite boolean true
+d-i partman-auto/method string lvm
+d-i partman-auto-lvm/guided_size string max
+d-i partman-partitioning/confirm_write_new_label boolean true
+
+# Package selection
+d-i pkgsel/include string curl
+
+# No verbose output and no boot splash screen.
+d-i debian-installer/quiet boolean true
+d-i debian-installer/splash boolean false
+
+# Bootloader
+d-i grub-installer/timeout string 0
+d-i grub-installer/only_debian boolean true
+d-i grub-installer/with_other_os boolean true
+
+# Command to run before finishing install
+d-i preseed/late_command string /cdrom/custom-data/wrap-install.sh
+
+# Poweroff the machine after install
+d-i finish-install/reboot_in_progress note
+d-i finish-install/keep-consoles boolean false
+d-i cdrom-detect/eject boolean false
+d-i debian-installer/exit/poweroff boolean true
new file mode 100644
--- /dev/null
+++ b/taskcluster/qemu/docker-image-builder/custom-data/wrap-install.sh
@@ -0,0 +1,6 @@
+#!/bin/sh
+
+cp -r /cdrom/custom-data /target/tmp/custom-data
+chmod +x /target/tmp/custom-data/install.sh
+chroot /target /bin/bash --login /tmp/custom-data/install.sh | /cdrom/custom-data/taskcluster-worker qemu-guest-tools post-log -
+rm -rf /target/tmp/custom-data
new file mode 100644
--- /dev/null
+++ b/taskcluster/qemu/docker-image-builder/image.yml
@@ -0,0 +1,82 @@
+symbol: 'I(docker-image-builder)'
+private: false
+description: |
+ Automatically built Ubuntu image for building in-tree docker images.
+ This image contains
+ * python 2.7
+ * mercurial
+ * docker
+ * run-task (from taskcluster/docker/recipes/run-task)
+disksize: 30 # GB
+machine: # virtual machine definition
+ uuid: 52bab607-10f1-4049-a0f8-ee4725cb715b
+ chipset: pc-i440fx-2.8
+ usb: nec-usb-xhci
+ network: e1000
+ mac: aa:54:1a:30:5c:de
+ storage: virtio-blk-pci
+ graphics: qxl-vga
+ sound: none
+ keyboard: usb-kbd
+ keyboardLayout: en-us
+ mouse: usb-mouse
+ tablet: usb-tablet
+cdromA:
+ # Download an extract ubuntu 16.04 server install media
+ - extract:
+ url: http://releases.ubuntu.com/16.04.2/ubuntu-16.04.2-server-amd64.iso
+ sha256: 737ae7041212c628de5751d15c3016058b0e833fdc32e7420209b76ca3d0a535
+ format: iso
+ target: /
+
+ # Modify the grub boot menu to have a timeout of 1 second
+ - sed: 's/timeout\s\+[0-9]\+/timeout 1/g'
+ target: /isolinux/isolinux.cfg
+
+ # Modify boot options to specify preseed in kernel parameters
+ - sed: '/label install/ilabel autoinstall\nmenu label ^Autoinstall Ubuntu Worker\nkernel /install/vmlinuz\nappend file=/cdrom/custom-data/worker.seed initrd=/install/initrd.gz auto=true priority=high preseed/file=/cdrom/custom-data/worker.seed --'
+ target: /isolinux/txt.cfg
+
+ # Add custom-data which contains preseed, install scripts, etc
+ - copy: ./custom-data
+ target: /custom-data
+ - chmod: +x
+ target: /custom-data/wrap-install.sh
+
+ # Add run-task and install-mercurial.sh as needed by custom-data/install.sh
+ - copy: /taskcluster/docker/recipes/run-task
+ target: /custom-data/run-task
+ - copy: /python/mozbuild/mozbuild/action/tooltool.py
+ target: /custom-data/tooltool.py
+ - copy: /taskcluster/docker/recipes/common.sh
+ target: /custom-data/common.sh
+ - copy: /testing/mozharness/external_tools/robustcheckout.py
+ target: /custom-data/robustcheckout.py
+ - copy: /taskcluster/docker/recipes/install-mercurial.sh
+ target: /custom-data/install-mercurial.sh
+
+ # Download taskcluster-worker and add it to custom-data
+ - copy:
+ url: https://github.com/taskcluster/taskcluster-worker/releases/download/v0.1.9/taskcluster-worker-0.1.9-linux-amd64
+ sha256: dc184f3c741ed4098c05350c531739f176d6e2146d38314311e6b019b5f84727
+ target: /custom-data/taskcluster-worker
+
+ # Make taskcluster-worker binary executable
+ - chmod: +x
+ target: /custom-data/taskcluster-worker
+
+ # Package everything into an ISO
+ - genisoimage:
+ - '-JlDrV'
+ - 'UBUNTU_INSTALLER'
+ - '-input-charset'
+ - 'utf-8'
+ - '-cache-inodes'
+ - '-b'
+ - 'isolinux/isolinux.bin'
+ - '-c'
+ - 'isolinux/boot.cat'
+ - '-no-emul-boot'
+ - '-boot-load-size'
+ - '4'
+ - '-boot-info-table'