Bug 1433033 - Create a base Debian docker image. r?dustin draft
authorMike Hommey <mh+mozilla@glandium.org>
Thu, 25 Jan 2018 22:15:19 +0900
changeset 747488 82744fbeb87f379af11013aabab77a291e6100ce
parent 747487 bc4db259b31a2df58ba24dcd5b7d0f00f2b70588
child 747489 7fb5eee317f5245be4597e76f3f8f0f372a17503
push id96908
push userbmo:mh+mozilla@glandium.org
push dateFri, 26 Jan 2018 02:31:20 +0000
reviewersdustin
bugs1433033
milestone60.0a1
Bug 1433033 - Create a base Debian docker image. r?dustin That image is used to derive all the debian7-* images, and its definition is parametrized, which will allow to create other images based on other versions of Debian, from the same definition. XZ_OPT is kept in each of those because we don't want to automatically set it in all further derived images.
taskcluster/ci/docker-image/kind.yml
taskcluster/docker/debian-base/Dockerfile
taskcluster/docker/debian-base/setup_packages.sh
taskcluster/docker/debian7-build/Dockerfile
taskcluster/docker/toolchain-build/Dockerfile
--- a/taskcluster/ci/docker-image/kind.yml
+++ b/taskcluster/ci/docker-image/kind.yml
@@ -20,48 +20,49 @@ jobs:
   image_builder:
     symbol: I(ib)
   desktop1604-test:
     symbol: I(dt16t)
   desktop-build:
     symbol: I(db)
   valgrind-build:
     symbol: I(vb)
-  toolchain-build:
-    symbol: I(toolchain)
-    packages:
-      - deb7-cmake
-      - deb7-git
-      - deb7-make
-      - deb7-mercurial
-      - deb7-ninja
-      - deb7-python
-      - deb7-xz-utils
-  debian7-amd64-build:
-    symbol: I(deb7)
-    definition: debian7-build
+  debian7-base:
+    symbol: I(deb7-base)
+    definition: debian-base
+    args:
+      DIST: wheezy
+      BASE_TAG: '20171210'
+      SNAPSHOT: '20171210T214726Z'
     packages:
       - deb7-git
       - deb7-make
       - deb7-mercurial
       - deb7-python
+      - deb7-xz-utils
+  toolchain-build:
+    symbol: I(toolchain)
+    parent: debian7-base
+    packages:
+      - deb7-cmake
+      - deb7-ninja
+  debian7-amd64-build:
+    symbol: I(deb7)
+    parent: debian7-base
+    definition: debian7-build
+    packages:
       - deb7-valgrind
-      - deb7-xz-utils
     args:
       ARCH: amd64
   debian7-i386-build:
     symbol: I(deb7-32)
+    parent: debian7-base
     definition: debian7-build
     packages:
-      - deb7-git
-      - deb7-make
-      - deb7-mercurial
-      - deb7-python
       - deb7-valgrind
-      - deb7-xz-utils
     args:
       ARCH: i386
   lint:
     symbol: I(lnt)
   android-build:
     symbol: I(agb)
   index-task:
     symbol: I(idx)
new file mode 100644
--- /dev/null
+++ b/taskcluster/docker/debian-base/Dockerfile
@@ -0,0 +1,69 @@
+# %ARG DIST
+# %ARG BASE_TAG
+FROM debian:$DIST-$BASE_TAG
+MAINTAINER Mike Hommey <mhommey@mozilla.com>
+
+### Add worker user and setup its workspace.
+RUN mkdir /builds && \
+    groupadd -g 500 worker && \
+    useradd -u 500 -g 500 -d /builds/worker -s /bin/bash -m worker && \
+    mkdir -p /builds/worker/workspace && \
+    chown -R worker:worker /builds
+
+# Declare default working folder
+WORKDIR /builds/worker
+
+# Set variable normally configured at login, by the shells parent process, these
+# are taken from GNU su manual
+ENV HOME=/builds/worker \
+    SHELL=/bin/bash \
+    USER=worker \
+    LOGNAME=worker \
+    HOSTNAME=taskcluster-worker \
+    DEBIAN_FRONTEND=noninteractive
+
+# Set a default command useful for debugging
+CMD ["/bin/bash", "--login"]
+
+# %ARG SNAPSHOT
+# Set apt sources list to a snapshot.
+RUN for s in debian_$DIST debian_$DIST-updates debian_$DIST-backports debian-security_$DIST/updates; do \
+      echo "deb http://snapshot.debian.org/archive/${s%_*}/$SNAPSHOT/ ${s#*_} main"; \
+    done > /etc/apt/sources.list && \
+    ( echo 'quiet "true";'; \
+      echo 'APT::Get::Assume-Yes "true";'; \
+      echo 'APT::Install-Recommends "false";'; \
+      echo 'Acquire::Check-Valid-Until "false";'; \
+    ) > /etc/apt/apt.conf.d/99taskcluster
+
+RUN apt-get update && \
+    apt-get install \
+      apt-transport-https \
+      ca-certificates
+
+COPY setup_packages.sh /usr/local/sbin/
+
+# %ARG DOCKER_IMAGE_PACKAGES
+RUN /usr/local/sbin/setup_packages.sh $DOCKER_IMAGE_PACKAGES && \
+    apt-get update && \
+    apt-get install \
+      git \
+      make \
+      mercurial \
+      python \
+      xz-utils
+
+# %include testing/mozharness/external_tools/robustcheckout.py
+COPY topsrcdir/testing/mozharness/external_tools/robustcheckout.py /usr/local/mercurial/robustcheckout.py
+
+# %include taskcluster/docker/recipes/hgrc
+COPY topsrcdir/taskcluster/docker/recipes/hgrc /etc/mercurial/hgrc.d/mozilla.rc
+
+# Add pip configuration, among other things.
+# %include taskcluster/docker/recipes/dot-config
+COPY topsrcdir/taskcluster/docker/recipes/dot-config /builds/worker/.config
+
+# %include taskcluster/docker/recipes/run-task
+COPY topsrcdir/taskcluster/docker/recipes/run-task /builds/worker/bin/run-task
+
+RUN chown -R worker:worker /builds/worker/bin && chmod 755 /builds/worker/bin/*
new file mode 100755
--- /dev/null
+++ b/taskcluster/docker/debian-base/setup_packages.sh
@@ -0,0 +1,5 @@
+#!/bin/sh
+
+for task in "$@"; do
+  echo "deb [trusted=yes] https://queue.taskcluster.net/v1/task/$task/runs/0/artifacts/public/build/ debian/"
+done > /etc/apt/sources.list.d/99packages.list
--- a/taskcluster/docker/debian7-build/Dockerfile
+++ b/taskcluster/docker/debian7-build/Dockerfile
@@ -1,93 +1,51 @@
-FROM debian:wheezy-20171210
+# %ARG DOCKER_IMAGE_PARENT
+FROM $DOCKER_IMAGE_PARENT
 MAINTAINER Mike Hommey <mhommey@mozilla.com>
 
-### Add worker user and setup its workspace.
-RUN mkdir /builds && \
-    groupadd -g 500 worker && \
-    useradd -u 500 -g 500 -d /builds/worker -s /bin/bash -m worker && \
-    mkdir -p /builds/worker/workspace && \
-    chown -R worker:worker /builds
-
-# Declare default working folder
-WORKDIR /builds/worker
-
 VOLUME /builds/worker/checkouts
 VOLUME /builds/worker/workspace
 VOLUME /builds/worker/tooltool-cache
 
-# Set variable normally configured at login, by the shells parent process, these
-# are taken from GNU su manual
-ENV HOME=/builds/worker \
-    SHELL=/bin/bash \
-    USER=worker \
-    LOGNAME=worker \
-    HOSTNAME=taskcluster-worker \
-    XZ_OPT=-T0 \
-    DEBIAN_FRONTEND=noninteractive
-
-# Set a default command useful for debugging
-CMD ["/bin/bash", "--login"]
-
-# Set apt sources list to a snapshot.
-RUN for s in debian_wheezy debian_wheezy-updates debian_wheezy-backports debian-security_wheezy/updates; do \
-      echo "deb http://snapshot.debian.org/archive/${s%_*}/20171210T214726Z/ ${s#*_} main"; \
-    done > /etc/apt/sources.list && \
-    ( echo 'quiet "true";'; \
-      echo 'APT::Get::Assume-Yes "true";'; \
-      echo 'APT::Install-Recommends "false";'; \
-      echo 'Acquire::Check-Valid-Until "false";'; \
-    ) > /etc/apt/apt.conf.d/99taskcluster
-
-RUN apt-get update && \
-    apt-get install \
-      apt-transport-https \
-      ca-certificates
+ENV XZ_OPT=-T0
 
 # %ARG DOCKER_IMAGE_PACKAGES
-RUN for task in $DOCKER_IMAGE_PACKAGES; do \
-      echo "deb [trusted=yes] https://queue.taskcluster.net/v1/task/$task/runs/0/artifacts/public/build/ debian/" >> /etc/apt/sources.list; \
-    done
+RUN /usr/local/sbin/setup_packages.sh $DOCKER_IMAGE_PACKAGES
 
 # %ARG ARCH
 RUN dpkg --add-architecture $ARCH
 
 # Ideally, we wouldn't need gcc-multilib and the extra linux-libc-dev,
 # but the latter is required to make the former installable, and the former
 # because of bug 1409276.
 RUN apt-get update && \
     apt-get install \
       autoconf2.13 \
       automake \
       bzip2 \
       curl \
       file \
       gawk \
       gcc-multilib \
-      git \
       gnupg \
-      make \
-      mercurial \
       p7zip-full \
       procps \
-      python \
       python-pip \
       python-setuptools \
       python-virtualenv \
       rsync \
       screen \
       tar \
       unzip \
       uuid \
       valgrind \
       wget \
       x11-utils \
       xvfb \
-      xz-utils \
       yasm/wheezy-backports \
       zip \
       linux-libc-dev/wheezy-backports \
       linux-libc-dev:$ARCH/wheezy-backports \
       pkg-config:$ARCH \
       libdbus-glib-1-dev:$ARCH \
       libfontconfig1-dev:$ARCH \
       libfreetype6-dev:$ARCH \
@@ -95,39 +53,24 @@ RUN apt-get update && \
       libgtk-3-dev:$ARCH \
       libgtk2.0-dev:$ARCH \
       libpango1.0-dev:$ARCH \
       libpulse-dev:$ARCH \
       libx11-xcb-dev:$ARCH \
       libxss-dev:$ARCH \
       libxt-dev:$ARCH
 
-# %include testing/mozharness/external_tools/robustcheckout.py
-COPY topsrcdir/testing/mozharness/external_tools/robustcheckout.py /usr/local/mercurial/robustcheckout.py
-
-# %include taskcluster/docker/recipes/hgrc
-COPY topsrcdir/taskcluster/docker/recipes/hgrc /etc/mercurial/hgrc.d/mozilla.rc
-
-# Add pip configuration, among other things.
-# %include taskcluster/docker/recipes/dot-config
-COPY topsrcdir/taskcluster/docker/recipes/dot-config /builds/worker/.config
-
 # Add wrapper scripts for xvfb allowing tasks to easily retry starting up xvfb
 # %include taskcluster/docker/recipes/xvfb.sh
 COPY topsrcdir/taskcluster/docker/recipes/xvfb.sh /builds/worker/scripts/xvfb.sh
 
-# %include taskcluster/docker/recipes/run-task
-COPY topsrcdir/taskcluster/docker/recipes/run-task /builds/worker/bin/run-task
-
 # Stubbed out credentials; mozharness looks for this file an issues a WARNING
 # if it's not found, which causes the build to fail.  Note that this needs to
 # be in the parent of the workspace directory and in the directory where
 # mozharness is run (not its --work-dir).  See Bug 1169652.
 # %include taskcluster/docker/desktop-build/oauth.txt
 COPY topsrcdir/taskcluster/docker/desktop-build/oauth.txt /builds/worker/
 
 # stubbed out buildprops, which keeps mozharness from choking
 # Note that this needs to be in the parent of the workspace directory and in
 # the directory where mozharness is run (not its --work-dir)
 # %include taskcluster/docker/desktop-build/buildprops.json
 COPY topsrcdir/taskcluster/docker/desktop-build/buildprops.json /builds/worker/
-
-RUN chown -R worker:worker /builds/worker/bin && chmod 755 /builds/worker/bin/*
--- a/taskcluster/docker/toolchain-build/Dockerfile
+++ b/taskcluster/docker/toolchain-build/Dockerfile
@@ -1,98 +1,42 @@
-FROM debian:wheezy-20171210
+# %ARG DOCKER_IMAGE_PARENT
+FROM $DOCKER_IMAGE_PARENT
 MAINTAINER Mike Hommey <mhommey@mozilla.com>
 
-### Add worker user and setup its workspace.
-RUN mkdir /builds && \
-    groupadd -g 500 worker && \
-    useradd -u 500 -g 500 -d /builds/worker -s /bin/bash -m worker && \
-    mkdir -p /builds/worker/workspace && \
-    chown -R worker:worker /builds
-
-# Declare default working folder
-WORKDIR /builds/worker
-
 VOLUME /builds/worker/checkouts
 VOLUME /builds/worker/workspace
 VOLUME /builds/worker/tooltool-cache
 
-# Set variable normally configured at login, by the shells parent process, these
-# are taken from GNU su manual
-ENV HOME=/builds/worker \
-    SHELL=/bin/bash \
-    USER=worker \
-    LOGNAME=worker \
-    HOSTNAME=taskcluster-worker \
-    XZ_OPT=-T0 \
-    DEBIAN_FRONTEND=noninteractive
-
-# Set a default command useful for debugging
-CMD ["/bin/bash", "--login"]
-
-# Set apt sources list to a snapshot.
-RUN for s in debian_wheezy debian_wheezy-updates debian-security_wheezy/updates; do \
-      echo "deb http://snapshot.debian.org/archive/${s%_*}/20171210T214726Z/ ${s#*_} main"; \
-    done > /etc/apt/sources.list && \
-    ( echo 'quiet "true";'; \
-      echo 'APT::Get::Assume-Yes "true";'; \
-      echo 'APT::Install-Recommends "false";'; \
-      echo 'Acquire::Check-Valid-Until "false";'; \
-    ) > /etc/apt/apt.conf.d/99taskcluster
-
-RUN apt-get update && \
-    apt-get install \
-      apt-transport-https \
-      ca-certificates
+ENV XZ_OPT=-T0
 
 # %ARG DOCKER_IMAGE_PACKAGES
-RUN for task in $DOCKER_IMAGE_PACKAGES; do \
-      echo "deb [trusted=yes] https://queue.taskcluster.net/v1/task/$task/runs/0/artifacts/public/build/ debian/" >> /etc/apt/sources.list; \
-    done
+RUN /usr/local/sbin/setup_packages.sh $DOCKER_IMAGE_PACKAGES
 
 RUN apt-get update && \
     apt-get install \
       autoconf \
       automake \
       bison \
       build-essential \
       curl \
       cmake \
       flex \
       gawk \
       gcc-multilib \
-      git \
       gnupg \
       libbz2-dev \
       libssl-dev \
       libtool \
-      make \
-      mercurial \
       ninja-build \
       p7zip-full \
       procps \
       python-pip \
       python-setuptools \
       python-virtualenv \
       subversion \
       tar \
       unzip \
       uuid \
       uuid-dev \
       wget \
-      xz-utils \
       zip \
       zlib1g-dev
-
-# %include testing/mozharness/external_tools/robustcheckout.py
-COPY topsrcdir/testing/mozharness/external_tools/robustcheckout.py /usr/local/mercurial/robustcheckout.py
-
-# %include taskcluster/docker/recipes/hgrc
-COPY topsrcdir/taskcluster/docker/recipes/hgrc /etc/mercurial/hgrc.d/mozilla.rc
-
-# Add pip configuration, among other things.
-# %include taskcluster/docker/recipes/dot-config
-COPY topsrcdir/taskcluster/docker/recipes/dot-config /builds/worker/.config
-
-# %include taskcluster/docker/recipes/run-task
-COPY topsrcdir/taskcluster/docker/recipes/run-task /builds/worker/bin/run-task
-
-RUN chown -R worker:worker /builds/worker/bin && chmod 755 /builds/worker/bin/*