Bug 1432395 - Switch the valgrind build to a Debian-based docker image. r?dustin draft
authorMike Hommey <mh+mozilla@glandium.org>
Sat, 27 Jan 2018 10:25:58 +0900
changeset 747946 034de5765014a071bcd3102ccb3a15ffee390e36
parent 747945 5fe4e707b176d8cc51697f11872d29a36ce5f914
child 747947 322085ffe15662e2f5f19dfc273826192cadf14d
push id97035
push userbmo:mh+mozilla@glandium.org
push dateSat, 27 Jan 2018 02:22:33 +0000
reviewersdustin
bugs1432395
milestone60.0a1
Bug 1432395 - Switch the valgrind build to a Debian-based docker image. r?dustin We start from the image used for Firefox builds, and add the debug packages for all the system libraries.
taskcluster/ci/docker-image/kind.yml
taskcluster/docker/recipes/centos-install-debug-symbols.sh
taskcluster/docker/valgrind-build/Dockerfile
--- a/taskcluster/ci/docker-image/kind.yml
+++ b/taskcluster/ci/docker-image/kind.yml
@@ -18,18 +18,16 @@ transforms:
 # knowledge that unnecessary images will be omitted from the target task graph
 jobs:
   image_builder:
     symbol: I(ib)
   desktop1604-test:
     symbol: I(dt16t)
   desktop-build:
     symbol: I(db)
-  valgrind-build:
-    symbol: I(vb)
   debian7-base:
     symbol: I(deb7-base)
     definition: debian-base
     args:
       DIST: wheezy
       BASE_TAG: '20171210'
       SNAPSHOT: '20171210T214726Z'
     packages:
@@ -60,16 +58,19 @@ jobs:
       - deb7-valgrind
     args:
       ARCH: i386
   debian7-mozjs-rust-build:
     symbol: I(deb7jsrs)
     parent: debian7-amd64-build
     packages:
       - deb7-cmake
+  valgrind-build:
+    symbol: I(vb)
+    parent: debian7-amd64-build
   lint:
     symbol: I(lnt)
   debian9-base:
     symbol: I(deb9-base)
     definition: debian-base
     args:
       DIST: stretch
       BASE_TAG: '20170620'
deleted file mode 100644
--- a/taskcluster/docker/recipes/centos-install-debug-symbols.sh
+++ /dev/null
@@ -1,95 +0,0 @@
-#!/bin/bash
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-# Install debuginfo packages
-
-# enable debug info repository
-yum-config-manager enable debuginfo
-
-install_options="-q -t -y --skip-broken"
-debuginfo_install="debuginfo-install $install_options"
-
-install_debuginfo_for_installed_packages() {
-    yum list installed \
-        | awk 'NF >= 3 && $1 !~ /debuginfo/ {
-                # Remove arch suffix
-                print gensub(/\.(i.86|x86_64|noarch)/, "", "", $1)
-            }' \
-        | xargs "$debuginfo_install" \
-        || : # ignore errors
-}
-
-echo "Installing debuginfo packages..."
-install_debuginfo_for_installed_packages > /dev/null
-
-# Now search for debuginfo for individual libraries in the system
-
-# Get the length of a string in bytes.
-# We have to set LANG=C to get the length in bytes, not chars.
-strlen() {
-    local old_lang byteslen
-    old_lang=$LANG
-    LANG=C
-    byteslen=${#1}
-    LANG=$old_lang
-    echo "$byteslen"
-}
-
-echo "Searching for additional debuginfo packages..."
-
-# libraries contains the list of libraries found in the system
-libraries=""
-
-# As we accumulate libraries in the $libraries variable, we have
-# to constantly check we didn't extrapolate the command line
-# argument length limit. arg_max stores the argument limit in
-# bytes, discounting the $debuginfo_install command plus one
-# space.
-arg_max=$(( $(getconf ARG_MAX)-$(strlen "$debuginfo_install")-$(strlen " ") ))
-
-to_debuginfo() {
-    # extracted from debuginfo-install script
-    if [[ $1 == *-rpms ]]; then
-        echo "${1%*-rpms}-debug-rpms"
-    else
-        echo "$1-debuginfo"
-    fi
-}
-
-get_debuginfo_package() {
-    local package=${1%.so*}
-
-    # Remove version suffix because some libraries have their debuginfo
-    # package without it in the name.
-    local unversioned_package=${package%-*}
-    if [ "$unversioned_package" != "$package" ]; then
-        package="$package $unversioned_package"
-    fi
-
-    echo "$package"
-}
-
-walk_dir() {
-    local lib
-    for i in $1/*; do
-        # if we found a library...
-        if [[ $i == *.so ]]; then
-            lib="$(get_debuginfo_package "$(basename "$i")")"
-            if [ "$(strlen "$debuginfo_install $libraries $lib")" -ge $arg_max ]; then
-                $debuginfo_install $libraries > /dev/null
-                libraries=""
-            fi
-            libraries="$libraries $lib"
-        fi
-    done
-}
-
-for i in /usr/lib /usr/lib64 /lib /lib64; do
-    walk_dir $i
-done
-
-if [ ${#libraries} -gt 0 ]; then
-    $debuginfo_install "$libraries" > /dev/null
-fi
--- a/taskcluster/docker/valgrind-build/Dockerfile
+++ b/taskcluster/docker/valgrind-build/Dockerfile
@@ -1,81 +1,51 @@
-# TODO remove VOLUME below when the base image is updated next.
-FROM          taskcluster/centos6-build-upd:0.1.8.20170808150401
-MAINTAINER    Dustin J. Mitchell <dustin@mozilla.com>
+# %ARG DOCKER_IMAGE_PARENT
+FROM $DOCKER_IMAGE_PARENT
+MAINTAINER Mike Hommey <mhommey@mozilla.com>
 
 VOLUME /builds/worker/checkouts
 VOLUME /builds/worker/workspace
 VOLUME /builds/worker/tooltool-cache
 
-# %include python/mozbuild/mozbuild/action/tooltool.py
-ADD topsrcdir/python/mozbuild/mozbuild/action/tooltool.py /builds/tooltool.py
-ADD topsrcdir/python/mozbuild/mozbuild/action/tooltool.py /setup/tooltool.py
-
-# %include testing/mozharness/external_tools/robustcheckout.py
-ADD topsrcdir/testing/mozharness/external_tools/robustcheckout.py /usr/local/mercurial/robustcheckout.py
-
-# %include taskcluster/docker/recipes/hgrc
-COPY topsrcdir/taskcluster/docker/recipes/hgrc /etc/mercurial/hgrc.d/mozilla.rc
-
-# %include taskcluster/docker/recipes/common.sh
-ADD topsrcdir/taskcluster/docker/recipes/common.sh /setup/common.sh
-
-# %include taskcluster/docker/recipes/install-mercurial.sh
-ADD topsrcdir/taskcluster/docker/recipes/install-mercurial.sh /setup/install-mercurial.sh
-
-# %include taskcluster/docker/recipes/install-make.sh
-ADD topsrcdir/taskcluster/docker/recipes/install-make.sh /setup/install-make.sh
-
-# %include taskcluster/docker/recipes/install-cmake.sh
-ADD topsrcdir/taskcluster/docker/recipes/install-cmake.sh /setup/install-cmake.sh
-
-# %include taskcluster/docker/recipes/centos6-build-system-setup.sh
-ADD topsrcdir/taskcluster/docker/recipes/centos6-build-system-setup.sh /setup/system-setup.sh
-
-# %include taskcluster/docker/recipes/centos-install-debug-symbols.sh
-ADD topsrcdir/taskcluster/docker/recipes/centos-install-debug-symbols.sh /setup/install-debug-symbols.sh
-
-# %include build/unix/build-gtk3/build-gtk3.sh
-ADD topsrcdir/build/unix/build-gtk3/build-gtk3.sh /setup/build-gtk3.sh
-
-# TODO remove once base image doesn't install Mercurial
-RUN pip uninstall -y Mercurial
-
-RUN bash /setup/build-gtk3.sh && bash /setup/system-setup.sh
-
-# Add wrapper scripts for xvfb allowing tasks to easily retry starting up xvfb
-# %include taskcluster/docker/recipes/xvfb.sh
-ADD topsrcdir/taskcluster/docker/recipes/xvfb.sh /builds/worker/scripts/xvfb.sh
-
-# %include taskcluster/docker/recipes/run-task
-ADD topsrcdir/taskcluster/docker/recipes/run-task /builds/worker/bin/run-task
-
-# Add configuration
-# %include taskcluster/docker/recipes/dot-config
-ADD topsrcdir/taskcluster/docker/recipes/dot-config /builds/worker/.config
-
-# Generate machine uuid file
-RUN dbus-uuidgen --ensure=/var/lib/dbus/machine-id
-
-# Stubbed out credentials; mozharness looks for this file an issues a WARNING
-# if it's not found, which causes the build to fail.  Note that this needs to
-# be in the parent of the workspace directory and in the directory where
-# mozharness is run (not its --work-dir).  See Bug 1169652.
-# %include taskcluster/docker/desktop-build/oauth.txt
-ADD topsrcdir/taskcluster/docker/desktop-build/oauth.txt /builds/worker/
-
-# stubbed out buildprops, which keeps mozharness from choking
-# Note that this needs to be in the parent of the workspace directory and in
-# the directory where mozharness is run (not its --work-dir)
-# %include taskcluster/docker/desktop-build/buildprops.json
-ADD topsrcdir/taskcluster/docker/desktop-build/buildprops.json /builds/worker/
-
-# Move installation to base centos6-build image once Bug 1272629 is fixed
-# Install the screen package here to use with xvfb.
-# Install bison to build binutils.
-RUN yum install -y bison screen
-
-# Install libtool.
-RUN yum install -y libtool
-
-# Set a default command useful for debugging
-CMD ["/bin/bash", "--login"]
+# We could try to be smart and install all the -dbg packages corresponding to
+# the installed packages, but, not all of them are actually for libraries used
+# by Firefox, leading to a larger docker image. Moreover, some of the -dbg
+# packages for unnecessary libraries pull other packages through dependencies,
+# that make for even larger docker images.
+RUN apt-get install \
+    dbus-1-dbg \
+    libatk1.0-dbg \
+    libcairo2-dbg \
+    libdbus-glib-1-2-dbg \
+    libdrm2-dbg \
+    libffi5-dbg \
+    libfontconfig1-dbg \
+    libgl1-mesa-glx-dbg \
+    libglapi-mesa-dbg \
+    libglib2.0-0-dbg \
+    libgtk-3-0-dbg \
+    libice6-dbg \
+    libpango1.0-0-dbg \
+    libpcre3-dbg \
+    libpixman-1-0-dbg \
+    libsm6-dbg \
+    libx11-6-dbg \
+    libx11-xcb1-dbg \
+    libxau6-dbg \
+    libxcb-glx0-dbg \
+    libxcb-render0-dbg \
+    libxcb-shm0-dbg \
+    libxcb1-dbg \
+    libxcomposite1-dbg \
+    libxcursor1-dbg \
+    libxdamage1-dbg \
+    libxdmcp6-dbg \
+    libxext6-dbg \
+    libxfixes3-dbg \
+    libxi6-dbg \
+    libxinerama1-dbg \
+    libxrandr2-dbg \
+    libxrender1-dbg \
+    libxt6-dbg \
+    libxxf86vm1-dbg \
+    valgrind-dbg \
+    zlib1g-dbg