Bug 1229588: Update taskcluster desktop-test image to include node and eslint modules. f?dustin draft
authorDave Townsend <dtownsend@oxymoronical.com>
Wed, 06 Jan 2016 13:33:30 -0800
changeset 319809 641ef71a5a8a8da580a818d393046d211e016c5e
parent 319805 f08d1e52d609f5c785d53e39be1f07bedcaa1e2e
child 319810 6506e7bef4b31e58eaecc31011137d8b9908cb31
push id9098
push userdtownsend@mozilla.com
push dateThu, 07 Jan 2016 23:13:02 +0000
bugs1229588
milestone46.0a1
Bug 1229588: Update taskcluster desktop-test image to include node and eslint modules. f?dustin This updates the desktop-build image to include eslint and the plugins we require but I suspect it isn't right so while I get some feedback I've left it pointing to my docker hub. The newer version of node is built with a newer C++ compiler than Ubuntu 12.04 includes and since desktop-build installs a couple of native node modules we need to install that newer compiler and use it when building those modules. https://github.com/nodesource/distributions/blob/master/OLDER_DISTROS.md has the details. The newer clang compiler and node are installed from their respective repositories. It looks like for anything except Ubuntu packages we prefer instead using repositories or files from tooltool for security but it isn't clear to me whether that is what I need to do or if having the public keys for the signed repositories is enough. I don't know if adding the new C++ compiler is a larger change than we want for the base image, I wonder if at this point it might be better to just build a new image from ubuntu1204-test-upd and add the new node and C++ compiler there and skip using desktop-test. Or we could base it off a newer ubuntu image and then we wouldn't need the newer compiler at all.
testing/docker/desktop-test/Dockerfile
testing/docker/desktop-test/REGISTRY
testing/docker/desktop-test/VERSION
testing/docker/desktop-test/bin/checkout-sources.sh
testing/docker/desktop-test/bin/npm.sh
testing/docker/ubuntu1204-test-upd/Dockerfile
testing/docker/ubuntu1204-test-upd/REGISTRY
testing/docker/ubuntu1204-test-upd/VERSION
testing/docker/ubuntu1204-test/REGISTRY
testing/docker/ubuntu1204-test/VERSION
testing/docker/ubuntu1204-test/system-setup.sh
--- a/testing/docker/desktop-test/Dockerfile
+++ b/testing/docker/desktop-test/Dockerfile
@@ -1,17 +1,17 @@
-FROM          taskcluster/ubuntu1204-test-upd:0.1.5.20151222110000
+FROM          mossop/ubuntu1204-test-upd:0.1.7.20160107110000
 MAINTAINER    Jonas Finnemann Jensen <jopsen@gmail.com>
 
 # Add utilities and configuration
 COPY           dot-config                    /home/worker/.config
 COPY           dot-pulse                     /home/worker/.pulse
 COPY           bin                           /home/worker/bin
 # This removes the requirement of a developer to call it with bash
-RUN            chmod +x bin/test.sh
+RUN            chmod +x bin/*
 # TODO: remove this when buildbot is gone
 COPY           buildprops.json               /home/worker/buildprops.json
 COPY           tc-vcs-config.yml /etc/taskcluster-vcs.yml
 
 # TODO: remove
 ADD            https://raw.githubusercontent.com/taskcluster/buildbot-step/master/buildbot_step /home/worker/bin/buildbot_step
 RUN chmod u+x /home/worker/bin/buildbot_step
 
@@ -21,18 +21,21 @@ RUN chmod +x /usr/local/bin/linux64-mini
 
 # allow the worker user to access video devices
 RUN usermod -a -G video worker
 
 RUN mkdir Documents; mkdir Pictures; mkdir Music; mkdir Videos; mkdir artifacts
 
 # install a new enough npm, plus tc-vcs and tc-npm-cache
 RUN npm install -g npm@^2.0.0
-RUN npm install -g taskcluster-vcs@2.3.12
-RUN npm install -g taskcluster-npm-cache@1.1.14
+RUN bin/npm.sh install -g taskcluster-vcs@2.3.12
+RUN bin/npm.sh install -g taskcluster-npm-cache@1.1.14
+RUN npm install -g eslint@1.10.3
+RUN npm install -g eslint-plugin-html@1.1.0
+RUN npm install -g eslint-plugin-react@3.13.1
 ENV PATH $PATH:/home/worker/bin
 
 # Remove once running under 'worker' user.  This is necessary for pulseaudio to start
 # XXX: change this back to worker:worker once permissions issues are resolved
 RUN            chown -R root:root /home/worker
 
 
 # TODO Re-enable worker when bug 1093833 lands
--- a/testing/docker/desktop-test/REGISTRY
+++ b/testing/docker/desktop-test/REGISTRY
@@ -1,1 +1,1 @@
-taskcluster
+mossop
--- a/testing/docker/desktop-test/VERSION
+++ b/testing/docker/desktop-test/VERSION
@@ -1,1 +1,1 @@
-0.5.2
+0.5.4
new file mode 100644
--- /dev/null
+++ b/testing/docker/desktop-test/bin/checkout-sources.sh
@@ -0,0 +1,55 @@
+#! /bin/bash -vex
+
+set -x -e
+
+# Inputs, with defaults
+
+# mozharness builds use three repositories: gecko (source), mozharness (build
+# scripts) and tools (miscellaneous) for each, specify *_REPOSITORY.  If the
+# revision is not in the standard repo for the codebase, specify *_BASE_REPO as
+# the canonical repo to clone and *_HEAD_REPO as the repo containing the
+# desired revision.  For Mercurial clones, only *_HEAD_REV is required; for Git
+# clones, specify the branch name to fetch as *_HEAD_REF and the desired sha1
+# as *_HEAD_REV.
+
+: GECKO_REPOSITORY              ${GECKO_REPOSITORY:=https://hg.mozilla.org/mozilla-central}
+: GECKO_BASE_REPOSITORY         ${GECKO_BASE_REPOSITORY:=${GECKO_REPOSITORY}}
+: GECKO_HEAD_REPOSITORY         ${GECKO_HEAD_REPOSITORY:=${GECKO_REPOSITORY}}
+: GECKO_HEAD_REV                ${GECKO_HEAD_REV:=default}
+: GECKO_HEAD_REF                ${GECKO_HEAD_REF:=${GECKO_HEAD_REV}}
+
+: TOOLS_REPOSITORY              ${TOOLS_REPOSITORY:=https://hg.mozilla.org/build/tools}
+: TOOLS_BASE_REPOSITORY         ${TOOLS_BASE_REPOSITORY:=${TOOLS_REPOSITORY}}
+: TOOLS_HEAD_REPOSITORY         ${TOOLS_HEAD_REPOSITORY:=${TOOLS_REPOSITORY}}
+: TOOLS_HEAD_REV                ${TOOLS_HEAD_REV:=default}
+: TOOLS_HEAD_REF                ${TOOLS_HEAD_REF:=${TOOLS_HEAD_REV}}
+: TOOLS_DISABLE                 ${TOOLS_DISABLE:=false}
+
+: WORKSPACE                     ${WORKSPACE:=/home/worker/workspace}
+
+set -v
+
+# check out tools where mozharness expects it to be ($PWD/build/tools and $WORKSPACE/build/tools)
+if [ ! "$TOOLS_DISABLE" = true ]
+then
+    tc-vcs checkout $WORKSPACE/build/tools $TOOLS_BASE_REPOSITORY $TOOLS_HEAD_REPOSITORY $TOOLS_HEAD_REV $TOOLS_HEAD_REF
+
+    if [ ! -d build ]; then
+        mkdir -p build
+        ln -s $WORKSPACE/build/tools build/tools
+    fi
+fi
+
+# TODO - include tools repository in EXTRA_CHECKOUT_REPOSITORIES list
+for extra_repo in $EXTRA_CHECKOUT_REPOSITORIES; do
+    BASE_REPO="${extra_repo}_BASE_REPOSITORY"
+    HEAD_REPO="${extra_repo}_HEAD_REPOSITORY"
+    HEAD_REV="${extra_repo}_HEAD_REV"
+    HEAD_REF="${extra_repo}_HEAD_REF"
+    DEST_DIR="${extra_repo}_DEST_DIR"
+
+    tc-vcs checkout ${!DEST_DIR} ${!BASE_REPO} ${!HEAD_REPO} ${!HEAD_REV} ${!HEAD_REF}
+done
+
+export GECKO_DIR=$WORKSPACE/build/src
+tc-vcs checkout $GECKO_DIR $GECKO_BASE_REPOSITORY $GECKO_HEAD_REPOSITORY $GECKO_HEAD_REV $GECKO_HEAD_REF
new file mode 100644
--- /dev/null
+++ b/testing/docker/desktop-test/bin/npm.sh
@@ -0,0 +1,8 @@
+#! /bin/bash
+
+CC=/usr/bin/clang-3.4
+export CC
+CXX=/usr/bin/clang++-3.4
+export CXX
+
+npm $@
--- a/testing/docker/ubuntu1204-test-upd/Dockerfile
+++ b/testing/docker/ubuntu1204-test-upd/Dockerfile
@@ -1,9 +1,9 @@
-FROM          taskcluster/ubuntu1204-test:0.1.5
+FROM          mossop/ubuntu1204-test:0.1.7
 MAINTAINER    Dustin J. Mitchell <dustin@mozilla.com>
 
 ### update to latest from upstream repositories
 # if this becomes a long list of packages, consider bumping the
 # ubunt1204-test version
 RUN apt-get update && apt-get upgrade -y --force-yes
 
 # Set a default command useful for debugging
--- a/testing/docker/ubuntu1204-test-upd/REGISTRY
+++ b/testing/docker/ubuntu1204-test-upd/REGISTRY
@@ -1,1 +1,1 @@
-taskcluster
+mossop
--- a/testing/docker/ubuntu1204-test-upd/VERSION
+++ b/testing/docker/ubuntu1204-test-upd/VERSION
@@ -1,1 +1,1 @@
-0.1.5.20151222110000
+0.1.7.20160107110000
--- a/testing/docker/ubuntu1204-test/REGISTRY
+++ b/testing/docker/ubuntu1204-test/REGISTRY
@@ -1,1 +1,1 @@
-taskcluster
+mossop
--- a/testing/docker/ubuntu1204-test/VERSION
+++ b/testing/docker/ubuntu1204-test/VERSION
@@ -1,1 +1,1 @@
-0.1.5
+0.1.7
--- a/testing/docker/ubuntu1204-test/system-setup.sh
+++ b/testing/docker/ubuntu1204-test/system-setup.sh
@@ -6,16 +6,17 @@ test `whoami` == 'root'
 
 mkdir -p /setup
 cd /setup
 
 apt_packages=()
 
 apt_packages+=('alsa-base')
 apt_packages+=('alsa-utils')
+apt_packages+=('apt-transport-https')
 apt_packages+=('autoconf2.13')
 apt_packages+=('bluez-alsa')
 apt_packages+=('bluez-alsa:i386')
 apt_packages+=('bluez-cups')
 apt_packages+=('bluez-gstreamer')
 apt_packages+=('build-essential')
 apt_packages+=('ca-certificates')
 apt_packages+=('ccache')
@@ -119,16 +120,31 @@ apt_packages+=('python-pip')
 
 apt-get update
 # This allows ubuntu-desktop to be installed without human interaction
 export DEBIAN_FRONTEND=noninteractive
 apt-get install -y --force-yes ${apt_packages[@]}
 
 dpkg-reconfigure locales
 
+# Add NodeSource's repository for the 4.x Argon release of nodejs
+curl -s https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add -
+echo "deb https://deb.nodesource.com/node_4.x precise main" > /etc/apt/sources.list.d/nodesource.list
+echo "deb-src https://deb.nodesource.com/node_4.x precise main" >> /etc/apt/sources.list.d/nodesource.list
+
+# Add repositories for clang which node packages depend on to build on Ubuntu Precise
+echo "deb http://ppa.launchpad.net/ubuntu-toolchain-r/test/ubuntu precise main" >> /etc/apt/sources.list
+echo "deb http://llvm.org/apt/precise/ llvm-toolchain-precise-3.4 main" >> /etc/apt/sources.list
+curl -s http://llvm.org/apt/llvm-snapshot.gpg.key | apt-key add -
+apt-key adv --keyserver keyserver.ubuntu.com --recv-keys BA9EF27F
+
+apt-get update
+
+apt-get install -y --force-yes clang-3.4 nodejs
+
 # set up tooltool (temporarily)
 curl https://raw.githubusercontent.com/mozilla/build-tooltool/master/tooltool.py > /setup/tooltool.py
 tooltool_fetch() {
     cat >manifest.tt
     python /setup/tooltool.py fetch
     rm manifest.tt
 }
 
@@ -155,31 +171,16 @@ cat >requirements.txt <<'EOF'
 # sha256: qryO8YzdvYoqnH-SvEPi_qVLEUczDWXbkg7zzpgS49w
 virtualenv==13.1.2
 
 # sha256: tQ9peOfTn-DLKY-j-j6c5B0jVnIdFV5SiPnFfl8T6ac
 mercurial==3.5
 EOF
 peep install -r requirements.txt
 
-# Install node
-tooltool_fetch <<'EOF'
-[
-{
-    "size": 5676610, 
-    "digest": "ce27b788dfd141a5ba7674332825fc136fe2c4f49a319dd19b3a87c8fffa7a97d86cbb8535661c9a68c9122719aa969fc6a8c886458a0df9fc822eec99ed130b", 
-    "algorithm": "sha512", 
-    "filename": "node-v0.10.36-linux-x64.tar.gz"
-}
-]
-
-EOF
-tar -C /usr/local -xz --strip-components 1 < node-*.tar.gz
-node -v  # verify
-
 # Install custom-built Debian packages.  These come from a set of repositories
 # packaged in tarballs on tooltool to make them replicable.  Because they have
 # inter-dependenices, we install all repositories first, then perform the
 # installation.
 cp /etc/apt/sources.list sources.list.orig
 
 # Install a slightly newer version of libxcb
 # See bug 975216 for the original build of these packages