Bug 1383973 - Introduce single build script for `run: using: mozharness`. r=dustin
The old process ran "before" and "after" steps as root. The
mozharness script doesn't run as root, which required some small
changes to not run Sonatype Nexus as root. Everything else is a
straight-forward move of the scripts out of the `android-gradle-build`
image and into `taskcluster/scripts`.
MozReview-Commit-ID: CqnNI33OKmb
--- a/taskcluster/docker/android-gradle-build/Dockerfile
+++ b/taskcluster/docker/android-gradle-build/Dockerfile
@@ -3,21 +3,16 @@ FROM taskcluster/centos6-build-
MAINTAINER Nick Alexander <nalexander@mozilla.com>
# BEGIN ../desktop-build/Dockerfile
# TODO remove when base image is updated
VOLUME /home/worker/workspace
VOLUME /home/worker/tooltool-cache
-# Add build scripts; these are the entry points from the taskcluster worker, and
-# operate on environment variables
-COPY bin /home/worker/bin
-RUN chmod +x /home/worker/bin/*
-
# %include python/mozbuild/mozbuild/action/tooltool.py
COPY topsrcdir/python/mozbuild/mozbuild/action/tooltool.py /build/tooltool.py
COPY topsrcdir/python/mozbuild/mozbuild/action/tooltool.py /builds/tooltool.py
COPY topsrcdir/python/mozbuild/mozbuild/action/tooltool.py /setup/tooltool.py
COPY topsrcdir/python/mozbuild/mozbuild/action/tooltool.py /tmp/tooltool.py
# %include testing/mozharness/external_tools/robustcheckout.py
COPY topsrcdir/testing/mozharness/external_tools/robustcheckout.py /usr/local/mercurial/robustcheckout.py
@@ -110,15 +105,18 @@ RUN curl --fail --silent --location --re
RUN echo "${NEXUS_SHA1SUM} nexus-${NEXUS_VERSION}-bundle.tar.gz" > nexus-${NEXUS_VERSION}-bundle.tar.gz.sha1
RUN sha1sum --check nexus-${NEXUS_VERSION}-bundle.tar.gz.sha1
RUN tar zxf nexus-${NEXUS_VERSION}-bundle.tar.gz \
&& mv /tmp/nexus-${NEXUS_VERSION}/* /opt/sonatype/nexus/ \
&& rm -rf /tmp/nexus-${NEXUS_VERSION} \
&& rm -rf /tmp/nexus-${NEXUS_VERSION}-bundle.tar.gz
-COPY nexus.xml /workspace/nexus/conf/nexus.xml
+# So that we don't have to RUN_AS_USER=root.
+RUN chown -R worker:worker /opt/sonatype/nexus/
+
+COPY nexus.xml /home/worker/workspace/nexus/conf/nexus.xml
# Back to the centos6-build workdir, matching desktop-build.
WORKDIR /home/worker
# Set a default command useful for debugging
CMD ["/bin/bash", "--login"]
deleted file mode 100644
--- a/taskcluster/docker/android-gradle-build/bin/after-checkout.sh
+++ /dev/null
@@ -1,16 +0,0 @@
-#!/bin/bash -vex
-
-set -x -e
-
-: WORKSPACE ${WORKSPACE:=/workspace}
-
-set -v
-
-# Populate /home/worker/workspace/build/src/java_home.
-cp -R /workspace/java/usr/lib/jvm/java_home /home/worker/workspace/build/src
-
-export JAVA_HOME=/home/worker/workspace/build/src/java_home
-export PATH=$PATH:$JAVA_HOME/bin
-
-# Populate /home/worker/.mozbuild/android-sdk-linux.
-python2.7 /home/worker/workspace/build/src/python/mozboot/mozboot/android.py --artifact-mode --no-interactive
deleted file mode 100644
--- a/taskcluster/docker/android-gradle-build/bin/build.sh
+++ /dev/null
@@ -1,31 +0,0 @@
-#! /bin/bash -vex
-
-set -x -e -v
-
-# TODO: when bug 1093833 is solved and tasks can run as non-root, reduce this
-# to a simple fail-if-root check
-if [ $(id -u) = 0 ]; then
- # each of the caches we have mounted are owned by root, so update that ownership
- # to 'worker'
- for cache in /home/worker/.tc-vcs /home/worker/workspace /home/worker/tooltool-cache; do
- if [ -d $cache ]; then
- # -R probably isn't necessary forever, but it fixes some poisoned
- # caches for now
- chown -R worker:worker $cache
- fi
- done
-
- # ..then drop privileges by re-running this script
- exec su worker /home/worker/bin/build.sh
-fi
-
-####
-# The default build works for any fx_desktop_build based mozharness job:
-# via linux-build.sh
-####
-
-. $HOME/bin/checkout-sources.sh
-
-. $HOME/bin/after-checkout.sh
-
-. $WORKSPACE/build/src/taskcluster/scripts/builder/build-linux.sh
deleted file mode 100644
--- a/taskcluster/docker/android-gradle-build/bin/checkout-script.sh
+++ /dev/null
@@ -1,17 +0,0 @@
-#! /bin/bash -vex
-
-set -x -e
-
-# Inputs, with defaults
-
-: GECKO_HEAD_REPOSITORY ${GECKO_HEAD_REPOSITORY:=https://hg.mozilla.org/mozilla-central}
-: GECKO_HEAD_REV ${GECKO_HEAD_REV:=default}
-
-: SCRIPT_DOWNLOAD_PATH ${SCRIPT_DOWNLOAD_PATH:=$PWD}
-: SCRIPT_PATH ${SCRIPT_PATH:?"script path must be set"}
-set -v
-
-# download script from the gecko repository
-url=${GECKO_HEAD_REPOSITORY}/raw-file/${GECKO_HEAD_REV}/${SCRIPT_PATH}
-wget --directory-prefix=${SCRIPT_DOWNLOAD_PATH} $url
-chmod +x `basename ${SCRIPT_PATH}`
deleted file mode 100644
--- a/taskcluster/docker/android-gradle-build/bin/checkout-sources.sh
+++ /dev/null
@@ -1,55 +0,0 @@
-#! /bin/bash -vex
-
-set -x -e
-
-# Inputs, with defaults
-
-# mozharness builds use three repositories: gecko (source), mozharness (build
-# scripts) and tools (miscellaneous) for each, specify *_REPOSITORY. If the
-# revision is not in the standard repo for the codebase, specify *_BASE_REPO as
-# the canonical repo to clone and *_HEAD_REPO as the repo containing the
-# desired revision. For Mercurial clones, only *_HEAD_REV is required; for Git
-# clones, specify the branch name to fetch as *_HEAD_REF and the desired sha1
-# as *_HEAD_REV.
-
-: GECKO_REPOSITORY ${GECKO_REPOSITORY:=https://hg.mozilla.org/mozilla-central}
-: GECKO_BASE_REPOSITORY ${GECKO_BASE_REPOSITORY:=${GECKO_REPOSITORY}}
-: GECKO_HEAD_REPOSITORY ${GECKO_HEAD_REPOSITORY:=${GECKO_REPOSITORY}}
-: GECKO_HEAD_REV ${GECKO_HEAD_REV:=default}
-: GECKO_HEAD_REF ${GECKO_HEAD_REF:=${GECKO_HEAD_REV}}
-
-: TOOLS_REPOSITORY ${TOOLS_REPOSITORY:=https://hg.mozilla.org/build/tools}
-: TOOLS_BASE_REPOSITORY ${TOOLS_BASE_REPOSITORY:=${TOOLS_REPOSITORY}}
-: TOOLS_HEAD_REPOSITORY ${TOOLS_HEAD_REPOSITORY:=${TOOLS_REPOSITORY}}
-: TOOLS_HEAD_REV ${TOOLS_HEAD_REV:=default}
-: TOOLS_HEAD_REF ${TOOLS_HEAD_REF:=${TOOLS_HEAD_REV}}
-: TOOLS_DISABLE ${TOOLS_DISABLE:=false}
-
-: WORKSPACE ${WORKSPACE:=/home/worker/workspace}
-
-set -v
-
-# check out tools where mozharness expects it to be ($PWD/build/tools and $WORKSPACE/build/tools)
-if [ ! "$TOOLS_DISABLE" = true ]
-then
- tc-vcs checkout $WORKSPACE/build/tools $TOOLS_BASE_REPOSITORY $TOOLS_HEAD_REPOSITORY $TOOLS_HEAD_REV $TOOLS_HEAD_REF
-
- if [ ! -d build ]; then
- mkdir -p build
- ln -s $WORKSPACE/build/tools build/tools
- fi
-fi
-
-# TODO - include tools repository in EXTRA_CHECKOUT_REPOSITORIES list
-for extra_repo in $EXTRA_CHECKOUT_REPOSITORIES; do
- BASE_REPO="${extra_repo}_BASE_REPOSITORY"
- HEAD_REPO="${extra_repo}_HEAD_REPOSITORY"
- HEAD_REV="${extra_repo}_HEAD_REV"
- HEAD_REF="${extra_repo}_HEAD_REF"
- DEST_DIR="${extra_repo}_DEST_DIR"
-
- tc-vcs checkout ${!DEST_DIR} ${!BASE_REPO} ${!HEAD_REPO} ${!HEAD_REV} ${!HEAD_REF}
-done
-
-export GECKO_DIR=$WORKSPACE/build/src
-tc-vcs checkout $GECKO_DIR $GECKO_BASE_REPOSITORY $GECKO_HEAD_REPOSITORY $GECKO_HEAD_REV $GECKO_HEAD_REF
new file mode 100755
--- /dev/null
+++ b/taskcluster/scripts/builder/build-android-dependencies.sh
@@ -0,0 +1,15 @@
+#!/bin/bash -vex
+
+set -x -e
+
+echo "running as" $(id)
+
+: WORKSPACE ${WORKSPACE:=/home/worker/workspace}
+
+set -v
+
+. $WORKSPACE/build/src/taskcluster/scripts/builder/build-android-dependencies/before.sh
+
+. $WORKSPACE/build/src/taskcluster/scripts/builder/build-linux.sh
+
+. $WORKSPACE/build/src/taskcluster/scripts/builder/build-android-dependencies/after.sh
rename from taskcluster/docker/android-gradle-build/bin/after.sh
rename to taskcluster/scripts/builder/build-android-dependencies/after.sh
--- a/taskcluster/docker/android-gradle-build/bin/after.sh
+++ b/taskcluster/scripts/builder/build-android-dependencies/after.sh
@@ -1,38 +1,40 @@
#!/bin/bash -vex
set -x -e
-: WORKSPACE ${WORKSPACE:=/workspace}
+echo "running as" $(id)
+
+: WORKSPACE ${WORKSPACE:=/home/worker/workspace}
: GRADLE_VERSION ${GRADLE_VERSION:=2.14.1}
set -v
# Package everything up.
-pushd ${WORKSPACE}
+pushd $WORKSPACE
cp -R /home/worker/.mozbuild/android-sdk-linux android-sdk-linux
tar cJf android-sdk-linux.tar.xz android-sdk-linux
# We can't redistribute the Android SDK publicly.
mkdir -p /home/worker/private/android-sdk
mv android-sdk-linux.tar.xz /home/worker/private/android-sdk
-cp -R /home/worker/workspace/build/src/java_home java_home
+cp -R $WORKSPACE/build/src/java_home java_home
tar cJf java_home.tar.xz java_home
# We can't redistribute Java publicly.
mkdir -p /home/worker/private/java_home
mv java_home.tar.xz /home/worker/private/java_home
-cp -R /workspace/nexus/storage/jcenter jcenter
+cp -R $WORKSPACE/nexus/storage/jcenter jcenter
tar cJf jcenter.tar.xz jcenter
-cp -R /workspace/nexus/storage/google google
+cp -R $WORKSPACE/nexus/storage/google google
tar cJf google.tar.xz google
# The Gradle wrapper will have downloaded and verified the hash of exactly one
# Gradle distribution. It will be located in $GRADLE_USER_HOME, like
# ~/.gradle/wrapper/dists/gradle-2.7-all/$PROJECT_HASH/gradle-2.7-all.zip. We
# want to remove the version from the internal directory for use via tooltool in
# a mozconfig.
cp $GRADLE_USER_HOME/wrapper/dists/gradle-${GRADLE_VERSION}-all/*/gradle-${GRADLE_VERSION}-all.zip gradle-${GRADLE_VERSION}-all.zip
@@ -40,20 +42,8 @@ unzip -q gradle-${GRADLE_VERSION}-all.zi
mv gradle-${GRADLE_VERSION} gradle-dist
tar cJf gradle-dist.tar.xz gradle-dist
mkdir -p /home/worker/artifacts
mv jcenter.tar.xz /home/worker/artifacts
mv google.tar.xz /home/worker/artifacts
mv gradle-dist.tar.xz /home/worker/artifacts
popd
-
-# Bug 1245170: at some point in the future, we'll be able to upload
-# things directly to tooltool.
-# pushd /home/worker/artifacts
-# /build/tooltool.py add --visibility=public jcenter.tar.xz
-# /build/tooltool.py add --visibility=public google.tar.xz
-# /build/tooltool.py add --visibility=public gradle-dist.tar.xz
-# /build/tooltool.py add --visibility=internal android-sdk-linux.tar.xz
-# /build/tooltool.py add --visibility=internal java_home.tar.xz
-# /build/tooltool.py upload -v --url=http://relengapi/tooltool/ \
-# --message="No message - Archives uploaded from taskcluster."
-# popd
rename from taskcluster/docker/android-gradle-build/bin/before.sh
rename to taskcluster/scripts/builder/build-android-dependencies/before.sh
--- a/taskcluster/docker/android-gradle-build/bin/before.sh
+++ b/taskcluster/scripts/builder/build-android-dependencies/before.sh
@@ -1,27 +1,33 @@
#!/bin/bash -vex
set -x -e
-: WORKSPACE ${WORKSPACE:=/workspace}
-: GRADLE_VERSION ${GRADLE_VERSION:=2.7}
+echo "running as" $(id)
+
+: WORKSPACE ${WORKSPACE:=/home/worker/workspace}
set -v
-# Populate $WORKSPACE/java/urs/lib/jvm/java_home. $topsrcdir hasn't
-# been checked out yet, so we can't put this directly into
-# $topsrcdir/java_home.
-. $HOME/bin/repackage-jdk-centos.sh
+# Populate /home/worker/workspace/build/src/java_home.
+. $WORKSPACE/build/src/taskcluster/scripts/builder/build-android-dependencies/repackage-jdk-centos.sh
-export JAVA_HOME=$WORKSPACE/java/usr/lib/jvm/java_home
+mv $WORKSPACE/java/usr/lib/jvm/java_home $WORKSPACE/build/src/java_home
+
+export JAVA_HOME=$WORKSPACE/build/src/java_home
export PATH=$PATH:$JAVA_HOME/bin
-# Frowned upon, but simplest.
-RUN_AS_USER=root NEXUS_WORK=${WORKSPACE}/nexus /opt/sonatype/nexus/bin/nexus restart
+# Populate /home/worker/.mozbuild/android-sdk-linux.
+python2.7 /home/worker/workspace/build/src/python/mozboot/mozboot/android.py --artifact-mode --no-interactive
+
+RUN_AS_USER=worker NEXUS_WORK=$WORKSPACE/nexus /opt/sonatype/nexus/bin/nexus restart
# Wait "a while" for Nexus to actually start. Don't fail if this fails.
wget --quiet --retry-connrefused --waitretry=2 --tries=100 \
http://localhost:8081/nexus/service/local/status || true
rm -rf status
+# It's helpful when debugging to see the "latest state".
+curl http://localhost:8081/nexus/service/local/status || true
+
# Verify Nexus has actually started. Fail if this fails.
curl --fail --silent --location http://localhost:8081/nexus/service/local/status | grep '<state>STARTED</state>'
rename from taskcluster/docker/android-gradle-build/bin/repackage-jdk-centos.sh
rename to taskcluster/scripts/builder/build-android-dependencies/repackage-jdk-centos.sh
--- a/taskcluster/docker/android-gradle-build/bin/repackage-jdk-centos.sh
+++ b/taskcluster/scripts/builder/build-android-dependencies/repackage-jdk-centos.sh
@@ -1,13 +1,13 @@
-#! /bin/bash
+#!/bin/bash -vex
set -e -x
-: WORKSPACE ${WORKSPACE:=/workspace}
+: WORKSPACE ${WORKSPACE:=/home/worker/workspace}
set -v
mkdir -p $WORKSPACE/java
pushd $WORKSPACE/java
# change these variables when updating java version
mirror_url_base="http://mirror.centos.org/centos/6/os/x86_64/Packages"