Bug 1383973 - Introduce single build script for `run: using: mozharness`. r=dustin draft
authorNick Alexander <nalexander@mozilla.com>
Wed, 02 Aug 2017 16:41:15 -0700
changeset 645256 f94e6b9b780f96038c60d3825039a0f94add0404
parent 645255 265937bc9ba3bc4c18756b6c675100a62929bafe
child 645257 2b51e02cd4f77675c42435708c3a579539e6108e
push id73719
push usernalexander@mozilla.com
push dateSat, 12 Aug 2017 03:10:52 +0000
reviewersdustin
bugs1383973
milestone57.0a1
Bug 1383973 - Introduce single build script for `run: using: mozharness`. r=dustin The old process ran "before" and "after" steps as root. The mozharness script doesn't run as root, which required some small changes to not run Sonatype Nexus as root. Everything else is a straight-forward move of the scripts out of the `android-gradle-build` image and into `taskcluster/scripts`. MozReview-Commit-ID: CqnNI33OKmb
taskcluster/docker/android-gradle-build/Dockerfile
taskcluster/docker/android-gradle-build/bin/after-checkout.sh
taskcluster/docker/android-gradle-build/bin/after.sh
taskcluster/docker/android-gradle-build/bin/before.sh
taskcluster/docker/android-gradle-build/bin/build.sh
taskcluster/docker/android-gradle-build/bin/checkout-script.sh
taskcluster/docker/android-gradle-build/bin/checkout-sources.sh
taskcluster/docker/android-gradle-build/bin/repackage-jdk-centos.sh
taskcluster/scripts/builder/build-android-dependencies.sh
taskcluster/scripts/builder/build-android-dependencies/after.sh
taskcluster/scripts/builder/build-android-dependencies/before.sh
taskcluster/scripts/builder/build-android-dependencies/repackage-jdk-centos.sh
--- a/taskcluster/docker/android-gradle-build/Dockerfile
+++ b/taskcluster/docker/android-gradle-build/Dockerfile
@@ -3,21 +3,16 @@ FROM          taskcluster/centos6-build-
 MAINTAINER    Nick Alexander <nalexander@mozilla.com>
 
 # BEGIN ../desktop-build/Dockerfile
 
 # TODO remove when base image is updated
 VOLUME /home/worker/workspace
 VOLUME /home/worker/tooltool-cache
 
-# Add build scripts; these are the entry points from the taskcluster worker, and
-# operate on environment variables
-COPY bin /home/worker/bin
-RUN chmod +x /home/worker/bin/*
-
 # %include python/mozbuild/mozbuild/action/tooltool.py
 COPY topsrcdir/python/mozbuild/mozbuild/action/tooltool.py /build/tooltool.py
 COPY topsrcdir/python/mozbuild/mozbuild/action/tooltool.py /builds/tooltool.py
 COPY topsrcdir/python/mozbuild/mozbuild/action/tooltool.py /setup/tooltool.py
 COPY topsrcdir/python/mozbuild/mozbuild/action/tooltool.py /tmp/tooltool.py
 
 # %include testing/mozharness/external_tools/robustcheckout.py
 COPY topsrcdir/testing/mozharness/external_tools/robustcheckout.py /usr/local/mercurial/robustcheckout.py
@@ -110,15 +105,18 @@ RUN curl --fail --silent --location --re
 RUN echo "${NEXUS_SHA1SUM}  nexus-${NEXUS_VERSION}-bundle.tar.gz" > nexus-${NEXUS_VERSION}-bundle.tar.gz.sha1
 RUN sha1sum --check nexus-${NEXUS_VERSION}-bundle.tar.gz.sha1
 
 RUN tar zxf nexus-${NEXUS_VERSION}-bundle.tar.gz \
   && mv /tmp/nexus-${NEXUS_VERSION}/* /opt/sonatype/nexus/ \
   && rm -rf /tmp/nexus-${NEXUS_VERSION} \
   && rm -rf /tmp/nexus-${NEXUS_VERSION}-bundle.tar.gz
 
-COPY nexus.xml /workspace/nexus/conf/nexus.xml
+# So that we don't have to RUN_AS_USER=root.
+RUN chown -R worker:worker /opt/sonatype/nexus/
+
+COPY nexus.xml /home/worker/workspace/nexus/conf/nexus.xml
 
 # Back to the centos6-build workdir, matching desktop-build.
 WORKDIR /home/worker
 
 # Set a default command useful for debugging
 CMD ["/bin/bash", "--login"]
deleted file mode 100644
--- a/taskcluster/docker/android-gradle-build/bin/after-checkout.sh
+++ /dev/null
@@ -1,16 +0,0 @@
-#!/bin/bash -vex
-
-set -x -e
-
-: WORKSPACE ${WORKSPACE:=/workspace}
-
-set -v
-
-# Populate /home/worker/workspace/build/src/java_home.
-cp -R /workspace/java/usr/lib/jvm/java_home /home/worker/workspace/build/src
-
-export JAVA_HOME=/home/worker/workspace/build/src/java_home
-export PATH=$PATH:$JAVA_HOME/bin
-
-# Populate /home/worker/.mozbuild/android-sdk-linux.
-python2.7 /home/worker/workspace/build/src/python/mozboot/mozboot/android.py --artifact-mode --no-interactive
deleted file mode 100644
--- a/taskcluster/docker/android-gradle-build/bin/build.sh
+++ /dev/null
@@ -1,31 +0,0 @@
-#! /bin/bash -vex
-
-set -x -e -v
-
-# TODO: when bug 1093833 is solved and tasks can run as non-root, reduce this
-# to a simple fail-if-root check
-if [ $(id -u) = 0 ]; then
-    # each of the caches we have mounted are owned by root, so update that ownership
-    # to 'worker'
-    for cache in /home/worker/.tc-vcs /home/worker/workspace /home/worker/tooltool-cache; do
-        if [ -d $cache ]; then
-            # -R probably isn't necessary forever, but it fixes some poisoned
-            # caches for now
-            chown -R worker:worker $cache
-        fi
-    done
-
-    # ..then drop privileges by re-running this script
-    exec su worker /home/worker/bin/build.sh
-fi
-
-####
-# The default build works for any fx_desktop_build based mozharness job:
-# via linux-build.sh
-####
-
-. $HOME/bin/checkout-sources.sh
-
-. $HOME/bin/after-checkout.sh
-
-. $WORKSPACE/build/src/taskcluster/scripts/builder/build-linux.sh
deleted file mode 100644
--- a/taskcluster/docker/android-gradle-build/bin/checkout-script.sh
+++ /dev/null
@@ -1,17 +0,0 @@
-#! /bin/bash -vex
-
-set -x -e
-
-# Inputs, with defaults
-
-: GECKO_HEAD_REPOSITORY              ${GECKO_HEAD_REPOSITORY:=https://hg.mozilla.org/mozilla-central}
-: GECKO_HEAD_REV                ${GECKO_HEAD_REV:=default}
-
-: SCRIPT_DOWNLOAD_PATH          ${SCRIPT_DOWNLOAD_PATH:=$PWD}
-: SCRIPT_PATH                   ${SCRIPT_PATH:?"script path must be set"}
-set -v
-
-# download script from the gecko repository
-url=${GECKO_HEAD_REPOSITORY}/raw-file/${GECKO_HEAD_REV}/${SCRIPT_PATH}
-wget --directory-prefix=${SCRIPT_DOWNLOAD_PATH} $url
-chmod +x `basename ${SCRIPT_PATH}`
deleted file mode 100644
--- a/taskcluster/docker/android-gradle-build/bin/checkout-sources.sh
+++ /dev/null
@@ -1,55 +0,0 @@
-#! /bin/bash -vex
-
-set -x -e
-
-# Inputs, with defaults
-
-# mozharness builds use three repositories: gecko (source), mozharness (build
-# scripts) and tools (miscellaneous) for each, specify *_REPOSITORY.  If the
-# revision is not in the standard repo for the codebase, specify *_BASE_REPO as
-# the canonical repo to clone and *_HEAD_REPO as the repo containing the
-# desired revision.  For Mercurial clones, only *_HEAD_REV is required; for Git
-# clones, specify the branch name to fetch as *_HEAD_REF and the desired sha1
-# as *_HEAD_REV.
-
-: GECKO_REPOSITORY              ${GECKO_REPOSITORY:=https://hg.mozilla.org/mozilla-central}
-: GECKO_BASE_REPOSITORY         ${GECKO_BASE_REPOSITORY:=${GECKO_REPOSITORY}}
-: GECKO_HEAD_REPOSITORY         ${GECKO_HEAD_REPOSITORY:=${GECKO_REPOSITORY}}
-: GECKO_HEAD_REV                ${GECKO_HEAD_REV:=default}
-: GECKO_HEAD_REF                ${GECKO_HEAD_REF:=${GECKO_HEAD_REV}}
-
-: TOOLS_REPOSITORY              ${TOOLS_REPOSITORY:=https://hg.mozilla.org/build/tools}
-: TOOLS_BASE_REPOSITORY         ${TOOLS_BASE_REPOSITORY:=${TOOLS_REPOSITORY}}
-: TOOLS_HEAD_REPOSITORY         ${TOOLS_HEAD_REPOSITORY:=${TOOLS_REPOSITORY}}
-: TOOLS_HEAD_REV                ${TOOLS_HEAD_REV:=default}
-: TOOLS_HEAD_REF                ${TOOLS_HEAD_REF:=${TOOLS_HEAD_REV}}
-: TOOLS_DISABLE                 ${TOOLS_DISABLE:=false}
-
-: WORKSPACE                     ${WORKSPACE:=/home/worker/workspace}
-
-set -v
-
-# check out tools where mozharness expects it to be ($PWD/build/tools and $WORKSPACE/build/tools)
-if [ ! "$TOOLS_DISABLE" = true ]
-then
-    tc-vcs checkout $WORKSPACE/build/tools $TOOLS_BASE_REPOSITORY $TOOLS_HEAD_REPOSITORY $TOOLS_HEAD_REV $TOOLS_HEAD_REF
-
-    if [ ! -d build ]; then
-        mkdir -p build
-        ln -s $WORKSPACE/build/tools build/tools
-    fi
-fi
-
-# TODO - include tools repository in EXTRA_CHECKOUT_REPOSITORIES list
-for extra_repo in $EXTRA_CHECKOUT_REPOSITORIES; do
-    BASE_REPO="${extra_repo}_BASE_REPOSITORY"
-    HEAD_REPO="${extra_repo}_HEAD_REPOSITORY"
-    HEAD_REV="${extra_repo}_HEAD_REV"
-    HEAD_REF="${extra_repo}_HEAD_REF"
-    DEST_DIR="${extra_repo}_DEST_DIR"
-
-    tc-vcs checkout ${!DEST_DIR} ${!BASE_REPO} ${!HEAD_REPO} ${!HEAD_REV} ${!HEAD_REF}
-done
-
-export GECKO_DIR=$WORKSPACE/build/src
-tc-vcs checkout $GECKO_DIR $GECKO_BASE_REPOSITORY $GECKO_HEAD_REPOSITORY $GECKO_HEAD_REV $GECKO_HEAD_REF
new file mode 100755
--- /dev/null
+++ b/taskcluster/scripts/builder/build-android-dependencies.sh
@@ -0,0 +1,15 @@
+#!/bin/bash -vex
+
+set -x -e
+
+echo "running as" $(id)
+
+: WORKSPACE ${WORKSPACE:=/home/worker/workspace}
+
+set -v
+
+. $WORKSPACE/build/src/taskcluster/scripts/builder/build-android-dependencies/before.sh
+
+. $WORKSPACE/build/src/taskcluster/scripts/builder/build-linux.sh
+
+. $WORKSPACE/build/src/taskcluster/scripts/builder/build-android-dependencies/after.sh
rename from taskcluster/docker/android-gradle-build/bin/after.sh
rename to taskcluster/scripts/builder/build-android-dependencies/after.sh
--- a/taskcluster/docker/android-gradle-build/bin/after.sh
+++ b/taskcluster/scripts/builder/build-android-dependencies/after.sh
@@ -1,38 +1,40 @@
 #!/bin/bash -vex
 
 set -x -e
 
-: WORKSPACE ${WORKSPACE:=/workspace}
+echo "running as" $(id)
+
+: WORKSPACE ${WORKSPACE:=/home/worker/workspace}
 : GRADLE_VERSION ${GRADLE_VERSION:=2.14.1}
 
 set -v
 
 # Package everything up.
-pushd ${WORKSPACE}
+pushd $WORKSPACE
 
 cp -R /home/worker/.mozbuild/android-sdk-linux android-sdk-linux
 tar cJf android-sdk-linux.tar.xz android-sdk-linux
 
 # We can't redistribute the Android SDK publicly.
 mkdir -p /home/worker/private/android-sdk
 mv android-sdk-linux.tar.xz /home/worker/private/android-sdk
 
-cp -R /home/worker/workspace/build/src/java_home java_home
+cp -R $WORKSPACE/build/src/java_home java_home
 tar cJf java_home.tar.xz java_home
 
 # We can't redistribute Java publicly.
 mkdir -p /home/worker/private/java_home
 mv java_home.tar.xz /home/worker/private/java_home
 
-cp -R /workspace/nexus/storage/jcenter jcenter
+cp -R $WORKSPACE/nexus/storage/jcenter jcenter
 tar cJf jcenter.tar.xz jcenter
 
-cp -R /workspace/nexus/storage/google google
+cp -R $WORKSPACE/nexus/storage/google google
 tar cJf google.tar.xz google
 
 # The Gradle wrapper will have downloaded and verified the hash of exactly one
 # Gradle distribution.  It will be located in $GRADLE_USER_HOME, like
 # ~/.gradle/wrapper/dists/gradle-2.7-all/$PROJECT_HASH/gradle-2.7-all.zip.  We
 # want to remove the version from the internal directory for use via tooltool in
 # a mozconfig.
 cp $GRADLE_USER_HOME/wrapper/dists/gradle-${GRADLE_VERSION}-all/*/gradle-${GRADLE_VERSION}-all.zip gradle-${GRADLE_VERSION}-all.zip
@@ -40,20 +42,8 @@ unzip -q gradle-${GRADLE_VERSION}-all.zi
 mv gradle-${GRADLE_VERSION} gradle-dist
 tar cJf gradle-dist.tar.xz gradle-dist
 
 mkdir -p /home/worker/artifacts
 mv jcenter.tar.xz /home/worker/artifacts
 mv google.tar.xz /home/worker/artifacts
 mv gradle-dist.tar.xz /home/worker/artifacts
 popd
-
-# Bug 1245170: at some point in the future, we'll be able to upload
-# things directly to tooltool.
-# pushd /home/worker/artifacts
-# /build/tooltool.py add --visibility=public jcenter.tar.xz
-# /build/tooltool.py add --visibility=public google.tar.xz
-# /build/tooltool.py add --visibility=public gradle-dist.tar.xz
-# /build/tooltool.py add --visibility=internal android-sdk-linux.tar.xz
-# /build/tooltool.py add --visibility=internal java_home.tar.xz
-# /build/tooltool.py upload -v --url=http://relengapi/tooltool/ \
-#   --message="No message - Archives uploaded from taskcluster."
-# popd
rename from taskcluster/docker/android-gradle-build/bin/before.sh
rename to taskcluster/scripts/builder/build-android-dependencies/before.sh
--- a/taskcluster/docker/android-gradle-build/bin/before.sh
+++ b/taskcluster/scripts/builder/build-android-dependencies/before.sh
@@ -1,27 +1,33 @@
 #!/bin/bash -vex
 
 set -x -e
 
-: WORKSPACE ${WORKSPACE:=/workspace}
-: GRADLE_VERSION ${GRADLE_VERSION:=2.7}
+echo "running as" $(id)
+
+: WORKSPACE ${WORKSPACE:=/home/worker/workspace}
 
 set -v
 
-# Populate $WORKSPACE/java/urs/lib/jvm/java_home.  $topsrcdir hasn't
-# been checked out yet, so we can't put this directly into
-# $topsrcdir/java_home.
-. $HOME/bin/repackage-jdk-centos.sh
+# Populate /home/worker/workspace/build/src/java_home.
+. $WORKSPACE/build/src/taskcluster/scripts/builder/build-android-dependencies/repackage-jdk-centos.sh
 
-export JAVA_HOME=$WORKSPACE/java/usr/lib/jvm/java_home
+mv $WORKSPACE/java/usr/lib/jvm/java_home $WORKSPACE/build/src/java_home
+
+export JAVA_HOME=$WORKSPACE/build/src/java_home
 export PATH=$PATH:$JAVA_HOME/bin
 
-# Frowned upon, but simplest.
-RUN_AS_USER=root NEXUS_WORK=${WORKSPACE}/nexus /opt/sonatype/nexus/bin/nexus restart
+# Populate /home/worker/.mozbuild/android-sdk-linux.
+python2.7 /home/worker/workspace/build/src/python/mozboot/mozboot/android.py --artifact-mode --no-interactive
+
+RUN_AS_USER=worker NEXUS_WORK=$WORKSPACE/nexus /opt/sonatype/nexus/bin/nexus restart
 
 # Wait "a while" for Nexus to actually start.  Don't fail if this fails.
 wget --quiet --retry-connrefused --waitretry=2 --tries=100 \
   http://localhost:8081/nexus/service/local/status || true
 rm -rf status
 
+# It's helpful when debugging to see the "latest state".
+curl http://localhost:8081/nexus/service/local/status || true
+
 # Verify Nexus has actually started.  Fail if this fails.
 curl --fail --silent --location http://localhost:8081/nexus/service/local/status | grep '<state>STARTED</state>'
rename from taskcluster/docker/android-gradle-build/bin/repackage-jdk-centos.sh
rename to taskcluster/scripts/builder/build-android-dependencies/repackage-jdk-centos.sh
--- a/taskcluster/docker/android-gradle-build/bin/repackage-jdk-centos.sh
+++ b/taskcluster/scripts/builder/build-android-dependencies/repackage-jdk-centos.sh
@@ -1,13 +1,13 @@
-#! /bin/bash
+#!/bin/bash -vex
 
 set -e -x
 
-: WORKSPACE ${WORKSPACE:=/workspace}
+: WORKSPACE ${WORKSPACE:=/home/worker/workspace}
 
 set -v
 
 mkdir -p $WORKSPACE/java
 pushd $WORKSPACE/java
 
 # change these variables when updating java version
 mirror_url_base="http://mirror.centos.org/centos/6/os/x86_64/Packages"