Bug 1278990 - configure sccache for taskcluster ec2 builders; r?mshal draft
authorRob Thijssen <rthijssen@mozilla.com>
Mon, 11 Jul 2016 12:47:09 +0100
changeset 386655 0cf6c45fdc999d262962598489033fe60ace8bcd
parent 386238 e78cec726586d5f19f7219ed3c2e864da9ae194d
child 525162 20e29f1cdc0b36af03d77d91429f22ea7c854371
push id22759
push userrthijssen@mozilla.com
push dateTue, 12 Jul 2016 12:01:19 +0000
reviewersmshal
bugs1278990
milestone50.0a1
Bug 1278990 - configure sccache for taskcluster ec2 builders; r?mshal MozReview-Commit-ID: 1JlGZh1J7hZ
build/mozconfig.cache
--- a/build/mozconfig.cache
+++ b/build/mozconfig.cache
@@ -2,16 +2,18 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 # Setup for build cache
 
 # Avoid duplication if the file happens to be included twice.
 if test -z "$bucket" -a -z "$NO_CACHE"; then
 
+# buildbot (or builders that use buildprops.json):
+if [ -f $topsrcdir/../buildprops.json ]; then
 read branch platform master <<EOF
 $(python2.7 -c 'import json; p = json.loads(open("'"$topsrcdir"'/../buildprops.json").read())["properties"]; print p["branch"], p["platform"], p["master"]' 2> /dev/null)
 EOF
 
 bucket=
 if test -z "$SCCACHE_DISABLE" -a -z "$no_sccache" -a -z "$MOZ_PGO_IS_SET" -a -z "$MOZ_PGO"; then
     case "${branch}" in
     try)
@@ -35,30 +37,78 @@ if test -z "$SCCACHE_DISABLE" -a -z "$no
         *usw2.mozilla.com*)
             bucket=mozilla-releng-s3-cache-us-west-2-prod
             ;;
         esac
         ;;
     esac
 fi
 
+# builds without buildprops (eg: taskcluster or non-buildbot) and without ccache env config and without sccache disabled:
+elif test -z "$CCACHE_DIR" -a -z "$SCCACHE_DISABLE" -a -z "$no_sccache" -a -z "$MOZ_PGO_IS_SET" -a -z "$MOZ_PGO"; then
+
+    # prevent rerun if az is set, or wget is not available
+    if test -z "$availability_zone" -a -x "$(command -v wget)"; then
+        # timeout after 1 second, and don't retry (failure indicates instance is not in ec2 or network issue)
+        # availability_zone is of the form <region><letter> where region is e.g. us-west-2, and az is us-west-2a
+        availability_zone=$(wget -T 1 -t 1 -q -O - http://169.254.169.254/latest/meta-data/placement/availability-zone || true)
+        if test -z "$availability_zone" -o "$availability_zone" = "not-ec2"; then
+            availability_zone=not-ec2
+        else
+            # region is az with last letter trimmed
+            region=${availability_zone%?}
+            # set S3 bucket according to tree (level)
+            case "${GECKO_HEAD_REPOSITORY}" in
+            *hg.mozilla.org/try*)
+                bucket=taskcluster-level-1-sccache-${region}
+                ;;
+            *hg.mozilla.org/integration/mozilla-inbound*|*hg.mozilla.org/integration/fx-team*)
+                bucket=taskcluster-level-3-sccache-${region}
+                ;;
+            esac
+
+            # set a dummy master
+            case "${region}" in
+            us-east-1)
+                master=dummy.use1.mozilla.com
+                ;;
+            us-west-1)
+                master=dummy.usw1.mozilla.com
+                ;;
+            us-west-2)
+                master=dummy.usw2.mozilla.com
+                ;;
+            esac
+        fi
+        export availability_zone
+    fi
+
+
+    # set platform based on the SYSTEMROOT env var
+    case "${SYSTEMROOT}" in
+    *Windows)
+        platform=windows
+        ;;
+    esac
+fi
+
 if test -z "$bucket"; then
     case "$platform" in
     win*) : ;;
     *)
         ac_add_options --with-ccache
     esac
 else
     if ! test -e $topsrcdir/sccache/sccache.py; then
         echo "Sccache missing in the tooltool manifest" >&2
         exit 1
     fi
     mk_add_options "export SCCACHE_BUCKET=$bucket"
     case "$master" in
-    *use1.mozilla.com*|*usw2.mozilla.com*)
+    *us[ew][12].mozilla.com*)
         mk_add_options "export SCCACHE_NAMESERVER=169.254.169.253"
         ;;
     esac
     ac_add_options "--with-compiler-wrapper=python2.7 $topsrcdir/sccache/sccache.py"
     mk_add_options MOZ_PREFLIGHT_ALL+=build/sccache.mk
     mk_add_options MOZ_POSTFLIGHT_ALL+=build/sccache.mk
     mk_add_options "UPLOAD_EXTRA_FILES+=sccache.log.gz"
     case "$platform" in