Bug 1255227 - Part 2: Bootstrap and upload android-sdk-linux.tar.xz. r=sebastian,dustin draft
authorNick Alexander <nalexander@mozilla.com>
Wed, 19 Jul 2017 15:45:31 -0700
changeset 617642 ef2fb11cf9d75532d5a0c74404998cc450d2719d
parent 617641 6cccb53e2ebc2642ee6c61ef13fcb6d8321b67cf
child 617643 5057463540a26ef9926d63ee4c548437406e01ce
push id71104
push usernalexander@mozilla.com
push dateFri, 28 Jul 2017 19:13:32 +0000
reviewerssebastian, dustin
bugs1255227
milestone56.0a1
Bug 1255227 - Part 2: Bootstrap and upload android-sdk-linux.tar.xz. r=sebastian,dustin Before this patch, we used the Gradle sdk-manager-plugin to download and install Android SDKs and other dependencies. This plugin is now deprecated; the main dependency downloading functionality has been incorporated into the Android-Gradle build plugin. Unfortunately, it's been incorporated into newer versions that in turn require newer toolchains than we currently support, so we can't use the new functionality immediately. Rather than replace sdk-manager-plugin with equivalent Gradle-based functionality, this ticket uses recently added bootstrap functionality to bootstrap the Android SDK during the dependencies task. It then _uses_ that SDK to run the dependency fetching task, _produces_ an android-sdk-linux.tar.xz, and then _uploads_ the new artifact as a private artifact, ready to be pushed into tooltool. This avoids engineers building this critical part of the toolchain locally themselves, and will also feed into ongoing work to push toolchain artifacts into build jobs in Task Cluster. MozReview-Commit-ID: B6FC0ugaCef
mobile/android/config/mozconfigs/android-api-15-gradle-dependencies/nightly
mobile/android/config/tooltool-manifests/android-gradle-dependencies/releng.manifest
taskcluster/ci/android-stuff/kind.yml
taskcluster/docker/android-gradle-build/bin/after-checkout.sh
taskcluster/docker/android-gradle-build/bin/after.sh
taskcluster/docker/android-gradle-build/bin/build.sh
--- a/mobile/android/config/mozconfigs/android-api-15-gradle-dependencies/nightly
+++ b/mobile/android/config/mozconfigs/android-api-15-gradle-dependencies/nightly
@@ -39,8 +39,14 @@ ac_add_options --with-branding=mobile/an
 # This will overwrite the default of stripping everything and keep the symbol table.
 # This is useful for profiling with eideticker. See bug 788680
 STRIP_FLAGS="--strip-debug"
 
 export MOZILLA_OFFICIAL=1
 export MOZ_TELEMETRY_REPORTING=1
 
 . "$topsrcdir/mobile/android/config/mozconfigs/common.override"
+
+# End ../android-api-15-frontend/nightly.
+
+# Populated after checking out the sources and before building the
+# tree as part of the dependencies task bin/ scripts.
+ac_add_options --with-android-sdk="/home/worker/.mozbuild/android-sdk-linux"
--- a/mobile/android/config/tooltool-manifests/android-gradle-dependencies/releng.manifest
+++ b/mobile/android/config/tooltool-manifests/android-gradle-dependencies/releng.manifest
@@ -1,25 +1,10 @@
 [
   {
-    "versions": [
-      "Android SDK 6.0 / API 23",
-      "Android tools r24.4",
-      "Android build tools 23.0.3",
-      "Android Support Repository (Support Library 23.0.1)",
-      "Google Support Repository (Google Play Services 8.1.0)"
-    ],
-    "size": 573952124,
-    "visibility": "internal",
-    "digest": "1d495d7a7386af3f27b14982e0ff7b0963fd1a63a08040b9b1db0e94c9681fa3704c195ba8be23b5f73e15101b2b767293bc8f96e0584e17867ef13b074e5038",
-    "algorithm": "sha512",
-    "filename": "android-sdk-linux.tar.xz",
-    "unpack": true
-  },
-  {
     "version": "rustc 1.19.0 (0ade33941 2017-07-17) repack",
     "size": 151322128,
     "digest": "1d6d6e8bbf7535a4f1c8f1292562e30c868bfd3ac323992fc287be09f7fa3167c2f9e01dc98143967983e30488fe3ecaebff530ba4ec9113033d60af936c4f4d",
     "algorithm": "sha512",
     "filename": "rustc.tar.xz",
     "unpack": true
   },
   {
--- a/taskcluster/ci/android-stuff/kind.yml
+++ b/taskcluster/ci/android-stuff/kind.yml
@@ -44,16 +44,19 @@ jobs:
                     platform_supports_post_upload_to_latest.py
                 MOZHARNESS_SCRIPT: "mozharness/scripts/fx_desktop_build.py"
                 TOOLTOOL_CACHE: "/home/worker/tooltool-cache"
                 TOOLTOOL_MANIFEST: "mobile/android/config/tooltool-manifests/android-gradle-dependencies/releng.manifest"
             artifacts:
               - name: public/build
                 path: /home/worker/artifacts/
                 type: directory
+              - name: private/android-sdk
+                path: /home/worker/private/android-sdk
+                type: directory
             caches:
               - name: tooltool-cache
                 mount-point: /home/worker/tooltool-cache
                 type: persistent
             relengapi-proxy: true
             command:
               - "/bin/bash"
               - "-c"
new file mode 100644
--- /dev/null
+++ b/taskcluster/docker/android-gradle-build/bin/after-checkout.sh
@@ -0,0 +1,10 @@
+#!/bin/bash -vex
+
+set -x -e
+
+: WORKSPACE ${WORKSPACE:=/workspace}
+
+set -v
+
+# Populate /home/worker/.mozbuild/android-sdk-linux.
+python2.7 $WORKSPACE/build/src/python/mozboot/mozboot/android.py --artifact-mode --no-interactive
--- a/taskcluster/docker/android-gradle-build/bin/after.sh
+++ b/taskcluster/docker/android-gradle-build/bin/after.sh
@@ -4,37 +4,38 @@ set -x -e
 
 : WORKSPACE ${WORKSPACE:=/workspace}
 : GRADLE_VERSION ${GRADLE_VERSION:=2.14.1}
 
 set -v
 
 # Package everything up.
 pushd ${WORKSPACE}
-# Not yet.  See notes on tooltool below.
-# cp -R /root/.android-sdk android-sdk-linux
-# tar cJf android-sdk-linux.tar.xz android-sdk-linux
+
+cp -R /home/worker/.mozbuild/android-sdk-linux android-sdk-linux
+tar cJf android-sdk-linux.tar.xz android-sdk-linux
+
+# We can't redistribute the Android SDK publicly.
+mkdir -p /home/worker/private/android-sdk
+mv android-sdk-linux.tar.xz /home/worker/private/android-sdk
 
 cp -R /workspace/nexus/storage/jcenter jcentral
 tar cJf jcentral.tar.xz jcentral
 
 # The Gradle wrapper will have downloaded and verified the hash of exactly one
 # Gradle distribution.  It will be located in $GRADLE_USER_HOME, like
 # ~/.gradle/wrapper/dists/gradle-2.7-all/$PROJECT_HASH/gradle-2.7-all.zip.  We
 # want to remove the version from the internal directory for use via tooltool in
 # a mozconfig.
 cp $GRADLE_USER_HOME/wrapper/dists/gradle-${GRADLE_VERSION}-all/*/gradle-${GRADLE_VERSION}-all.zip gradle-${GRADLE_VERSION}-all.zip
 unzip -q gradle-${GRADLE_VERSION}-all.zip
 mv gradle-${GRADLE_VERSION} gradle-dist
 tar cJf gradle-dist.tar.xz gradle-dist
 
 mkdir -p /home/worker/artifacts
-# We can't redistribute the Android SDK publicly just yet.  We'll
-# upload to (internal) tooltool eventually.  mv
-# android-sdk-linux.tar.xz /home/worker/artifacts
 mv jcentral.tar.xz /home/worker/artifacts
 mv gradle-dist.tar.xz /home/worker/artifacts
 popd
 
 # Bug 1245170: at some point in the future, we'll be able to upload
 # things directly to tooltool.
 # pushd /home/worker/artifacts
 # /build/tooltool.py add --visibility=public jcentral.tar.xz
--- a/taskcluster/docker/android-gradle-build/bin/build.sh
+++ b/taskcluster/docker/android-gradle-build/bin/build.sh
@@ -21,9 +21,11 @@ fi
 
 ####
 # The default build works for any fx_desktop_build based mozharness job:
 # via linux-build.sh
 ####
 
 . $HOME/bin/checkout-sources.sh
 
+. $HOME/bin/after-checkout.sh
+
 . $WORKSPACE/build/src/taskcluster/scripts/builder/build-linux.sh