Bug 1405413 - Migrate Gradle dependency fetching to toolchain dependencies. r=dustin
MozReview-Commit-ID: 6OOD4mhICG1
--- a/mobile/android/config/mozconfigs/android-api-16-frontend/nightly
+++ b/mobile/android/config/mozconfigs/android-api-16-frontend/nightly
@@ -9,18 +9,18 @@ MOZ_AUTOMATION_UPLOAD=0
MOZ_AUTOMATION_UPLOAD_SYMBOLS=0
MOZ_AUTOMATION_PACKAGE_GENERATED_SOURCES=0
NO_CACHE=1
NO_NDK=1
. "$topsrcdir/mobile/android/config/mozconfigs/common"
-ac_add_options --with-gradle="$topsrcdir/gradle-dist/bin/gradle"
-export GRADLE_MAVEN_REPOSITORIES="file://$topsrcdir/jcenter","file://$topsrcdir/google"
+ac_add_options --with-gradle="$topsrcdir/android-gradle-dependencies/gradle-dist/bin/gradle"
+export GRADLE_MAVEN_REPOSITORIES="file://$topsrcdir/android-gradle-dependencies/jcenter","file://$topsrcdir/android-gradle-dependencies/google"
unset HOST_CC
unset HOST_CXX
unset RUSTC
unset CARGO
ac_add_options --disable-compile-environment
ac_add_options --disable-tests
--- a/mobile/android/config/mozconfigs/android-api-16-gradle-dependencies/nightly
+++ b/mobile/android/config/mozconfigs/android-api-16-gradle-dependencies/nightly
@@ -48,8 +48,17 @@ STRIP_FLAGS="--strip-debug"
export MOZILLA_OFFICIAL=1
export MOZ_TELEMETRY_REPORTING=1
export MOZ_ANDROID_POCKET=1
. "$topsrcdir/mobile/android/config/mozconfigs/common.override"
# End ../android-api-16-frontend/nightly.
+
+# Disable Keyfile Loading (and checks) since dependency fetching doesn't need these keys.
+# This overrides the settings in the common android mozconfig
+ac_add_options --without-mozilla-api-keyfile
+ac_add_options --without-google-api-keyfile
+# We need dummy Keyfiles in order to enable features we care about.
+ac_add_options --with-adjust-sdk-keyfile="$topsrcdir/mobile/android/base/adjust-sdk-sandbox.token"
+ac_add_options --with-leanplum-sdk-keyfile="$topsrcdir/mobile/android/base/leanplum-sdk-sandbox.token"
+ac_add_options --with-pocket-api-keyfile="$topsrcdir/mobile/android/base/pocket-api-sandbox.token"
--- a/mobile/android/config/mozconfigs/android-api-16-gradle/nightly
+++ b/mobile/android/config/mozconfigs/android-api-16-gradle/nightly
@@ -1,8 +1,8 @@
. "$topsrcdir/mobile/android/config/mozconfigs/common"
. "$topsrcdir/mobile/android/config/mozconfigs/android-api-16/nightly"
-ac_add_options --with-gradle="$topsrcdir/gradle-dist/bin/gradle"
-export GRADLE_MAVEN_REPOSITORIES="file://$topsrcdir/jcenter","file://$topsrcdir/google"
+ac_add_options --with-gradle="$topsrcdir/android-gradle-dependencies/gradle-dist/bin/gradle"
+export GRADLE_MAVEN_REPOSITORIES="file://$topsrcdir/android-gradle-dependencies/jcenter","file://$topsrcdir/android-gradle-dependencies/google"
. "$topsrcdir/mobile/android/config/mozconfigs/common.override"
--- a/mobile/android/config/tooltool-manifests/android-frontend/releng.manifest
+++ b/mobile/android/config/tooltool-manifests/android-frontend/releng.manifest
@@ -1,34 +1,2 @@
[
- {
- "size": 37130176,
- "visibility": "internal",
- "digest": "254eecc06eea407c098417c1dbedf89f59883cdde7e173b0473fa5b34719c967830afa11189c3a9f69dc3f3b9e5fb43469434340e1f8feb378609af965783b48",
- "algorithm": "sha512",
- "filename": "java_home.tar.xz",
- "unpack": true
- },
- {
- "size": 7797928,
- "visibility": "public",
- "digest": "7bb5c7b3586179a7805ae60b025b7856cb8dbed66da1b7962bc446d94ea6fa45ed55ebfe065d2d82690c68bf71bdab613e97f53da79b85e3185955461f751256",
- "algorithm": "sha512",
- "filename": "google.tar.xz",
- "unpack": true
- },
- {
- "size": 52693504,
- "visibility": "public",
- "digest": "d8c0d91ff4039dd8a22e34634c0f72b21b1c272d60987f20a709fbfd919ba62eab4c3bddbaf96af1c6caf85320b4bf0676589071fe3b31724c51782a55da83e0",
- "algorithm": "sha512",
- "filename": "jcenter.tar.xz",
- "unpack": true
- },
- {
- "algorithm": "sha512",
- "visibility": "public",
- "filename": "gradle-dist.tar.xz",
- "unpack": true,
- "digest": "e3cfe7f8259ad97722243d4e873d5a05c014bfc24d637427f89d804bf5073290229c778ea303142cf06c2dc79e0492f23521f57d3a73825f55b8db587317646f",
- "size": 51753660
- }
]
--- a/mobile/android/config/tooltool-manifests/android-x86/releng.manifest
+++ b/mobile/android/config/tooltool-manifests/android-x86/releng.manifest
@@ -10,42 +10,10 @@
},
{
"size": 6856444,
"visibility": "public",
"unpack": true,
"digest": "d68dd7d31b0153095ecf5cde5837fb1f95dc6e3f799d496fb764f7afeb9c6095c332467177c3aa54d3749b1901e0d6fa84c42162526e764e8a9d2196a0189861",
"algorithm": "sha512",
"filename": "jsshell.tar.xz"
- },
- {
- "size": 7797928,
- "visibility": "public",
- "digest": "7bb5c7b3586179a7805ae60b025b7856cb8dbed66da1b7962bc446d94ea6fa45ed55ebfe065d2d82690c68bf71bdab613e97f53da79b85e3185955461f751256",
- "algorithm": "sha512",
- "filename": "google.tar.xz",
- "unpack": true
- },
- {
- "size": 52693504,
- "visibility": "public",
- "digest": "d8c0d91ff4039dd8a22e34634c0f72b21b1c272d60987f20a709fbfd919ba62eab4c3bddbaf96af1c6caf85320b4bf0676589071fe3b31724c51782a55da83e0",
- "algorithm": "sha512",
- "filename": "jcenter.tar.xz",
- "unpack": true
- },
- {
- "algorithm": "sha512",
- "visibility": "public",
- "filename": "gradle-dist.tar.xz",
- "unpack": true,
- "digest": "e3cfe7f8259ad97722243d4e873d5a05c014bfc24d637427f89d804bf5073290229c778ea303142cf06c2dc79e0492f23521f57d3a73825f55b8db587317646f",
- "size": 51753660
- },
- {
- "size": 37130176,
- "visibility": "internal",
- "digest": "254eecc06eea407c098417c1dbedf89f59883cdde7e173b0473fa5b34719c967830afa11189c3a9f69dc3f3b9e5fb43469434340e1f8feb378609af965783b48",
- "algorithm": "sha512",
- "filename": "java_home.tar.xz",
- "unpack": true
}
]
--- a/mobile/android/config/tooltool-manifests/android/releng.manifest
+++ b/mobile/android/config/tooltool-manifests/android/releng.manifest
@@ -10,42 +10,10 @@
},
{
"size": 6856444,
"visibility": "public",
"unpack": true,
"digest": "d68dd7d31b0153095ecf5cde5837fb1f95dc6e3f799d496fb764f7afeb9c6095c332467177c3aa54d3749b1901e0d6fa84c42162526e764e8a9d2196a0189861",
"algorithm": "sha512",
"filename": "jsshell.tar.xz"
- },
- {
- "size": 37130176,
- "visibility": "internal",
- "digest": "254eecc06eea407c098417c1dbedf89f59883cdde7e173b0473fa5b34719c967830afa11189c3a9f69dc3f3b9e5fb43469434340e1f8feb378609af965783b48",
- "algorithm": "sha512",
- "filename": "java_home.tar.xz",
- "unpack": true
- },
- {
- "size": 7797928,
- "visibility": "public",
- "digest": "7bb5c7b3586179a7805ae60b025b7856cb8dbed66da1b7962bc446d94ea6fa45ed55ebfe065d2d82690c68bf71bdab613e97f53da79b85e3185955461f751256",
- "algorithm": "sha512",
- "filename": "google.tar.xz",
- "unpack": true
- },
- {
- "size": 52693504,
- "visibility": "public",
- "digest": "d8c0d91ff4039dd8a22e34634c0f72b21b1c272d60987f20a709fbfd919ba62eab4c3bddbaf96af1c6caf85320b4bf0676589071fe3b31724c51782a55da83e0",
- "algorithm": "sha512",
- "filename": "jcenter.tar.xz",
- "unpack": true
- },
- {
- "algorithm": "sha512",
- "visibility": "public",
- "filename": "gradle-dist.tar.xz",
- "unpack": true,
- "digest": "e3cfe7f8259ad97722243d4e873d5a05c014bfc24d637427f89d804bf5073290229c778ea303142cf06c2dc79e0492f23521f57d3a73825f55b8db587317646f",
- "size": 51753660
}
]
--- a/taskcluster/ci/build/android-stuff.yml
+++ b/taskcluster/ci/build/android-stuff.yml
@@ -1,53 +1,8 @@
-android-dependencies/opt:
- description: "Android armv7 api-16+ gradle dependencies"
- index:
- product: mobile
- job-name: android-dependencies
- treeherder:
- platform: android-4-0-armv7-api16/opt
- kind: build
- tier: 2
- symbol: tc(Deps)
- worker-type: aws-provisioner-v1/gecko-{level}-b-android
- worker:
- docker-image: {in-tree: android-build}
- env:
- GRADLE_USER_HOME: "/builds/worker/workspace/build/src/mobile/android/gradle/dotgradle-online"
- PERFHERDER_EXTRA_OPTIONS: android-dependencies
- TOOLTOOL_MANIFEST: "mobile/android/config/tooltool-manifests/android-gradle-dependencies/releng.manifest"
- WORKSPACE: "/builds/worker/workspace"
- artifacts:
- - name: public/build
- path: /builds/worker/artifacts/
- type: directory
- max-run-time: 36000
- run:
- using: mozharness
- actions: [get-secrets build multi-l10n update]
- config:
- - builds/releng_base_android_64_builds.py
- - disable_signing.py
- - platform_supports_post_upload_to_latest.py
- script: "mozharness/scripts/fx_desktop_build.py"
- secrets: true
- custom-build-variant-cfg: api-16-gradle-dependencies
- tooltool-downloads: internal
- job-script: taskcluster/scripts/builder/build-android-dependencies.sh
- toolchains:
- - android-sdk-linux
- - proguard-jar
- optimization:
- skip-unless-changed:
- - "mobile/android/config/**"
- - "testing/mozharness/configs/builds/releng_sub_android_configs/*gradle_dependencies.py"
- - "**/*.gradle"
- - "taskcluster/docker/android-build/**"
-
android-test/opt:
description: "Android armv7 unit tests"
index:
product: mobile
job-name: android-test
treeherder:
platform: android-4-0-armv7-api16/opt
kind: build
@@ -75,16 +30,17 @@ android-test/opt:
- builds/releng_base_android_64_builds.py
- disable_signing.py
- platform_supports_post_upload_to_latest.py
script: "mozharness/scripts/fx_desktop_build.py"
secrets: true
custom-build-variant-cfg: android-test
tooltool-downloads: internal
toolchains:
+ - android-gradle-dependencies
- android-sdk-linux
- proguard-jar
optimization:
skip-unless-changed:
- "mobile/android/base/**"
- "mobile/android/config/**"
- "mobile/android/tests/background/junit4/**"
- "**/*.gradle"
@@ -136,16 +92,17 @@ android-lint/opt:
- builds/releng_base_android_64_builds.py
- disable_signing.py
- platform_supports_post_upload_to_latest.py
script: "mozharness/scripts/fx_desktop_build.py"
secrets: true
custom-build-variant-cfg: android-lint
tooltool-downloads: internal
toolchains:
+ - android-gradle-dependencies
- android-sdk-linux
- proguard-jar
optimization:
skip-unless-changed:
- "mobile/android/**/*.java"
- "mobile/android/**/*.jpeg"
- "mobile/android/**/*.jpg"
- "mobile/android/**/*.png"
@@ -191,16 +148,17 @@ android-checkstyle/opt:
- builds/releng_base_android_64_builds.py
- disable_signing.py
- platform_supports_post_upload_to_latest.py
script: "mozharness/scripts/fx_desktop_build.py"
secrets: true
custom-build-variant-cfg: android-checkstyle
tooltool-downloads: internal
toolchains:
+ - android-gradle-dependencies
- android-sdk-linux
- proguard-jar
optimization:
skip-unless-changed:
- "mobile/android/**/checkstyle.xml"
- "mobile/android/**/*.java"
- "mobile/android/**/Makefile.in"
- "mobile/android/config/**"
@@ -248,16 +206,17 @@ android-findbugs/opt:
- builds/releng_base_android_64_builds.py
- disable_signing.py
- platform_supports_post_upload_to_latest.py
script: "mozharness/scripts/fx_desktop_build.py"
secrets: true
custom-build-variant-cfg: android-findbugs
tooltool-downloads: internal
toolchains:
+ - android-gradle-dependencies
- android-sdk-linux
- proguard-jar
optimization:
skip-unless-changed:
- "mobile/android/**/*.java"
- "mobile/android/**/Makefile.in"
- "mobile/android/config/**"
- "mobile/android/**/moz.build"
--- a/taskcluster/ci/build/android.yml
+++ b/taskcluster/ci/build/android.yml
@@ -330,16 +330,17 @@ android-api-16-gradle/opt:
- builds/releng_base_android_64_builds.py
- disable_signing.py
- platform_supports_post_upload_to_latest.py
script: "mozharness/scripts/fx_desktop_build.py"
secrets: true
custom-build-variant-cfg: api-16-gradle
tooltool-downloads: internal
toolchains:
+ - android-gradle-dependencies
- android-sdk-linux
- linux64-rust-android
- linux64-sccache
- proguard-jar
android-aarch64/opt:
description: "Android 5.0 AArch64 Opt"
index:
--- a/taskcluster/ci/toolchain/linux.yml
+++ b/taskcluster/ci/toolchain/linux.yml
@@ -221,16 +221,48 @@ linux64-android-sdk-linux-repack:
using: toolchain-script
script: repack-android-sdk-linux.sh
tc-vcs: false
resources:
- 'python/mozboot/**/*android*'
toolchain-artifact: project/gecko/android-sdk/android-sdk-linux.tar.xz
toolchain-alias: android-sdk-linux
+linux64-android-gradle-dependencies:
+ description: "Android Gradle dependencies toolchain task"
+ treeherder:
+ kind: build
+ platform: toolchains/opt
+ symbol: TL(gradle-dependencies)
+ tier: 1
+ worker-type: aws-provisioner-v1/gecko-{level}-b-linux
+ worker:
+ docker-image: {in-tree: android-build}
+ env:
+ GRADLE_USER_HOME: "/builds/worker/workspace/build/src/mobile/android/gradle/dotgradle-online"
+ max-run-time: 36000
+ run:
+ using: toolchain-script
+ script: android-gradle-dependencies.sh
+ sparse-profile: null
+
+ tc-vcs: false
+ resources:
+ - 'taskcluster/scripts/misc/tooltool-download.sh'
+ - 'taskcluster/scripts/misc/android-gradle-dependencies/**'
+ - '**/*.gradle'
+ - 'mobile/android/config/mozconfigs/android-api-16-gradle-dependencies/**'
+ - 'mobile/android/config/mozconfigs/common*'
+ toolchain-artifact: public/build/android-gradle-dependencies.tar.xz
+ toolchain-alias: android-gradle-dependencies
+ toolchains:
+ # Aliases aren't allowed for toolchains depending on toolchains.
+ - linux64-android-sdk-linux-repack
+ - linux64-proguard-jar-repack
+
linux64-rust-1.19:
description: "rust repack"
treeherder:
kind: build
platform: toolchains/opt
symbol: TL(rust)
tier: 1
worker-type: aws-provisioner-v1/gecko-{level}-b-linux
rename from taskcluster/scripts/builder/build-android-dependencies.sh
rename to taskcluster/scripts/misc/android-gradle-dependencies.sh
--- a/taskcluster/scripts/builder/build-android-dependencies.sh
+++ b/taskcluster/scripts/misc/android-gradle-dependencies.sh
@@ -3,13 +3,20 @@
set -x -e
echo "running as" $(id)
: WORKSPACE ${WORKSPACE:=/builds/worker/workspace}
set -v
-. $WORKSPACE/build/src/taskcluster/scripts/builder/build-android-dependencies/before.sh
+cd $WORKSPACE/build/src
+
+# Download toolchain artifacts.
+. taskcluster/scripts/misc/tooltool-download.sh
-. $WORKSPACE/build/src/taskcluster/scripts/builder/build-linux.sh
+. taskcluster/scripts/misc/android-gradle-dependencies/before.sh
-. $WORKSPACE/build/src/taskcluster/scripts/builder/build-android-dependencies/after.sh
+export MOZCONFIG=mobile/android/config/mozconfigs/android-api-16-gradle-dependencies/nightly
+./mach build
+./mach android gradle-dependencies
+
+. taskcluster/scripts/misc/android-gradle-dependencies/after.sh
rename from taskcluster/scripts/builder/build-android-dependencies/after.sh
rename to taskcluster/scripts/misc/android-gradle-dependencies/after.sh
--- a/taskcluster/scripts/builder/build-android-dependencies/after.sh
+++ b/taskcluster/scripts/misc/android-gradle-dependencies/after.sh
@@ -6,30 +6,25 @@ echo "running as" $(id)
: WORKSPACE ${WORKSPACE:=/builds/worker/workspace}
: GRADLE_VERSION ${GRADLE_VERSION:=2.14.1}
set -v
# Package everything up.
pushd $WORKSPACE
+mkdir -p android-gradle-dependencies /builds/worker/artifacts
-cp -R ${NEXUS_WORK}/storage/jcenter jcenter
-tar cJf jcenter.tar.xz jcenter
-
-cp -R ${NEXUS_WORK}/storage/google google
-tar cJf google.tar.xz google
+cp -R ${NEXUS_WORK}/storage/jcenter android-gradle-dependencies
+cp -R ${NEXUS_WORK}/storage/google android-gradle-dependencies
# The Gradle wrapper will have downloaded and verified the hash of exactly one
# Gradle distribution. It will be located in $GRADLE_USER_HOME, like
# ~/.gradle/wrapper/dists/gradle-2.7-all/$PROJECT_HASH/gradle-2.7-all.zip. We
# want to remove the version from the internal directory for use via tooltool in
# a mozconfig.
-cp $GRADLE_USER_HOME/wrapper/dists/gradle-${GRADLE_VERSION}-all/*/gradle-${GRADLE_VERSION}-all.zip gradle-${GRADLE_VERSION}-all.zip
+cp ${GRADLE_USER_HOME}/wrapper/dists/gradle-${GRADLE_VERSION}-all/*/gradle-${GRADLE_VERSION}-all.zip gradle-${GRADLE_VERSION}-all.zip
unzip -q gradle-${GRADLE_VERSION}-all.zip
-mv gradle-${GRADLE_VERSION} gradle-dist
-tar cJf gradle-dist.tar.xz gradle-dist
+mv gradle-${GRADLE_VERSION} android-gradle-dependencies/gradle-dist
-mkdir -p /builds/worker/artifacts
-mv jcenter.tar.xz /builds/worker/artifacts
-mv google.tar.xz /builds/worker/artifacts
-mv gradle-dist.tar.xz /builds/worker/artifacts
+tar cf - android-gradle-dependencies | xz > /builds/worker/artifacts/android-gradle-dependencies.tar.xz
+
popd
rename from taskcluster/scripts/builder/build-android-dependencies/before.sh
rename to taskcluster/scripts/misc/android-gradle-dependencies/before.sh
--- a/taskcluster/scripts/builder/build-android-dependencies/before.sh
+++ b/taskcluster/scripts/misc/android-gradle-dependencies/before.sh
@@ -4,17 +4,17 @@ set -x -e
echo "running as" $(id)
: WORKSPACE ${WORKSPACE:=/builds/worker/workspace}
set -v
mkdir -p ${NEXUS_WORK}/conf
-cp /builds/worker/workspace/build/src/taskcluster/scripts/builder/build-android-dependencies/nexus.xml ${NEXUS_WORK}/conf/nexus.xml
+cp /builds/worker/workspace/build/src/taskcluster/scripts/misc/android-gradle-dependencies/nexus.xml ${NEXUS_WORK}/conf/nexus.xml
RUN_AS_USER=worker /opt/sonatype/nexus/bin/nexus restart
# Wait "a while" for Nexus to actually start. Don't fail if this fails.
wget --quiet --retry-connrefused --waitretry=2 --tries=100 \
http://localhost:8081/nexus/service/local/status || true
rm -rf status
rename from taskcluster/scripts/builder/build-android-dependencies/nexus.xml
rename to taskcluster/scripts/misc/android-gradle-dependencies/nexus.xml