Bug 1352599 - Part 2: Add PROGUARD_JAR configure option. r=chmanchester,rillian draft
authorNick Alexander <nalexander@mozilla.com>
Thu, 12 Oct 2017 14:28:31 -0700
changeset 680318 c62434c4068d31f5a42610de28656b1714969a81
parent 680303 63824f309d216eab06ea34a6db244eb299139f58
child 680319 87dec4fdb32290ef396cda7f92da6bd688bc5b7b
push id84459
push usernalexander@mozilla.com
push dateFri, 13 Oct 2017 20:34:20 +0000
reviewerschmanchester, rillian
bugs1352599
milestone58.0a1
Bug 1352599 - Part 2: Add PROGUARD_JAR configure option. r=chmanchester,rillian MozReview-Commit-ID: KOzcGY6txI6
build/moz.configure/java.configure
mobile/android/base/Makefile.in
mobile/android/config/mozconfigs/common
python/mozbuild/mozbuild/test/configure/common.py
python/mozbuild/mozbuild/test/configure/test_checks_configure.py
taskcluster/ci/l10n/kind.yml
taskcluster/ci/nightly-l10n/kind.yml
--- a/build/moz.configure/java.configure
+++ b/build/moz.configure/java.configure
@@ -39,17 +39,17 @@ def check_java_tool(tool):
             die("The program %s was not found.  Set $JAVA_HOME to your Java "
                 "SDK directory or use '--with-java-bin-path={java-bin-dir}'"
                 % tool)
         return result
 
     return require_tool
 
 
-check_java_tool('java')
+java = check_java_tool('java')
 check_java_tool('javah')
 check_java_tool('jar')
 check_java_tool('jarsigner')
 check_java_tool('keytool')
 javac = check_java_tool('javac')
 
 
 @depends(javac)
@@ -61,8 +61,63 @@ def javac_version(javac):
                                          stderr=subprocess.STDOUT).rstrip()
         version = Version(output.split(' ')[-1])
         if version < '1.8':
             die('javac 1.8 or higher is required (found %s). '
                 'Check the JAVA_HOME environment variable.' % version)
         return version
     except subprocess.CalledProcessError as e:
         die('Failed to get javac version: %s', e.output)
+
+
+# Proguard detection
+# ========================================================
+@depends('--help')
+@imports('os')
+def proguard_jar_default(_):
+    # By default, look for proguard.jar in the location to which `mach
+    # bootstrap` or `mach artifact toolchain` will install Proguard.
+    default = os.path.expanduser(os.path.join('~', '.mozbuild'))
+    mozbuild_state_dir = os.environ.get('MOZBUILD_STATE_PATH', default)
+    return os.path.join(mozbuild_state_dir, 'proguard', 'lib', 'proguard.jar')
+
+
+# Proguard is really required; we provide a good error message when
+# validating.
+option(env='PROGUARD_JAR', nargs=1, default=proguard_jar_default,
+       help='Path to proguard.jar')
+
+
+@depends(java, 'PROGUARD_JAR')
+@checking('for proguard.jar version')
+# Do not change, this is fragile!  This form works with the test
+# configure sandbox.
+@imports(_from='os', _import='path')
+@imports('subprocess')
+def valid_proguard(java, proguard_jar):
+    if not proguard_jar or not path.isfile(proguard_jar[0]):
+        die('proguard.jar 5.3.3 or higher is required (looked for {}). '
+            'Run |mach artifact install --from-build proguard-jar| or add '
+            '`export PROGUARD_JAR=/path/to/proguard.jar` to your mozconfig.'
+            .format(proguard_jar[0]))
+
+    try:
+        output = subprocess.check_output([java, '-jar', proguard_jar[0]])
+        # Exit code zero shouldn't happen.
+        die('Expected `java -jar {}` to fail (with version in output) '
+            'but got exit code 0'
+            .format(proguard_jar[0]))
+
+    except subprocess.CalledProcessError as e:
+        # Exit code is non zero and output is like
+        # ProGuard, version 5.3.3
+        # Usage: java proguard.ProGuard [options ...]
+        output = e.output
+
+    version = Version(output.splitlines()[0].split(' ')[-1])
+    if version < '5.3.3':
+        die('proguard.jar 5.3.3 or higher is required (found %s). '
+            'Run |mach bootstrap| to upgrade. ' % version)
+
+    return proguard_jar[0]
+
+
+set_config('PROGUARD_JAR', valid_proguard)
--- a/mobile/android/base/Makefile.in
+++ b/mobile/android/base/Makefile.in
@@ -280,37 +280,37 @@ classycle_jar := $(topsrcdir)/mobile/and
 # sometimes corrupt this information if present (which it does for some of the
 # included libraries). This corruption prevents dex from completing, so we need
 # to get rid of it.  This prevents us from seeing line numbers in stack traces
 # for stack frames inside libraries.
 #
 # This step can occur much earlier than the main Proguard pass: it needs only
 # gecko-R.jar to have been compiled (as that's where the library R.java files
 # end up), but it does block the main Proguard pass.
-.bundled.proguard.deps: gecko-R.jar $(proguard_config_dir)/strip-libs.cfg
+.bundled.proguard.deps: gecko-R.jar $(proguard_config_dir)/strip-libs.cfg $(PROGUARD_JAR)
 	$(REPORT_BUILD)
 	@$(TOUCH) $@
 	$(JAVA) \
 		-Xmx512m -Xms128m \
-		-jar $(ANDROID_SDK_ROOT)/tools/proguard/lib/proguard.jar \
+		-jar $(PROGUARD_JAR) \
 		@$(proguard_config_dir)/strip-libs.cfg \
 		-injars $(subst ::,:,$(java_bundled_libs))\
 		-outjars bundled-jars-nodebug \
 		-libraryjars $(library_jars):gecko-R.jar
 
 # We touch the target file before invoking Proguard so that Proguard's
 # outputs are fresher than the target, preventing a subsequent
 # invocation from thinking Proguard's outputs are stale.  This is safe
 # because Make removes the target file if any recipe command fails.
-.proguard.deps: .geckoview.deps .bundled.proguard.deps $(ALL_JARS) $(proguard_config_dir)/proguard.cfg
+.proguard.deps: .geckoview.deps .bundled.proguard.deps $(ALL_JARS) $(proguard_config_dir)/proguard.cfg $(PROGUARD_JAR)
 	$(REPORT_BUILD)
 	@$(TOUCH) $@
 	$(JAVA) \
 		-Xmx512m -Xms128m \
-		-jar $(ANDROID_SDK_ROOT)/tools/proguard/lib/proguard.jar \
+		-jar $(PROGUARD_JAR) \
 		@$(proguard_config_dir)/proguard.cfg \
 		-optimizationpasses $(PROGUARD_PASSES) \
 		-injars $(subst ::,:,$(all_jars_classpath)):bundled-jars-nodebug \
 		-outjars jars-proguarded \
 		-libraryjars $(library_jars)
 
 ANNOTATION_PROCESSOR_JAR_FILES := $(abspath $(DEPTH)/build/annotationProcessors/annotationProcessors.jar)
 
--- a/mobile/android/config/mozconfigs/common
+++ b/mobile/android/config/mozconfigs/common
@@ -103,8 +103,10 @@ fi
 export SOCORRO_SYMBOL_UPLOAD_TOKEN_FILE=/builds/crash-stats-api.token
 
 # Package js shell.
 export MOZ_PACKAGE_JSSHELL=1
 
 . "$topsrcdir/build/unix/mozconfig.stdcxx"
 
 JS_BINARY="$topsrcdir/mobile/android/config/js_wrapper.sh"
+
+export PROGUARD_JAR="$topsrcdir/proguard/lib/proguard.jar"
--- a/python/mozbuild/mozbuild/test/configure/common.py
+++ b/python/mozbuild/mozbuild/test/configure/common.py
@@ -37,17 +37,16 @@ def ensure_exe_extension(path):
     return path
 
 
 class ConfigureTestVFS(object):
     def __init__(self, paths):
         self._paths = set(mozpath.abspath(p) for p in paths)
 
     def exists(self, path):
-        path = mozpath.abspath(path)
         if path in self._paths:
             return True
         if mozpath.basedir(path, [topsrcdir, topobjdir]):
             return os.path.exists(path)
         return False
 
     def isfile(self, path):
         path = mozpath.abspath(path)
@@ -121,16 +120,25 @@ class ConfigureTestSandbox(ConfigureSand
             return ReadOnlyNamespace(
                 CalledProcessError=subprocess.CalledProcessError,
                 check_output=self.check_output,
                 PIPE=subprocess.PIPE,
                 STDOUT=subprocess.STDOUT,
                 Popen=self.Popen,
             )
 
+        if what == 'os.path':
+            return self.imported_os.path
+
+        if what == 'os.path.exists':
+            return self.imported_os.path.exists
+
+        if what == 'os.path.isfile':
+            return self.imported_os.path.isfile
+
         if what == 'os.environ':
             return self._environ
 
         if what == 'ctypes.wintypes':
             return ReadOnlyNamespace(
                 LPCWSTR=0,
                 LPWSTR=1,
                 DWORD=2,
--- a/python/mozbuild/mozbuild/test/configure/test_checks_configure.py
+++ b/python/mozbuild/mozbuild/test/configure/test_checks_configure.py
@@ -476,199 +476,258 @@ class TestChecksConfigure(unittest.TestC
             checking for a... 
             DEBUG: a: Trying known-a
             ERROR: Paths provided to find_program must be a list of strings, not %r
         ''' % mozpath.dirname(self.OTHER_A)))
 
     def test_java_tool_checks(self):
         includes = ('util.configure', 'checks.configure', 'java.configure')
 
-        def mock_valid_javac(_, args):
-            if len(args) == 1 and args[0] == '-version':
-                return 0, '1.8', ''
-            self.fail("Unexpected arguments to mock_valid_javac: %s" % args)
-
         # A valid set of tools in a standard location.
         java = mozpath.abspath('/usr/bin/java')
         javah = mozpath.abspath('/usr/bin/javah')
         javac = mozpath.abspath('/usr/bin/javac')
         jar = mozpath.abspath('/usr/bin/jar')
         jarsigner = mozpath.abspath('/usr/bin/jarsigner')
         keytool = mozpath.abspath('/usr/bin/keytool')
+        proguard_jar = mozpath.abspath('/path/to/proguard.jar')
+        old_proguard_jar = mozpath.abspath('/path/to/old_proguard.jar')
+
+        def mock_valid_java(_, args):
+            # Yield valid proguard.jar output with a version based on the given path.
+            stdout = \
+                 'ProGuard, version {version}' + \
+                 'Usage: java proguard.ProGuard [options ...]'
+            args = tuple(args)
+            if args == ('-jar', proguard_jar):
+                return 1, stdout.format(version="5.3.3"), ''
+            elif args == ('-jar', old_proguard_jar):
+                return 1, stdout.format(version="4.2"), ''
+            self.fail("Unexpected arguments to mock_valid_java: %s" % args)
+
+        def mock_valid_javac(_, args):
+            if len(args) == 1 and args[0] == '-version':
+                return 0, '1.8', ''
+            self.fail("Unexpected arguments to mock_valid_javac: %s" % args)
 
         paths = {
-            java: None,
+            java: mock_valid_java,
             javah: None,
             javac: mock_valid_javac,
             jar: None,
             jarsigner: None,
             keytool: None,
+            proguard_jar: mock_valid_java,
         }
 
-        config, out, status = self.get_result(includes=includes, extra_paths=paths)
+        config, out, status = self.get_result(includes=includes, extra_paths=paths,
+                                              environ={
+                                                  'PROGUARD_JAR': proguard_jar,
+                                              })
+        self.assertEqual(out, textwrap.dedent('''\
+             checking for java... %s
+             checking for javah... %s
+             checking for jar... %s
+             checking for jarsigner... %s
+             checking for keytool... %s
+             checking for javac... %s
+             checking for javac version... 1.8
+             checking for proguard.jar version... %s
+        ''' % (java, javah, jar, jarsigner, keytool, javac, proguard_jar)))
         self.assertEqual(status, 0)
         self.assertEqual(config, {
             'JAVA': java,
             'JAVAH': javah,
             'JAVAC': javac,
             'JAR': jar,
             'JARSIGNER': jarsigner,
             'KEYTOOL': keytool,
+            'PROGUARD_JAR': proguard_jar,
         })
-        self.assertEqual(out, textwrap.dedent('''\
-             checking for java... %s
-             checking for javah... %s
-             checking for jar... %s
-             checking for jarsigner... %s
-             checking for keytool... %s
-             checking for javac... %s
-             checking for javac version... 1.8
-        ''' % (java, javah, jar, jarsigner, keytool, javac)))
 
         # An alternative valid set of tools referred to by JAVA_HOME.
         alt_java = mozpath.abspath('/usr/local/bin/java')
         alt_javah = mozpath.abspath('/usr/local/bin/javah')
         alt_javac = mozpath.abspath('/usr/local/bin/javac')
         alt_jar = mozpath.abspath('/usr/local/bin/jar')
         alt_jarsigner = mozpath.abspath('/usr/local/bin/jarsigner')
         alt_keytool = mozpath.abspath('/usr/local/bin/keytool')
         alt_java_home = mozpath.dirname(mozpath.dirname(alt_java))
 
         paths.update({
-            alt_java: None,
+            alt_java: mock_valid_java,
             alt_javah: None,
             alt_javac: mock_valid_javac,
             alt_jar: None,
             alt_jarsigner: None,
             alt_keytool: None,
         })
 
         config, out, status = self.get_result(includes=includes,
                                               extra_paths=paths,
                                               environ={
                                                   'JAVA_HOME': alt_java_home,
-                                                  'PATH': mozpath.dirname(java)
+                                                  'PATH': mozpath.dirname(java),
+                                                  'PROGUARD_JAR': proguard_jar,
                                               })
-        self.assertEqual(status, 0)
-        self.assertEqual(config, {
-            'JAVA': alt_java,
-            'JAVAH': alt_javah,
-            'JAVAC': alt_javac,
-            'JAR': alt_jar,
-            'JARSIGNER': alt_jarsigner,
-            'KEYTOOL': alt_keytool,
-        })
         self.assertEqual(out, textwrap.dedent('''\
              checking for java... %s
              checking for javah... %s
              checking for jar... %s
              checking for jarsigner... %s
              checking for keytool... %s
              checking for javac... %s
              checking for javac version... 1.8
+             checking for proguard.jar version... %s
         ''' % (alt_java, alt_javah, alt_jar, alt_jarsigner,
-               alt_keytool, alt_javac)))
+               alt_keytool, alt_javac, proguard_jar)))
+        self.assertEqual(status, 0)
+        self.assertEqual(config, {
+            'JAVA': alt_java,
+            'JAVAH': alt_javah,
+            'JAVAC': alt_javac,
+            'JAR': alt_jar,
+            'JARSIGNER': alt_jarsigner,
+            'KEYTOOL': alt_keytool,
+            'PROGUARD_JAR': proguard_jar,
+        })
 
         # We can use --with-java-bin-path instead of JAVA_HOME to similar
         # effect.
         config, out, status = self.get_result(
             args=['--with-java-bin-path=%s' % mozpath.dirname(alt_java)],
             includes=includes,
             extra_paths=paths,
             environ={
-                'PATH': mozpath.dirname(java)
+                'PATH': mozpath.dirname(java),
+                'PROGUARD_JAR': proguard_jar,
             })
+        self.assertEqual(out, textwrap.dedent('''\
+             checking for java... %s
+             checking for javah... %s
+             checking for jar... %s
+             checking for jarsigner... %s
+             checking for keytool... %s
+             checking for javac... %s
+             checking for javac version... 1.8
+             checking for proguard.jar version... %s
+        ''' % (alt_java, alt_javah, alt_jar, alt_jarsigner,
+               alt_keytool, alt_javac, proguard_jar)))
         self.assertEqual(status, 0)
         self.assertEqual(config, {
             'JAVA': alt_java,
             'JAVAH': alt_javah,
             'JAVAC': alt_javac,
             'JAR': alt_jar,
             'JARSIGNER': alt_jarsigner,
             'KEYTOOL': alt_keytool,
+            'PROGUARD_JAR': proguard_jar,
         })
-        self.assertEqual(out, textwrap.dedent('''\
-             checking for java... %s
-             checking for javah... %s
-             checking for jar... %s
-             checking for jarsigner... %s
-             checking for keytool... %s
-             checking for javac... %s
-             checking for javac version... 1.8
-        ''' % (alt_java, alt_javah, alt_jar, alt_jarsigner,
-               alt_keytool, alt_javac)))
 
         # If --with-java-bin-path and JAVA_HOME are both set,
         # --with-java-bin-path takes precedence.
         config, out, status = self.get_result(
             args=['--with-java-bin-path=%s' % mozpath.dirname(alt_java)],
             includes=includes,
             extra_paths=paths,
             environ={
                 'PATH': mozpath.dirname(java),
                 'JAVA_HOME': mozpath.dirname(mozpath.dirname(java)),
+                'PROGUARD_JAR': proguard_jar,
             })
+        self.assertEqual(out, textwrap.dedent('''\
+             checking for java... %s
+             checking for javah... %s
+             checking for jar... %s
+             checking for jarsigner... %s
+             checking for keytool... %s
+             checking for javac... %s
+             checking for javac version... 1.8
+             checking for proguard.jar version... %s
+        ''' % (alt_java, alt_javah, alt_jar, alt_jarsigner,
+               alt_keytool, alt_javac, proguard_jar)))
         self.assertEqual(status, 0)
         self.assertEqual(config, {
             'JAVA': alt_java,
             'JAVAH': alt_javah,
             'JAVAC': alt_javac,
             'JAR': alt_jar,
             'JARSIGNER': alt_jarsigner,
             'KEYTOOL': alt_keytool,
+            'PROGUARD_JAR': proguard_jar,
         })
+
+        def mock_old_javac(_, args):
+            if len(args) == 1 and args[0] == '-version':
+                return 0, '1.6.9', ''
+            self.fail("Unexpected arguments to mock_old_javac: %s" % args)
+
+        # An old proguard JAR is fatal.
+        config, out, status = self.get_result(includes=includes,
+                                              extra_paths=paths,
+                                              environ={
+                                                  'PATH': mozpath.dirname(java),
+                                                  'PROGUARD_JAR': old_proguard_jar,
+                                              })
         self.assertEqual(out, textwrap.dedent('''\
              checking for java... %s
              checking for javah... %s
              checking for jar... %s
              checking for jarsigner... %s
              checking for keytool... %s
              checking for javac... %s
              checking for javac version... 1.8
-        ''' % (alt_java, alt_javah, alt_jar, alt_jarsigner,
-               alt_keytool, alt_javac)))
-
-        def mock_old_javac(_, args):
-            if len(args) == 1 and args[0] == '-version':
-                return 0, '1.6.9', ''
-            self.fail("Unexpected arguments to mock_old_javac: %s" % args)
-
-        # An old javac is fatal.
-        paths[javac] = mock_old_javac
-        config, out, status = self.get_result(includes=includes,
-                                              extra_paths=paths,
-                                              environ={
-                                                  'PATH': mozpath.dirname(java)
-                                              })
+             checking for proguard.jar version... 
+             ERROR: proguard.jar 5.3.3 or higher is required (looked for %s). Run |mach artifact install --from-build proguard-jar| or add `export PROGUARD_JAR=/path/to/proguard.jar` to your mozconfig.
+        ''' % (java, javah, jar, jarsigner, keytool, javac, old_proguard_jar)))
         self.assertEqual(status, 1)
         self.assertEqual(config, {
             'JAVA': java,
             'JAVAH': javah,
             'JAVAC': javac,
             'JAR': jar,
             'JARSIGNER': jarsigner,
             'KEYTOOL': keytool,
         })
+
+        # An old javac is fatal.
+        paths[javac] = mock_old_javac
+        config, out, status = self.get_result(includes=includes,
+                                              extra_paths=paths,
+                                              environ={
+                                                  'PATH': mozpath.dirname(java),
+                                                  'PROGUARD_JAR': proguard_jar,
+                                              })
         self.assertEqual(out, textwrap.dedent('''\
              checking for java... %s
              checking for javah... %s
              checking for jar... %s
              checking for jarsigner... %s
              checking for keytool... %s
              checking for javac... %s
              checking for javac version... 
              ERROR: javac 1.8 or higher is required (found 1.6.9). Check the JAVA_HOME environment variable.
         ''' % (java, javah, jar, jarsigner, keytool, javac)))
+        self.assertEqual(status, 1)
+        self.assertEqual(config, {
+            'JAVA': java,
+            'JAVAH': javah,
+            'JAVAC': javac,
+            'JAR': jar,
+            'JARSIGNER': jarsigner,
+            'KEYTOOL': keytool,
+        })
 
         # Any missing tool is fatal when these checks run.
         del paths[jarsigner]
         config, out, status = self.get_result(includes=includes,
                                               extra_paths=paths,
                                               environ={
-                                                  'PATH': mozpath.dirname(java)
+                                                  'PATH': mozpath.dirname(java),
+                                                  'PROGUARD_JAR': proguard_jar,
                                               })
         self.assertEqual(status, 1)
         self.assertEqual(config, {
             'JAVA': java,
             'JAVAH': javah,
             'JAR': jar,
             'JARSIGNER': ':',
         })
--- a/taskcluster/ci/l10n/kind.yml
+++ b/taskcluster/ci/l10n/kind.yml
@@ -41,16 +41,21 @@ job-template:
          android-api-16-l10n: 18000
    docker-image:
       by-build-platform:
          default:
            in-tree: desktop-build
          android-api-16-l10n:
            in-tree: android-build
          win.*: null
+   toolchains:
+      by-build-platform:
+         default: []
+         android-api-16-l10n:
+            - proguard-jar
    tooltool:
       by-build-platform:
          default: public
          android-api-16-l10n: internal
          macosx64-nightly: internal
    worker-type:
       by-build-platform:
          default: aws-provisioner-v1/gecko-{level}-b-linux
--- a/taskcluster/ci/nightly-l10n/kind.yml
+++ b/taskcluster/ci/nightly-l10n/kind.yml
@@ -44,16 +44,21 @@ job-template:
          android-api-16-nightly: 18000
    docker-image:
       by-build-platform:
          default:
            in-tree: desktop-build
          android-api-16-nightly:
            in-tree: android-build
          win.*: null
+   toolchains:
+      by-build-platform:
+         default: []
+         android-api-16-nightly:
+            - proguard-jar
    tooltool:
       by-build-platform:
          default: public
          android-api-16-nightly: internal
          macosx64-nightly: internal
          win32-nightly: internal
          win64-nightly: internal
    index: