Bug 1355625 - Part 1: Invoke aapt using py_action. r=mshal draft
authorNick Alexander <nalexander@mozilla.com>
Mon, 08 May 2017 14:19:24 -0700
changeset 596952 21a60780841f1fbfa8410a2ba8d8612fe0823849
parent 593475 91134c95d68cbcfe984211fa3cbd28d610361ef1
child 596953 b672ed502ce23edbec2ae89b2310590c047feaca
push id64783
push usernalexander@mozilla.com
push dateMon, 19 Jun 2017 22:44:01 +0000
reviewersmshal
bugs1355625
milestone56.0a1
Bug 1355625 - Part 1: Invoke aapt using py_action. r=mshal This is partial work, which I will fold down before landing. This adds a py_action invocation wrapping aapt, and lets me compare the output easily. With this, the R.txt produced by the aapt invocation is the same as the R.txt produced by the py_action invocation. The reason to do this is to be able to process the various inputs before aapt gets to them. The rest of this ticket will implement a hacky implementation of the Gradle build system's resource merging algorithm; once we have the moz.build and Gradle resources identical, we'll be one big step closer to producing bit-identical builds and flipping the switch in favour of Gradle. Originally I wrote this to use GENERATED_FILES, but it produced a world of pain. Since Android's aapt tool is fundamentally directory oriented, not file oriented, it required adding support for FORCE to GENERATED_FILES and required directory crawling and FileAvoidWrite in the wrapper. After getting that working I was eventually stymied by the arcane requirements of the Android re-packaging system, which interacts with the l10n system. I would have required support for building GENERATED_FILES in the libs tier rather than the misc tier. After that realization I gave up and turned to py_action: the dependencies on branding are just too entangled with l10n to use GENERATED_FILES. And, in the not-so-distant future, all of this moz.build and Makefile.in chicanery will be deleted in favour of invoking Gradle at the appropriate points! MozReview-Commit-ID: 4ueVNa7gzgs
mobile/android/base/Makefile.in
python/mozbuild/mozbuild/action/aapt_package.py
--- a/mobile/android/base/Makefile.in
+++ b/mobile/android/base/Makefile.in
@@ -461,16 +461,28 @@ define aapt_command
 		$(if $(ANDROID_EXTRA_PACKAGES),--extra-packages $$(subst $$(NULL) ,:,$$(strip $$(ANDROID_EXTRA_PACKAGES)))) \
 		$(if $(ANDROID_EXTRA_RES_DIRS),$$(addprefix -S ,$$(ANDROID_EXTRA_RES_DIRS))) \
 		--custom-package org.mozilla.gecko \
 		--no-version-vectors \
 		-F $(3) \
 		-J $(4) \
 		--output-text-symbols $(5) \
 		--ignore-assets "$$(ANDROID_AAPT_IGNORE)"
+	mkdir -p $(subst /,-2/,$(4))
+	mkdir -p aapt
+	$$(call py_action,aapt_package,-f -m \
+		-M AndroidManifest.xml \
+		$$(addprefix -A ,$$(ANDROID_ASSETS_DIRS)) \
+		$$(addprefix -S ,$$(ANDROID_RES_DIRS)) \
+		$(if $(ANDROID_EXTRA_RES_DIRS),$$(addprefix -S ,$$(ANDROID_EXTRA_RES_DIRS))) \
+		$(if $(ANDROID_EXTRA_PACKAGES),--extra-packages $$(subst $$(NULL) ,:,$$(strip $$(ANDROID_EXTRA_PACKAGES)))) \
+		-F $(subst .ap_,-2.ap_,$(3)) \
+		-J $(subst /,-2/,$(4)) \
+		--output-text-symbols aapt/ \
+	  --verbose)
 endef
 
 # [Comment 3/3] The first of these rules is used during regular
 # builds.  The second writes an ap_ file that is only used during
 # packaging.  It doesn't write the normal ap_, or R.java, since we
 # don't want the packaging step to write anything that would make a
 # further no-op build do work.  See also
 # toolkit/mozapps/installer/packager.mk.
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozbuild/action/aapt_package.py
@@ -0,0 +1,125 @@
+#!/bin/python
+
+# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+'''
+Invoke Android `aapt package`.
+
+Right now, this passes arguments through.  Eventually it will
+implement a much restricted version of the Gradle build system's
+resource merging algorithm before invoking aapt.
+'''
+
+from __future__ import (
+    print_function,
+    unicode_literals,
+)
+
+import argparse
+from collections import OrderedDict
+import os
+import subprocess
+import sys
+
+import buildconfig
+from mozbuild.preprocessor import Preprocessor
+from mozbuild.util import ensureParentDir 
+import mozpack.path as mozpath
+
+
+def uniqify(iterable):
+    """Remove duplicates from iterable, preserving order."""
+    # Cribbed from https://thingspython.wordpress.com/2011/03/09/snippet-uniquify-a-sequence-preserving-order/.
+    seen = set()
+    return [item for item in iterable if not (item in seen or seen.add(item))]
+
+
+def main(*argv):
+    parser = argparse.ArgumentParser(
+        description='Invoke Android `aapt package`.')
+
+    # These serve to order build targets; they're otherwise ignored.
+    parser.add_argument('ignored_inputs', nargs='*')
+    parser.add_argument('-f', action='store_true', default=False,
+                        help='force overwrite of existing files')
+    parser.add_argument('-m', action='store_true', default=False,
+                        help='make package directories under location specified by -J')
+    parser.add_argument('-F', required=True,
+                        help='specify the apk file to output')
+    parser.add_argument('-M', required=True,
+                        help='specify full path to AndroidManifest.xml to include in zip')
+    parser.add_argument('-J', required=True,
+                        help='specify where to output R.java resource constant definitions')
+    parser.add_argument('-S', action='append', dest='res_dirs',
+                        default=[],
+                        help='directory in which to find resources. ' +
+                        'Multiple directories will be scanned and the first ' +
+                        'match found (left to right) will take precedence.')
+    parser.add_argument('-A', action='append', dest='assets_dirs',
+                        default=[],
+                        help='additional directory in which to find raw asset files')
+    parser.add_argument('--extra-packages', action='append',
+                        default=[],
+                        help='generate R.java for libraries')
+    parser.add_argument('--output-text-symbols', required=True,
+                        help='Generates a text file containing the resource ' +
+                             'symbols of the R class in the specified folder.')
+    parser.add_argument('--verbose', action='store_true', default=False,
+                        help='provide verbose output')
+
+    args = parser.parse_args(argv)
+
+    args.res_dirs = uniqify(args.res_dirs)
+    args.assets_dirs = uniqify(args.assets_dirs)
+    args.extra_packages = uniqify(args.extra_packages)
+
+    import itertools
+
+    debug = False
+    if (not buildconfig.substs['MOZILLA_OFFICIAL']) or \
+       (buildconfig.substs['NIGHTLY_BUILD'] and buildconfig.substs['MOZ_DEBUG']):
+        debug = True
+
+    cmd = [
+        buildconfig.substs['AAPT'],
+        'package',
+    ] + \
+    (['-f'] if args.f else []) + \
+    [
+        '-m',
+        '-M', args.M,
+	'-I', mozpath.join(buildconfig.substs['ANDROID_SDK'], 'android.jar'),
+        '--auto-add-overlay',
+    ] + \
+    list(itertools.chain(*(('-A', x) for x in args.assets_dirs))) + \
+    list(itertools.chain(*(('-S', x) for x in args.res_dirs))) + \
+    (['--extra-packages', ':'.join(args.extra_packages)] if args.extra_packages else []) + \
+    ['--custom-package', 'org.mozilla.gecko'] + \
+    ['--no-version-vectors'] + \
+    (['--debug-mode'] if debug else []) + \
+    [
+        '-F', args.F,
+        '-J', args.J,
+        '--output-text-symbols', args.output_text_symbols,
+        '--ignore-assets', '!.svn:!.git:.*:<dir>_*:!CVS:!thumbs.db:!picasa.ini:!*.scc:*~:#*:*.rej:*.orig',
+    ]
+
+    # We run aapt to produce gecko.ap_ and gecko-nodeps.ap_; it's
+    # helpful to tag logs with the file being produced.
+    logtag = os.path.basename(args.F)
+
+    if args.verbose:
+        print('[aapt {}] {}'.format(logtag, ' '.join(cmd)))
+
+    try:
+        subprocess.check_output(cmd, stderr=subprocess.STDOUT)
+    except subprocess.CalledProcessError as e:
+        print('\n'.join(['[aapt {}] {}'.format(logtag, line) for line in e.output.splitlines()]))
+        return 1
+
+
+if __name__ == '__main__':
+    sys.exit(main(*sys.argv[1:]))