Bug 1355625 - Part 1: Invoke aapt using py_action. r=mshal
This adds a py_action invocation wrapping aapt and implements a hacky
implementation of the Gradle build system's resource merging
algorithm; once we have the moz.build and Gradle resources identical,
we'll be one big step closer to producing bit-identical builds and
flipping the switch in favour of Gradle. With this, the R.txt
produced by the aapt invocation is the same as the R.txt produced by
the py_action invocation.
Originally I wrote this to use GENERATED_FILES, but it produced a
world of pain. Since Android's aapt tool is fundamentally directory
oriented, not file oriented, it required adding support for FORCE to
GENERATED_FILES and required directory crawling and FileAvoidWrite in
the wrapper. After getting that working I was eventually stymied by
the arcane requirements of the Android re-packaging system, which
interacts with the l10n system. I would have required support for
building GENERATED_FILES in the libs tier rather than the misc tier.
After that realization I gave up and turned to py_action: the
dependencies on branding are just too entangled with l10n to use
GENERATED_FILES.
And, in the not-so-distant future, all of this moz.build and
Makefile.in chicanery will be deleted in favour of invoking Gradle at
the appropriate points!
MozReview-Commit-ID: 4ueVNa7gzgs
--- a/mobile/android/base/Makefile.in
+++ b/mobile/android/base/Makefile.in
@@ -444,33 +444,28 @@ ANDROID_AAPT_IGNORE := !.svn:!.git:.*:<d
# 4: directory to write R.java into.
# 5: directory to write R.txt into.
# We touch the target file before invoking aapt so that aapt's outputs
# are fresher than the target, preventing a subsequent invocation from
# thinking aapt's outputs are stale. This is safe because Make
# removes the target file if any recipe command fails.
define aapt_command
-$(1): $$(call mkdir_deps,$(filter-out ./,$(dir $(3) $(4) $(5)))) $(2)
+$(1): $(topsrcdir)/python/mozbuild/mozbuild/action/aapt_package.py $$(call mkdir_deps,$(filter-out ./,$(dir $(3) $(4) $(5)))) $(2)
@$$(TOUCH) $$@
- $$(AAPT) package -f -m \
+ $$(call py_action,aapt_package,-f \
-M AndroidManifest.xml \
- -I $(ANDROID_SDK)/android.jar \
- $(if $(MOZ_ANDROID_MAX_SDK_VERSION),--max-res-version $(MOZ_ANDROID_MAX_SDK_VERSION),) \
- --auto-add-overlay \
+ $$(addprefix -A ,$$(ANDROID_ASSETS_DIRS)) \
$$(addprefix -S ,$$(ANDROID_RES_DIRS)) \
- $$(addprefix -A ,$$(ANDROID_ASSETS_DIRS)) \
+ $(if $(ANDROID_EXTRA_RES_DIRS),$$(addprefix -S ,$$(ANDROID_EXTRA_RES_DIRS))) \
$(if $(ANDROID_EXTRA_PACKAGES),--extra-packages $$(subst $$(NULL) ,:,$$(strip $$(ANDROID_EXTRA_PACKAGES)))) \
- $(if $(ANDROID_EXTRA_RES_DIRS),$$(addprefix -S ,$$(ANDROID_EXTRA_RES_DIRS))) \
- --custom-package org.mozilla.gecko \
- --no-version-vectors \
-F $(3) \
-J $(4) \
--output-text-symbols $(5) \
- --ignore-assets "$$(ANDROID_AAPT_IGNORE)"
+ --verbose)
endef
# [Comment 3/3] The first of these rules is used during regular
# builds. The second writes an ap_ file that is only used during
# packaging. It doesn't write the normal ap_, or R.java, since we
# don't want the packaging step to write anything that would make a
# further no-op build do work. See also
# toolkit/mozapps/installer/packager.mk.
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozbuild/action/aapt_package.py
@@ -0,0 +1,129 @@
+#!/bin/python
+
+# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+'''
+Invoke Android `aapt package`.
+
+Right now, this passes arguments through. Eventually it will
+implement a much restricted version of the Gradle build system's
+resource merging algorithm before invoking aapt.
+'''
+
+from __future__ import (
+ print_function,
+ unicode_literals,
+)
+
+import argparse
+import os
+import subprocess
+import sys
+
+import buildconfig
+import mozpack.path as mozpath
+
+import merge_resources
+
+
+def uniqify(iterable):
+ """Remove duplicates from iterable, preserving order."""
+ # Cribbed from
+ # https://thingspython.wordpress.com/2011/03/09/snippet-uniquify-a-sequence-preserving-order/.
+ seen = set()
+ return [item for item in iterable if not (item in seen or seen.add(item))]
+
+
+def main(*argv):
+ parser = argparse.ArgumentParser(
+ description='Invoke Android `aapt package`.')
+
+ # These serve to order build targets; they're otherwise ignored.
+ parser.add_argument('ignored_inputs', nargs='*')
+ parser.add_argument('-f', action='store_true', default=False,
+ help='force overwrite of existing files')
+ parser.add_argument('-F', required=True,
+ help='specify the apk file to output')
+ parser.add_argument('-M', required=True,
+ help='specify full path to AndroidManifest.xml to include in zip')
+ parser.add_argument('-J', required=True,
+ help='specify where to output R.java resource constant definitions')
+ parser.add_argument('-S', action='append', dest='res_dirs',
+ default=[],
+ help='directory in which to find resources. ' +
+ 'Multiple directories will be scanned and the first ' +
+ 'match found (left to right) will take precedence.')
+ parser.add_argument('-A', action='append', dest='assets_dirs',
+ default=[],
+ help='additional directory in which to find raw asset files')
+ parser.add_argument('--extra-packages', action='append',
+ default=[],
+ help='generate R.java for libraries')
+ parser.add_argument('--output-text-symbols', required=True,
+ help='Generates a text file containing the resource ' +
+ 'symbols of the R class in the specified folder.')
+ parser.add_argument('--verbose', action='store_true', default=False,
+ help='provide verbose output')
+
+ args = parser.parse_args(argv)
+
+ args.res_dirs = uniqify(args.res_dirs)
+ args.assets_dirs = uniqify(args.assets_dirs)
+ args.extra_packages = uniqify(args.extra_packages)
+
+ import itertools
+
+ debug = False
+ if (not buildconfig.substs['MOZILLA_OFFICIAL']) or \
+ (buildconfig.substs['NIGHTLY_BUILD'] and buildconfig.substs['MOZ_DEBUG']):
+ debug = True
+
+ merge_resources.main('merged', True, *args.res_dirs)
+
+ cmd = [
+ buildconfig.substs['AAPT'],
+ 'package',
+ ] + \
+ (['-f'] if args.f else []) + \
+ [
+ '-m',
+ '-M', args.M,
+ '-I', mozpath.join(buildconfig.substs['ANDROID_SDK'], 'android.jar'),
+ '--auto-add-overlay',
+ ] + \
+ list(itertools.chain(*(('-A', x) for x in args.assets_dirs))) + \
+ ['-S', os.path.abspath('merged')] + \
+ (['--extra-packages', ':'.join(args.extra_packages)] if args.extra_packages else []) + \
+ ['--custom-package', 'org.mozilla.gecko'] + \
+ ['--no-version-vectors'] + \
+ (['--debug-mode'] if debug else []) + \
+ [
+ '-F',
+ args.F,
+ '-J',
+ args.J,
+ '--output-text-symbols',
+ args.output_text_symbols,
+ '--ignore-assets',
+ '!.svn:!.git:.*:<dir>_*:!CVS:!thumbs.db:!picasa.ini:!*.scc:*~:#*:*.rej:*.orig',
+ ]
+
+ # We run aapt to produce gecko.ap_ and gecko-nodeps.ap_; it's
+ # helpful to tag logs with the file being produced.
+ logtag = os.path.basename(args.F)
+
+ if args.verbose:
+ print('[aapt {}] {}'.format(logtag, ' '.join(cmd)))
+
+ try:
+ subprocess.check_output(cmd, stderr=subprocess.STDOUT)
+ except subprocess.CalledProcessError as e:
+ print('\n'.join(['[aapt {}] {}'.format(logtag, line) for line in e.output.splitlines()]))
+ return 1
+
+
+if __name__ == '__main__':
+ sys.exit(main(*sys.argv[1:]))
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozbuild/action/merge_resources.py
@@ -0,0 +1,301 @@
+#!/bin/python
+
+# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+'''
+A hacked together clone of the Android Gradle plugin's resource
+merging algorithm. To be abandoned in favour of --with-gradle as soon
+as possible!
+'''
+
+from __future__ import (
+ print_function,
+ unicode_literals,
+)
+
+from collections import defaultdict
+import re
+import os
+import sys
+
+from mozbuild.util import ensureParentDir
+from mozpack.copier import (
+ FileCopier,
+)
+from mozpack.manifests import (
+ InstallManifest,
+)
+import mozpack.path as mozpath
+from mozpack.files import (
+ FileFinder,
+)
+
+import xml.etree.cElementTree as ET
+
+
+# From https://github.com/miracle2k/android-platform_sdk/blob/master/common/src/com/android/resources/ResourceType.java.
+# TODO: find a more authoritative source!
+resource_type = {
+ "anim": 0,
+ "animator": 1,
+ # The only interesting ones.
+ "string-array": 2,
+ "integer-array": 2,
+ "attr": 3,
+ "bool": 4,
+ "color": 5,
+ "declare-styleable": 6,
+ "dimen": 7,
+ "drawable": 8,
+ "fraction": 9,
+ "id": 10,
+ "integer": 11,
+ "interpolator": 12,
+ "layout": 13,
+ "menu": 14,
+ "mipmap": 15,
+ "plurals": 16,
+ "raw": 17,
+ "string": 18,
+ "style": 19,
+ "styleable": 20,
+ "xml": 21,
+ # "public": 0,
+}
+
+
+def uniqify(iterable):
+ """Remove duplicates from iterable, preserving order."""
+ # Cribbed from https://thingspython.wordpress.com/2011/03/09/snippet-uniquify-a-sequence-preserving-order/.
+ seen = set()
+ return [item for item in iterable if not (item in seen or seen.add(item))]
+
+
+# Exclusions, arising in appcompat-v7-23.4.0.
+MANIFEST_EXCLUSIONS = (
+ 'color/abc_background_cache_hint_selector_material_dark.xml',
+ 'color/abc_background_cache_hint_selector_material_light.xml',
+)
+
+SMALLEST_SCREEN_WIDTH_QUALIFIER_RE = re.compile(r"(^|-)w(\d+)dp($|-)")
+SCREEN_WIDTH_QUALIFIER_RE = re.compile(r"(^|-)sw(\d+)dp($|-)")
+# Different densities were introduced in different Android versions.
+# However, earlier versions of aapt (like the one we are building
+# with) don't have fine-grained versions; they're all lumped into v4.
+DENSITIES = [
+ (re.compile(r"(^|-)xxxhdpi($|-)"), 18),
+ (re.compile(r"(^|-)560dpi($|-)"), 1),
+ (re.compile(r"(^|-)xxhdpi($|-)"), 16),
+ (re.compile(r"(^|-)400dpi($|-)"), 1),
+ (re.compile(r"(^|-)360dpi($|-)"), 23),
+ (re.compile(r"(^|-)xhdpi($|-)"), 8),
+ (re.compile(r"(^|-)280dpi($|-)"), 22),
+ (re.compile(r"(^|-)hdpi($|-)"), 4),
+ (re.compile(r"(^|-)tvdpi($|-)"), 13),
+ (re.compile(r"(^|-)mdpi($|-)"), 4),
+ (re.compile(r"(^|-)ldpi($|-)"), 4),
+ (re.compile(r"(^|-)anydpi($|-)"), 21),
+ (re.compile(r"(^|-)nodpi($|-)"), 4),
+]
+SCREEN_SIZE_RE = re.compile(r"(^|-)(small|normal|large|xlarge)($|-)")
+
+def with_version(dir):
+ """Resources directories without versions (like values-large) that
+correspond to resource filters added to Android in vN (like large,
+which was added in v4) automatically get a -vN added (so values-large
+becomes values-large-v4, since Android versions before v4 will not
+recognize values-large)."""
+ # Order matters! We need to check for later features before
+ # earlier features, so that "ldrtl-sw-large" will be v17, not v13
+ # or v4.
+ if '-ldrtl' in dir and '-v' not in dir:
+ return '{}-v17'.format(dir)
+
+ if re.search(SMALLEST_SCREEN_WIDTH_QUALIFIER_RE, dir) and '-v' not in dir:
+ return '{}-v13'.format(dir)
+
+ if re.search(SCREEN_WIDTH_QUALIFIER_RE, dir) and '-v' not in dir:
+ return '{}-v13'.format(dir)
+
+ for (density, _since) in DENSITIES:
+ if re.search(density, dir) and '-v' not in dir:
+ return '{}-v{}'.format(dir, 4)
+
+ if re.search(SCREEN_SIZE_RE, dir) and '-v' not in dir:
+ return '{}-v4'.format(dir)
+
+ return dir
+
+
+def classify(path):
+ """Return `(resource, version)` for a given path.
+
+`resource` is of the form `unversioned/name` where `unversionsed` is a resource
+type (like "drawable" or "strings"), and `version` is an
+integer version number or `None`."""
+ dir, name = path.split('/')
+ segments = dir.split('-')
+ version = None
+ for segment in segments[1:]:
+ if segment.startswith('v'):
+ version = int(segment[1:])
+ break
+ segments = [segment for segment in segments if not segment.startswith('v')]
+ resource = '{}/{}'.format('-'.join(segments), name)
+ return (resource, version)
+
+
+def main(output_dirname, verbose, *input_dirs):
+ # Map directories to source paths, like
+ # `{'values-large-v11': ['/path/to/values-large-v11/strings.xml',
+ # '/path/to/values-large-v11/colors.xml', ...], ...}`.
+ values = defaultdict(list)
+ # Map unversioned resource names to maps from versions to source paths, like:
+ # `{'drawable-large/icon.png':
+ # {None: '/path/to/drawable-large/icon.png',
+ # 11: '/path/to/drawable-large-v11/icon.png', ...}, ...}`.
+ resources = defaultdict(dict)
+
+ manifest = InstallManifest()
+
+ for p in uniqify(input_dirs):
+ finder = FileFinder(p, find_executables=False)
+
+ values_pattern = 'values*/*.xml'
+ for path, _ in finder.find('*/*'):
+ if path in MANIFEST_EXCLUSIONS:
+ continue
+
+ source_path = mozpath.join(finder.base, path)
+
+ if mozpath.match(path, values_pattern):
+ dir, _name = path.split('/')
+ dir = with_version(dir)
+ values[dir].append(source_path)
+ continue
+
+ (resource, version) = classify(path)
+
+ # Earlier paths are taken in preference to later paths.
+ # This agrees with aapt.
+ if version not in resources:
+ resources[resource][version] = source_path
+
+ # Step 1: merge all XML values into one single, sorted
+ # per-configuration values.xml file. This apes what the Android
+ # Gradle resource merging algorithm does.
+ merged_values = defaultdict(list)
+
+ for dir, files in values.items():
+ for file in files:
+ values = ET.ElementTree(file=file).getroot()
+ merged_values[dir].extend(values)
+
+ values = ET.Element('resources')
+ # Sort by <type> tag, and then by name. Note that <item
+ # type="type"> is equivalent to <type>.
+ key = lambda x: (resource_type.get(x.get('type', x.tag)), x.get('name'))
+ values[:] = sorted(merged_values[dir], key=key)
+
+ for value in values:
+ if value.get('name') == 'TextAppearance.Design.Snackbar.Message':
+ if value.get('{http://schemas.android.com/tools}override', False):
+ values.remove(value)
+ break
+
+ merged_values[dir] = values
+
+ for dir, values in merged_values.items():
+ o = mozpath.join(output_dirname, dir, '{}.xml'.format(dir))
+ ensureParentDir(o)
+ ET.ElementTree(values).write(o)
+
+ manifest.add_required_exists(mozpath.join(dir, '{}.xml'.format(dir)))
+
+ # Step 2a: add version numbers for unversioned features
+ # corresponding to when the feature was introduced. Resource
+ # qualifiers will never be recognized by Android versions before
+ # they were introduced. For example, density qualifiers are
+ # supported only in Android v4 and above. Therefore
+ # "drawable-hdpi" is implicitly "drawable-hdpi-v4". We version
+ # such unversioned resources here.
+ for (resource, versions) in resources.items():
+ if None in versions:
+ dir, name = resource.split('/')
+ new_dir = with_version(dir)
+ (new_resource, new_version) = classify('{}/{}'.format(new_dir, name))
+ if new_resource != resource:
+ raise ValueError('this is bad')
+
+ # `new_version` might be None: for example, `dir` might be "drawable".
+ source_path = versions.pop(None)
+ versions[new_version] = source_path
+
+ if verbose:
+ if new_version:
+ print("Versioning unversioned resource {} as {}-v{}/{}".format(source_path, dir, new_version, name))
+
+ # TODO: make this a command line argument that takes MOZ_ANDROID_MIN_SDK_VERSION.
+ min_sdk = 15
+ retained = defaultdict(dict)
+
+ # Step 2b: drop resource directories that will never be used by
+ # Android on device. This depends on the minimum supported
+ # Android SDK version. Suppose the minimum SDK is 15 and we have
+ # drawable-v4/icon.png and drawable-v11/icon.png. The v4 version
+ # will never be chosen, since v15 is always greater than v11.
+ for (resource, versions) in resources.items():
+ def key(v):
+ return 0 if v is None else v
+ # Versions in descending order.
+ version_list = sorted(versions.keys(), key=key, reverse=True)
+ for version in version_list:
+ retained[resource][version] = versions[version]
+ if version is not None and version <= min_sdk:
+ break
+
+ if set(retained.keys()) != set(resources.keys()):
+ raise ValueError('Something terrible has happened; retained '
+ 'resource names do not match input resources '
+ 'names')
+
+ if verbose:
+ for resource in resources:
+ if resources[resource] != retained[resource]:
+ for version in sorted(resources[resource].keys(), reverse=True):
+ if version in retained[resource]:
+ print("Keeping reachable resource {}".format(resources[resource][version]))
+ else:
+ print("Dropping unreachable resource {}".format(resources[resource][version]))
+
+ # Populate manifest.
+ for (resource, versions) in retained.items():
+ for version in sorted(versions.keys(), reverse=True):
+ path = resource
+ if version:
+ dir, name = resource.split('/')
+ path = '{}-v{}/{}'.format(dir, version, name)
+ manifest.add_copy(versions[version], path)
+
+
+ copier = FileCopier()
+ manifest.populate_registry(copier)
+ print('mr', os.getcwd())
+ result = copier.copy(output_dirname,
+ remove_unaccounted=True,
+ remove_all_directory_symlinks=False,
+ remove_empty_directories=True)
+
+ if verbose:
+ print('Updated:', result.updated_files_count)
+ print('Removed:', result.removed_files_count + result.removed_directories_count)
+ print('Existed:', result.existing_files_count)
+
+ return 0
+
+
+if __name__ == '__main__':
+ sys.exit(main(*sys.argv[1:]))