Bug 1355625 - Part 2: Merge Android resources before invoking aapt. r=sebastian draft
authorNick Alexander <nalexander@mozilla.com>
Mon, 19 Jun 2017 15:42:22 -0700
changeset 596953 b672ed502ce23edbec2ae89b2310590c047feaca
parent 596952 21a60780841f1fbfa8410a2ba8d8612fe0823849
child 596954 e44f8664860f841b0cfd151f23f123344a348d49
push id64783
push usernalexander@mozilla.com
push dateMon, 19 Jun 2017 22:44:01 +0000
reviewerssebastian
bugs1355625
milestone56.0a1
Bug 1355625 - Part 2: Merge Android resources before invoking aapt. r=sebastian This is a restricted version of the Android Gradle build tool resource merging algorithm. The goal is for the R.java produced by moz.build to have exactly the same resource ID map as the R.java produced by Gradle. This implementation does two things: 1) Copy all resources into a single directory. There's the potential for conflicts based on order here; Android's resource merging algorithm is intended to support product flavors and build types; it's non-trivial. Our moz.build "algorithm" is based on the order the directories are provided to aapt, and to the best of my knowledge Fennec does not depend on this ordering; it's trivial. 2) Merge all XML values into a single values.xml (depending on resource configurations). This is the crucial piece: we need to collect and order the resulting XML nodes exactly as the Gradle resource merging implementation does. Luckily, it's not hard! In both these places there's are two wrinkles around versioning. The first is that the Gradle resource merging algorithm assigns versions to unversioned resources depending on what Android SDK version added the features that the resource is filtered on. For example, "xlarge" was added in v4 and "ldrtl" was added in v13, so "drawable-xlarge" is really "drawable-xlarge-v4" and "drawable-ldrtl" is really "drawable-ldrtl-v13". The second wrinkle is that some versions of a resource may be unreachable: this happens if, for example, a v4 and a v11 version is provided. Since the minimum SDK is (currently) v15, the v11 version can always be used, meaning that the v14 version will never be used and can be culled. With this work, the R.txt produced by moz.build is identical to the R.txt produced by Gradle, paving the way to get other key parts of the build -- including the compiled Java .class files -- identical! MozReview-Commit-ID: G97MM64gksQ
mobile/android/app/build.gradle
mobile/android/base/Makefile.in
python/mozbuild/mozbuild/action/aapt_package.py
python/mozbuild/mozbuild/action/merge_resources.py
--- a/mobile/android/app/build.gradle
+++ b/mobile/android/app/build.gradle
@@ -31,16 +31,20 @@ android {
         vectorDrawables.useSupportLibrary = true
     }
 
     compileOptions {
         sourceCompatibility JavaVersion.VERSION_1_7
         targetCompatibility JavaVersion.VERSION_1_7
     }
 
+    aaptOptions {
+        cruncherEnabled = false
+    }
+
     dexOptions {
         javaMaxHeapSize "2g"
     }
 
     lintOptions {
         abortOnError true
     }
 
--- a/mobile/android/base/Makefile.in
+++ b/mobile/android/base/Makefile.in
@@ -444,44 +444,27 @@ ANDROID_AAPT_IGNORE := !.svn:!.git:.*:<d
 # 4: directory to write R.java into.
 # 5: directory to write R.txt into.
 # We touch the target file before invoking aapt so that aapt's outputs
 # are fresher than the target, preventing a subsequent invocation from
 # thinking aapt's outputs are stale.  This is safe because Make
 # removes the target file if any recipe command fails.
 
 define aapt_command
-$(1): $$(call mkdir_deps,$(filter-out ./,$(dir $(3) $(4) $(5)))) $(2)
+$(1): $(topsrcdir)/python/mozbuild/mozbuild/action/aapt_package.py $$(call mkdir_deps,$(filter-out ./,$(dir $(3) $(4) $(5)))) $(2)
 	@$$(TOUCH) $$@
-	$$(AAPT) package -f -m \
-		-M AndroidManifest.xml \
-		-I $(ANDROID_SDK)/android.jar \
-		$(if $(MOZ_ANDROID_MAX_SDK_VERSION),--max-res-version $(MOZ_ANDROID_MAX_SDK_VERSION),) \
-		--auto-add-overlay \
-		$$(addprefix -S ,$$(ANDROID_RES_DIRS)) \
-		$$(addprefix -A ,$$(ANDROID_ASSETS_DIRS)) \
-		$(if $(ANDROID_EXTRA_PACKAGES),--extra-packages $$(subst $$(NULL) ,:,$$(strip $$(ANDROID_EXTRA_PACKAGES)))) \
-		$(if $(ANDROID_EXTRA_RES_DIRS),$$(addprefix -S ,$$(ANDROID_EXTRA_RES_DIRS))) \
-		--custom-package org.mozilla.gecko \
-		--no-version-vectors \
-		-F $(3) \
-		-J $(4) \
-		--output-text-symbols $(5) \
-		--ignore-assets "$$(ANDROID_AAPT_IGNORE)"
-	mkdir -p $(subst /,-2/,$(4))
-	mkdir -p aapt
 	$$(call py_action,aapt_package,-f -m \
 		-M AndroidManifest.xml \
 		$$(addprefix -A ,$$(ANDROID_ASSETS_DIRS)) \
 		$$(addprefix -S ,$$(ANDROID_RES_DIRS)) \
 		$(if $(ANDROID_EXTRA_RES_DIRS),$$(addprefix -S ,$$(ANDROID_EXTRA_RES_DIRS))) \
 		$(if $(ANDROID_EXTRA_PACKAGES),--extra-packages $$(subst $$(NULL) ,:,$$(strip $$(ANDROID_EXTRA_PACKAGES)))) \
-		-F $(subst .ap_,-2.ap_,$(3)) \
-		-J $(subst /,-2/,$(4)) \
-		--output-text-symbols aapt/ \
+		-F $(3) \
+		-J $(4) \
+		--output-text-symbols $(5) \
 	  --verbose)
 endef
 
 # [Comment 3/3] The first of these rules is used during regular
 # builds.  The second writes an ap_ file that is only used during
 # packaging.  It doesn't write the normal ap_, or R.java, since we
 # don't want the packaging step to write anything that would make a
 # further no-op build do work.  See also
--- a/python/mozbuild/mozbuild/action/aapt_package.py
+++ b/python/mozbuild/mozbuild/action/aapt_package.py
@@ -14,30 +14,30 @@ resource merging algorithm before invoki
 '''
 
 from __future__ import (
     print_function,
     unicode_literals,
 )
 
 import argparse
-from collections import OrderedDict
 import os
 import subprocess
 import sys
 
 import buildconfig
-from mozbuild.preprocessor import Preprocessor
-from mozbuild.util import ensureParentDir 
 import mozpack.path as mozpath
 
+import merge_resources
+
 
 def uniqify(iterable):
     """Remove duplicates from iterable, preserving order."""
-    # Cribbed from https://thingspython.wordpress.com/2011/03/09/snippet-uniquify-a-sequence-preserving-order/.
+    # Cribbed from
+    # https://thingspython.wordpress.com/2011/03/09/snippet-uniquify-a-sequence-preserving-order/.
     seen = set()
     return [item for item in iterable if not (item in seen or seen.add(item))]
 
 
 def main(*argv):
     parser = argparse.ArgumentParser(
         description='Invoke Android `aapt package`.')
 
@@ -78,38 +78,44 @@ def main(*argv):
 
     import itertools
 
     debug = False
     if (not buildconfig.substs['MOZILLA_OFFICIAL']) or \
        (buildconfig.substs['NIGHTLY_BUILD'] and buildconfig.substs['MOZ_DEBUG']):
         debug = True
 
+    merge_resources.main('merged', True, *args.res_dirs)
+
     cmd = [
         buildconfig.substs['AAPT'],
         'package',
     ] + \
     (['-f'] if args.f else []) + \
     [
         '-m',
         '-M', args.M,
-	'-I', mozpath.join(buildconfig.substs['ANDROID_SDK'], 'android.jar'),
+        '-I', mozpath.join(buildconfig.substs['ANDROID_SDK'], 'android.jar'),
         '--auto-add-overlay',
     ] + \
     list(itertools.chain(*(('-A', x) for x in args.assets_dirs))) + \
-    list(itertools.chain(*(('-S', x) for x in args.res_dirs))) + \
+    ['-S', os.path.abspath('merged')] + \
     (['--extra-packages', ':'.join(args.extra_packages)] if args.extra_packages else []) + \
     ['--custom-package', 'org.mozilla.gecko'] + \
     ['--no-version-vectors'] + \
     (['--debug-mode'] if debug else []) + \
     [
-        '-F', args.F,
-        '-J', args.J,
-        '--output-text-symbols', args.output_text_symbols,
-        '--ignore-assets', '!.svn:!.git:.*:<dir>_*:!CVS:!thumbs.db:!picasa.ini:!*.scc:*~:#*:*.rej:*.orig',
+        '-F',
+        args.F,
+        '-J',
+        args.J,
+        '--output-text-symbols',
+        args.output_text_symbols,
+        '--ignore-assets',
+        '!.svn:!.git:.*:<dir>_*:!CVS:!thumbs.db:!picasa.ini:!*.scc:*~:#*:*.rej:*.orig',
     ]
 
     # We run aapt to produce gecko.ap_ and gecko-nodeps.ap_; it's
     # helpful to tag logs with the file being produced.
     logtag = os.path.basename(args.F)
 
     if args.verbose:
         print('[aapt {}] {}'.format(logtag, ' '.join(cmd)))
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozbuild/action/merge_resources.py
@@ -0,0 +1,300 @@
+#!/bin/python
+
+# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+'''
+A hacked together clone of the Android Gradle plugin's resource
+merging algorithm.  To be abandoned in favour of --with-gradle as soon
+as possible!
+'''
+
+from __future__ import (
+    print_function,
+    unicode_literals,
+)
+
+from collections import defaultdict
+import re
+import os
+import sys
+
+from mozbuild.util import ensureParentDir 
+from mozpack.copier import (
+    FileCopier,
+)
+from mozpack.manifests import (
+    InstallManifest,
+)
+import mozpack.path as mozpath
+from mozpack.files import (
+    FileFinder,
+)
+
+import xml.etree.cElementTree as ET
+
+
+# From https://github.com/miracle2k/android-platform_sdk/blob/master/common/src/com/android/resources/ResourceType.java.
+# TODO: find a more authoritative source!
+resource_type = {
+    "anim": 0,
+    "animator": 1,
+    # The only interesting ones.
+    "string-array": 2,
+    "integer-array": 2,
+    "attr": 3,
+    "bool": 4,
+    "color": 5,
+    "declare-styleable": 6,
+    "dimen": 7,
+    "drawable": 8,
+    "fraction": 9,
+    "id": 10,
+    "integer": 11,
+    "interpolator": 12,
+    "layout": 13,
+    "menu": 14,
+    "mipmap": 15,
+    "plurals": 16,
+    "raw": 17,
+    "string": 18,
+    "style": 19,
+    "styleable": 20,
+    "xml": 21,
+    # "public": 0,
+}
+
+
+def uniqify(iterable):
+    """Remove duplicates from iterable, preserving order."""
+    # Cribbed from https://thingspython.wordpress.com/2011/03/09/snippet-uniquify-a-sequence-preserving-order/.
+    seen = set()
+    return [item for item in iterable if not (item in seen or seen.add(item))]
+
+
+# Exclusions, arising in appcompat-v7-23.4.0.
+MANIFEST_EXCLUSIONS = (
+    'color/abc_background_cache_hint_selector_material_dark.xml',
+    'color/abc_background_cache_hint_selector_material_light.xml',
+)
+
+SMALLEST_SCREEN_WIDTH_QUALIFIER_RE = re.compile(r"(^|-)w(\d+)dp($|-)")
+SCREEN_WIDTH_QUALIFIER_RE = re.compile(r"(^|-)sw(\d+)dp($|-)")
+# Different densities were introduced in different Android versions.
+# However, earlier versions of aapt (like the one we are building
+# with) don't have fine-grained versions; they're all lumped into v4.
+DENSITIES = [
+    (re.compile(r"(^|-)xxxhdpi($|-)"), 18),
+    (re.compile(r"(^|-)560dpi($|-)"),  1),
+    (re.compile(r"(^|-)xxhdpi($|-)"),  16),
+    (re.compile(r"(^|-)400dpi($|-)"),  1),
+    (re.compile(r"(^|-)360dpi($|-)"),  23),
+    (re.compile(r"(^|-)xhdpi($|-)"),   8),
+    (re.compile(r"(^|-)280dpi($|-)"),  22),
+    (re.compile(r"(^|-)hdpi($|-)"),    4),
+    (re.compile(r"(^|-)tvdpi($|-)"),   13),
+    (re.compile(r"(^|-)mdpi($|-)"),    4),
+    (re.compile(r"(^|-)ldpi($|-)"),    4),
+    (re.compile(r"(^|-)anydpi($|-)"),  21),
+    (re.compile(r"(^|-)nodpi($|-)"),   4),
+]
+SCREEN_SIZE_RE = re.compile(r"(^|-)(small|normal|large|xlarge)($|-)")
+
+def with_version(dir):
+    """Resources directories without versions (like values-large) that
+correspond to resource filters added to Android in vN (like large,
+which was added in v4) automatically get a -vN added (so values-large
+becomes values-large-v4, since Android versions before v4 will not
+recognize values-large)."""
+    # Order matters!  We need to check for later features before
+    # earlier features, so that "ldrtl-sw-large" will be v17, not v13
+    # or v4.
+    if '-ldrtl' in dir and '-v' not in dir:
+        return '{}-v17'.format(dir)
+
+    if re.search(SMALLEST_SCREEN_WIDTH_QUALIFIER_RE, dir) and '-v' not in dir:
+        return '{}-v13'.format(dir)
+
+    if re.search(SCREEN_WIDTH_QUALIFIER_RE, dir) and '-v' not in dir:
+        return '{}-v13'.format(dir)
+
+    for (density, _since) in DENSITIES:
+        if re.search(density, dir) and '-v' not in dir:
+            return '{}-v{}'.format(dir, 4)
+
+    if re.search(SCREEN_SIZE_RE, dir) and '-v' not in dir:
+        return '{}-v4'.format(dir)
+
+    return dir
+
+
+def classify(path):
+    """Return `(resource, version)` for a given path.
+
+`resource` is of the form `unversioned/name` where `unversionsed` is a resource
+type (like "drawable" or "strings"), and `version` is an
+integer version number or `None`."""
+    dir, name = path.split('/')
+    segments = dir.split('-')
+    version = None
+    for segment in segments[1:]:
+        if segment.startswith('v'):
+            version = int(segment[1:])
+            break
+    segments = [segment for segment in segments if not segment.startswith('v')]
+    resource = '{}/{}'.format('-'.join(segments), name)
+    return (resource, version)
+
+
+def main(output_dirname, verbose, *input_dirs):
+    # Map directories to source paths, like
+    # `{'values-large-v11': ['/path/to/values-large-v11/strings.xml',
+    #                        '/path/to/values-large-v11/colors.xml', ...], ...}`.
+    values = defaultdict(list)
+    # Map unversioned resource names to maps from versions to source paths, like:
+    # `{'drawable-large/icon.png':
+    #     {None: '/path/to/drawable-large/icon.png',
+    #      11: '/path/to/drawable-large-v11/icon.png', ...}, ...}`.
+    resources = defaultdict(dict)
+
+    manifest = InstallManifest()
+
+    for p in uniqify(input_dirs):
+        finder = FileFinder(p, find_executables=False)
+
+        values_pattern = 'values*/*.xml'
+        for path, _ in finder.find('*/*'):
+            if path in MANIFEST_EXCLUSIONS:
+                continue
+
+            source_path = mozpath.join(finder.base, path)
+
+            if mozpath.match(path, values_pattern):
+                dir, _name = path.split('/')
+                dir = with_version(dir)
+                values[dir].append(source_path)
+                continue
+
+            (resource, version) = classify(path)
+
+            # Earlier paths are taken in preference to later paths.
+            # This agrees with aapt.
+            if version not in resources:
+                resources[resource][version] = source_path
+
+    # Step 1: merge all XML values into one single, sorted
+    # per-configuration values.xml file.  This apes what the Android
+    # Gradle resource merging algorithm does.
+    merged_values = defaultdict(list)
+
+    for dir, files in values.items():
+        for file in files:
+            values = ET.ElementTree(file=file).getroot()
+            merged_values[dir].extend(values)
+
+        values = ET.Element('resources')
+        # Sort by <type> tag, and then by name.  Note that <item
+        # type="type"> is equivalent to <type>.
+        key = lambda x: (resource_type.get(x.get('type', x.tag)), x.get('name'))
+        values[:] = sorted(merged_values[dir], key=key)
+
+        for value in values:
+            if value.get('name') == 'TextAppearance.Design.Snackbar.Message':
+                if value.get('{http://schemas.android.com/tools}override', False):
+                    values.remove(value)
+                    break
+
+        merged_values[dir] = values
+
+    for dir, values in merged_values.items():
+        o = mozpath.join(output_dirname, dir, '{}.xml'.format(dir))
+        ensureParentDir(o)
+        ET.ElementTree(values).write(o)
+
+        manifest.add_required_exists(mozpath.join(dir, '{}.xml'.format(dir)))
+
+    # Step 2a: add version numbers for unversioned features
+    # corresponding to when the feature was introduced.  Resource
+    # qualifiers will never be recognized by Android versions before
+    # they were introduced.  For example, density qualifiers are
+    # supported only in Android v4 and above.  Therefore
+    # "drawable-hdpi" is implicitly "drawable-hdpi-v4".  We version
+    # such unversioned resources here.
+    for (resource, versions) in resources.items():
+        if None in versions:
+            dir, name = resource.split('/')
+            new_dir = with_version(dir)
+            (new_resource, new_version) = classify('{}/{}'.format(new_dir, name))
+            if new_resource != resource:
+                raise ValueError('this is bad')
+            
+            # `new_version` might be None: for example, `dir` might be "drawable". 
+            source_path = versions.pop(None)
+            versions[new_version] = source_path
+
+            if verbose:
+                if new_version:
+                    print("Versioning unversioned resource {} as {}-v{}/{}".format(source_path, dir, new_version, name))
+
+    min_sdk = 15 # TODO: command line argument!
+    retained = defaultdict(dict)
+
+    # Step 2b: drop resource directories that will never be used by
+    # Android on device.  This depends on the minimum supported
+    # Android SDK version.  Suppose the minimum SDK is 15 and we have
+    # drawable-v4/icon.png and drawable-v11/icon.png.  The v4 version
+    # will never be chosen, since v15 is always greater than v11.
+    for (resource, versions) in resources.items():
+        def key(v):
+            return 0 if v is None else v
+        # Versions in descending order.
+        version_list = sorted(versions.keys(), key=key, reverse=True)
+        for version in version_list:
+            retained[resource][version] = versions[version]
+            if version is not None and version <= min_sdk:
+                break
+
+    if set(retained.keys()) != set(resources.keys()):
+        raise ValueError('Something terrible has happened; retained '
+                         'resource names do not match input resources '
+                         'names')
+
+    if verbose:
+        for resource in resources:
+            if resources[resource] != retained[resource]:
+                for version in sorted(resources[resource].keys(), reverse=True):
+                    if version in retained[resource]:
+                        print("Keeping reachable resource {}".format(resources[resource][version]))
+                    else:
+                        print("Dropping unreachable resource {}".format(resources[resource][version]))
+
+    # Populate manifest.
+    for (resource, versions) in retained.items():
+        for version in sorted(versions.keys(), reverse=True):
+            path = resource
+            if version:
+                dir, name = resource.split('/')
+                path = '{}-v{}/{}'.format(dir, version, name)
+            manifest.add_copy(versions[version], path)
+
+
+    copier = FileCopier()
+    manifest.populate_registry(copier)
+    print('mr', os.getcwd())
+    result = copier.copy(output_dirname,
+                         remove_unaccounted=True,
+                         remove_all_directory_symlinks=False,
+                         remove_empty_directories=True)
+
+    if verbose: 
+        print('Updated:', result.updated_files_count)
+        print('Removed:', result.removed_files_count + result.removed_directories_count)
+        print('Existed:', result.existing_files_count)
+
+    return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main(*sys.argv[1:]))