Bug 1459004: Generate built_in_addons.json from moz.build definitions. r?mshal draft
authorKris Maglione <maglione.k@gmail.com>
Wed, 06 Jun 2018 16:43:00 -0700
changeset 810557 051bda9ef0f55854a0e5ec181695f72205543527
parent 810556 4a20ed6e2fee8c64419134ad5b6778f26bfffaec
push id114033
push usermaglione.k@gmail.com
push dateTue, 26 Jun 2018 04:21:11 +0000
reviewersmshal
bugs1459004
milestone63.0a1
Bug 1459004: Generate built_in_addons.json from moz.build definitions. r?mshal MozReview-Commit-ID: 1HRLG5tSlFr
browser/app/Makefile.in
config/faster/rules.mk
mobile/android/base/Makefile.in
python/mozbuild/mozbuild/action/generate_builtin_addons.py
python/mozbuild/mozbuild/backend/tup.py
toolkit/mozapps/extensions/gen_built_in_addons.py
toolkit/mozapps/extensions/moz.build
--- a/browser/app/Makefile.in
+++ b/browser/app/Makefile.in
@@ -74,17 +74,17 @@ endif
 LPROJ := Contents/Resources/$(LPROJ_ROOT).lproj
 
 clean clobber repackage::
 	$(RM) -r $(dist_dest)
 
 MAC_BUNDLE_VERSION = $(shell $(PYTHON) $(srcdir)/macversion.py --version=$(MOZ_APP_VERSION) --buildid=$(DEPTH)/buildid.h)
 
 .PHONY: repackage
-tools repackage:: $(DIST)/bin/$(MOZ_APP_NAME) features
+tools repackage:: $(DIST)/bin/$(MOZ_APP_NAME)
 	rm -rf $(dist_dest)
 	$(MKDIR) -p '$(dist_dest)/Contents/MacOS'
 	$(MKDIR) -p '$(dist_dest)/$(LPROJ)'
 	rsync -a --exclude '*.in' $(srcdir)/macbuild/Contents '$(dist_dest)' --exclude English.lproj
 	rsync -a --exclude '*.in' $(srcdir)/macbuild/Contents/Resources/English.lproj/ '$(dist_dest)/$(LPROJ)'
 	sed -e 's/%APP_VERSION%/$(MOZ_APP_VERSION)/' -e 's/%MAC_APP_NAME%/$(MAC_APP_NAME)/' -e 's/%MOZ_MACBUNDLE_ID%/$(MOZ_MACBUNDLE_ID)/' -e 's/%MAC_BUNDLE_VERSION%/$(MAC_BUNDLE_VERSION)/' -e 's|%MOZ_DEVELOPER_REPO_PATH%|$(topsrcdir)|' -e 's|%MOZ_DEVELOPER_OBJ_PATH%|$(topobjdir)|' $(srcdir)/macbuild/Contents/Info.plist.in > '$(dist_dest)/Contents/Info.plist'
 	sed -e 's/%MAC_APP_NAME%/$(MAC_APP_NAME)/' $(srcdir)/macbuild/Contents/Resources/English.lproj/InfoPlist.strings.in | iconv -f UTF-8 -t UTF-16 > '$(dist_dest)/$(LPROJ)/InfoPlist.strings'
 	rsync -a --exclude-from='$(srcdir)/macbuild/Contents/MacOS-files.in' $(DIST)/bin/ '$(dist_dest)/Contents/Resources'
@@ -95,12 +95,8 @@ tools repackage:: $(DIST)/bin/$(MOZ_APP_
 	cp -RL $(topsrcdir)/$(MOZ_BRANDING_DIRECTORY)/document.icns '$(dist_dest)/Contents/Resources/document.icns'
 	$(MKDIR) -p '$(dist_dest)/Contents/Library/LaunchServices'
 ifdef MOZ_UPDATER
 	mv -f '$(dist_dest)/Contents/MacOS/updater.app/Contents/MacOS/org.mozilla.updater' '$(dist_dest)/Contents/Library/LaunchServices'
 	ln -s ../../../../Library/LaunchServices/org.mozilla.updater '$(dist_dest)/Contents/MacOS/updater.app/Contents/MacOS/org.mozilla.updater'
 endif
 	printf APPLMOZB > '$(dist_dest)/Contents/PkgInfo'
 endif
-
-.PHONY: features
-tools features::
-	$(call py_action,generate_builtin_addons,--features=browser/features browser/chrome/browser/content/browser/built_in_addons.json)
--- a/config/faster/rules.mk
+++ b/config/faster/rules.mk
@@ -91,28 +91,8 @@ ACDEFINES += -DBUILD_FASTER
 		$(ACDEFINES) \
 		install_$(subst /,_,$*)
 
 # ============================================================================
 # Below is a set of additional dependencies and variables used to build things
 # that are not supported by data in moz.build.
 
 $(TOPOBJDIR)/build/application.ini: $(TOPOBJDIR)/buildid.h $(TOPOBJDIR)/source-repo.h
-
-# The manifest of allowed system add-ons should be re-built when using
-# "build faster".
-#
-# Note the dependency on install-dist/bin.  The form of this
-# dependency is critical: it's triggering the stem rule (install-%)
-# above to force the dist/bin manifest to be processed.  The more
-# obvious `$(TOPOBJDIR)/install-dist_bin` doesn't work because
-# dist/bin isn't in $(INSTALL_MANIFESTS) in the
-# FasterMake+RecursiveMake (artifact build) situation.
-ifeq ($(MOZ_BUILD_APP),browser)
-$(TOPOBJDIR)/browser/app/features: install-dist/bin
-
-default: $(TOPOBJDIR)/browser/app/features
-endif
-ifeq ($(MOZ_BUILD_APP),mobile/android)
-$(TOPOBJDIR)/mobile/android/base/features: install-dist/bin
-
-default: $(TOPOBJDIR)/mobile/android/base/features
-endif
--- a/mobile/android/base/Makefile.in
+++ b/mobile/android/base/Makefile.in
@@ -41,20 +41,16 @@ include $(topsrcdir)/config/android-comm
 $(ABS_DIST)/fennec/$(OMNIJAR_NAME): FORCE
 	$(REPORT_BUILD)
 	$(MAKE) -C ../../../faster
 	$(MAKE) -C ../installer stage-package
 	$(MKDIR) -p $(@D)
 	rsync --update $(DIST)/fennec/$(notdir $(OMNIJAR_NAME)) $@
 	$(RM) $(DIST)/fennec/$(notdir $(OMNIJAR_NAME))
 
-.PHONY: features
-features::
-	$(call py_action,generate_builtin_addons,--features=features chrome/chrome/content/built_in_addons.json)
-
 ifndef MOZILLA_OFFICIAL
 # Targets built very early during a Gradle build.  In automation,
 # these are built before Gradle is invoked, and gradle-targets is not
 # made at all.  This is required to avoid building gradle-targets with
 # AB_CD=multi during multi-l10n builds.
 gradle-targets: $(generated_resources) $(generated_files)
 
 # Local developers update omni.ja during their builds.  There's a
--- a/python/mozbuild/mozbuild/backend/tup.py
+++ b/python/mozbuild/mozbuild/backend/tup.py
@@ -94,16 +94,17 @@ class BackendTupfile(object):
         self.programs = []
         self.host_programs = []
         self.host_library = None
         self.exports = set()
 
         # These files are special, ignore anything that generates them or
         # depends on them.
         self._skip_files = [
+            'built_in_addons.json',
             'signmar',
         ]
 
         self.fh = FileAvoidWrite(self.name, capture_diff=True, dry_run=dry_run)
         self.fh.write('# THIS FILE WAS AUTOMATICALLY GENERATED. DO NOT EDIT.\n')
         self.fh.write('\n')
 
     def write(self, buf):
rename from python/mozbuild/mozbuild/action/generate_builtin_addons.py
rename to toolkit/mozapps/extensions/gen_built_in_addons.py
--- a/python/mozbuild/mozbuild/action/generate_builtin_addons.py
+++ b/toolkit/mozapps/extensions/gen_built_in_addons.py
@@ -1,46 +1,94 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-from __future__ import absolute_import, print_function, unicode_literals
+from __future__ import print_function, unicode_literals
 
 import argparse
 import json
 import os.path
 import sys
 
 import buildconfig
+import mozpack.path as mozpath
+
+from mozpack.copier import FileRegistry
+from mozpack.manifests import InstallManifest
+
+
+# A list of build manifests, and their relative base paths, from which to
+# extract lists of install files. These vary depending on which backend we're
+# using, so nonexistent manifests are ignored.
+manifest_paths = (
+    ('', '_build_manifests/install/dist_bin'),
+    ('', 'faster/install_dist_bin'),
+    ('browser', 'faster/install_dist_bin_browser'),
+)
 
 
-def main(argv):
+def get_registry(paths):
+    used_paths = set()
+
+    registry = FileRegistry()
+    for base, path in paths:
+        full_path = mozpath.join(buildconfig.topobjdir, path)
+        if not os.path.exists(full_path):
+            continue
+
+        used_paths.add(full_path)
+
+        reg = FileRegistry()
+        InstallManifest(full_path).populate_registry(reg)
+
+        for p, f in reg:
+            path = mozpath.join(base, p)
+            if not registry.contains(path):
+                registry.add(path, f)
+
+    return registry, used_paths
+
+
+def get_child(base, path):
+    """Returns the nearest parent of `path` which is an immediate child of
+    `base`"""
+
+    dirname = mozpath.dirname(path)
+    while dirname != base:
+        path = dirname
+        dirname = mozpath.dirname(path)
+    return path
+
+
+def main(output, *args):
     parser = argparse.ArgumentParser(
         description='Produces a JSON manifest of built-in add-ons')
     parser.add_argument('--features', type=str, dest='featuresdir',
                         action='store', help=('The distribution sub-directory '
                                               'containing feature add-ons'))
-    parser.add_argument('outputfile', help='File to write output to')
-    args = parser.parse_args(argv)
+    args = parser.parse_args(args)
 
-    bindir = os.path.join(buildconfig.topobjdir, 'dist/bin')
+    registry, inputs = get_registry(manifest_paths)
 
-    def find_dictionaries(path):
-        dicts = {}
-        for filename in os.listdir(os.path.join(bindir, path)):
-            base, ext = os.path.splitext(filename)
-            if ext == '.dic':
-                dicts[base] = '%s/%s' % (path, filename)
-        return dicts
+    dicts = {}
+    for path in registry.match('dictionaries/*.dic'):
+        base, ext = os.path.splitext(mozpath.basename(path))
+        dicts[base] = path
 
     listing = {
-        "dictionaries": find_dictionaries("dictionaries"),
+        "dictionaries": dicts,
     }
+
     if args.featuresdir:
-        listing["system"] = sorted(os.listdir(os.path.join(bindir,
-                                                           args.featuresdir)))
+        features = set()
+        for p in registry.match('%s/*' % args.featuresdir):
+            features.add(mozpath.basename(get_child(args.featuresdir, p)))
 
-    with open(os.path.join(bindir, args.outputfile), 'w') as fh:
-        json.dump(listing, fh, sort_keys=True)
+        listing["system"] = sorted(features)
+
+    json.dump(listing, output)
+
+    return inputs
 
 
 if __name__ == '__main__':
-    sys.exit(main(sys.argv[1:]))
+    main(sys.stdout, *sys.argv[1:])
--- a/toolkit/mozapps/extensions/moz.build
+++ b/toolkit/mozapps/extensions/moz.build
@@ -20,16 +20,34 @@ TEST_DIRS += ['test']
 
 XPIDL_SOURCES += [
     'amIAddonManagerStartup.idl',
     'amIWebInstallPrompt.idl',
 ]
 
 XPIDL_MODULE = 'extensions'
 
+built_in_addons = 'built_in_addons.json'
+GENERATED_FILES += [built_in_addons]
+manifest = GENERATED_FILES[built_in_addons]
+manifest.script = 'gen_built_in_addons.py'
+
+if CONFIG['MOZ_BUILD_APP'] == 'browser':
+    manifest.flags = ['--features=browser/features']
+
+    FINAL_TARGET_FILES.browser.chrome.browser.content.browser += [
+        '!%s' % built_in_addons,
+    ]
+elif CONFIG['MOZ_BUILD_APP'] == 'mobile/android':
+    manifest.flags = ['--features=features']
+
+    FINAL_TARGET_FILES.chrome.chrome.content += [
+        '!%s' % built_in_addons,
+    ]
+
 EXTRA_COMPONENTS += [
     'addonManager.js',
     'amContentHandler.js',
     'amInstallTrigger.js',
     'amWebAPI.js',
 ]
 
 EXTRA_PP_COMPONENTS += [