Bug 1390916 - Always use --track instead of --no-remove for install manifests; r?gps draft
authorMike Shal <mshal@mozilla.com>
Tue, 08 Aug 2017 17:07:07 -0400
changeset 649960 faf7a061e951571a8af24ecc64d4d05ad5c8e46a
parent 649959 b62a49c2bac4846912f2701ba2f8df7f0225be25
child 727251 221c1e17aa5c2d33ae5d20a4b3a05c065ca6ac3f
push id75219
push userbmo:mshal@mozilla.com
push dateMon, 21 Aug 2017 15:45:59 +0000
reviewersgps
bugs1390916
milestone57.0a1
Bug 1390916 - Always use --track instead of --no-remove for install manifests; r?gps The --track flag provides a more accurate accounting of what files were installed by the manifest, so they can be appropriately removed. For example, test files are now removed from _tests if an entry in a test file is deleted. The --no-remove flag is removed as an alternative, and the --track flag is now mandatory. MozReview-Commit-ID: Wiup4Gzwkb
Makefile.in
js/src/build/Makefile.in
python/mozbuild/mozbuild/action/process_install_manifest.py
xpcom/xpidl/Makefile.in
--- a/Makefile.in
+++ b/Makefile.in
@@ -155,51 +155,35 @@ ifdef MOZ_ARTIFACT_BUILDS
 endif
 	$(call BUILDSTATUS,TIER_START make)
 	$(MAKE) buildid.h source-repo.h
 	$(call BUILDSTATUS,TIER_FINISH make)
 	$(call BUILDSTATUS,TIER_START tup)
 	@$(TUP) $(if $(findstring s,$(filter-out --%,$(MAKEFLAGS))),,--verbose)
 	$(call BUILDSTATUS,TIER_FINISH tup)
 
-# process_install_manifest needs to be invoked with --no-remove when building
-# js as standalone because automated builds are building nspr separately and
-# that would remove the resulting files.
-# Eventually, a standalone js build would just be able to build nspr itself,
-# removing the need for the former.
-ifdef JS_STANDALONE
-NO_REMOVE=1
-endif
-
-# For an artifact build, _tests will already be partly populated, so run
-# this install manifest with NO_REMOVE set in this case.
-ifdef MOZ_ARTIFACT_BUILDS
-install-_tests: NO_REMOVE=1
-endif
-
 .PHONY: $(addprefix install-,$(install_manifests))
 $(addprefix install-,$(install_manifests)): install-%: $(install_manifest_depends)
 ifneq (,$(filter FasterMake+RecursiveMake,$(BUILD_BACKENDS)))
 	@# If we're using the hybrid FasterMake/RecursiveMake backend, we want
 	@# to ensure the FasterMake end doesn't have install manifests for the
 	@# same directory, because that would blow up
 	$(if $(wildcard _build_manifests/install/$(subst /,_,$*)),$(if $(wildcard faster/install_$(subst /,_,$*)*),$(error FasterMake and RecursiveMake ends of the hybrid build system want to handle $*)))
 endif
-	$(addprefix $(call py_action,process_install_manifest,$(if $(NO_REMOVE),--no-remove )$*) ,$(wildcard _build_manifests/install/$(subst /,_,$*)))
+	$(addprefix $(call py_action,process_install_manifest,--track install_$(subst /,_,$*).track $*) ,$(wildcard _build_manifests/install/$(subst /,_,$*)))
 
 # Dummy wrapper rule to allow the faster backend to piggy back
 $(addprefix install-,$(subst /,_,$(filter dist/%,$(install_manifests)))): install-dist_%: install-dist/% ;
 
 .PHONY: install-tests
 install-tests: install-test-files
 
-# Force --no-remove, because $objdir/_tests is handled by multiple manifests.
 .PHONY: install-test-files
 install-test-files:
-	$(call py_action,process_install_manifest,--no-remove _tests _build_manifests/install/_test_files)
+	$(call py_action,process_install_manifest,--track install__test_files.track _tests _build_manifests/install/_test_files)
 
 include $(topsrcdir)/build/moz-automation.mk
 
 # dist and _tests should be purged during cleaning. However, we don't want them
 # purged during PGO builds because they contain some auto-generated files.
 ifneq ($(filter-out maybe_clobber_profiledbuild,$(MAKECMDGOALS)),)
 GARBAGE_DIRS += dist _tests
 endif
@@ -212,23 +196,23 @@ recurse_pre-export::
 # process the install manifests as part of export.
 # For the binaries rule, not all the install manifests matter, so force only
 # the interesting ones to be done.
 ifdef MOZ_PROFILE_USE
 ifndef NO_PROFILE_GUIDED_OPTIMIZE
 ifneq ($(OS_ARCH)_$(GNU_CC), WINNT_)
 recurse_pre-export:: install-manifests
 binaries::
-	@$(MAKE) install-manifests NO_REMOVE=1 install_manifests=dist/include
+	@$(MAKE) install-manifests install_manifests=dist/include
 endif
 endif
 else # !MOZ_PROFILE_USE (normal build)
 recurse_pre-export:: install-manifests
 binaries::
-	@$(MAKE) install-manifests NO_REMOVE=1 install_manifests=dist/include
+	@$(MAKE) install-manifests install_manifests=dist/include
 endif
 
 recurse_artifact:
 	$(topsrcdir)/mach --log-no-times artifact install
 
 ifndef JS_STANDALONE
 ifdef ENABLE_TESTS
 # Additional makefile targets to call automated test suites
--- a/js/src/build/Makefile.in
+++ b/js/src/build/Makefile.in
@@ -65,17 +65,17 @@ install:: ../js-config.h
 #          js/src/ds/* public headers
 #        gc/
 #          js/src/gc/* public headers
 #        mozilla/
 #          mfbt headers
 #
 
 install::
-	$(call py_action,process_install_manifest,--no-remove --no-symlinks $(DESTDIR)$(includedir) $(DEPTH)/_build_manifests/install/dist_include)
+	$(call py_action,process_install_manifest,--track install_dist_include.track --no-symlinks $(DESTDIR)$(includedir) $(DEPTH)/_build_manifests/install/dist_include)
 
 #
 # END SpiderMonkey header installation
 #############################################
 
 # Install versioned script, for parallel installability in Linux distributions
 install:: js-config
 	cp $^ js$(MOZJS_MAJOR_VERSION)-config
--- a/python/mozbuild/mozbuild/action/process_install_manifest.py
+++ b/python/mozbuild/mozbuild/action/process_install_manifest.py
@@ -23,42 +23,41 @@ from mozpack.manifests import (
 from mozbuild.util import DefinesAction
 
 
 COMPLETE = 'Elapsed: {elapsed:.2f}s; From {dest}: Kept {existing} existing; ' \
     'Added/updated {updated}; ' \
     'Removed {rm_files} files and {rm_dirs} directories.'
 
 
-def process_manifest(destdir, paths, track=None,
-        remove_unaccounted=True,
-        remove_all_directory_symlinks=True,
-        remove_empty_directories=True,
+def process_manifest(destdir, paths, track,
         no_symlinks=False,
         defines={}):
 
-    if track:
-        if os.path.exists(track):
-            # We use the same format as install manifests for the tracking
-            # data.
-            manifest = InstallManifest(path=track)
-            remove_unaccounted = FileRegistry()
-            dummy_file = BaseFile()
+    if os.path.exists(track):
+        # We use the same format as install manifests for the tracking
+        # data.
+        manifest = InstallManifest(path=track)
+        remove_unaccounted = FileRegistry()
+        dummy_file = BaseFile()
 
-            finder = FileFinder(destdir, find_dotfiles=True)
-            for dest in manifest._dests:
-                for p, f in finder.find(dest):
-                    remove_unaccounted.add(p, dummy_file)
+        finder = FileFinder(destdir, find_dotfiles=True)
+        for dest in manifest._dests:
+            for p, f in finder.find(dest):
+                remove_unaccounted.add(p, dummy_file)
 
-        else:
-            # If tracking is enabled and there is no file, we don't want to
-            # be removing anything.
-            remove_unaccounted=False
-            remove_empty_directories=False
-            remove_all_directory_symlinks=False
+        remove_empty_directories=True
+        remove_all_directory_symlinks=True
+
+    else:
+        # If tracking is enabled and there is no file, we don't want to
+        # be removing anything.
+        remove_unaccounted = False
+        remove_empty_directories=False
+        remove_all_directory_symlinks=False
 
     manifest = InstallManifest()
     for path in paths:
         manifest |= InstallManifest(path=path)
 
     copier = FileCopier()
     link_policy = "copy" if no_symlinks else "symlink"
     manifest.populate_registry(
@@ -76,38 +75,30 @@ def process_manifest(destdir, paths, tra
 
 
 def main(argv):
     parser = argparse.ArgumentParser(
         description='Process install manifest files.')
 
     parser.add_argument('destdir', help='Destination directory.')
     parser.add_argument('manifests', nargs='+', help='Path to manifest file(s).')
-    parser.add_argument('--no-remove', action='store_true',
-        help='Do not remove unaccounted files from destination.')
-    parser.add_argument('--no-remove-all-directory-symlinks', action='store_true',
-        help='Do not remove all directory symlinks from destination.')
-    parser.add_argument('--no-remove-empty-directories', action='store_true',
-        help='Do not remove empty directories from destination.')
     parser.add_argument('--no-symlinks', action='store_true',
         help='Do not install symbolic links. Always copy files')
-    parser.add_argument('--track', metavar="PATH",
+    parser.add_argument('--track', metavar="PATH", required=True,
         help='Use installed files tracking information from the given path.')
     parser.add_argument('-D', action=DefinesAction,
         dest='defines', metavar="VAR[=VAL]",
         help='Define a variable to override what is specified in the manifest')
 
     args = parser.parse_args(argv)
 
     start = time.time()
 
     result = process_manifest(args.destdir, args.manifests,
-        track=args.track, remove_unaccounted=not args.no_remove,
-        remove_all_directory_symlinks=not args.no_remove_all_directory_symlinks,
-        remove_empty_directories=not args.no_remove_empty_directories,
+        track=args.track,
         no_symlinks=args.no_symlinks,
         defines=args.defines)
 
     elapsed = time.time() - start
 
     print(COMPLETE.format(
         elapsed=elapsed,
         dest=args.destdir,
--- a/xpcom/xpidl/Makefile.in
+++ b/xpcom/xpidl/Makefile.in
@@ -1,10 +1,10 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 export::
-	$(call py_action,process_install_manifest,$(DIST)/idl $(DEPTH)/_build_manifests/install/dist_idl)
+	$(call py_action,process_install_manifest,--track install-xpidl.track $(DIST)/idl $(DEPTH)/_build_manifests/install/dist_idl)
 	$(call SUBMAKE,xpidl,$(DEPTH)/config/makefiles/xpidl)
 
 clean clobber realclean clobber_all distclean::
 	$(call SUBMAKE,$@,$(DEPTH)/config/makefiles/xpidl)