Bug 1355630 - Memoize tests_defaults_for_path call; r?chmanchester draft
authorGregory Szorc <gps@mozilla.com>
Tue, 11 Apr 2017 16:24:42 -0700
changeset 561605 004466bd4b9a25151b73d9e52cfe00ec43a94c7e
parent 560842 58739c053019a4dc56cfb4e4255fa89bcaeae2c0
child 561606 9bc23f7645772eca410cfd48ddd8b279f2bca938
push id53789
push userbmo:gps@mozilla.com
push dateWed, 12 Apr 2017 23:08:05 +0000
reviewerschmanchester
bugs1355630
milestone55.0a1
Bug 1355630 - Memoize tests_defaults_for_path call; r?chmanchester Calling self.test_defaults_for_path() from files_info() with tens of thousands of paths resulted in a CPU explosion in various path normalization functions. I don't think it was so much the complexity of the operations as much as the volume. For an input with 9131 elements, this reduces execution time of a mach command from ~25.7s to ~8.8s. With ~42,000 inputs, execution time drops from <it took too long and I gave up> to ~90s. MozReview-Commit-ID: pjQQByi2Bc
python/mozbuild/mozbuild/frontend/reader.py
--- a/python/mozbuild/mozbuild/frontend/reader.py
+++ b/python/mozbuild/mozbuild/frontend/reader.py
@@ -1367,16 +1367,29 @@ class BuildReader(object):
         2. Evaluate moz.build files starting with the most distant.
         3. Iterate over Files sub-contexts.
         4. If the file pattern matches the file we're seeking info on,
            apply attribute updates.
         5. Return the most recent value of attributes.
         """
         paths, _ = self.read_relevant_mozbuilds(paths)
 
+        # For thousands of inputs (say every file in a sub-tree),
+        # test_defaults_for_path() gets called with the same contexts multiple
+        # times (once for every path in a directory that doesn't have any
+        # test metadata). So, we cache the function call.
+        defaults_cache = {}
+        def test_defaults_for_path(ctxs):
+            key = tuple(ctx.current_path or ctx.main_path for ctx in ctxs)
+
+            if key not in defaults_cache:
+                defaults_cache[key] = self.test_defaults_for_path(ctxs)
+
+            return defaults_cache[key]
+
         r = {}
 
         for path, ctxs in paths.items():
             flags = Files(Context())
 
             for ctx in ctxs:
                 if not isinstance(ctx, Files):
                     continue
@@ -1387,17 +1400,17 @@ class BuildReader(object):
                 # Only do wildcard matching if the '*' character is present.
                 # Otherwise, mozpath.match will match directories, which we've
                 # arbitrarily chosen to not allow.
                 if pattern == relpath or \
                         ('*' in pattern and mozpath.match(relpath, pattern)):
                     flags += ctx
 
             if not any([flags.test_tags, flags.test_files, flags.test_flavors]):
-                flags += self.test_defaults_for_path(ctxs)
+                flags += test_defaults_for_path(ctxs)
 
             r[path] = flags
 
         return r
 
     def test_defaults_for_path(self, ctxs):
         # This names the context keys that will end up emitting a test
         # manifest.