Bug 1355630 - Memoize tests_defaults_for_path call; r?chmanchester
Calling self.test_defaults_for_path() from files_info() with tens
of thousands of paths resulted in a CPU explosion in various path
normalization functions. I don't think it was so much the complexity
of the operations as much as the volume.
For an input with 9131 elements, this reduces execution time of a
mach command from ~25.7s to ~8.8s. With ~42,000 inputs, execution time
drops from <it took too long and I gave up> to ~90s.
MozReview-Commit-ID: pjQQByi2Bc
--- a/python/mozbuild/mozbuild/frontend/reader.py
+++ b/python/mozbuild/mozbuild/frontend/reader.py
@@ -1367,16 +1367,29 @@ class BuildReader(object):
2. Evaluate moz.build files starting with the most distant.
3. Iterate over Files sub-contexts.
4. If the file pattern matches the file we're seeking info on,
apply attribute updates.
5. Return the most recent value of attributes.
"""
paths, _ = self.read_relevant_mozbuilds(paths)
+ # For thousands of inputs (say every file in a sub-tree),
+ # test_defaults_for_path() gets called with the same contexts multiple
+ # times (once for every path in a directory that doesn't have any
+ # test metadata). So, we cache the function call.
+ defaults_cache = {}
+ def test_defaults_for_path(ctxs):
+ key = tuple(ctx.current_path or ctx.main_path for ctx in ctxs)
+
+ if key not in defaults_cache:
+ defaults_cache[key] = self.test_defaults_for_path(ctxs)
+
+ return defaults_cache[key]
+
r = {}
for path, ctxs in paths.items():
flags = Files(Context())
for ctx in ctxs:
if not isinstance(ctx, Files):
continue
@@ -1387,17 +1400,17 @@ class BuildReader(object):
# Only do wildcard matching if the '*' character is present.
# Otherwise, mozpath.match will match directories, which we've
# arbitrarily chosen to not allow.
if pattern == relpath or \
('*' in pattern and mozpath.match(relpath, pattern)):
flags += ctx
if not any([flags.test_tags, flags.test_files, flags.test_flavors]):
- flags += self.test_defaults_for_path(ctxs)
+ flags += test_defaults_for_path(ctxs)
r[path] = flags
return r
def test_defaults_for_path(self, ctxs):
# This names the context keys that will end up emitting a test
# manifest.