Bug 1242051 - Extract support files processing from the emitter. draft
authorChris Manchester <cmanchester@mozilla.com>
Fri, 25 Mar 2016 18:51:38 -0700
changeset 345227 567e559ea1ce91d73022228d1d2516f56081a0c2
parent 344388 24c5fbde4488e06ef79905e1c520027cddcd1189
child 345228 2db5d3ad9e2e4f26718ec36e551b334c2c8bfec7
push id14026
push usercmanchester@mozilla.com
push dateMon, 28 Mar 2016 15:57:01 +0000
bugs1242051
milestone48.0a1
Bug 1242051 - Extract support files processing from the emitter. This extracts the logic from the emitter that handles support files in ini manifests to a seperate function in testing.py, so that this logic can be re-used to determine how to install all the files necessary to run a particular test fon the corresponding object in all-tests.json. MozReview-Commit-ID: Gao2w9OG550
python/mozbuild/mozbuild/frontend/emitter.py
python/mozbuild/mozbuild/testing.py
--- a/python/mozbuild/mozbuild/frontend/emitter.py
+++ b/python/mozbuild/mozbuild/frontend/emitter.py
@@ -76,16 +76,17 @@ from mozpack.chrome.manifest import (
 )
 
 from .reader import SandboxValidationError
 
 from ..testing import (
     TEST_MANIFESTS,
     REFTEST_FLAVORS,
     WEB_PLATFORM_TESTS_FLAVORS,
+    convert_support_files,
 )
 
 from .context import (
     Context,
     SourcePath,
     ObjDirPath,
     Path,
     SubContext,
@@ -1085,73 +1086,33 @@ class TreeMetadataEmitter(LoggingMixin):
             out_dir = mozpath.join(install_prefix, manifest_reldir)
             if 'install-to-subdir' in defaults:
                 # This is terrible, but what are you going to do?
                 out_dir = mozpath.join(out_dir, defaults['install-to-subdir'])
                 obj.manifest_obj_relpath = mozpath.join(manifest_reldir,
                                                         defaults['install-to-subdir'],
                                                         mozpath.basename(path))
 
-
-            # "head" and "tail" lists.
-            # All manifests support support-files.
-            #
             # Keep a set of already seen support file patterns, because
             # repeatedly processing the patterns from the default section
             # for every test is quite costly (see bug 922517).
             extras = (('head', set()),
                       ('tail', set()),
-                      ('support-files', set()))
+                      ('support-files', set()),
+                      ('generated-files', set()))
             def process_support_files(test):
-                for thing, seen in extras:
-                    value = test.get(thing, '')
-                    if value in seen:
-                        continue
-                    seen.add(value)
-                    for pattern in value.split():
-                        # We only support globbing on support-files because
-                        # the harness doesn't support * for head and tail.
-                        if '*' in pattern and thing == 'support-files':
-                            obj.pattern_installs.append(
-                                (manifest_dir, pattern, out_dir))
-                        # "absolute" paths identify files that are to be
-                        # placed in the install_root directory (no globs)
-                        elif pattern[0] == '/':
-                            full = mozpath.normpath(mozpath.join(manifest_dir,
-                                mozpath.basename(pattern)))
-                            obj.installs[full] = (mozpath.join(install_root,
-                                pattern[1:]), False)
-                        else:
-                            full = mozpath.normpath(mozpath.join(manifest_dir,
-                                pattern))
+                patterns, installs, external = convert_support_files(extras, test,
+                                                                     install_root,
+                                                                     manifest_dir,
+                                                                     out_dir)
 
-                            dest_path = mozpath.join(out_dir, pattern)
-
-                            # If the path resolves to a different directory
-                            # tree, we take special behavior depending on the
-                            # entry type.
-                            if not full.startswith(manifest_dir):
-                                # If it's a support file, we install the file
-                                # into the current destination directory.
-                                # This implementation makes installing things
-                                # with custom prefixes impossible. If this is
-                                # needed, we can add support for that via a
-                                # special syntax later.
-                                if thing == 'support-files':
-                                    dest_path = mozpath.join(out_dir,
-                                        os.path.basename(pattern))
-                                # If it's not a support file, we ignore it.
-                                # This preserves old behavior so things like
-                                # head files doesn't get installed multiple
-                                # times.
-                                else:
-                                    continue
-
-                            obj.installs[full] = (mozpath.normpath(dest_path),
-                                False)
+                obj.pattern_installs.extend(patterns)
+                for source, dest in installs:
+                    obj.installs[source] = (dest, False)
+                obj.external_installs |= external
 
             for test in filtered:
                 obj.tests.append(test)
 
                 # Some test files are compiled and should not be copied into the
                 # test package. They function as identifiers rather than files.
                 if package_tests:
                     manifest_relpath = mozpath.relpath(test['path'],
@@ -1174,20 +1135,18 @@ class TreeMetadataEmitter(LoggingMixin):
             # FUTURE we should be able to detect autogenerated files from
             # other build metadata. Once we do that, we can get rid of this.
             for f in defaults.get('generated-files', '').split():
                 # We re-raise otherwise the stack trace isn't informative.
                 try:
                     del obj.installs[mozpath.join(manifest_dir, f)]
                 except KeyError:
                     raise SandboxValidationError('Error processing test '
-                        'manifest %s: entry in generated-files not present '
-                        'elsewhere in manifest: %s' % (path, f), context)
-
-                obj.external_installs.add(mozpath.join(out_dir, f))
+                       'manifest %s: entry in generated-files not present '
+                       'elsewhere in manifest: %s' % (path, f), context)
 
             yield obj
         except (AssertionError, Exception):
             raise SandboxValidationError('Error processing test '
                 'manifest file %s: %s' % (path,
                     '\n'.join(traceback.format_exception(*sys.exc_info()))),
                 context)
 
--- a/python/mozbuild/mozbuild/testing.py
+++ b/python/mozbuild/mozbuild/testing.py
@@ -282,16 +282,80 @@ REFTEST_FLAVORS = ('crashtest', 'reftest
 WEB_PLATFORM_TESTS_FLAVORS = ('web-platform-tests',)
 
 def all_test_flavors():
     return ([v[0] for v in TEST_MANIFESTS.values()] +
             list(REFTEST_FLAVORS) +
             list(WEB_PLATFORM_TESTS_FLAVORS) +
             ['python'])
 
+def convert_support_files(extras, test, install_root, manifest_dir, out_dir):
+    # Processes a "support-files" entry from a test object and returns
+    # the installs to perform for this test object.
+    #
+    # Arguments:
+    #  extras - Tuples used for the basis of memoization (the same support-files
+    #           in the same manifest always have the same effect).
+    #  test - The test object to process.
+    #  install_root - The directory under $objdir/_tests that will contain
+    #                 the tests for this harness (examples are "testing/mochitest",
+    #                 "xpcshell").
+    #  manifest_dir - Absoulute path to the (srcdir) directory containing the
+    #                 manifest that included this test
+    #  out_dir - The path relative to $objdir/_tests used as the destination for the
+    #            test, based on the relative path to the manifest in the srcdir,
+    #            the install_root, and 'install-to-subdir', if present in the manifest.
+    pattern_installs, installs, external = [], [], set()
+    for thing, seen in extras:
+        value = test.get(thing, '')
+        # We need to memoize on the basis of both the path and the output
+        # directory for the benefit of tests specifying 'install-to-subdir'.
+        if (value, out_dir) in seen:
+            continue
+        seen.add((value, out_dir))
+        for pattern in value.split():
+            if thing == 'generated-files':
+                external.add(mozpath.join(out_dir, pattern))
+            # We only support globbing on support-files because
+            # the harness doesn't support * for head and tail.
+            elif '*' in pattern and thing == 'support-files':
+                pattern_installs.append((manifest_dir, pattern, out_dir))
+            # "absolute" paths identify files that are to be
+            # placed in the install_root directory (no globs)
+            elif pattern[0] == '/':
+                full = mozpath.normpath(mozpath.join(manifest_dir,
+                                                     mozpath.basename(pattern)))
+                installs.append((full, mozpath.join(install_root, pattern[1:])))
+            else:
+                full = mozpath.normpath(mozpath.join(manifest_dir, pattern))
+                dest_path = mozpath.join(out_dir, pattern)
+
+                # If the path resolves to a different directory
+                # tree, we take special behavior depending on the
+                # entry type.
+                if not full.startswith(manifest_dir):
+                    # If it's a support file, we install the file
+                    # into the current destination directory.
+                    # This implementation makes installing things
+                    # with custom prefixes impossible. If this is
+                    # needed, we can add support for that via a
+                    # special syntax later.
+                    if thing == 'support-files':
+                        dest_path = mozpath.join(out_dir,
+                                                 os.path.basename(pattern))
+                    # If it's not a support file, we ignore it.
+                    # This preserves old behavior so things like
+                    # head files doesn't get installed multiple
+                    # times.
+                    else:
+                        continue
+                installs.append((full, mozpath.normpath(dest_path)))
+
+    return pattern_installs, installs, external
+
 # Convenience methods for test manifest reading.
 def read_manifestparser_manifest(context, manifest_path):
     path = mozpath.normpath(mozpath.join(context.srcdir, manifest_path))
     return manifestparser.TestManifest(manifests=[path], strict=True,
                                        rootdir=context.config.topsrcdir,
                                        finder=context._finder)
 
 def read_reftest_manifest(context, manifest_path):