Bug 1330421 - Change the structure of test archives include generated test support files in common.tests.zip for the benefit of artifact builds. draft
authorChris Manchester <cmanchester@mozilla.com>
Thu, 12 Jan 2017 15:29:48 -0800
changeset 460324 98b912ebca4e31953d2deb30b19d9934be62c2f4
parent 458610 e68cbc3b5b3d3fba4fe3e17e234713020f44e4a0
child 460325 e79dd299628806750827f8fc6ac115b117bc8691
push id41371
push userbmo:cmanchester@mozilla.com
push dateThu, 12 Jan 2017 23:31:11 +0000
bugs1330421
milestone53.0a1
Bug 1330421 - Change the structure of test archives include generated test support files in common.tests.zip for the benefit of artifact builds. Generated support files for specific tests generally end up in a harness specific test archive, but artifact builds, which might not want to generate these files themselves, get all of their test related files from the common.tests.zip. This effectively moves a small number of these support files from harness specific archives to the common test archive, making it possible to run tests that depend on these files against artifact builds. MozReview-Commit-ID: BvEj1ot1OTw
python/mozbuild/mozbuild/action/test_archive.py
--- a/python/mozbuild/mozbuild/action/test_archive.py
+++ b/python/mozbuild/mozbuild/action/test_archive.py
@@ -15,17 +15,19 @@ import itertools
 import os
 import sys
 import time
 
 from manifestparser import TestManifest
 from reftest import ReftestManifest
 
 from mozbuild.util import ensureParentDir
-from mozpack.files import FileFinder
+from mozpack.copier import FileRegistry
+from mozpack.files import ExistingFile, FileFinder
+from mozpack.manifests import InstallManifest
 from mozpack.mozjar import JarWriter
 import mozpack.path as mozpath
 
 import buildconfig
 
 STAGE = mozpath.join(buildconfig.topobjdir, 'dist', 'test-stage')
 
 TEST_HARNESS_BINS = [
@@ -67,16 +69,32 @@ TEST_PLUGIN_DIRS = [
 ]
 
 GMP_TEST_PLUGIN_DIRS = [
     'gmp-clearkey/**',
     'gmp-fake/**',
     'gmp-fakeopenh264/**',
 ]
 
+# These entries will be used by artifact builds to re-construct an
+# objdir with the appropriate generated support files.
+OBJDIR_TEST_FILES = {
+    'xpcshell': {
+        'source': buildconfig.topobjdir,
+        'base': '_tests/xpcshell',
+        'pattern': '**',
+        'dest': 'xpcshell/tests',
+    },
+    'mochitest': {
+        'source': buildconfig.topobjdir,
+        'base': '_tests/testing',
+        'pattern': 'mochitest/**',
+    },
+}
+
 
 ARCHIVE_FILES = {
     'common': [
         {
             'source': STAGE,
             'base': '',
             'pattern': '**',
             'ignore': [
@@ -297,16 +315,17 @@ ARCHIVE_FILES = {
     'gtest': [
         {
             'source': STAGE,
             'base': '',
             'pattern': 'gtest/**',
         },
     ],
     'mochitest': [
+        OBJDIR_TEST_FILES['mochitest'],
         {
             'source': buildconfig.topobjdir,
             'base': '_tests/testing',
             'pattern': 'mochitest/**',
         },
         {
             'source': STAGE,
             'base': '',
@@ -379,22 +398,17 @@ ARCHIVE_FILES = {
         {
             'source': buildconfig.topobjdir,
             'base': '',
             'pattern': 'mozinfo.json',
             'dest': 'web-platform',
         },
     ],
     'xpcshell': [
-        {
-            'source': buildconfig.topobjdir,
-            'base': '_tests/xpcshell',
-            'pattern': '**',
-            'dest': 'xpcshell/tests',
-        },
+        OBJDIR_TEST_FILES['xpcshell'],
         {
             'source': buildconfig.topsrcdir,
             'base': 'testing/xpcshell',
             'patterns': [
                 'head.js',
                 'mach_test_package_commands.py',
                 'moz-http2/**',
                 'moz-spdy/**',
@@ -438,18 +452,61 @@ for k, v in ARCHIVE_FILES.items():
 
     ignores = set(itertools.chain(*(e.get('ignore', [])
                                   for e in ARCHIVE_FILES['common'])))
 
     if not any(p.startswith('%s/' % k) for p in ignores):
         raise Exception('"common" ignore list probably should contain %s' % k)
 
 
+def find_generated_harness_files():
+    # TEST_HARNESS_FILES end up in an install manifest at
+    # $topsrcdir/_build_manifests/install/_tests.
+    manifest = InstallManifest(mozpath.join(buildconfig.topobjdir,
+                                            '_build_manifests',
+                                            'install',
+                                            '_tests'))
+    registry = FileRegistry()
+    manifest.populate_registry(registry)
+    # Conveniently, the generated files we care about will already
+    # exist in the objdir, so we can identify relevant files if
+    # they're an `ExistingFile` instance.
+    return [mozpath.join('_tests', p) for p in registry.paths()
+            if isinstance(registry[p], ExistingFile)]
+
+
 def find_files(archive):
-    for entry in ARCHIVE_FILES[archive]:
+    extra_entries = []
+    generated_harness_files = find_generated_harness_files()
+
+    if archive == 'common':
+        # Construct entries ensuring all our generated harness files are
+        # packaged in the common tests zip.
+        packaged_paths = set()
+        for entry in OBJDIR_TEST_FILES.values():
+            pat = mozpath.join(entry['base'], entry['pattern'])
+            del entry['pattern']
+            patterns = []
+            for path in generated_harness_files:
+                if mozpath.match(path, pat):
+                    patterns.append(path[len(entry['base']) + 1:])
+                    packaged_paths.add(path)
+            if patterns:
+                entry['patterns'] = patterns
+                extra_entries.append(entry)
+        entry = {
+            'source': buildconfig.topobjdir,
+            'base': '_tests',
+            'patterns': [],
+        }
+        for path in set(generated_harness_files) - packaged_paths:
+            entry['patterns'].append(path[len('_tests') + 1:])
+        extra_entries.append(entry)
+
+    for entry in ARCHIVE_FILES[archive] + extra_entries:
         source = entry['source']
         dest = entry.get('dest')
         base = entry.get('base', '')
 
         pattern = entry.get('pattern')
         patterns = entry.get('patterns', [])
         if pattern:
             patterns.append(pattern)
@@ -464,16 +521,22 @@ def find_files(archive):
 
         ignore = list(entry.get('ignore', []))
         ignore.extend([
             '**/.flake8',
             '**/.mkdir.done',
             '**/*.pyc',
         ])
 
+        if archive != 'common' and base.startswith('_tests'):
+            # We may have generated_harness_files to exclude from this entry.
+            for path in generated_harness_files:
+                if path.startswith(base):
+                    ignore.append(path[len(base) + 1:])
+
         common_kwargs = {
             'find_executables': False,
             'find_dotfiles': True,
             'ignore': ignore,
         }
 
         finder = FileFinder(os.path.join(source, base), **common_kwargs)