Bug 1283919 - Improve test package archiver for collecting files from directories referenced by a root manifest draft
authorHenrik Skupin <mail@hskupin.info>
Wed, 07 Sep 2016 23:39:45 +0200
changeset 411278 1abc9b83dbbdef84f047805f3d9669782f77318b
parent 411277 ab70808cd4b6c6ad9a57a9f71cfa495fcea0aecd
child 530706 7dca5905efc2910074da8ff9ea7e946a40724007
push id28870
push userbmo:hskupin@gmail.com
push dateWed, 07 Sep 2016 21:42:08 +0000
bugs1283919
milestone51.0a1
Bug 1283919 - Improve test package archiver for collecting files from directories referenced by a root manifest MozReview-Commit-ID: EuGK3OS8XLj
python/mozbuild/mozbuild/action/test_archive.py
python/mozbuild/mozpack/mozjar.py
--- a/python/mozbuild/mozbuild/action/test_archive.py
+++ b/python/mozbuild/mozbuild/action/test_archive.py
@@ -11,16 +11,19 @@
 from __future__ import absolute_import, print_function, unicode_literals
 
 import argparse
 import itertools
 import os
 import sys
 import time
 
+from manifestparser import TestManifest
+from reftest import ReftestManifest
+
 from mozbuild.util import ensureParentDir
 from mozpack.files import FileFinder
 from mozpack.mozjar import JarWriter
 import mozpack.path as mozpath
 
 import buildconfig
 
 STAGE = mozpath.join(buildconfig.topobjdir, 'dist', 'test-stage')
@@ -310,16 +313,25 @@ ARCHIVE_FILES = {
             'pattern': 'reftest/**',
         },
         {
             'source': buildconfig.topobjdir,
             'base': '',
             'pattern': 'mozinfo.json',
             'dest': 'reftest',
         },
+        {
+            'source': buildconfig.topsrcdir,
+            'base': '',
+            'manifests': [
+                'layout/reftests/reftest.list',
+                'testing/crashtest/crashtests.list',
+            ],
+            'dest': 'reftest/tests',
+        }
     ],
     'talos': [
         {
             'source': buildconfig.topsrcdir,
             'base': 'testing',
             'pattern': 'talos/**',
         },
     ],
@@ -409,49 +421,77 @@ for k, v in ARCHIVE_FILES.items():
 
     if not any(p.startswith('%s/' % k) for p in ignores):
         raise Exception('"common" ignore list probably should contain %s' % k)
 
 
 def find_files(archive):
     for entry in ARCHIVE_FILES[archive]:
         source = entry['source']
+        dest = entry.get('dest')
         base = entry.get('base', '')
+
         pattern = entry.get('pattern')
         patterns = entry.get('patterns', [])
         if pattern:
             patterns.append(pattern)
-        dest = entry.get('dest')
+
+        manifest = entry.get('manifest')
+        manifests = entry.get('manifests', [])
+        if manifest:
+            manifests.append(manifest)
+        if manifests:
+            dirs = find_manifest_dirs(buildconfig.topsrcdir, manifests)
+            patterns.extend({'{}/**'.format(d) for d in dirs})
+
         ignore = list(entry.get('ignore', []))
-        ignore.append('**/.mkdir.done')
-        ignore.append('**/*.pyc')
+        ignore.extend([
+            '**/.flake8',
+            '**/.mkdir.done',
+            '**/*.pyc',
+        ])
 
         common_kwargs = {
             'find_executables': False,
             'find_dotfiles': True,
             'ignore': ignore,
         }
 
         finder = FileFinder(os.path.join(source, base), **common_kwargs)
 
         for pattern in patterns:
             for p, f in finder.find(pattern):
                 if dest:
                     p = mozpath.join(dest, p)
                 yield p, f
 
 
-def find_reftest_dirs(topsrcdir, manifests):
-    from reftest import ReftestManifest
+def find_manifest_dirs(topsrcdir, manifests):
+    """Routine to retrieve directories specified in a manifest, relative to topsrcdir.
 
+    It does not recurse into manifests, as we currently have no need for that.
+    """
     dirs = set()
+
     for p in manifests:
-        m = ReftestManifest()
-        m.load(os.path.join(topsrcdir, p))
-        dirs |= m.dirs
+        p = os.path.join(topsrcdir, p)
+
+        if p.endswith('.ini'):
+            test_manifest = TestManifest()
+            test_manifest.read(p)
+            dirs |= set([os.path.dirname(m) for m in test_manifest.manifests()])
+
+        elif p.endswith('.list'):
+            m = ReftestManifest()
+            m.load(p)
+            dirs |= m.dirs
+
+        else:
+            raise Exception('"{}" is not a supported manifest format.'.format(
+                os.path.splitext(p)[1]))
 
     dirs = {mozpath.normpath(d[len(topsrcdir):]).lstrip('/') for d in dirs}
 
     # Filter out children captured by parent directories because duplicates
     # will confuse things later on.
     def parents(p):
         while True:
             p = mozpath.dirname(p)
@@ -462,66 +502,40 @@ def find_reftest_dirs(topsrcdir, manifes
     seen = set()
     for d in sorted(dirs, key=len):
         if not any(p in seen for p in parents(d)):
             seen.add(d)
 
     return sorted(seen)
 
 
-def insert_reftest_entries(entries):
-    """Reftests have their own mechanism for defining tests and locations.
-
-    This function is called when processing the reftest archive to process
-    reftest test manifests and insert the results into the existing list of
-    archive entries.
-    """
-    manifests = (
-        'layout/reftests/reftest.list',
-        'testing/crashtest/crashtests.list',
-    )
-
-    for base in find_reftest_dirs(buildconfig.topsrcdir, manifests):
-        entries.append({
-            'source': buildconfig.topsrcdir,
-            'base': '',
-            'pattern': '%s/**' % base,
-            'dest': 'reftest/tests',
-        })
-
-
 def main(argv):
     parser = argparse.ArgumentParser(
         description='Produce test archives')
     parser.add_argument('archive', help='Which archive to generate')
     parser.add_argument('outputfile', help='File to write output to')
 
     args = parser.parse_args(argv)
 
     if not args.outputfile.endswith('.zip'):
         raise Exception('expected zip output file')
 
-    # Adjust reftest entries only if processing reftests (because it is
-    # unnecessary overhead otherwise).
-    if args.archive == 'reftest':
-        insert_reftest_entries(ARCHIVE_FILES['reftest'])
-
     file_count = 0
     t_start = time.time()
     ensureParentDir(args.outputfile)
     with open(args.outputfile, 'wb') as fh:
         # Experimentation revealed that level 5 is significantly faster and has
         # marginally larger sizes than higher values and is the sweet spot
         # for optimal compression. Read the detailed commit message that
         # introduced this for raw numbers.
         with JarWriter(fileobj=fh, optimize=False, compress_level=5) as writer:
             res = find_files(args.archive)
             for p, f in res:
+                writer.add(p.encode('utf-8'), f.read(), mode=f.mode, skip_duplicates=True)
                 file_count += 1
-                writer.add(p.encode('utf-8'), f.read(), mode=f.mode)
 
     duration = time.time() - t_start
     zip_size = os.path.getsize(args.outputfile)
     basename = os.path.basename(args.outputfile)
     print('Wrote %d files in %d bytes to %s in %.2fs' % (
           file_count, zip_size, basename, duration))
 
 
--- a/python/mozbuild/mozpack/mozjar.py
+++ b/python/mozbuild/mozpack/mozjar.py
@@ -565,35 +565,37 @@ class JarWriter(object):
         if not self._optimize:
             end['cdir_offset'] = offset
             for entry, _ in self._contents.itervalues():
                 self._data.write(entry.serialize())
         # Store the end of central directory.
         self._data.write(end.serialize())
         self._data.close()
 
-    def add(self, name, data, compress=None, mode=None):
+    def add(self, name, data, compress=None, mode=None, skip_duplicates=False):
         '''
         Add a new member to the jar archive, with the given name and the given
         data.
         The compress option indicates if the given data should be compressed
         (True), not compressed (False), or compressed according to the default
         defined when creating the JarWriter (None).
         When the data should be compressed (True or None with self.compress ==
         True), it is only really compressed if the compressed size is smaller
         than the uncompressed size.
         The mode option gives the unix permissions that should be stored
         for the jar entry.
+        If a duplicated member is found skip_duplicates will prevent raising
+        an exception if set to True.
         The given data may be a buffer, a file-like instance, a Deflater or a
         JarFileReader instance. The latter two allow to avoid uncompressing
         data to recompress it.
         '''
         name = mozpath.normsep(name)
 
-        if name in self._contents:
+        if name in self._contents and not skip_duplicates:
             raise JarWriterError("File %s already in JarWriter" % name)
         if compress is None:
             compress = self._compress
         if (isinstance(data, JarFileReader) and data.compressed == compress) \
                 or (isinstance(data, Deflater) and data.compress == compress):
             deflater = data
         else:
             deflater = Deflater(compress, compress_level=self._compress_level)