TODO Bug 1240134 - Extract XPIDL components files from omni.ja draft
authorGregory Szorc <gps@mozilla.com>
Sat, 20 Feb 2016 19:23:51 -0800
changeset 333937 d988bd2c434df66c299b6878d57b07a723b00a71
parent 333936 266928b5a7615fa054c70adf0f649cbb3f085e8d
child 333938 e8fc85ba94d7de61d31df48eb91c59b632390f6b
push id11397
push usergszorc@mozilla.com
push dateTue, 23 Feb 2016 22:51:52 +0000
bugs1240134
milestone47.0a1
TODO Bug 1240134 - Extract XPIDL components files from omni.ja We don't need to build XPIDL files during artifact builds because the build artifacts have the interfaces.xpt and components.manifest files in them. Unlike executables and other files currently handled by artifact builds, these files are in omni.ja files within the main archive. This commit teaches the artifact processing code to extract XPIDL related files from omni.ja files. By installing these files from the artifacts, we'll be able to stop processing XPIDL source files as part of artifact builds. This commit does introduce some overhead to extract and read omni.ja files. However, this still takes less CPU than processing 1000+ XPIDL source files. TODO OS X support MozReview-Commit-ID: 4tkUJdsFDTU
python/mozbuild/mozbuild/artifacts.py
--- a/python/mozbuild/mozbuild/artifacts.py
+++ b/python/mozbuild/mozbuild/artifacts.py
@@ -106,16 +106,23 @@ class ArtifactJob(object):
         ('bin/certutil', ('bin', 'bin')),
         ('bin/fileid', ('bin', 'bin')),
         ('bin/pk12util', ('bin', 'bin')),
         ('bin/ssltunnel', ('bin', 'bin')),
         ('bin/xpcshell', ('bin', 'bin')),
         ('bin/plugins/*', ('bin/plugins', 'plugins'))
     }
 
+    # Files from the omni.ja to extract.
+    omni_ja_files = {
+        # XPIDL output.
+        'components/components.manifest',
+        'components/interfaces.xpt',
+    }
+
     # We can tell our input is a test archive by this suffix, which happens to
     # be the same across platforms.
     _test_archive_suffix = '.common.tests.zip'
 
     def __init__(self, package_re, tests_re, log=None):
         self._package_re = re.compile(package_re)
         self._tests_re = None
         if tests_re:
@@ -170,16 +177,32 @@ class ArtifactJob(object):
                     writer.add(destpath.encode('utf-8'), reader[filename], mode=mode)
                     added_entry = True
 
         if not added_entry:
             raise ValueError('Archive format changed! No pattern from "{patterns}"'
                              'matched an archive path.'.format(
                                  patterns=LinuxArtifactJob.test_artifact_patterns))
 
+    def _process_omni_ja(self, base_dir, data, writer):
+        """Process an omni.ja file for files to extract."""
+        reader = JarReader(data=data)
+        # This assumes omni_ja_files is small. If not, iterating over
+        # the contents of the JAR might be faster.
+        for f in self.omni_ja_files:
+            if f not in reader:
+                continue
+
+            dest_path = mozpath.join('bin', base_dir, f).encode('utf-8')
+            self.log(logging.INFO, 'artifact',
+                     {'dest_path': dest_path},
+                     'Adding {dest_path} to processed archive')
+            writer.add(dest_path, reader[f])
+
+
 class AndroidArtifactJob(ArtifactJob):
     def process_artifact(self, filename, processed_filename):
         # Extract all .so files into the root, which will get copied into dist/bin.
         with JarWriter(file=processed_filename, optimize=False, compress_level=5) as writer:
             for f in JarReader(filename):
                 if not f.filename.endswith('.so') and \
                    not f.filename in ('platform.ini', 'application.ini'):
                     continue
@@ -212,16 +235,22 @@ class LinuxArtifactJob(ArtifactJob):
         added_entry = False
 
         with JarWriter(file=processed_filename, optimize=False, compress_level=5) as writer:
             with tarfile.open(filename) as reader:
                 for f in reader:
                     if not f.isfile():
                         continue
 
+                    if mozpath.basename(f.name) == 'omni.ja':
+                        self._process_omni_ja(mozpath.dirname(f.name).lstrip('firefox/'),
+                                              reader.extractfile(f).read(),
+                                              writer)
+                        continue
+
                     if not any(mozpath.match(f.name, p) for p in self.package_artifact_patterns):
                         continue
 
                     # We strip off the relative "firefox/" bit from the path,
                     # but otherwise preserve it.
                     destpath = mozpath.join('bin',
                                             mozpath.relpath(f.name, "firefox"))
                     self.log(logging.INFO, 'artifact',
@@ -351,16 +380,20 @@ class WinArtifactJob(ArtifactJob):
         ('bin/xpcshell.exe', ('bin', 'bin')),
         ('bin/plugins/*', ('bin/plugins', 'plugins'))
     }
 
     def process_package_artifact(self, filename, processed_filename):
         added_entry = False
         with JarWriter(file=processed_filename, optimize=False, compress_level=5) as writer:
             for f in JarReader(filename):
+                if mozpath.basename(f.name) == 'omni.ja':
+                    self._process_omni_ja(f.read(), writer)
+                    continue
+
                 if not any(mozpath.match(f.filename, p) for p in self.package_artifact_patterns):
                     continue
 
                 # strip off the relative "firefox/" bit from the path:
                 basename = mozpath.relpath(f.filename, "firefox")
                 basename = mozpath.join('bin', basename)
                 self.log(logging.INFO, 'artifact',
                     {'basename': basename},