Bug 1253110 - Produce binary artifact zip files for OS X builds; r?mshal, glandium draft
authorGregory Szorc <gps@mozilla.com>
Tue, 03 May 2016 17:11:36 -0700
changeset 363135 822dbcb3f04ddb5d3b8e1c6dfefdde422f954c76
parent 363134 8988e990dda08c2329d87eb39de8a075f617a0ef
child 363136 446375da6c76c354b1dd15640831b50546a89feb
push id17107
push usergszorc@mozilla.com
push dateWed, 04 May 2016 00:54:14 +0000
reviewersmshal, glandium
bugs1253110
milestone49.0a1
Bug 1253110 - Produce binary artifact zip files for OS X builds; r?mshal, glandium Currently, artifact builds download DMGs and process them into JAR files containing a subset of the files they care about. This adds overhead. This commit unblocks future work to stop doing that. We introduce an "artifact_archive" build action. Given an output filename, it reads the build configuration and produces a zip file containing files relevant to artifact builds. This file is uploaded as part of the automation results. In the future, artifact builds can search for this file, download it, and extract it verbatim. A goal is to make the client-side of artifact builds as dumb as possible: just extract an archive. I'm not very happy with the amount of hackery in artifact_archive.py regarding searching for binary files. I really wish these things came more from moz.build files. I would appreciate alternative solutions so we don't have to hard-code lists of files. (It's worth noting that the client side of artifact builds currently codes a similar files list. Once we have the client side of artifacts doing a simple archive extract, we will have effectively moved that files list from mach to packaging, which is closer to where it should be, but not actually in moz.build, which is where I think it should belong.) MozReview-Commit-ID: KTt90kBDxkb
python/mozbuild/mozbuild/action/artifact_archive.py
python/mozbuild/mozbuild/backend/common.py
toolkit/mozapps/installer/packager.mk
toolkit/mozapps/installer/upload-files.mk
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozbuild/action/artifact_archive.py
@@ -0,0 +1,195 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, unicode_literals
+
+import collections
+import json
+import os
+import sys
+
+from mozbuild.base import (
+    MozbuildObject,
+)
+from mozpack.files import (
+    File,
+    FileFinder,
+    GeneratedFile,
+)
+from mozpack.mozjar import (
+    JarWriter,
+)
+import mozpack.path as mozpath
+
+
+def get_binaries(build_app, objdir):
+    """Obtain binaries that should be packaged in the archive."""
+    with open(os.path.join(objdir, 'binaries.json'), 'rb') as fh:
+        binaries = json.load(fh)
+
+    # Ideally the data below comes from moz.build or some other source.
+
+    # Directories we don't package outright.
+    IGNORE_RELDIRS = {
+        'toolkit/library/gtest',
+    }
+
+    # This binary is quite large (~10 MB) and doesn't provide much value.
+    # Ignore it for now.
+    IGNORE_PROGRAMS = {
+        'ReadNTLM',
+    }
+
+    IGNORE_LIB_BASENAMES = {
+        'dummy_replace_malloc',
+    }
+
+    EXTRAS = {
+        'dependentlibs.list',
+    }
+
+    # TODO take DIST_INSTALL from moz.build into account. This likely requires
+    # changing binaries.json to record this.
+
+    for program in binaries['programs']:
+        if program['program'] in IGNORE_PROGRAMS:
+            continue
+
+        # Don't package C++ unit test binaries.
+        if program['program'].startswith(('Test', 'test_')):
+            continue
+        if program['program'].endswith(('Test', 'test', '_unittest', '_unittests', '-tests')):
+            continue
+
+        # Host binaries are used by the build and aren't needed for
+        # artifact builds.
+        if program['kind'] == 'host':
+            continue
+
+        inpath = mozpath.join(program['relobjdir'], program['program'])
+
+        # Bug 1266252 - install_target contains the app directory when it
+        # shouldn't.
+        target = program['install_target']
+        if target.endswith('/%s' % build_app):
+            target = target[0:-(len(build_app) + 1)]
+
+        outpath = mozpath.join(target, program['program'])
+        yield inpath, outpath
+
+    for lib in binaries['shared_libraries']:
+        if lib['relobjdir'] in IGNORE_RELDIRS:
+            continue
+
+        if lib['basename'] in IGNORE_LIB_BASENAMES:
+            continue
+
+        # Plugin files are handled below.
+        if lib['install_target'].startswith('dist/plugins/'):
+            continue
+
+        # Don't package libraries for tests.
+        if lib['basename'].endswith('test'):
+            continue
+        if lib['install_target'].startswith('_tests/'):
+            continue
+
+        inpath = mozpath.join(lib['relobjdir'], lib['lib_name'])
+        outpath = mozpath.join(lib['install_target'], lib['lib_name'])
+        yield inpath, outpath
+
+    # Copy all plugin files.
+    finder = FileFinder(os.path.join(objdir, 'dist', 'plugins'), find_executables=False)
+    for p, f in finder.find('*'):
+        yield 'dist/plugins/%s' % p, 'dist/plugins/%s' % p
+
+    # Copy all files in gmp-* directories.
+    finder = FileFinder(os.path.join(objdir, 'dist', 'bin'), find_executables=False)
+    for p, f in finder.find('*'):
+        if p.startswith('gmp-'):
+            yield 'dist/bin/%s' % p, 'dist/bin/%s' % p
+
+    for p in EXTRAS:
+        yield 'dist/bin/%s' % p, 'dist/bin/%s' % p
+
+
+def get_xpidl_files(objdir):
+    # Find XPIDL files. These are .xpt and .manifest files in components/
+    # directories. We need to also emit manifest files containing references
+    # to these component manifests.
+    components_manifests = collections.defaultdict(set)
+    finder = FileFinder(os.path.join(objdir, 'dist', 'bin'), find_executables=False)
+    for p, f in finder.find('**/components/*'):
+        if not p.endswith(('.xpt', '.manifest')):
+            continue
+
+        relpath = 'bin/%s' % p
+        yield relpath.encode('utf-8'), f
+
+        if not p.endswith('.manifest'):
+            continue
+
+        comp_index = p.index('components/')
+        if comp_index:
+            key = '%s/chrome.manifest' % p[0:comp_index - 1]
+            manifest_path = p[comp_index:]
+        else:
+            key = 'chrome.manifest'
+            manifest_path = p
+
+        components_manifests[key].add(manifest_path)
+
+    for path, manifests in components_manifests.items():
+        content = b'\n'.join(b'manifest %s' % m.encode('utf-8') for m in sorted(manifests))
+        p = 'bin/%s' % path
+        yield p.encode('utf-8'), GeneratedFile(content)
+
+
+def get_osx_files(objdir, build_app):
+    """Obtain the files for an OS X artifact archive.
+
+    Yields tuples of (path, file) that can be added to a JarWriter.
+    """
+    all_files = {}
+    for inpath, outpath in get_binaries(build_app, objdir):
+        assert outpath.startswith(('dist/bin/', 'dist/plugins/'))
+        outpath = outpath[5:]
+        all_files[outpath] = inpath
+
+    for dest, source in all_files.items():
+        yield dest.encode('utf-8'), File(os.path.join(objdir, source))
+
+        if dest == 'bin/firefox':
+            yield b'bin/firefox-bin', File(os.path.join(objdir, source))
+
+    for t in get_xpidl_files(objdir):
+        yield t
+
+
+def create_archive(build, output_path):
+    if build.defines.get('XP_MACOSX'):
+        build_app = build.substs['MOZ_BUILD_APP']
+        files = dict(get_osx_files(build.topobjdir, build_app))
+    else:
+        raise Exception('platform not yet supported')
+
+    with JarWriter(file=output_path, optimize=False, compress_level=5) as writer:
+        for p, f in sorted(files.items()):
+            writer.add(p, f, mode=f.mode)
+
+
+def main(args):
+    import argparse
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument('destfile', help='Archive file to write')
+
+    args = parser.parse_args(args)
+    build = MozbuildObject.from_environment()
+
+    create_archive(build, args.destfile)
+
+
+if __name__ == '__main__':
+    sys.exit(main(sys.argv[1:]))
--- a/python/mozbuild/mozbuild/backend/common.py
+++ b/python/mozbuild/mozbuild/backend/common.py
@@ -27,16 +27,17 @@ from mozbuild.frontend.data import (
     IPDLFile,
     FinalTargetPreprocessedFiles,
     FinalTargetFiles,
     GeneratedEventWebIDLFile,
     GeneratedWebIDLFile,
     PreprocessedTestWebIDLFile,
     PreprocessedWebIDLFile,
     RustRlibLibrary,
+    Program,
     SharedLibrary,
     TestManifest,
     TestWebIDLFile,
     UnifiedSources,
     XPIDLFile,
     WebIDLFile,
 )
 from mozbuild.jar import (
@@ -384,16 +385,18 @@ class CommonBackend(BuildBackend):
         path = mozpath.join(self.environment.topobjdir, 'test-installs.json')
         with self._write_file(path) as fh:
             json.dump({k: v for k, v in self._test_manager.installs_by_path.items()
                        if k in self._test_manager.deferred_installs},
                       fh,
                       sort_keys=True,
                       indent=4)
 
+        self._add_missing_binaries()
+
         # Write out a machine-readable file describing binaries.
         with self._write_file(mozpath.join(topobjdir, 'binaries.json')) as fh:
             d = {
                 'shared_libraries': [s.to_dict() for s in self._binaries.shared_libraries],
                 'programs': [p.to_dict() for p in self._binaries.programs],
             }
             json.dump(d, fh, sort_keys=True, indent=4)
 
@@ -563,8 +566,61 @@ class CommonBackend(BuildBackend):
                     FinalTargetPreprocessedFiles(jar_context, files_pp))
 
             for m in jarinfo.chrome_manifests:
                 entry = parse_manifest_line(
                     mozpath.dirname(jarinfo.name),
                     m.replace('%', mozpath.basename(jarinfo.name) + '/'))
                 self.consume_object(ChromeManifestEntry(
                     jar_context, '%s.manifest' % jarinfo.name, entry))
+
+    def _add_missing_binaries(self):
+        """Add missing binaries to the binaries collection.
+
+        Binaries from some 3rd party build systems aren't exposed to moz.build.
+        This should eventually be fixed as we rewrite build systems for 3rd
+        party projects or come up with a separate mechanism for declaring the
+        binaries. For now, work around the problem by defining the binaries
+        manually.
+        """
+        # No-op if no binaries defined. This handles cases where we aren't
+        # building binaries, which includes some test cases.
+        if not self._binaries.programs or not self._binaries.shared_libraries:
+            return
+
+        NEW_PROGRAMS = [
+            ('certutil', 'security/nss/cmd/certutil'),
+            ('pk12util', 'security/nss/cmd/pk12util'),
+        ]
+
+        NEW_SHARED_LIBRARIES = [
+            ('freebl3', 'security/nss/lib/freebl'),
+            ('nssckbi', 'security/nss/lib/ckfw/builtins'),
+            ('nssdbm3', 'security/nss/lib/softoken/legacydb'),
+            ('softokn3', 'security/nss/lib/softoken'),
+        ]
+
+        # Verify the binaries we're adding aren't already there.
+        existing = {p.program for p in self._binaries.programs}
+        new_programs = {p[0] for p in NEW_PROGRAMS}
+        if existing & new_programs:
+            raise Exception('attempting to add missing program that is already '
+                            'defined: %s' % ', '.join(existing & new_programs))
+
+        existing = {s.basename for s in self._binaries.shared_libraries}
+        new_basenames = {s[0] for s in NEW_SHARED_LIBRARIES}
+        if existing & new_basenames:
+            raise Exception('attempting to add missing binary that is already '
+                            'defined: %s' % ', '.join(existing & new_basenames))
+
+        for m in NEW_PROGRAMS:
+            context = Context(VARIABLES, self.environment)
+            context.push_source(mozpath.join(
+                self.environment.topsrcdir, m[1], 'moz.build'))
+            self._binaries.programs.append(
+                Program(context, m[0]))
+
+        for m in NEW_SHARED_LIBRARIES:
+            context = Context(VARIABLES, self.environment)
+            context.push_source(mozpath.join(
+                self.environment.topsrcdir, m[1], 'moz.build'))
+            self._binaries.shared_libraries.append(
+                SharedLibrary(context, m[0]))
--- a/toolkit/mozapps/installer/packager.mk
+++ b/toolkit/mozapps/installer/packager.mk
@@ -81,16 +81,19 @@ ifdef MOZ_ASAN
 endif # MOZ_ASAN
 endif # Darwin
 
 prepare-package: stage-package
 
 make-package-internal: prepare-package make-sourcestamp-file make-buildinfo-file make-mozinfo-file
 	@echo 'Compressing...'
 	cd $(DIST) && $(MAKE_PACKAGE)
+ifeq (cocoa,$(MOZ_WIDGET_TOOLKIT))
+	$(call py_action,artifact_archive,$(DIST)/$(BINARY_ARTIFACT_PACKAGE))
+endif
 
 make-package: FORCE
 	$(MAKE) make-package-internal
 	$(TOUCH) $@
 
 GARBAGE += make-package
 
 make-sourcestamp-file::
--- a/toolkit/mozapps/installer/upload-files.mk
+++ b/toolkit/mozapps/installer/upload-files.mk
@@ -43,16 +43,17 @@ ifndef _RESPATH
 endif
 ifdef UNIVERSAL_BINARY
 STAGEPATH = universal/
 endif
 endif
 
 PACKAGE_BASE_DIR = $(ABS_DIST)
 PACKAGE       = $(PKG_PATH)$(PKG_BASENAME)$(PKG_SUFFIX)
+BINARY_ARTIFACT_PACKAGE = $(PKG_PATH)$(PKG_BASENAME).binary-artifacts.zip
 
 # By default, the SDK uses the same packaging type as the main bundle,
 # but on mac it is a .tar.bz2
 SDK_SUFFIX    = $(PKG_SUFFIX)
 SDK           = $(SDK_PATH)$(PKG_BASENAME).sdk$(SDK_SUFFIX)
 ifdef UNIVERSAL_BINARY
 SDK           = $(SDK_PATH)$(PKG_BASENAME)-$(TARGET_CPU).sdk$(SDK_SUFFIX)
 endif
@@ -492,16 +493,22 @@ ifdef MOZ_SIGN_CMD
   UPLOAD_FILES += $(call QUOTED_WILDCARD,$(INSTALLER_PACKAGE).asc)
   UPLOAD_FILES += $(call QUOTED_WILDCARD,$(DIST)/$(PACKAGE).asc)
 endif
 
 ifdef MOZ_STUB_INSTALLER
   UPLOAD_FILES += $(call QUOTED_WILDCARD,$(DIST)/$(PKG_INST_PATH)$(PKG_STUB_BASENAME).exe)
 endif
 
+# Upload binary artifact archive.
+# (Only works on OS X so far)
+ifeq (cocoa,$(MOZ_WIDGET_TOOLKIT))
+  UPLOAD_FILES += $(call QUOTED_WILDCARD,$(DIST)/$(BINARY_ARTIFACT_PACKAGE))
+endif
+
 ifndef MOZ_PKG_SRCDIR
   MOZ_PKG_SRCDIR = $(topsrcdir)
 endif
 
 SRC_TAR_PREFIX = $(MOZ_APP_NAME)-$(MOZ_PKG_VERSION)
 SRC_TAR_EXCLUDE_PATHS += \
   --exclude='.hg*' \
   --exclude='CVS' \