Bug 1253110 - Produce binary artifact zip files for OS X builds; r?mshal, glandium
Currently, artifact builds download DMGs and process them into JAR files
containing a subset of the files they care about. This adds overhead.
This commit unblocks future work to stop doing that.
We introduce an "artifact_archive" build action. Given an output
filename, it reads the build configuration and produces a zip file
containing files relevant to artifact builds. This file is uploaded
as part of the automation results. In the future, artifact builds
can search for this file, download it, and extract it verbatim.
A goal is to make the client-side of artifact builds as dumb as
possible: just extract an archive.
I'm not very happy with the amount of hackery in artifact_archive.py
regarding searching for binary files. I really wish these things came
more from moz.build files. I would appreciate alternative solutions
so we don't have to hard-code lists of files. (It's worth noting that
the client side of artifact builds currently codes a similar files
list. Once we have the client side of artifacts doing a simple
archive extract, we will have effectively moved that files list
from mach to packaging, which is closer to where it should be,
but not actually in moz.build, which is where I think it should
belong.)
MozReview-Commit-ID: KTt90kBDxkb
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozbuild/action/artifact_archive.py
@@ -0,0 +1,195 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, unicode_literals
+
+import collections
+import json
+import os
+import sys
+
+from mozbuild.base import (
+ MozbuildObject,
+)
+from mozpack.files import (
+ File,
+ FileFinder,
+ GeneratedFile,
+)
+from mozpack.mozjar import (
+ JarWriter,
+)
+import mozpack.path as mozpath
+
+
+def get_binaries(build_app, objdir):
+ """Obtain binaries that should be packaged in the archive."""
+ with open(os.path.join(objdir, 'binaries.json'), 'rb') as fh:
+ binaries = json.load(fh)
+
+ # Ideally the data below comes from moz.build or some other source.
+
+ # Directories we don't package outright.
+ IGNORE_RELDIRS = {
+ 'toolkit/library/gtest',
+ }
+
+ # This binary is quite large (~10 MB) and doesn't provide much value.
+ # Ignore it for now.
+ IGNORE_PROGRAMS = {
+ 'ReadNTLM',
+ }
+
+ IGNORE_LIB_BASENAMES = {
+ 'dummy_replace_malloc',
+ }
+
+ EXTRAS = {
+ 'dependentlibs.list',
+ }
+
+ # TODO take DIST_INSTALL from moz.build into account. This likely requires
+ # changing binaries.json to record this.
+
+ for program in binaries['programs']:
+ if program['program'] in IGNORE_PROGRAMS:
+ continue
+
+ # Don't package C++ unit test binaries.
+ if program['program'].startswith(('Test', 'test_')):
+ continue
+ if program['program'].endswith(('Test', 'test', '_unittest', '_unittests', '-tests')):
+ continue
+
+ # Host binaries are used by the build and aren't needed for
+ # artifact builds.
+ if program['kind'] == 'host':
+ continue
+
+ inpath = mozpath.join(program['relobjdir'], program['program'])
+
+ # Bug 1266252 - install_target contains the app directory when it
+ # shouldn't.
+ target = program['install_target']
+ if target.endswith('/%s' % build_app):
+ target = target[0:-(len(build_app) + 1)]
+
+ outpath = mozpath.join(target, program['program'])
+ yield inpath, outpath
+
+ for lib in binaries['shared_libraries']:
+ if lib['relobjdir'] in IGNORE_RELDIRS:
+ continue
+
+ if lib['basename'] in IGNORE_LIB_BASENAMES:
+ continue
+
+ # Plugin files are handled below.
+ if lib['install_target'].startswith('dist/plugins/'):
+ continue
+
+ # Don't package libraries for tests.
+ if lib['basename'].endswith('test'):
+ continue
+ if lib['install_target'].startswith('_tests/'):
+ continue
+
+ inpath = mozpath.join(lib['relobjdir'], lib['lib_name'])
+ outpath = mozpath.join(lib['install_target'], lib['lib_name'])
+ yield inpath, outpath
+
+ # Copy all plugin files.
+ finder = FileFinder(os.path.join(objdir, 'dist', 'plugins'), find_executables=False)
+ for p, f in finder.find('*'):
+ yield 'dist/plugins/%s' % p, 'dist/plugins/%s' % p
+
+ # Copy all files in gmp-* directories.
+ finder = FileFinder(os.path.join(objdir, 'dist', 'bin'), find_executables=False)
+ for p, f in finder.find('*'):
+ if p.startswith('gmp-'):
+ yield 'dist/bin/%s' % p, 'dist/bin/%s' % p
+
+ for p in EXTRAS:
+ yield 'dist/bin/%s' % p, 'dist/bin/%s' % p
+
+
+def get_xpidl_files(objdir):
+ # Find XPIDL files. These are .xpt and .manifest files in components/
+ # directories. We need to also emit manifest files containing references
+ # to these component manifests.
+ components_manifests = collections.defaultdict(set)
+ finder = FileFinder(os.path.join(objdir, 'dist', 'bin'), find_executables=False)
+ for p, f in finder.find('**/components/*'):
+ if not p.endswith(('.xpt', '.manifest')):
+ continue
+
+ relpath = 'bin/%s' % p
+ yield relpath.encode('utf-8'), f
+
+ if not p.endswith('.manifest'):
+ continue
+
+ comp_index = p.index('components/')
+ if comp_index:
+ key = '%s/chrome.manifest' % p[0:comp_index - 1]
+ manifest_path = p[comp_index:]
+ else:
+ key = 'chrome.manifest'
+ manifest_path = p
+
+ components_manifests[key].add(manifest_path)
+
+ for path, manifests in components_manifests.items():
+ content = b'\n'.join(b'manifest %s' % m.encode('utf-8') for m in sorted(manifests))
+ p = 'bin/%s' % path
+ yield p.encode('utf-8'), GeneratedFile(content)
+
+
+def get_osx_files(objdir, build_app):
+ """Obtain the files for an OS X artifact archive.
+
+ Yields tuples of (path, file) that can be added to a JarWriter.
+ """
+ all_files = {}
+ for inpath, outpath in get_binaries(build_app, objdir):
+ assert outpath.startswith(('dist/bin/', 'dist/plugins/'))
+ outpath = outpath[5:]
+ all_files[outpath] = inpath
+
+ for dest, source in all_files.items():
+ yield dest.encode('utf-8'), File(os.path.join(objdir, source))
+
+ if dest == 'bin/firefox':
+ yield b'bin/firefox-bin', File(os.path.join(objdir, source))
+
+ for t in get_xpidl_files(objdir):
+ yield t
+
+
+def create_archive(build, output_path):
+ if build.defines.get('XP_MACOSX'):
+ build_app = build.substs['MOZ_BUILD_APP']
+ files = dict(get_osx_files(build.topobjdir, build_app))
+ else:
+ raise Exception('platform not yet supported')
+
+ with JarWriter(file=output_path, optimize=False, compress_level=5) as writer:
+ for p, f in sorted(files.items()):
+ writer.add(p, f, mode=f.mode)
+
+
+def main(args):
+ import argparse
+
+ parser = argparse.ArgumentParser()
+ parser.add_argument('destfile', help='Archive file to write')
+
+ args = parser.parse_args(args)
+ build = MozbuildObject.from_environment()
+
+ create_archive(build, args.destfile)
+
+
+if __name__ == '__main__':
+ sys.exit(main(sys.argv[1:]))
--- a/python/mozbuild/mozbuild/backend/common.py
+++ b/python/mozbuild/mozbuild/backend/common.py
@@ -27,16 +27,17 @@ from mozbuild.frontend.data import (
IPDLFile,
FinalTargetPreprocessedFiles,
FinalTargetFiles,
GeneratedEventWebIDLFile,
GeneratedWebIDLFile,
PreprocessedTestWebIDLFile,
PreprocessedWebIDLFile,
RustRlibLibrary,
+ Program,
SharedLibrary,
TestManifest,
TestWebIDLFile,
UnifiedSources,
XPIDLFile,
WebIDLFile,
)
from mozbuild.jar import (
@@ -384,16 +385,18 @@ class CommonBackend(BuildBackend):
path = mozpath.join(self.environment.topobjdir, 'test-installs.json')
with self._write_file(path) as fh:
json.dump({k: v for k, v in self._test_manager.installs_by_path.items()
if k in self._test_manager.deferred_installs},
fh,
sort_keys=True,
indent=4)
+ self._add_missing_binaries()
+
# Write out a machine-readable file describing binaries.
with self._write_file(mozpath.join(topobjdir, 'binaries.json')) as fh:
d = {
'shared_libraries': [s.to_dict() for s in self._binaries.shared_libraries],
'programs': [p.to_dict() for p in self._binaries.programs],
}
json.dump(d, fh, sort_keys=True, indent=4)
@@ -563,8 +566,61 @@ class CommonBackend(BuildBackend):
FinalTargetPreprocessedFiles(jar_context, files_pp))
for m in jarinfo.chrome_manifests:
entry = parse_manifest_line(
mozpath.dirname(jarinfo.name),
m.replace('%', mozpath.basename(jarinfo.name) + '/'))
self.consume_object(ChromeManifestEntry(
jar_context, '%s.manifest' % jarinfo.name, entry))
+
+ def _add_missing_binaries(self):
+ """Add missing binaries to the binaries collection.
+
+ Binaries from some 3rd party build systems aren't exposed to moz.build.
+ This should eventually be fixed as we rewrite build systems for 3rd
+ party projects or come up with a separate mechanism for declaring the
+ binaries. For now, work around the problem by defining the binaries
+ manually.
+ """
+ # No-op if no binaries defined. This handles cases where we aren't
+ # building binaries, which includes some test cases.
+ if not self._binaries.programs or not self._binaries.shared_libraries:
+ return
+
+ NEW_PROGRAMS = [
+ ('certutil', 'security/nss/cmd/certutil'),
+ ('pk12util', 'security/nss/cmd/pk12util'),
+ ]
+
+ NEW_SHARED_LIBRARIES = [
+ ('freebl3', 'security/nss/lib/freebl'),
+ ('nssckbi', 'security/nss/lib/ckfw/builtins'),
+ ('nssdbm3', 'security/nss/lib/softoken/legacydb'),
+ ('softokn3', 'security/nss/lib/softoken'),
+ ]
+
+ # Verify the binaries we're adding aren't already there.
+ existing = {p.program for p in self._binaries.programs}
+ new_programs = {p[0] for p in NEW_PROGRAMS}
+ if existing & new_programs:
+ raise Exception('attempting to add missing program that is already '
+ 'defined: %s' % ', '.join(existing & new_programs))
+
+ existing = {s.basename for s in self._binaries.shared_libraries}
+ new_basenames = {s[0] for s in NEW_SHARED_LIBRARIES}
+ if existing & new_basenames:
+ raise Exception('attempting to add missing binary that is already '
+ 'defined: %s' % ', '.join(existing & new_basenames))
+
+ for m in NEW_PROGRAMS:
+ context = Context(VARIABLES, self.environment)
+ context.push_source(mozpath.join(
+ self.environment.topsrcdir, m[1], 'moz.build'))
+ self._binaries.programs.append(
+ Program(context, m[0]))
+
+ for m in NEW_SHARED_LIBRARIES:
+ context = Context(VARIABLES, self.environment)
+ context.push_source(mozpath.join(
+ self.environment.topsrcdir, m[1], 'moz.build'))
+ self._binaries.shared_libraries.append(
+ SharedLibrary(context, m[0]))
--- a/toolkit/mozapps/installer/packager.mk
+++ b/toolkit/mozapps/installer/packager.mk
@@ -81,16 +81,19 @@ ifdef MOZ_ASAN
endif # MOZ_ASAN
endif # Darwin
prepare-package: stage-package
make-package-internal: prepare-package make-sourcestamp-file make-buildinfo-file make-mozinfo-file
@echo 'Compressing...'
cd $(DIST) && $(MAKE_PACKAGE)
+ifeq (cocoa,$(MOZ_WIDGET_TOOLKIT))
+ $(call py_action,artifact_archive,$(DIST)/$(BINARY_ARTIFACT_PACKAGE))
+endif
make-package: FORCE
$(MAKE) make-package-internal
$(TOUCH) $@
GARBAGE += make-package
make-sourcestamp-file::
--- a/toolkit/mozapps/installer/upload-files.mk
+++ b/toolkit/mozapps/installer/upload-files.mk
@@ -43,16 +43,17 @@ ifndef _RESPATH
endif
ifdef UNIVERSAL_BINARY
STAGEPATH = universal/
endif
endif
PACKAGE_BASE_DIR = $(ABS_DIST)
PACKAGE = $(PKG_PATH)$(PKG_BASENAME)$(PKG_SUFFIX)
+BINARY_ARTIFACT_PACKAGE = $(PKG_PATH)$(PKG_BASENAME).binary-artifacts.zip
# By default, the SDK uses the same packaging type as the main bundle,
# but on mac it is a .tar.bz2
SDK_SUFFIX = $(PKG_SUFFIX)
SDK = $(SDK_PATH)$(PKG_BASENAME).sdk$(SDK_SUFFIX)
ifdef UNIVERSAL_BINARY
SDK = $(SDK_PATH)$(PKG_BASENAME)-$(TARGET_CPU).sdk$(SDK_SUFFIX)
endif
@@ -492,16 +493,22 @@ ifdef MOZ_SIGN_CMD
UPLOAD_FILES += $(call QUOTED_WILDCARD,$(INSTALLER_PACKAGE).asc)
UPLOAD_FILES += $(call QUOTED_WILDCARD,$(DIST)/$(PACKAGE).asc)
endif
ifdef MOZ_STUB_INSTALLER
UPLOAD_FILES += $(call QUOTED_WILDCARD,$(DIST)/$(PKG_INST_PATH)$(PKG_STUB_BASENAME).exe)
endif
+# Upload binary artifact archive.
+# (Only works on OS X so far)
+ifeq (cocoa,$(MOZ_WIDGET_TOOLKIT))
+ UPLOAD_FILES += $(call QUOTED_WILDCARD,$(DIST)/$(BINARY_ARTIFACT_PACKAGE))
+endif
+
ifndef MOZ_PKG_SRCDIR
MOZ_PKG_SRCDIR = $(topsrcdir)
endif
SRC_TAR_PREFIX = $(MOZ_APP_NAME)-$(MOZ_PKG_VERSION)
SRC_TAR_EXCLUDE_PATHS += \
--exclude='.hg*' \
--exclude='CVS' \