Bug 1250991 - Produce binary artifact zip files for OS X builds; r?mshal
Currently, artifact builds download DMGs and process them into JAR files
containing a subset of the files they care about. This adds overhead.
This commit unblocks future work to stop doing that.
We introduce an "artifact_archive" build action. Given an output
filename, it reads the build configuration and produces a zip file
containing files relevant to artifact builds. This file is uploaded
as part of the automation results. In the future, artifact builds
can search for this file, download it, and extract it verbatim.
A goal is to make the client-side of artifact builds as dumb as
possible: just extract an archive.
MozReview-Commit-ID: KTt90kBDxkb
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozbuild/action/artifact_archive.py
@@ -0,0 +1,77 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, unicode_literals
+
+import os
+import sys
+
+from mozbuild.base import MozbuildObject
+from mozpack.files import (
+ File,
+)
+from mozpack.mozjar import (
+ JarWriter,
+)
+
+
+def get_osx_files(objdir):
+ """Obtain the files for an OS X artifact archive.
+
+ Yields tuples of (path, file) that can be added to a JarWriter.
+ """
+ IGNORE_BIN_DYLIBS = {
+ 'liblogalloc.dylib',
+ 'libreplace_jemalloc.dylib',
+ 'libreplace_malloc.dylib',
+ }
+
+ bin_path = os.path.join(objdir, 'dist', 'bin')
+
+ bin_files = {
+ 'browser/components/libbrowsercomps.dylib',
+ 'crashreporter',
+ 'dependentlibs.list',
+ 'firefox',
+ 'firefox-bin',
+ 'gmp-clearkey/0.1/libclearkey.dylib',
+ 'plugin-container',
+ 'updater',
+ 'xpcshell',
+ 'XUL',
+ }
+
+ bin_files |= {p for p in os.listdir(bin_path)
+ if p.endswith('.dylib') and p not in IGNORE_BIN_DYLIBS}
+
+ all_files = {p: 'bin/%s' % p for p in bin_files}
+
+ for source, dest in sorted(all_files.items()):
+ yield dest.encode('utf-8'), File(os.path.join(bin_path, source))
+
+
+def create_archive(build, output_path):
+ if build.defines.get('XP_MACOSX'):
+ files = get_osx_files(build.topobjdir)
+ else:
+ raise Exception('platform not yet supported')
+
+ with JarWriter(file=output_path, optimize=False, compress_level=5) as writer:
+ for p, f in files:
+ writer.add(p, f, mode=f.mode)
+
+
+def main(args):
+ import argparse
+
+ parser = argparse.ArgumentParser()
+ parser.add_argument('destfile', help='Archive file to write')
+
+ args = parser.parse_args(args)
+ build = MozbuildObject.from_environment()
+
+ create_archive(build, args.destfile)
+
+if __name__ == '__main__':
+ sys.exit(main(sys.argv[1:]))
--- a/toolkit/mozapps/installer/packager.mk
+++ b/toolkit/mozapps/installer/packager.mk
@@ -81,16 +81,19 @@ ifdef MOZ_ASAN
endif # MOZ_ASAN
endif # Darwin
prepare-package: stage-package
make-package-internal: prepare-package make-sourcestamp-file make-buildinfo-file make-mozinfo-file
@echo 'Compressing...'
cd $(DIST) && $(MAKE_PACKAGE)
+ifeq (cocoa,$(MOZ_WIDGET_TOOLKIT))
+ $(call py_action,artifact_archive,$(DIST)/$(BINARY_ARTIFACT_PACKAGE))
+endif
make-package: FORCE
$(MAKE) make-package-internal
$(TOUCH) $@
GARBAGE += make-package
make-sourcestamp-file::
--- a/toolkit/mozapps/installer/upload-files.mk
+++ b/toolkit/mozapps/installer/upload-files.mk
@@ -43,16 +43,17 @@ ifndef _RESPATH
endif
ifdef UNIVERSAL_BINARY
STAGEPATH = universal/
endif
endif
PACKAGE_BASE_DIR = $(ABS_DIST)
PACKAGE = $(PKG_PATH)$(PKG_BASENAME)$(PKG_SUFFIX)
+BINARY_ARTIFACT_PACKAGE = $(PKG_PATH)$(PKG_BASENAME)-binary-artifacts.zip
# By default, the SDK uses the same packaging type as the main bundle,
# but on mac it is a .tar.bz2
SDK_SUFFIX = $(PKG_SUFFIX)
SDK = $(SDK_PATH)$(PKG_BASENAME).sdk$(SDK_SUFFIX)
ifdef UNIVERSAL_BINARY
SDK = $(SDK_PATH)$(PKG_BASENAME)-$(TARGET_CPU).sdk$(SDK_SUFFIX)
endif
@@ -517,16 +518,22 @@ ifdef MOZ_SIGN_CMD
UPLOAD_FILES += $(call QUOTED_WILDCARD,$(INSTALLER_PACKAGE).asc)
UPLOAD_FILES += $(call QUOTED_WILDCARD,$(DIST)/$(PACKAGE).asc)
endif
ifdef MOZ_STUB_INSTALLER
UPLOAD_FILES += $(call QUOTED_WILDCARD,$(DIST)/$(PKG_INST_PATH)$(PKG_STUB_BASENAME).exe)
endif
+# Upload binary artifact archive.
+# (Only works on OS X so far)
+ifeq (cocoa,$(MOZ_WIDGET_TOOLKIT))
+ UPLOAD_FILES += $(call QUOTED_WILDCARD,$(DIST)/$(BINARY_ARTIFACT_PACKAGE))
+endif
+
ifndef MOZ_PKG_SRCDIR
MOZ_PKG_SRCDIR = $(topsrcdir)
endif
SRC_TAR_PREFIX = $(MOZ_APP_NAME)-$(MOZ_PKG_VERSION)
SRC_TAR_EXCLUDE_PATHS += \
--exclude='.hg*' \
--exclude='CVS' \