Bug 1250991 - Produce binary artifact zip files for OS X builds; r?mshal draft
authorGregory Szorc <gps@mozilla.com>
Wed, 24 Feb 2016 15:36:57 -0800
changeset 334353 e7040a640a48b1da2058f9569417c99e51b7be36
parent 334352 ed8dde56f394c99406a0705c343bf2926fe45e26
child 514888 a76e1953441ea14d795597b9b4548fb92f9cc7e8
push id11519
push usergszorc@mozilla.com
push dateThu, 25 Feb 2016 00:03:35 +0000
reviewersmshal
bugs1250991
milestone47.0a1
Bug 1250991 - Produce binary artifact zip files for OS X builds; r?mshal Currently, artifact builds download DMGs and process them into JAR files containing a subset of the files they care about. This adds overhead. This commit unblocks future work to stop doing that. We introduce an "artifact_archive" build action. Given an output filename, it reads the build configuration and produces a zip file containing files relevant to artifact builds. This file is uploaded as part of the automation results. In the future, artifact builds can search for this file, download it, and extract it verbatim. A goal is to make the client-side of artifact builds as dumb as possible: just extract an archive. MozReview-Commit-ID: KTt90kBDxkb
python/mozbuild/mozbuild/action/artifact_archive.py
toolkit/mozapps/installer/packager.mk
toolkit/mozapps/installer/upload-files.mk
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozbuild/action/artifact_archive.py
@@ -0,0 +1,77 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, unicode_literals
+
+import os
+import sys
+
+from mozbuild.base import MozbuildObject
+from mozpack.files import (
+    File,
+)
+from mozpack.mozjar import (
+    JarWriter,
+)
+
+
+def get_osx_files(objdir):
+    """Obtain the files for an OS X artifact archive.
+
+    Yields tuples of (path, file) that can be added to a JarWriter.
+    """
+    IGNORE_BIN_DYLIBS = {
+        'liblogalloc.dylib',
+        'libreplace_jemalloc.dylib',
+        'libreplace_malloc.dylib',
+    }
+
+    bin_path = os.path.join(objdir, 'dist', 'bin')
+
+    bin_files = {
+        'browser/components/libbrowsercomps.dylib',
+        'crashreporter',
+        'dependentlibs.list',
+        'firefox',
+        'firefox-bin',
+        'gmp-clearkey/0.1/libclearkey.dylib',
+        'plugin-container',
+        'updater',
+        'xpcshell',
+        'XUL',
+    }
+
+    bin_files |= {p for p in os.listdir(bin_path)
+                  if p.endswith('.dylib') and p not in IGNORE_BIN_DYLIBS}
+
+    all_files = {p: 'bin/%s' % p for p in bin_files}
+
+    for source, dest in sorted(all_files.items()):
+        yield dest.encode('utf-8'), File(os.path.join(bin_path, source))
+
+
+def create_archive(build, output_path):
+    if build.defines.get('XP_MACOSX'):
+        files = get_osx_files(build.topobjdir)
+    else:
+        raise Exception('platform not yet supported')
+
+    with JarWriter(file=output_path, optimize=False, compress_level=5) as writer:
+        for p, f in files:
+            writer.add(p, f, mode=f.mode)
+
+
+def main(args):
+    import argparse
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument('destfile', help='Archive file to write')
+
+    args = parser.parse_args(args)
+    build = MozbuildObject.from_environment()
+
+    create_archive(build, args.destfile)
+
+if __name__ == '__main__':
+    sys.exit(main(sys.argv[1:]))
--- a/toolkit/mozapps/installer/packager.mk
+++ b/toolkit/mozapps/installer/packager.mk
@@ -81,16 +81,19 @@ ifdef MOZ_ASAN
 endif # MOZ_ASAN
 endif # Darwin
 
 prepare-package: stage-package
 
 make-package-internal: prepare-package make-sourcestamp-file make-buildinfo-file make-mozinfo-file
 	@echo 'Compressing...'
 	cd $(DIST) && $(MAKE_PACKAGE)
+ifeq (cocoa,$(MOZ_WIDGET_TOOLKIT))
+	$(call py_action,artifact_archive,$(DIST)/$(BINARY_ARTIFACT_PACKAGE))
+endif
 
 make-package: FORCE
 	$(MAKE) make-package-internal
 	$(TOUCH) $@
 
 GARBAGE += make-package
 
 make-sourcestamp-file::
--- a/toolkit/mozapps/installer/upload-files.mk
+++ b/toolkit/mozapps/installer/upload-files.mk
@@ -43,16 +43,17 @@ ifndef _RESPATH
 endif
 ifdef UNIVERSAL_BINARY
 STAGEPATH = universal/
 endif
 endif
 
 PACKAGE_BASE_DIR = $(ABS_DIST)
 PACKAGE       = $(PKG_PATH)$(PKG_BASENAME)$(PKG_SUFFIX)
+BINARY_ARTIFACT_PACKAGE = $(PKG_PATH)$(PKG_BASENAME)-binary-artifacts.zip
 
 # By default, the SDK uses the same packaging type as the main bundle,
 # but on mac it is a .tar.bz2
 SDK_SUFFIX    = $(PKG_SUFFIX)
 SDK           = $(SDK_PATH)$(PKG_BASENAME).sdk$(SDK_SUFFIX)
 ifdef UNIVERSAL_BINARY
 SDK           = $(SDK_PATH)$(PKG_BASENAME)-$(TARGET_CPU).sdk$(SDK_SUFFIX)
 endif
@@ -517,16 +518,22 @@ ifdef MOZ_SIGN_CMD
   UPLOAD_FILES += $(call QUOTED_WILDCARD,$(INSTALLER_PACKAGE).asc)
   UPLOAD_FILES += $(call QUOTED_WILDCARD,$(DIST)/$(PACKAGE).asc)
 endif
 
 ifdef MOZ_STUB_INSTALLER
   UPLOAD_FILES += $(call QUOTED_WILDCARD,$(DIST)/$(PKG_INST_PATH)$(PKG_STUB_BASENAME).exe)
 endif
 
+# Upload binary artifact archive.
+# (Only works on OS X so far)
+ifeq (cocoa,$(MOZ_WIDGET_TOOLKIT))
+  UPLOAD_FILES += $(call QUOTED_WILDCARD,$(DIST)/$(BINARY_ARTIFACT_PACKAGE))
+endif
+
 ifndef MOZ_PKG_SRCDIR
   MOZ_PKG_SRCDIR = $(topsrcdir)
 endif
 
 SRC_TAR_PREFIX = $(MOZ_APP_NAME)-$(MOZ_PKG_VERSION)
 SRC_TAR_EXCLUDE_PATHS += \
   --exclude='.hg*' \
   --exclude='CVS' \