bug 1442306 - generate a buildhub.json file during packaging. r?build
buildhub is a service that stores a list of nightly and release builds and
can be queried to find specific builds. Currently it ingests data by scraping
info from ftp.mozilla.org. This patch makes it so we generate a buildhub.json
during packaging with the data that buildhub wants.
There are a few pieces of data that we can't accurately provide from the
build system such as the URL to the build, so we provide some stub data
there with the expectation that a release engineering process will fill
them in later.
MozReview-Commit-ID: 266BnZZBFoL
--- a/toolkit/mozapps/installer/informulate.py
+++ b/toolkit/mozapps/installer/informulate.py
@@ -4,28 +4,34 @@
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
# Generate build info files for use by other tools.
# This script assumes it is being run in a Mozilla CI build.
from __future__ import unicode_literals
from argparse import ArgumentParser
+import datetime
+import buildconfig
import json
-import buildconfig
+import mozinfo
import os
def main():
parser = ArgumentParser()
parser.add_argument('output_json', help='Output JSON file')
+ parser.add_argument('buildhub_json', help='Output buildhub JSON file')
parser.add_argument('output_txt', help='Output text file')
# TODO: Move package-name.mk variables into moz.configure.
parser.add_argument('pkg_platform', help='Package platform identifier')
+ parser.add_argument('--package', help='Path to application package file')
+ parser.add_argument('--installer', help='Path to application installer file')
args = parser.parse_args()
+ mozinfo.find_and_update_from_json()
important_substitutions = [
'target_alias', 'target_cpu', 'target_os', 'target_vendor',
'host_alias', 'host_cpu', 'host_os', 'host_vendor',
'MOZ_UPDATE_CHANNEL', 'MOZ_APP_VENDOR', 'MOZ_APP_NAME',
'MOZ_APP_VERSION', 'MOZ_APP_MAXVERSION', 'MOZ_APP_ID',
'CC', 'CXX', 'AS', 'MOZ_SOURCE_REPO',
]
@@ -38,14 +44,58 @@ def main():
'moz_source_stamp': buildconfig.substs['MOZ_SOURCE_CHANGESET'],
'moz_pkg_platform': args.pkg_platform,
})
with open(args.output_json, 'wb') as f:
json.dump(all_key_value_pairs, f, indent=2, sort_keys=True)
f.write('\n')
+ with open(args.buildhub_json, 'wb') as f:
+ if args.installer and os.path.exists(args.installer):
+ package = args.installer
+ else:
+ package = args.package
+ build_time = datetime.datetime.strptime(build_id, '%Y%m%d%H%M%S')
+ st = os.stat(package)
+ mtime = datetime.datetime.fromtimestamp(st.st_mtime)
+ s = buildconfig.substs
+ record = {
+ 'build': {
+ 'id': build_id,
+ 'date': build_time.isoformat() + 'Z',
+ 'as': s['AS'],
+ 'cc': s['CC'],
+ 'cxx': s['CXX'],
+ 'host': s['host_alias'],
+ 'target': s['target_alias'],
+ },
+ 'source': {
+ 'product': s['MOZ_APP_NAME'],
+ 'repository': s['MOZ_SOURCE_REPO'],
+ 'tree': os.environ['MH_BRANCH'],
+ 'revision': s['MOZ_SOURCE_CHANGESET'],
+ },
+ 'target': {
+ 'platform': args.pkg_platform,
+ 'os': mozinfo.info['os'],
+ # This would be easier if the locale was specified at configure time.
+ 'locale': os.environ.get('AB_CD', 'en-US'),
+ 'version': s['MOZ_APP_VERSION'],
+ 'channel': s['MOZ_UPDATE_CHANNEL'],
+ },
+ 'download': {
+ # The release pipeline will update these keys.
+ 'url': os.path.basename(package),
+ 'mimetype': 'application/octet-stream',
+ 'date': mtime.isoformat() + 'Z',
+ 'size': st.st_size,
+ }
+ }
+ json.dump(record, f, indent=2, sort_keys=True)
+ f.write('\n')
+
with open(args.output_txt, 'wb') as f:
f.write('buildID={}\n'.format(build_id))
if __name__ == '__main__':
main()
--- a/toolkit/mozapps/installer/package-name.mk
+++ b/toolkit/mozapps/installer/package-name.mk
@@ -107,16 +107,17 @@ GTEST_PACKAGE = $(PKG_BASENAME).gtest.te
ifneq (,$(wildcard $(DIST)/bin/application.ini))
BUILDID = $(shell $(PYTHON) $(MOZILLA_DIR)/config/printconfigsetting.py $(DIST)/bin/application.ini App BuildID)
else
BUILDID = $(shell $(PYTHON) $(MOZILLA_DIR)/config/printconfigsetting.py $(DIST)/bin/platform.ini Build BuildID)
endif
MOZ_SOURCESTAMP_FILE = $(DIST)/$(PKG_PATH)/$(MOZ_INFO_BASENAME).txt
MOZ_BUILDINFO_FILE = $(DIST)/$(PKG_PATH)/$(MOZ_INFO_BASENAME).json
+MOZ_BUILDHUB_JSON = $(DIST)/$(PKG_PATH)/buildhub.json
MOZ_BUILDID_INFO_TXT_FILE = $(DIST)/$(PKG_PATH)/$(MOZ_INFO_BASENAME)_info.txt
MOZ_MOZINFO_FILE = $(DIST)/$(PKG_PATH)/$(MOZ_INFO_BASENAME).mozinfo.json
MOZ_TEST_PACKAGES_FILE = $(DIST)/$(PKG_PATH)/$(PKG_BASENAME).test_packages.json
# JavaScript Shell
ifdef MOZ_SIMPLE_PACKAGE_NAME
JSSHELL_NAME := $(MOZ_SIMPLE_PACKAGE_NAME).jsshell.zip
else
--- a/toolkit/mozapps/installer/packager.mk
+++ b/toolkit/mozapps/installer/packager.mk
@@ -89,17 +89,20 @@ make-package: FORCE
ifeq (WINNT,$(OS_ARCH))
ifeq ($(MOZ_PKG_FORMAT),ZIP)
$(MAKE) -C windows ZIP_IN='$(ABS_DIST)/$(PACKAGE)' installer
endif
endif
ifdef MOZ_AUTOMATION
cp $(DEPTH)/mozinfo.json $(MOZ_MOZINFO_FILE)
$(PYTHON) $(MOZILLA_DIR)/toolkit/mozapps/installer/informulate.py \
- $(MOZ_BUILDINFO_FILE) $(MOZ_BUILDID_INFO_TXT_FILE) $(MOZ_PKG_PLATFORM)
+ $(MOZ_BUILDINFO_FILE) $(MOZ_BUILDHUB_JSON) $(MOZ_BUILDID_INFO_TXT_FILE) \
+ $(MOZ_PKG_PLATFORM) \
+ --package=$(DIST)/$(PACKAGE) \
+ --installer=$(INSTALLER_PACKAGE)
endif
$(TOUCH) $@
GARBAGE += make-package
make-sourcestamp-file::
$(NSINSTALL) -D $(DIST)/$(PKG_PATH)
@echo '$(BUILDID)' > $(MOZ_SOURCESTAMP_FILE)
--- a/toolkit/mozapps/installer/upload-files.mk
+++ b/toolkit/mozapps/installer/upload-files.mk
@@ -385,16 +385,17 @@ UPLOAD_FILES= \
$(call QUOTED_WILDCARD,$(DIST)/$(COMPLETE_MAR)) \
$(call QUOTED_WILDCARD,$(DIST)/$(LANGPACK)) \
$(call QUOTED_WILDCARD,$(wildcard $(DIST)/$(PARTIAL_MAR))) \
$(call QUOTED_WILDCARD,$(DIST)/$(PKG_PATH)$(MOZHARNESS_PACKAGE)) \
$(call QUOTED_WILDCARD,$(DIST)/$(PKG_PATH)$(SYMBOL_ARCHIVE_BASENAME).zip) \
$(call QUOTED_WILDCARD,$(DIST)/$(PKG_PATH)$(GENERATED_SOURCE_FILE_PACKAGE)) \
$(call QUOTED_WILDCARD,$(MOZ_SOURCESTAMP_FILE)) \
$(call QUOTED_WILDCARD,$(MOZ_BUILDINFO_FILE)) \
+ $(MOZ_BUILDHUB_JSON) \
$(call QUOTED_WILDCARD,$(MOZ_BUILDID_INFO_TXT_FILE)) \
$(call QUOTED_WILDCARD,$(MOZ_MOZINFO_FILE)) \
$(call QUOTED_WILDCARD,$(MOZ_TEST_PACKAGES_FILE)) \
$(call QUOTED_WILDCARD,$(PKG_JSSHELL)) \
$(call QUOTED_WILDCARD,$(DIST)/$(PKG_PATH)$(SYMBOL_FULL_ARCHIVE_BASENAME).zip) \
$(call QUOTED_WILDCARD,$(topobjdir)/$(MOZ_BUILD_APP)/installer/windows/instgen/setup.exe) \
$(call QUOTED_WILDCARD,$(topobjdir)/$(MOZ_BUILD_APP)/installer/windows/instgen/setup-stub.exe) \
$(call QUOTED_WILDCARD,$(topsrcdir)/toolchains.json) \
