bug 1259832 - package generated sources and upload them along with other build artifacts. r=gps draft
authorTed Mielczarek <ted@mielczarek.org>
Mon, 31 Jul 2017 12:47:08 -0400
changeset 648516 6bc57d5e47a91265be853d8ba3dca1ecef2d065d
parent 648515 1937b559402b5c8de80d0b014dfe85cf4a24cad3
child 648517 c494bebe332e8c9fe1df7e25b6c23bbef20be289
push id74775
push userbmo:ted@mielczarek.org
push dateThu, 17 Aug 2017 21:15:41 +0000
reviewersgps
bugs1259832
milestone57.0a1
bug 1259832 - package generated sources and upload them along with other build artifacts. r=gps This change makes us upload an `$(PKG_BASENAME).generated-files.tar.gz` archive alongside other build artifacts which contains all the generated source files from the build. A change after this will introduce an `upload-generated-sources` task to take this artifact and upload the individual files to an S3 bucket. This will be used to provide links to generated source files when they appear in stack traces in crash reports. MozReview-Commit-ID: 6yQAdlZ5q3O
Makefile.in
build/moz-automation.mk
build/mozconfig.artifact.automation
build/mozconfig.automation
mobile/android/config/mozconfigs/android-api-15-frontend/nightly
mobile/android/config/mozconfigs/android-api-15-gradle-dependencies/nightly
mobile/android/config/mozconfigs/android-api-15-gradle/nightly-artifact
mobile/android/config/mozconfigs/android-api-15/debug-artifact
mobile/android/config/mozconfigs/android-api-15/nightly-artifact
mobile/android/config/mozconfigs/android-x86/nightly-artifact
python/mozbuild/mozbuild/action/package_generated_sources.py
python/mozbuild/mozbuild/generated_sources.py
toolkit/mozapps/installer/package-name.mk
toolkit/mozapps/installer/upload-files.mk
--- a/Makefile.in
+++ b/Makefile.in
@@ -300,16 +300,20 @@ ifdef SOCORRO_SYMBOL_UPLOAD_TOKEN_FILE
 	$(PYTHON) -u $(topsrcdir)/toolkit/crashreporter/tools/upload_symbols.py '$(DIST)/$(PKG_PATH)$(SYMBOL_FULL_ARCHIVE_BASENAME).zip'
 endif
 endif
 
 .PHONY: update-packaging
 update-packaging:
 	$(MAKE) -C tools/update-packaging
 
+.PHONY: package-generated-sources
+package-generated-sources:
+	$(call py_action,package_generated_sources,'$(DIST)/$(PKG_PATH)$(GENERATED_SOURCE_FILE_PACKAGE)')
+
 #XXX: this is a hack, since we don't want to clobber for MSVC
 # PGO support, but we can't do this test in client.mk
 ifneq ($(OS_ARCH)_$(GNU_CC), WINNT_)
 # No point in clobbering if PGO has been explicitly disabled.
 ifndef NO_PROFILE_GUIDED_OPTIMIZE
 maybe_clobber_profiledbuild: clean
 else
 maybe_clobber_profiledbuild:
--- a/build/moz-automation.mk
+++ b/build/moz-automation.mk
@@ -22,30 +22,32 @@ endif
 # Helper variables to convert from MOZ_AUTOMATION_* variables to the
 # corresponding the make target
 tier_MOZ_AUTOMATION_BUILD_SYMBOLS = buildsymbols
 tier_MOZ_AUTOMATION_L10N_CHECK = l10n-check
 tier_MOZ_AUTOMATION_INSTALLER = installer
 tier_MOZ_AUTOMATION_PACKAGE = package
 tier_MOZ_AUTOMATION_PACKAGE_TESTS = package-tests
 tier_MOZ_AUTOMATION_UPDATE_PACKAGING = update-packaging
+tier_MOZ_AUTOMATION_PACKAGE_GENERATED_SOURCES = package-generated-sources
 tier_MOZ_AUTOMATION_UPLOAD_SYMBOLS = uploadsymbols
 tier_MOZ_AUTOMATION_UPLOAD = upload
 
 # Automation build steps. Everything in MOZ_AUTOMATION_TIERS also gets used in
 # TIERS for mach display. As such, the MOZ_AUTOMATION_TIERS are roughly sorted
 # here in the order that they will be executed (since mach doesn't know of the
 # dependencies between them).
 moz_automation_symbols = \
   MOZ_AUTOMATION_PACKAGE_TESTS \
   MOZ_AUTOMATION_BUILD_SYMBOLS \
   MOZ_AUTOMATION_UPLOAD_SYMBOLS \
   MOZ_AUTOMATION_PACKAGE \
   MOZ_AUTOMATION_INSTALLER \
   MOZ_AUTOMATION_UPDATE_PACKAGING \
+  MOZ_AUTOMATION_PACKAGE_GENERATED_SOURCES \
   MOZ_AUTOMATION_L10N_CHECK \
   MOZ_AUTOMATION_UPLOAD \
   $(NULL)
 MOZ_AUTOMATION_TIERS := $(foreach sym,$(moz_automation_symbols),$(if $(filter 1,$($(sym))),$(tier_$(sym))))
 
 # Dependencies between automation build steps
 automation/uploadsymbols: automation/buildsymbols
 
@@ -55,16 +57,17 @@ automation/update-packaging: automation/
 automation/l10n-check: automation/package
 automation/l10n-check: automation/installer
 
 automation/upload: automation/installer
 automation/upload: automation/package
 automation/upload: automation/package-tests
 automation/upload: automation/buildsymbols
 automation/upload: automation/update-packaging
+automation/upload: automation/package-generated-sources
 
 # The installer and packager all run stage-package, and may conflict
 # with each other.
 automation/installer: automation/package
 
 automation/build: $(addprefix automation/,$(MOZ_AUTOMATION_TIERS))
 	@echo Automation steps completed.
 
--- a/build/mozconfig.artifact.automation
+++ b/build/mozconfig.artifact.automation
@@ -1,5 +1,6 @@
 # Common options for artifact builds to set automation steps.
 # This gets included before mozconfig.automation.
 
 MOZ_AUTOMATION_BUILD_SYMBOLS=0
 MOZ_AUTOMATION_L10N_CHECK=0
+MOZ_AUTOMATION_PACKAGE_GENERATED_SOURCES=0
--- a/build/mozconfig.automation
+++ b/build/mozconfig.automation
@@ -10,12 +10,13 @@
 # overridden by setting them earlier in the appropriate mozconfig.
 
 mk_add_options "export MOZ_AUTOMATION_BUILD_SYMBOLS=${MOZ_AUTOMATION_BUILD_SYMBOLS-1}"
 mk_add_options "export MOZ_AUTOMATION_L10N_CHECK=${MOZ_AUTOMATION_L10N_CHECK-1}"
 mk_add_options "export MOZ_AUTOMATION_PACKAGE=${MOZ_AUTOMATION_PACKAGE-1}"
 mk_add_options "export MOZ_AUTOMATION_PACKAGE_TESTS=${MOZ_AUTOMATION_PACKAGE_TESTS-1}"
 mk_add_options "export MOZ_AUTOMATION_INSTALLER=${MOZ_AUTOMATION_INSTALLER-0}"
 mk_add_options "export MOZ_AUTOMATION_UPDATE_PACKAGING=${MOZ_AUTOMATION_UPDATE_PACKAGING-0}"
+mk_add_options "export MOZ_AUTOMATION_PACKAGE_GENERATED_SOURCES=${MOZ_AUTOMATION_PACKAGE_GENERATED_SOURCES-1}"
 mk_add_options "export MOZ_AUTOMATION_UPLOAD=${MOZ_AUTOMATION_UPLOAD-1}"
 mk_add_options "export MOZ_AUTOMATION_UPLOAD_SYMBOLS=${MOZ_AUTOMATION_UPLOAD_SYMBOLS-0}"
 
 export MOZ_AUTOMATION_MOZCONFIG=1
--- a/mobile/android/config/mozconfigs/android-api-15-frontend/nightly
+++ b/mobile/android/config/mozconfigs/android-api-15-frontend/nightly
@@ -2,16 +2,17 @@
 MOZ_AUTOMATION_BUILD_SYMBOLS=0
 MOZ_AUTOMATION_INSTALLER=0
 MOZ_AUTOMATION_L10N_CHECK=0
 MOZ_AUTOMATION_PACKAGE=0
 MOZ_AUTOMATION_PACKAGE_TESTS=0
 MOZ_AUTOMATION_UPDATE_PACKAGING=0
 MOZ_AUTOMATION_UPLOAD=0
 MOZ_AUTOMATION_UPLOAD_SYMBOLS=0
+MOZ_AUTOMATION_PACKAGE_GENERATED_SOURCES=0
 
 NO_CACHE=1
 NO_NDK=1
 
 . "$topsrcdir/mobile/android/config/mozconfigs/common"
 
 ac_add_options --with-gradle="$topsrcdir/gradle-dist/bin/gradle"
 export GRADLE_MAVEN_REPOSITORIES="file://$topsrcdir/jcenter","file://$topsrcdir/google"
--- a/mobile/android/config/mozconfigs/android-api-15-gradle-dependencies/nightly
+++ b/mobile/android/config/mozconfigs/android-api-15-gradle-dependencies/nightly
@@ -2,16 +2,17 @@
 MOZ_AUTOMATION_BUILD_SYMBOLS=0
 MOZ_AUTOMATION_INSTALLER=0
 MOZ_AUTOMATION_L10N_CHECK=0
 MOZ_AUTOMATION_PACKAGE=0
 MOZ_AUTOMATION_PACKAGE_TESTS=0
 MOZ_AUTOMATION_UPDATE_PACKAGING=0
 MOZ_AUTOMATION_UPLOAD=0
 MOZ_AUTOMATION_UPLOAD_SYMBOLS=0
+MOZ_AUTOMATION_PACKAGE_GENERATED_SOURCES=0
 
 NO_CACHE=1
 NO_NDK=1
 
 . "$topsrcdir/mobile/android/config/mozconfigs/common"
 
 # We want to download Gradle.
 ac_add_options --with-gradle
--- a/mobile/android/config/mozconfigs/android-api-15-gradle/nightly-artifact
+++ b/mobile/android/config/mozconfigs/android-api-15-gradle/nightly-artifact
@@ -1,10 +1,9 @@
-MOZ_AUTOMATION_BUILD_SYMBOLS=0
-MOZ_AUTOMATION_L10N_CHECK=0
+. "$topsrcdir/build/mozconfig.artifact.automation"
 
 NO_CACHE=1
 NO_NDK=1
 
 . "$topsrcdir/mobile/android/config/mozconfigs/common"
 
 . "$topsrcdir/mobile/android/config/mozconfigs/android-api-15-gradle/nightly"
 
--- a/mobile/android/config/mozconfigs/android-api-15/debug-artifact
+++ b/mobile/android/config/mozconfigs/android-api-15/debug-artifact
@@ -1,10 +1,9 @@
-MOZ_AUTOMATION_BUILD_SYMBOLS=0
-MOZ_AUTOMATION_L10N_CHECK=0
+. "$topsrcdir/build/mozconfig.artifact.automation"
 
 NO_CACHE=1
 NO_NDK=1
 
 . "$topsrcdir/mobile/android/config/mozconfigs/common"
 
 # Global options
 ac_add_options --enable-debug
--- a/mobile/android/config/mozconfigs/android-api-15/nightly-artifact
+++ b/mobile/android/config/mozconfigs/android-api-15/nightly-artifact
@@ -1,10 +1,9 @@
-MOZ_AUTOMATION_BUILD_SYMBOLS=0
-MOZ_AUTOMATION_L10N_CHECK=0
+. "$topsrcdir/build/mozconfig.artifact.automation"
 
 NO_CACHE=1
 NO_NDK=1
 
 . "$topsrcdir/mobile/android/config/mozconfigs/common"
 
 . "$topsrcdir/mobile/android/config/mozconfigs/android-api-15/nightly"
 
--- a/mobile/android/config/mozconfigs/android-x86/nightly-artifact
+++ b/mobile/android/config/mozconfigs/android-x86/nightly-artifact
@@ -1,10 +1,9 @@
-MOZ_AUTOMATION_BUILD_SYMBOLS=0
-MOZ_AUTOMATION_L10N_CHECK=0
+. "$topsrcdir/build/mozconfig.artifact.automation"
 
 NO_CACHE=1
 NO_NDK=1
 
 . "$topsrcdir/mobile/android/config/mozconfigs/common"
 
 . "$topsrcdir/mobile/android/config/mozconfigs/android-x86/nightly"
 
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozbuild/action/package_generated_sources.py
@@ -0,0 +1,30 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import argparse
+import json
+import os.path
+import sys
+
+import buildconfig
+from mozpack.archive import create_tar_gz_from_files
+from mozbuild.generated_sources import get_generated_sources
+
+
+def main(argv):
+    parser = argparse.ArgumentParser(
+        description='Produce archive of generated sources')
+    parser.add_argument('outputfile', help='File to write output to')
+    args = parser.parse_args(argv)
+
+
+    files = dict(get_generated_sources())
+    with open(args.outputfile, 'wb') as fh:
+        create_tar_gz_from_files(fh, files, compresslevel=5)
+
+
+if __name__ == '__main__':
+    sys.exit(main(sys.argv[1:]))
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozbuild/generated_sources.py
@@ -0,0 +1,40 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import json
+import os
+
+from mozpack.files import FileFinder
+import mozpack.path as mozpath
+
+
+def get_generated_sources():
+    '''
+    Yield tuples of `(objdir-rel-path, file)` for generated source files
+    in this objdir, where `file` is either an absolute path to the file or
+    a `mozpack.File` instance.
+    '''
+    import buildconfig
+
+    # First, get the list of generated sources produced by the build backend.
+    gen_sources = os.path.join(buildconfig.topobjdir, 'generated-sources.json')
+    with open(gen_sources, 'rb') as f:
+        data = json.load(f)
+    for f in data['sources']:
+        yield f, mozpath.join(buildconfig.topobjdir, f)
+    # Next, return all the files in $objdir/ipc/ipdl/_ipdlheaders.
+    base = 'ipc/ipdl/_ipdlheaders'
+    finder = FileFinder(mozpath.join(buildconfig.topobjdir, base))
+    for p, f in finder.find('**/*.h'):
+        yield mozpath.join(base, p), f
+    # Next, return any Rust source files that were generated into the Rust
+    # object directory.
+    rust_build_kind = 'debug' if buildconfig.substs.get('MOZ_DEBUG_RUST') else 'release'
+    base = mozpath.join('toolkit/library',
+                        buildconfig.substs['RUST_TARGET'],
+                        rust_build_kind,
+                        'build')
+    finder = FileFinder(mozpath.join(buildconfig.topobjdir, base))
+    for p, f in finder.find('**/*.rs'):
+        yield mozpath.join(base, p), f
--- a/toolkit/mozapps/installer/package-name.mk
+++ b/toolkit/mozapps/installer/package-name.mk
@@ -75,16 +75,19 @@ LANGPACK = $(PKG_LANGPACK_PATH)$(PKG_LAN
 PKG_SRCPACK_BASENAME = $(MOZ_PKG_APPNAME)-$(MOZ_PKG_VERSION).source
 PKG_BUNDLE_BASENAME = $(MOZ_PKG_APPNAME)-$(MOZ_PKG_VERSION)
 PKG_SRCPACK_PATH =
 
 # Symbol package naming
 SYMBOL_FULL_ARCHIVE_BASENAME = $(PKG_BASENAME).crashreporter-symbols-full
 SYMBOL_ARCHIVE_BASENAME = $(PKG_BASENAME).crashreporter-symbols
 
+# Generated file package naming
+GENERATED_SOURCE_FILE_PACKAGE = $(PKG_BASENAME).generated-files.tar.gz
+
 # Code coverage package naming
 CODE_COVERAGE_ARCHIVE_BASENAME = $(PKG_BASENAME).code-coverage-gcno
 
 # Mozharness naming
 MOZHARNESS_PACKAGE = mozharness.zip
 
 # Test package naming
 TEST_PACKAGE = $(PKG_BASENAME).common.tests.zip
--- a/toolkit/mozapps/installer/upload-files.mk
+++ b/toolkit/mozapps/installer/upload-files.mk
@@ -394,16 +394,17 @@ UPLOAD_FILES= \
   $(call QUOTED_WILDCARD,$(DIST)/$(PKG_PATH)$(XPC_TEST_PACKAGE)) \
   $(call QUOTED_WILDCARD,$(DIST)/$(PKG_PATH)$(MOCHITEST_PACKAGE)) \
   $(call QUOTED_WILDCARD,$(DIST)/$(PKG_PATH)$(TALOS_PACKAGE)) \
   $(call QUOTED_WILDCARD,$(DIST)/$(PKG_PATH)$(AWSY_PACKAGE)) \
   $(call QUOTED_WILDCARD,$(DIST)/$(PKG_PATH)$(REFTEST_PACKAGE)) \
   $(call QUOTED_WILDCARD,$(DIST)/$(PKG_PATH)$(WP_TEST_PACKAGE)) \
   $(call QUOTED_WILDCARD,$(DIST)/$(PKG_PATH)$(GTEST_PACKAGE)) \
   $(call QUOTED_WILDCARD,$(DIST)/$(PKG_PATH)$(SYMBOL_ARCHIVE_BASENAME).zip) \
+  $(call QUOTED_WILDCARD,$(DIST)/$(PKG_PATH)$(GENERATED_SOURCE_FILE_PACKAGE)) \
   $(call QUOTED_WILDCARD,$(MOZ_SOURCESTAMP_FILE)) \
   $(call QUOTED_WILDCARD,$(MOZ_BUILDINFO_FILE)) \
   $(call QUOTED_WILDCARD,$(MOZ_BUILDID_INFO_TXT_FILE)) \
   $(call QUOTED_WILDCARD,$(MOZ_MOZINFO_FILE)) \
   $(call QUOTED_WILDCARD,$(MOZ_TEST_PACKAGES_FILE)) \
   $(call QUOTED_WILDCARD,$(PKG_JSSHELL)) \
   $(call QUOTED_WILDCARD,$(DIST)/$(PKG_PATH)$(SYMBOL_FULL_ARCHIVE_BASENAME).zip) \
   $(call QUOTED_WILDCARD,$(topobjdir)/browser/installer/windows/instgen/setup.exe) \