Bug 1305795 - Produce a bin.tests.zip with binary files; r?ted
Previously, common.tests.zip contained a bin/ directory with binaries.
common.tests.zip also contained a bunch of other test files (such as
thousands of JS JIT tests).
During artifact builds, we'd download all of common.tests.zip and
extract a whitelist of bin/ files to their own cached file. When
running WPT in automation, we'd download all of common.tests.zip
and extract all of bin/ files alongside the files from the main
package artifact. These operations were inefficient because they
required us to download a ~20 MB zip file when we only needed ~4MB
from it (at least those are the sizes on Linux). Multiplied by
tens of thousands of test jobs a day or even throw in a slow
internet connection, and that overhead could matter.
This commit extracts the bin/ directory of common.tests.zip to a
bin.tests.zip file. The generated test_packages.json file has
been updated to include this file everywhere common.tests.zip
is being used. Artifact builds have been updated to download
bin.tests.zip instead of common.tests.zip.
The change to artifact builds means that once this lands, there
will be a window where artifact builds won't work until
automation has produced the bin.tests.zip file for an ancestor
build. This should be OK for end-users. But it will likely cause
a perma failure on the landing that introduces this commit. We
can't avoid that unless we have redundant packaging of the
bin/ files. I don't think this is worth the complexity.
MozReview-Commit-ID: Dd2Yz9nGCZn
--- a/build/gen_test_packages_manifest.py
+++ b/build/gen_test_packages_manifest.py
@@ -37,16 +37,19 @@ OPTIONAL_PACKAGES = [
def parse_args():
parser = ArgumentParser(description='Generate a test_packages.json file to tell automation which harnesses require which test packages.')
parser.add_argument("--common", required=True,
action="store", dest="tests_common",
help="Name of the \"common\" archive, a package to be used by all harnesses.")
parser.add_argument("--jsshell", required=True,
action="store", dest="jsshell",
help="Name of the jsshell zip.")
+ parser.add_argument("--bin", required=True,
+ dest="bin",
+ help="Name of the 'bin' archive, containing test binaries shared by multiple harnesses")
for harness in PACKAGE_SPECIFIED_HARNESSES:
parser.add_argument("--%s" % harness, required=True,
action="store", dest=harness,
help="Name of the %s zip." % harness)
for harness in OPTIONAL_PACKAGES:
parser.add_argument("--%s" % harness, required=False,
action="store", dest=harness,
help="Name of the %s zip." % harness)
@@ -61,17 +64,17 @@ def generate_package_data(args):
# mozharness and testsuite-targets.mk, ideally) to the set of archive names that
# harness depends on to run.
# mozharness will use this file to determine what test zips to download,
# which will be an optimization once parts of the main zip are split to harness
# specific zips.
tests_common = args.tests_common
jsshell = args.jsshell
- harness_requirements = dict([(k, [tests_common]) for k in ALL_HARNESSES])
+ harness_requirements = dict([(k, [tests_common, args.bin]) for k in ALL_HARNESSES])
harness_requirements['jittest'].append(jsshell)
for harness in PACKAGE_SPECIFIED_HARNESSES + OPTIONAL_PACKAGES:
pkg_name = getattr(args, harness, None)
if pkg_name is None:
continue
harness_requirements[harness].append(pkg_name)
return harness_requirements
--- a/python/mozbuild/mozbuild/action/test_archive.py
+++ b/python/mozbuild/mozbuild/action/test_archive.py
@@ -77,16 +77,17 @@ GMP_TEST_PLUGIN_DIRS = [
ARCHIVE_FILES = {
'common': [
{
'source': STAGE,
'base': '',
'pattern': '**',
'ignore': [
+ 'bin/**',
'cppunittest/**',
'gtest/**',
'mochitest/**',
'reftest/**',
'talos/**',
'web-platform/**',
'xpcshell/**',
],
@@ -199,34 +200,42 @@ ARCHIVE_FILES = {
'dest': 'tools/wptserve',
},
{
'source': buildconfig.topobjdir,
'base': '',
'pattern': 'mozinfo.json',
},
{
+ 'source': buildconfig.topsrcdir,
+ 'base': 'build/pgo/certs',
+ 'pattern': '**',
+ 'dest': 'certs',
+ }
+ ],
+ 'bin': [
+ {
'source': buildconfig.topobjdir,
'base': 'dist/bin',
'patterns': [
- '%s%s' % (f, buildconfig.substs['BIN_SUFFIX'])
- for f in TEST_HARNESS_BINS
- ] + [
- '%s%s%s' % (buildconfig.substs['DLL_PREFIX'], f, buildconfig.substs['DLL_SUFFIX'])
- for f in TEST_HARNESS_DLLS
- ],
+ '%s%s' % (f, buildconfig.substs['BIN_SUFFIX'])
+ for f in TEST_HARNESS_BINS
+ ] + [
+ '%s%s%s' % (buildconfig.substs['DLL_PREFIX'], f, buildconfig.substs['DLL_SUFFIX'])
+ for f in TEST_HARNESS_DLLS
+ ],
'dest': 'bin',
},
{
'source': buildconfig.topobjdir,
'base': 'dist/plugins',
'patterns': [
'%s%s%s' % (buildconfig.substs['DLL_PREFIX'], f, buildconfig.substs['DLL_SUFFIX'])
for f in TEST_PLUGIN_DLLS
- ],
+ ],
'dest': 'bin/plugins',
},
{
'source': buildconfig.topobjdir,
'base': 'dist/plugins',
'patterns': TEST_PLUGIN_DIRS,
'dest': 'bin/plugins',
},
@@ -252,22 +261,16 @@ ARCHIVE_FILES = {
'base': 'dist/bin/components',
'patterns': [
'httpd.js',
'httpd.manifest',
'test_necko.xpt',
],
'dest': 'bin/components',
},
- {
- 'source': buildconfig.topsrcdir,
- 'base': 'build/pgo/certs',
- 'pattern': '**',
- 'dest': 'certs',
- }
],
'cppunittest': [
{
'source': STAGE,
'base': '',
'pattern': 'cppunittest/**',
},
# We don't ship these files if startup cache is disabled, which is--- a/python/mozbuild/mozbuild/artifacts.py
+++ b/python/mozbuild/mozbuild/artifacts.py
@@ -121,17 +121,17 @@ class ArtifactJob(object):
('bin/pk12util', ('bin', 'bin')),
('bin/ssltunnel', ('bin', 'bin')),
('bin/xpcshell', ('bin', 'bin')),
('bin/plugins/*', ('bin/plugins', 'plugins'))
}
# We can tell our input is a test archive by this suffix, which happens to
# be the same across platforms.
- _test_archive_suffix = '.common.tests.zip'
+ _test_archive_suffix = '.bin.tests.zip'
def __init__(self, package_re, tests_re, log=None):
self._package_re = re.compile(package_re)
self._tests_re = None
if tests_re:
self._tests_re = re.compile(tests_re)
self._log = log
@@ -160,16 +160,22 @@ class ArtifactJob(object):
def process_artifact(self, filename, processed_filename):
if filename.endswith(ArtifactJob._test_archive_suffix) and self._tests_re:
return self.process_tests_artifact(filename, processed_filename)
return self.process_package_artifact(filename, processed_filename)
def process_package_artifact(self, filename, processed_filename):
raise NotImplementedError("Subclasses must specialize process_package_artifact!")
+ # TODO reconsider doing this.
+ # When we introduced test artifact processing, the point was to extract
+ # bin/* files from common.tests.zip into a smaller zip. Now that we have
+ # bin.tests.zip, this extract step may not be necessary and we can perform
+ # any filtering (if we need to do it at all) during archive extraction
+ # time.
def process_tests_artifact(self, filename, processed_filename):
added_entry = False
with JarWriter(file=processed_filename, optimize=False, compress_level=5) as writer:
reader = JarReader(filename)
for filename, entry in reader.entries.iteritems():
for pattern, (src_prefix, dest_prefix) in self.test_artifact_patterns:
if not mozpath.match(filename, pattern):
@@ -402,35 +408,35 @@ class WinArtifactJob(ArtifactJob):
# https://tools.taskcluster.net/index/artifacts/#buildbot.branches.mozilla-central/buildbot.branches.mozilla-central.
# The values correpsond to a pair of (<package regex>, <test archive regex>).
JOB_DETAILS = {
'android-api-15': (AndroidArtifactJob, ('public/build/fennec-(.*)-arm\.apk',
None)),
'android-x86': (AndroidArtifactJob, ('public/build/fennec-(.*)-i386\.apk',
None)),
'linux': (LinuxArtifactJob, ('public/build/firefox-(.*)\.linux-i686\.tar\.bz2',
- 'public/build/firefox-(.*)\.common\.tests\.zip')),
+ 'public/build/firefox-(.*)\.bin\.tests\.zip')),
'linux-debug': (LinuxArtifactJob, ('public/build/firefox-(.*)\.linux-i686\.tar\.bz2',
- 'public/build/firefox-(.*)\.common\.tests\.zip')),
+ 'public/build/firefox-(.*)\.bin\.tests\.zip')),
'linux64': (LinuxArtifactJob, ('public/build/firefox-(.*)\.linux-x86_64\.tar\.bz2',
- 'public/build/firefox-(.*)\.common\.tests\.zip')),
+ 'public/build/firefox-(.*)\.bin\.tests\.zip')),
'linux64-debug': (LinuxArtifactJob, ('public/build/target\.tar\.bz2',
- 'public/build/target\.common\.tests\.zip')),
+ 'public/build/target\.bin\.tests\.zip')),
'macosx64': (MacArtifactJob, ('public/build/firefox-(.*)\.mac\.dmg',
- 'public/build/firefox-(.*)\.common\.tests\.zip')),
+ 'public/build/firefox-(.*)\.bin\.tests\.zip')),
'macosx64-debug': (MacArtifactJob, ('public/build/firefox-(.*)\.mac64\.dmg',
- 'public/build/firefox-(.*)\.common\.tests\.zip')),
+ 'public/build/firefox-(.*)\.bin\.tests\.zip')),
'win32': (WinArtifactJob, ('public/build/firefox-(.*)\.win32.zip',
- 'public/build/firefox-(.*)\.common\.tests\.zip')),
+ 'public/build/firefox-(.*)\.bin\.tests\.zip')),
'win32-debug': (WinArtifactJob, ('public/build/firefox-(.*)\.win32.zip',
- 'public/build/firefox-(.*)\.common\.tests\.zip')),
+ 'public/build/firefox-(.*)\.bin\.tests\.zip')),
'win64': (WinArtifactJob, ('public/build/firefox-(.*)\.win64.zip',
- 'public/build/firefox-(.*)\.common\.tests\.zip')),
+ 'public/build/firefox-(.*)\.bin\.tests\.zip')),
'win64-debug': (WinArtifactJob, ('public/build/firefox-(.*)\.win64.zip',
- 'public/build/firefox-(.*)\.common\.tests\.zip')),
+ 'public/build/firefox-(.*)\.bin\.tests\.zip')),
}
def get_job_details(job, log=None):
cls, (package_re, tests_re) = JOB_DETAILS[job]
return cls(package_re, tests_re, log=log)
--- a/testing/testsuite-targets.mk
+++ b/testing/testsuite-targets.mk
@@ -189,16 +189,17 @@ stage-all: stage-steeplechase
endif
ifdef COMPILE_ENVIRONMENT
stage-all: stage-cppunittests
endif
TEST_PKGS := \
common \
+ bin \
cppunittest \
mochitest \
reftest \
talos \
xpcshell \
$(NULL)
ifdef BUILD_GTEST
--- a/toolkit/mozapps/installer/package-name.mk
+++ b/toolkit/mozapps/installer/package-name.mk
@@ -132,16 +132,17 @@ SYMBOL_ARCHIVE_BASENAME = $(PKG_BASENAME
# Code coverage package naming
CODE_COVERAGE_ARCHIVE_BASENAME = $(PKG_BASENAME).code-coverage-gcno
# Mozharness naming
MOZHARNESS_PACKAGE = mozharness.zip
# Test package naming
TEST_PACKAGE = $(PKG_BASENAME).common.tests.zip
+BIN_TEST_PACKAGE = $(PKG_BASENAME).bin.tests.zip
CPP_TEST_PACKAGE = $(PKG_BASENAME).cppunittest.tests.zip
XPC_TEST_PACKAGE = $(PKG_BASENAME).xpcshell.tests.zip
MOCHITEST_PACKAGE = $(PKG_BASENAME).mochitest.tests.zip
REFTEST_PACKAGE = $(PKG_BASENAME).reftest.tests.zip
WP_TEST_PACKAGE = $(PKG_BASENAME).web-platform.tests.zip
TALOS_PACKAGE = $(PKG_BASENAME).talos.tests.zip
GTEST_PACKAGE = $(PKG_BASENAME).gtest.tests.zip
--- a/toolkit/mozapps/installer/upload-files.mk
+++ b/toolkit/mozapps/installer/upload-files.mk
@@ -441,16 +441,17 @@ endif
UPLOAD_FILES= \
$(call QUOTED_WILDCARD,$(DIST)/$(PACKAGE)) \
$(call QUOTED_WILDCARD,$(INSTALLER_PACKAGE)) \
$(call QUOTED_WILDCARD,$(DIST)/$(COMPLETE_MAR)) \
$(call QUOTED_WILDCARD,$(DIST)/$(LANGPACK)) \
$(call QUOTED_WILDCARD,$(wildcard $(DIST)/$(PARTIAL_MAR))) \
$(call QUOTED_WILDCARD,$(DIST)/$(PKG_PATH)$(MOZHARNESS_PACKAGE)) \
$(call QUOTED_WILDCARD,$(DIST)/$(PKG_PATH)$(TEST_PACKAGE)) \
+ $(call QUOTED_WILDCARD,$(DIST)/$(PKG_PATH)$(BIN_TEST_PACKAGE)) \
$(call QUOTED_WILDCARD,$(DIST)/$(PKG_PATH)$(CPP_TEST_PACKAGE)) \
$(call QUOTED_WILDCARD,$(DIST)/$(PKG_PATH)$(XPC_TEST_PACKAGE)) \
$(call QUOTED_WILDCARD,$(DIST)/$(PKG_PATH)$(MOCHITEST_PACKAGE)) \
$(call QUOTED_WILDCARD,$(DIST)/$(PKG_PATH)$(TALOS_PACKAGE)) \
$(call QUOTED_WILDCARD,$(DIST)/$(PKG_PATH)$(REFTEST_PACKAGE)) \
$(call QUOTED_WILDCARD,$(DIST)/$(PKG_PATH)$(WP_TEST_PACKAGE)) \
$(call QUOTED_WILDCARD,$(DIST)/$(PKG_PATH)$(GTEST_PACKAGE)) \
$(call QUOTED_WILDCARD,$(DIST)/$(PKG_PATH)$(SYMBOL_ARCHIVE_BASENAME).zip) \
