--- a/python/mozbuild/mozbuild/artifacts.py
+++ b/python/mozbuild/mozbuild/artifacts.py
@@ -141,21 +141,27 @@ class ArtifactJob(object):
('bin/plugins/*', ('bin/plugins', 'plugins')),
('bin/components/*.xpt', ('bin/components', 'bin/components')),
}
# We can tell our input is a test archive by this suffix, which happens to
# be the same across platforms.
_test_archive_suffix = '.common.tests.zip'
- def __init__(self, package_re, tests_re, log=None, download_symbols=False, substs=None):
+ def __init__(self, package_re, tests_re, log=None,
+ download_symbols=False,
+ download_host_bins=False,
+ substs=None):
self._package_re = re.compile(package_re)
self._tests_re = None
if tests_re:
self._tests_re = re.compile(tests_re)
+ self._host_bins_re = None
+ if download_host_bins:
+ self._host_bins_re = re.compile(r'public/build/host/bin/(mar|mbsdiff)(.exe)?')
self._log = log
self._substs = substs
self._symbols_archive_suffix = None
if download_symbols:
self._symbols_archive_suffix = 'crashreporter-symbols.zip'
def log(self, *args, **kwargs):
if self._log:
@@ -163,16 +169,18 @@ class ArtifactJob(object):
def find_candidate_artifacts(self, artifacts):
# TODO: Handle multiple artifacts, taking the latest one.
tests_artifact = None
for artifact in artifacts:
name = artifact['name']
if self._package_re and self._package_re.match(name):
yield name
+ elif self._host_bins_re and self._host_bins_re.match(name):
+ yield name
elif self._tests_re and self._tests_re.match(name):
tests_artifact = name
yield name
elif self._symbols_archive_suffix and name.endswith(self._symbols_archive_suffix):
yield name
else:
self.log(logging.DEBUG, 'artifact',
{'name': name},
@@ -181,16 +189,23 @@ class ArtifactJob(object):
raise ValueError('Expected tests archive matching "{re}", but '
'found none!'.format(re=self._tests_re))
def process_artifact(self, filename, processed_filename):
if filename.endswith(ArtifactJob._test_archive_suffix) and self._tests_re:
return self.process_tests_artifact(filename, processed_filename)
if self._symbols_archive_suffix and filename.endswith(self._symbols_archive_suffix):
return self.process_symbols_archive(filename, processed_filename)
+ if self._host_bins_re:
+ # Turn 'HASH-mar.exe' into 'mar.exe'. `filename` is a path on disk
+ # without the full path to the artifact, so we must reconstruct
+ # that path here.
+ orig_basename = os.path.basename(filename).split('-', 1)[1]
+ if self._host_bins_re.match('public/build/host/bin/{}'.format(orig_basename)):
+ return self.process_host_bin(filename, processed_filename)
return self.process_package_artifact(filename, processed_filename)
def process_package_artifact(self, filename, processed_filename):
raise NotImplementedError("Subclasses must specialize process_package_artifact!")
def process_tests_artifact(self, filename, processed_filename):
from mozbuild.action.test_archive import OBJDIR_TEST_FILES
added_entry = False
@@ -232,16 +247,26 @@ class ArtifactJob(object):
reader = JarReader(filename)
for filename in reader.entries:
destpath = mozpath.join('crashreporter-symbols', filename)
self.log(logging.INFO, 'artifact',
{'destpath': destpath},
'Adding {destpath} to processed archive')
writer.add(destpath.encode('utf-8'), reader[filename])
+ def process_host_bin(self, filename, processed_filename):
+ with JarWriter(file=processed_filename, optimize=False, compress_level=5) as writer:
+ # Turn 'HASH-mar.exe' into 'mar.exe'. `filename` is a path on disk
+ # without any of the path parts of the artifact, so we must inject
+ # the desired `host/bin` prefix here.
+ orig_basename = os.path.basename(filename).split('-', 1)[1]
+ destpath = mozpath.join('host/bin', orig_basename)
+ writer.add(destpath.encode('utf-8'), open(filename, 'rb'))
+
+
class AndroidArtifactJob(ArtifactJob):
product = 'mobile'
package_artifact_patterns = {
'application.ini',
'platform.ini',
'**/*.so',
@@ -496,21 +521,27 @@ JOB_DETAILS = {
'win64-opt': (WinArtifactJob, (r'public/build/firefox-(.*)\.win64\.zip|public/build/target\.zip',
r'public/build/firefox-(.*)\.common\.tests\.zip|public/build/target\.common\.tests\.zip')),
'win64-debug': (WinArtifactJob, (r'public/build/firefox-(.*)\.win64\.zip|public/build/target\.zip',
r'public/build/firefox-(.*)\.common\.tests\.zip|public/build/target\.common\.tests\.zip')),
}
-def get_job_details(job, log=None, download_symbols=False, substs=None):
+def get_job_details(job, log=None,
+ download_symbols=False,
+ download_host_bins=False,
+ substs=None):
cls, (package_re, tests_re) = JOB_DETAILS[job]
- return cls(package_re, tests_re, log=log, download_symbols=download_symbols,
+ return cls(package_re, tests_re, log=log,
+ download_symbols=download_symbols,
+ download_host_bins=download_host_bins,
substs=substs)
+
def cachedmethod(cachefunc):
'''Decorator to wrap a class or instance method with a memoizing callable that
saves results in a (possibly shared) cache.
'''
def decorator(method):
def wrapper(self, *args, **kwargs):
mapping = cachefunc(self)
if mapping is None:
@@ -627,19 +658,21 @@ class PushheadCache(CacheManager):
class TaskCache(CacheManager):
'''Map candidate pushheads to Task Cluster task IDs and artifact URLs.'''
def __init__(self, cache_dir, log=None, skip_cache=False):
CacheManager.__init__(self, cache_dir, 'artifact_url', MAX_CACHED_TASKS, log=log, skip_cache=skip_cache)
@cachedmethod(operator.attrgetter('_cache'))
- def artifact_urls(self, tree, job, rev, download_symbols):
+ def artifact_urls(self, tree, job, rev, download_symbols, download_host_bins):
try:
- artifact_job = get_job_details(job, log=self._log, download_symbols=download_symbols)
+ artifact_job = get_job_details(job, log=self._log,
+ download_symbols=download_symbols,
+ download_host_bins=download_host_bins)
except KeyError:
self.log(logging.INFO, 'artifact',
{'job': job},
'Unknown job {job}')
raise KeyError("Unknown job")
# Grab the second part of the repo name, which is generally how things
# are indexed. Eg: 'integration/mozilla-inbound' is indexed as
@@ -855,29 +888,35 @@ class Artifacts(object):
def __init__(self, tree, substs, defines, job=None, log=None,
cache_dir='.', hg=None, git=None, skip_cache=False,
topsrcdir=None):
if (hg and git) or (not hg and not git):
raise ValueError("Must provide path to exactly one of hg and git")
self._substs = substs
self._download_symbols = self._substs.get('MOZ_ARTIFACT_BUILD_SYMBOLS', False)
+ # Host binaries are not produced for macOS consumers: that is, there's
+ # no macOS-hosted job to produce them at this time. Therefore we
+ # enable this only for automation builds, which only require Linux and
+ # Windows host binaries.
+ self._download_host_bins = self._substs.get('MOZ_AUTOMATION', False)
self._defines = defines
self._tree = tree
self._job = job or self._guess_artifact_job()
self._log = log
self._hg = hg
self._git = git
self._cache_dir = cache_dir
self._skip_cache = skip_cache
self._topsrcdir = topsrcdir
try:
self._artifact_job = get_job_details(self._job, log=self._log,
download_symbols=self._download_symbols,
+ download_host_bins=self._download_host_bins,
substs=self._substs)
except KeyError:
self.log(logging.INFO, 'artifact',
{'job': self._job},
'Unknown job {job}')
raise KeyError("Unknown job")
self._task_cache = TaskCache(self._cache_dir, log=self._log, skip_cache=self._skip_cache)
@@ -1046,17 +1085,19 @@ class Artifacts(object):
if not count:
raise Exception('Could not find any candidate pushheads in the last {num} revisions.\n'
'Search started with {rev}, which must be known to Mozilla automation.\n\n'
'see https://developer.mozilla.org/en-US/docs/Artifact_builds'.format(
rev=last_revs[0], num=NUM_PUSHHEADS_TO_QUERY_PER_PARENT))
def find_pushhead_artifacts(self, task_cache, job, tree, pushhead):
try:
- urls = task_cache.artifact_urls(tree, job, pushhead, self._download_symbols)
+ urls = task_cache.artifact_urls(tree, job, pushhead,
+ self._download_symbols,
+ self._download_host_bins)
except ValueError:
return None
if urls:
self.log(logging.INFO, 'artifact',
{'pushhead': pushhead,
'tree': tree},
'Installing from remote pushhead {pushhead} on {tree}')
return urls