--- a/python/mozbuild/mozbuild/artifacts.py
+++ b/python/mozbuild/mozbuild/artifacts.py
@@ -88,26 +88,28 @@ MAX_CACHED_ARTIFACTS = 6
# Downloaded artifacts are cached, and a subset of their contents extracted for
# easy installation. This is most noticeable on Mac OS X: since mounting and
# copying from DMG files is very slow, we extract the desired binaries to a
# separate archive for fast re-installation.
PROCESSED_SUFFIX = '.processed.jar'
class ArtifactJob(object):
# These are a subset of TEST_HARNESS_BINS in testing/mochitest/Makefile.in.
- test_artifact_patterns = {
+ test_binary_patterns = {
'bin/BadCertServer',
'bin/GenerateOCSPResponse',
'bin/OCSPStaplingServer',
'bin/certutil',
'bin/fileid',
'bin/pk12util',
'bin/ssltunnel',
'bin/xpcshell',
}
+ test_plugin_pattern = 'bin/plugins/*'
+
# We can tell our input is a test archive by this suffix, which happens to
# be the same across platforms.
_test_archive_suffix = '.common.tests.zip'
def __init__(self, package_re, tests_re, log=None):
self._package_re = re.compile(package_re)
self._tests_re = None
if tests_re:
@@ -145,44 +147,55 @@ class ArtifactJob(object):
raise NotImplementedError("Subclasses must specialize process_package_artifact!")
def process_tests_artifact(self, filename, processed_filename):
added_entry = False
with JarWriter(file=processed_filename, optimize=False, compress_level=5) as writer:
reader = JarReader(filename)
for filename, entry in reader.entries.iteritems():
- if filename in self.test_artifact_patterns:
+ if filename in self.test_binary_patterns:
basename = mozpath.basename(filename)
self.log(logging.INFO, 'artifact',
{'basename': basename},
'Adding {basename} to processed archive')
mode = entry['external_attr'] >> 16
- writer.add(basename.encode('utf-8'), reader[filename], mode=mode)
+ destpath = mozpath.join('bin', basename)
+ writer.add(destpath.encode('utf-8'), reader[filename], mode=mode)
added_entry = True
+ elif mozpath.match(filename, self.test_plugin_pattern):
+ # We keep the full paths to plugin files after trimming off the
+ # leading "bin/".
+ destpath = mozpath.relpath(filename, 'bin')
+ self.log(logging.INFO, 'artifact',
+ {'filename': filename},
+ 'Adding {filename} to processed archive')
+ mode = entry['external_attr'] >> 16
+ writer.add(destpath.encode('utf-8'), reader[filename], mode=mode)
if not added_entry:
raise ValueError('Archive format changed! No pattern from "{patterns}"'
'matched an archive path.'.format(
- patterns=LinuxArtifactJob.test_artifact_patterns))
+ patterns=LinuxArtifactJob.test_binary_patterns))
class AndroidArtifactJob(ArtifactJob):
def process_artifact(self, filename, processed_filename):
# Extract all .so files into the root, which will get copied into dist/bin.
with JarWriter(file=processed_filename, optimize=False, compress_level=5) as writer:
for f in JarReader(filename):
if not f.filename.endswith('.so') and \
not f.filename in ('platform.ini', 'application.ini'):
continue
basename = os.path.basename(f.filename)
self.log(logging.INFO, 'artifact',
{'basename': basename},
'Adding {basename} to processed archive')
+ basename = mozpath.join('bin', basename)
writer.add(basename.encode('utf-8'), f)
class LinuxArtifactJob(ArtifactJob):
package_artifact_patterns = {
'firefox/application.ini',
'firefox/crashreporter',
@@ -205,17 +218,18 @@ class LinuxArtifactJob(ArtifactJob):
if not f.isfile():
continue
if not any(mozpath.match(f.name, p) for p in self.package_artifact_patterns):
continue
# We strip off the relative "firefox/" bit from the path,
# but otherwise preserve it.
- destpath = mozpath.relpath(f.name, "firefox")
+ destpath = mozpath.join('bin',
+ mozpath.relpath(f.name, "firefox"))
self.log(logging.INFO, 'artifact',
{'destpath': destpath},
'Adding {destpath} to processed archive')
writer.add(destpath.encode('utf-8'), reader.extractfile(f), mode=f.mode)
added_entry = True
if not added_entry:
raise ValueError('Archive format changed! No pattern from "{patterns}" '
@@ -290,26 +304,28 @@ class MacArtifactJob(ArtifactJob):
with JarWriter(file=processed_filename, optimize=False, compress_level=5) as writer:
root, paths = paths_no_keep_path
finder = FileFinder(mozpath.join(source, root))
for path in paths:
for p, f in finder.find(path):
self.log(logging.INFO, 'artifact',
{'path': path},
'Adding {path} to processed archive')
- writer.add(os.path.basename(p).encode('utf-8'), f, mode=os.stat(mozpath.join(finder.base, p)).st_mode)
+ destpath = mozpath.join('bin', os.path.basename(p))
+ writer.add(destpath.encode('utf-8'), f, mode=os.stat(mozpath.join(finder.base, p)).st_mode)
root, paths = paths_keep_path
finder = FileFinder(mozpath.join(source, root))
for path in paths:
for p, f in finder.find(path):
self.log(logging.INFO, 'artifact',
{'path': path},
'Adding {path} to processed archive')
- writer.add(p.encode('utf-8'), f, mode=os.stat(mozpath.join(finder.base, p)).st_mode)
+ destpath = mozpath.join('bin', p)
+ writer.add(destpath.encode('utf-8'), f, mode=os.stat(mozpath.join(finder.base, p)).st_mode)
finally:
try:
shutil.rmtree(tempdir)
except (OSError, IOError):
self.log(logging.WARN, 'artifact',
{'tempdir': tempdir},
'Unable to delete {tempdir}')
@@ -320,17 +336,17 @@ class WinArtifactJob(ArtifactJob):
package_artifact_patterns = {
'firefox/dependentlibs.list',
'firefox/platform.ini',
'firefox/application.ini',
'firefox/**/*.dll',
'firefox/*.exe',
}
# These are a subset of TEST_HARNESS_BINS in testing/mochitest/Makefile.in.
- test_artifact_patterns = {
+ test_binary_patterns = {
'bin/BadCertServer.exe',
'bin/GenerateOCSPResponse.exe',
'bin/OCSPStaplingServer.exe',
'bin/certutil.exe',
'bin/fileid.exe',
'bin/pk12util.exe',
'bin/ssltunnel.exe',
'bin/xpcshell.exe',
@@ -340,16 +356,17 @@ class WinArtifactJob(ArtifactJob):
added_entry = False
with JarWriter(file=processed_filename, optimize=False, compress_level=5) as writer:
for f in JarReader(filename):
if not any(mozpath.match(f.filename, p) for p in self.package_artifact_patterns):
continue
# strip off the relative "firefox/" bit from the path:
basename = mozpath.relpath(f.filename, "firefox")
+ basename = mozpath.join('bin', basename)
self.log(logging.INFO, 'artifact',
{'basename': basename},
'Adding {basename} to processed archive')
writer.add(basename.encode('utf-8'), f)
added_entry = True
if not added_entry:
raise ValueError('Archive format changed! No pattern from "{patterns}"'
@@ -659,17 +676,17 @@ class Artifacts(object):
self._pushhead_cache = PushHeadCache(self._hg, self._cache_dir, log=self._log)
self._task_cache = TaskCache(self._cache_dir, log=self._log)
self._artifact_cache = ArtifactCache(self._cache_dir, log=self._log)
def log(self, *args, **kwargs):
if self._log:
self._log(*args, **kwargs)
- def install_from_file(self, filename, bindir, install_callback=None):
+ def install_from_file(self, filename, distdir, install_callback=None):
self.log(logging.INFO, 'artifact',
{'filename': filename},
'Installing from {filename}')
# Do we need to post-process?
processed_filename = filename + PROCESSED_SUFFIX
if not os.path.exists(processed_filename):
self.log(logging.INFO, 'artifact',
@@ -680,48 +697,48 @@ class Artifacts(object):
'Writing processed {processed_filename}')
self._artifact_job.process_artifact(filename, processed_filename)
self.log(logging.INFO, 'artifact',
{'processed_filename': processed_filename},
'Installing from processed {processed_filename}')
# Copy all .so files, avoiding modification where possible.
- ensureParentDir(mozpath.join(bindir, '.dummy'))
+ ensureParentDir(mozpath.join(distdir, '.dummy'))
with zipfile.ZipFile(processed_filename) as zf:
for info in zf.infolist():
if info.filename.endswith('.ini'):
continue
- n = mozpath.join(bindir, info.filename)
+ n = mozpath.join(distdir, info.filename)
fh = FileAvoidWrite(n, mode='rb')
shutil.copyfileobj(zf.open(info), fh)
file_existed, file_updated = fh.close()
self.log(logging.INFO, 'artifact',
{'updating': 'Updating' if file_updated else 'Not updating', 'filename': n},
'{updating} {filename}')
if not file_existed or file_updated:
# Libraries and binaries may need to be marked executable,
# depending on platform.
perms = info.external_attr >> 16 # See http://stackoverflow.com/a/434689.
perms |= stat.S_IWUSR | stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH # u+w, a+r.
os.chmod(n, perms)
if install_callback:
install_callback(info.filename, file_existed, file_updated)
return 0
- def install_from_url(self, url, bindir, install_callback=None):
+ def install_from_url(self, url, distdir, install_callback=None):
self.log(logging.INFO, 'artifact',
{'url': url},
'Installing from {url}')
with self._artifact_cache as artifact_cache: # The with block handles persistence.
filename = artifact_cache.fetch(url)
- return self.install_from_file(filename, bindir, install_callback=install_callback)
+ return self.install_from_file(filename, distdir, install_callback=install_callback)
- def install_from_hg(self, revset, bindir, install_callback=None):
+ def install_from_hg(self, revset, distdir, install_callback=None):
if not revset:
revset = '.'
if len(revset) != 40:
revset = subprocess.check_output([self._hg, 'log', '--template', '{node}\n', '-r', revset]).strip()
if len(revset.split('\n')) != 1:
raise ValueError('hg revision specification must resolve to exactly one commit')
self.log(logging.INFO, 'artifact',
@@ -740,39 +757,39 @@ class Artifacts(object):
self.log(logging.INFO, 'artifact',
{'pushhead': pushhead},
'Installing from remote pushhead {pushhead}')
break
except ValueError:
pass
if urls:
for url in urls:
- if self.install_from_url(url, bindir, install_callback=install_callback):
+ if self.install_from_url(url, distdir, install_callback=install_callback):
return 1
return 0
self.log(logging.ERROR, 'artifact',
{'revset': revset},
'No built artifacts for {revset} found.')
return 1
- def install_from(self, source, bindir, install_callback=None):
- """Install artifacts from a ``source`` into the given ``bindir``.
+ def install_from(self, source, distdir, install_callback=None):
+ """Install artifacts from a ``source`` into the given ``distdir``.
If ``callback`` is given, it is called once with arguments ``(path,
existed, updated)``, where ``path`` is the file path written relative
- to ``bindir``; ``existed`` is a boolean indicating whether the file
+ to ``distdir``; ``existed`` is a boolean indicating whether the file
existed; and ``updated`` is a boolean indicating whether the file was
updated.
"""
if source and os.path.isfile(source):
- return self.install_from_file(source, bindir, install_callback=install_callback)
+ return self.install_from_file(source, distdir, install_callback=install_callback)
elif source and urlparse.urlparse(source).scheme:
- return self.install_from_url(source, bindir, install_callback=install_callback)
+ return self.install_from_url(source, distdir, install_callback=install_callback)
else:
- return self.install_from_hg(source, bindir, install_callback=install_callback)
+ return self.install_from_hg(source, distdir, install_callback=install_callback)
def print_last(self):
self.log(logging.INFO, 'artifact',
{},
'Printing last used artifact details.')
self._pushhead_cache.print_last()
self._task_cache.print_last()
self._artifact_cache.print_last()