--- a/python/mozbuild/mozbuild/artifacts.py
+++ b/python/mozbuild/mozbuild/artifacts.py
@@ -98,16 +98,20 @@ MAX_CACHED_TASKS = 400 # Number of push
MAX_CACHED_ARTIFACTS = 6
# Downloaded artifacts are cached, and a subset of their contents extracted for
# easy installation. This is most noticeable on Mac OS X: since mounting and
# copying from DMG files is very slow, we extract the desired binaries to a
# separate archive for fast re-installation.
PROCESSED_SUFFIX = '.processed.jar'
+# Trailing filename component of artifacts containing all binary files.
+BINARY_ARTIFACT_SUFFIX = 'binary-artifacts.zip'
+
+
class ArtifactJob(object):
# These are a subset of TEST_HARNESS_BINS in testing/mochitest/Makefile.in.
# Each item is a pair of (pattern, (src_prefix, dest_prefix), where src_prefix
# is the prefix of the pattern relevant to its location in the archive, and
# dest_prefix is the prefix to be added that will yield the final path relative
# to dist/.
test_artifact_patterns = {
('bin/BadCertServer', ('bin', 'bin')),
@@ -142,30 +146,44 @@ class ArtifactJob(object):
if self._log:
self._log(*args, **kwargs)
def find_candidate_artifacts(self, artifacts):
# TODO: Handle multiple artifacts, taking the latest one.
tests_artifact = None
for artifact in artifacts:
name = artifact['name']
- if self._package_re and self._package_re.match(name):
- yield name
- elif self._tests_re and self._tests_re.match(name):
- tests_artifact = name
- yield name
+ # Binary artifacts take precedence over package and tests artifacts.
+ # Only emit binary artifact if wanted.
+ if self._binary_artifacts_re:
+ if self._binary_artifacts_re.match(name):
+ yield name
+ continue
else:
- self.log(logging.DEBUG, 'artifact',
- {'name': name},
- 'Not yielding artifact named {name} as a candidate artifact')
+ if self._package_re and self._package_re.match(name):
+ yield name
+ continue
+ elif self._tests_re and self._tests_re.match(name):
+ tests_artifact = name
+ yield name
+ continue
+
+ self.log(logging.DEBUG, 'artifact',
+ {'name': name},
+ 'Not yielding artifact named {name} as a candidate artifact')
+
if self._tests_re and not tests_artifact:
raise ValueError('Expected tests archive matching "{re}", but '
'found none!'.format(re=self._tests_re))
def process_artifact(self, filename, processed_filename):
+ # Binary artifacts don't require any processing. Just return.
+ if filename.endswith(BINARY_ARTIFACT_SUFFIX):
+ return
+
if filename.endswith(ArtifactJob._test_archive_suffix) and self._tests_re:
return self.process_tests_artifact(filename, processed_filename)
return self.process_package_artifact(filename, processed_filename)
def process_package_artifact(self, filename, processed_filename):
raise NotImplementedError("Subclasses must specialize process_package_artifact!")
def process_tests_artifact(self, filename, processed_filename):
@@ -247,108 +265,17 @@ class LinuxArtifactJob(ArtifactJob):
if not added_entry:
raise ValueError('Archive format changed! No pattern from "{patterns}" '
'matched an archive path.'.format(
patterns=LinuxArtifactJob.package_artifact_patterns))
class MacArtifactJob(ArtifactJob):
- def process_package_artifact(self, filename, processed_filename):
- tempdir = tempfile.mkdtemp()
- try:
- self.log(logging.INFO, 'artifact',
- {'tempdir': tempdir},
- 'Unpacking DMG into {tempdir}')
- mozinstall.install(filename, tempdir) # Doesn't handle already mounted DMG files nicely:
-
- # InstallError: Failed to install "/Users/nalexander/.mozbuild/package-frontend/b38eeeb54cdcf744-firefox-44.0a1.en-US.mac.dmg (local variable 'appDir' referenced before assignment)"
-
- # File "/Users/nalexander/Mozilla/gecko/mobile/android/mach_commands.py", line 250, in artifact_install
- # return artifacts.install_from(source, self.distdir)
- # File "/Users/nalexander/Mozilla/gecko/python/mozbuild/mozbuild/artifacts.py", line 457, in install_from
- # return self.install_from_hg(source, distdir)
- # File "/Users/nalexander/Mozilla/gecko/python/mozbuild/mozbuild/artifacts.py", line 445, in install_from_hg
- # return self.install_from_url(url, distdir)
- # File "/Users/nalexander/Mozilla/gecko/python/mozbuild/mozbuild/artifacts.py", line 418, in install_from_url
- # return self.install_from_file(filename, distdir)
- # File "/Users/nalexander/Mozilla/gecko/python/mozbuild/mozbuild/artifacts.py", line 336, in install_from_file
- # mozinstall.install(filename, tempdir)
- # File "/Users/nalexander/Mozilla/gecko/objdir-dce/_virtualenv/lib/python2.7/site-packages/mozinstall/mozinstall.py", line 117, in install
- # install_dir = _install_dmg(src, dest)
- # File "/Users/nalexander/Mozilla/gecko/objdir-dce/_virtualenv/lib/python2.7/site-packages/mozinstall/mozinstall.py", line 261, in _install_dmg
- # subprocess.call('hdiutil detach %s -quiet' % appDir,
-
- bundle_dirs = glob.glob(mozpath.join(tempdir, '*.app'))
- if len(bundle_dirs) != 1:
- raise ValueError('Expected one source bundle, found: {}'.format(bundle_dirs))
- [source] = bundle_dirs
-
- # These get copied into dist/bin without the path, so "root/a/b/c" -> "dist/bin/c".
- paths_no_keep_path = ('Contents/MacOS', [
- 'crashreporter.app/Contents/MacOS/crashreporter',
- 'firefox',
- 'firefox-bin',
- 'libfreebl3.dylib',
- 'liblgpllibs.dylib',
- # 'liblogalloc.dylib',
- 'libmozglue.dylib',
- 'libnss3.dylib',
- 'libnssckbi.dylib',
- 'libnssdbm3.dylib',
- 'libplugin_child_interpose.dylib',
- # 'libreplace_jemalloc.dylib',
- # 'libreplace_malloc.dylib',
- 'libsoftokn3.dylib',
- 'plugin-container.app/Contents/MacOS/plugin-container',
- 'updater.app/Contents/MacOS/updater',
- # 'xpcshell',
- 'XUL',
- ])
-
- # These get copied into dist/bin with the path, so "root/a/b/c" -> "dist/bin/a/b/c".
- paths_keep_path = ('Contents/Resources', [
- 'browser/components/libbrowsercomps.dylib',
- 'dependentlibs.list',
- # 'firefox',
- 'gmp-clearkey/0.1/libclearkey.dylib',
- # 'gmp-fake/1.0/libfake.dylib',
- # 'gmp-fakeopenh264/1.0/libfakeopenh264.dylib',
- ])
-
- with JarWriter(file=processed_filename, optimize=False, compress_level=5) as writer:
- root, paths = paths_no_keep_path
- finder = FileFinder(mozpath.join(source, root))
- for path in paths:
- for p, f in finder.find(path):
- self.log(logging.INFO, 'artifact',
- {'path': path},
- 'Adding {path} to processed archive')
- destpath = mozpath.join('bin', os.path.basename(p))
- writer.add(destpath.encode('utf-8'), f, mode=os.stat(mozpath.join(finder.base, p)).st_mode)
-
- root, paths = paths_keep_path
- finder = FileFinder(mozpath.join(source, root))
- for path in paths:
- for p, f in finder.find(path):
- self.log(logging.INFO, 'artifact',
- {'path': path},
- 'Adding {path} to processed archive')
- destpath = mozpath.join('bin', p)
- writer.add(destpath.encode('utf-8'), f, mode=os.stat(mozpath.join(finder.base, p)).st_mode)
-
- finally:
- try:
- shutil.rmtree(tempdir)
- except (OSError, IOError):
- self.log(logging.WARN, 'artifact',
- {'tempdir': tempdir},
- 'Unable to delete {tempdir}')
- pass
-
+ """This class now no-ops since binary artifact zips contain all we need."""
class WinArtifactJob(ArtifactJob):
package_artifact_patterns = {
'firefox/dependentlibs.list',
'firefox/platform.ini',
'firefox/application.ini',
'firefox/**/*.dll',
'firefox/*.exe',
@@ -404,22 +331,19 @@ JOB_DETAILS = {
'tests_re': 'public/build/firefox-(.*)\.common\.tests\.zip'}),
'linux64': (LinuxArtifactJob, {
'package_re': 'public/build/firefox-(.*)\.linux-x86_64\.tar\.bz2',
'tests_re': 'public/build/firefox-(.*)\.common\.tests\.zip'}),
'linux64-debug': (LinuxArtifactJob, {
'package_re': 'public/build/firefox-(.*)\.linux-x86_64\.tar\.bz2',
'tests_re': 'public/build/firefox-(.*)\.common\.tests\.zip'}),
'macosx64': (MacArtifactJob, {
- 'binary_artifacts_re': 'public/build/firefox-(.*)\.binary-artifacts.zip',
- 'package_re': 'public/build/firefox-(.*)\.mac\.dmg',
- 'tests_re': 'public/build/firefox-(.*)\.common\.tests\.zip'}),
+ 'binary_artifacts_re': 'public/build/firefox-(.*)\.mac\.binary-artifacts.zip'}),
'macosx64-debug': (MacArtifactJob, {
- 'package_re': 'public/build/firefox-(.*)\.mac64\.dmg',
- 'tests_re': 'public/build/firefox-(.*)\.common\.tests\.zip'}),
+ 'binary_artifacts_re': 'public/build/firefox-(.*)\.mac64\.binary-artifacts.zip'}),
'win32': (WinArtifactJob, {
'package_re': 'public/build/firefox-(.*)\.win32.zip',
'tests_re': 'public/build/firefox-(.*)\.common\.tests\.zip'}),
'win32-debug': (WinArtifactJob, {
'package_re': 'public/build/firefox-(.*)\.win32.zip',
'tests_re': 'public/build/firefox-(.*)\.common\.tests\.zip'}),
'win64': (WinArtifactJob, {
'package_re': 'public/build/firefox-(.*)\.win64.zip',
@@ -924,40 +848,42 @@ class Artifacts(object):
def install_from_file(self, filename, distdir):
self.log(logging.INFO, 'artifact',
{'filename': filename},
'Installing from {filename}')
# Do we need to post-process?
processed_filename = filename + PROCESSED_SUFFIX
+ is_binary_artifact = filename.endswith(BINARY_ARTIFACT_SUFFIX)
+ extract_filename = filename if is_binary_artifact else processed_filename
if self._skip_cache and os.path.exists(processed_filename):
self.log(logging.DEBUG, 'artifact',
{'path': processed_filename},
'Skipping cache: removing cached processed artifact {path}')
os.remove(processed_filename)
- if not os.path.exists(processed_filename):
+ if not is_binary_artifact and not os.path.exists(processed_filename):
self.log(logging.INFO, 'artifact',
{'filename': filename},
'Processing contents of {filename}')
self.log(logging.INFO, 'artifact',
{'processed_filename': processed_filename},
'Writing processed {processed_filename}')
self._artifact_job.process_artifact(filename, processed_filename)
self.log(logging.INFO, 'artifact',
{'processed_filename': processed_filename},
'Installing from processed {processed_filename}')
# Copy all .so files, avoiding modification where possible.
ensureParentDir(mozpath.join(distdir, '.dummy'))
- with zipfile.ZipFile(processed_filename) as zf:
+ with zipfile.ZipFile(extract_filename) as zf:
for info in zf.infolist():
if info.filename.endswith('.ini'):
continue
n = mozpath.join(distdir, info.filename)
fh = FileAvoidWrite(n, mode='rb')
shutil.copyfileobj(zf.open(info), fh)
file_existed, file_updated = fh.close()
self.log(logging.INFO, 'artifact',