Bug 1382697 - Use mozpack for I/O when processing jar.mn files; r?glandium
jar.py predates mozpack. mozpack does the file management part of
what jar.py does only better. (For the most part.)
This commits converts jar.py to use mozpack for I/O.
Essentially, we populate an InstallManifest instead of doing I/O at
each entry time. Then once all entries are processed, we
instantiate a FileCopier and do all I/O in bulk.
A side-effect of this change is that we no longer create links on
Windows. This is slightly unfortunate. But mozpack should soon gain
hard link support. So this regression won't last for long.
There is still room to optimize this code: we currently process an
InstallManifest per jar.mn section. It should be possible to
consolidate down to a single InstallManifest. This optimization
is deferred to another time.
A lot of test code to support links on Windows was deleted. It
will likely be reborn as part of mozpack some day.
MozReview-Commit-ID: 4ed4SKeJ8VW
--- a/python/mozbuild/mozbuild/jar.py
+++ b/python/mozbuild/mozbuild/jar.py
@@ -8,39 +8,35 @@ processing jar.mn files.
See the documentation for jar.mn on MDC for further details on the format.
'''
from __future__ import absolute_import
import io
import sys
import os
-import errno
import re
import logging
-from time import localtime
from mozbuild.preprocessor import Preprocessor
from mozbuild.action.buildlist import addEntriesToListFile
-from mozpack.files import FileFinder
+from mozpack.copier import (
+ FileCopier,
+)
+from mozpack.files import (
+ FileFinder,
+)
+from mozpack.manifests import (
+ InstallManifest,
+)
import mozpack.path as mozpath
-if sys.platform == 'win32':
- from ctypes import windll, WinError
- CreateHardLink = windll.kernel32.CreateHardLinkA
__all__ = ['JarMaker']
-def getModTime(aPath):
- if not os.path.isfile(aPath):
- return 0
- mtime = os.stat(aPath).st_mtime
- return localtime(mtime)
-
-
class JarManifestEntry(object):
def __init__(self, output, source, is_locale=False, preprocess=False):
self.output = output
self.source = source
self.is_locale = is_locale
self.preprocess = preprocess
@@ -330,30 +326,35 @@ class JarMaker(object):
'''
# chromebasepath is used for chrome registration manifests
# {0} is getting replaced with chrome/ for chrome.manifest, and with
# an empty string for jarfile.manifest
chromebasepath = '{0}%s/' % os.path.basename(jarinfo.name)
- jarfile = os.path.join(jardir, jarinfo.base, jarinfo.name)
- outHelper = getattr(self, 'OutputHelper_'
- + self.outputFormat)(jarfile)
+ dest_path = os.path.join(jardir, jarinfo.base, jarinfo.name)
if jarinfo.relativesrcdir:
self.localedirs = self.generateLocaleDirs(jarinfo.relativesrcdir)
+ manifest = InstallManifest()
+
for e in jarinfo.entries:
- self._processEntryLine(e, outHelper)
+ self._processEntryLine(manifest, e)
+
+ if len(manifest):
+ copier = FileCopier()
+ manifest.populate_registry(copier)
+ copier.copy(dest_path, remove_unaccounted=False)
self.finalizeJar(jardir, jarinfo.base, jarinfo.name, chromebasepath,
jarinfo.chrome_manifests)
- def _processEntryLine(self, e, outHelper):
+ def _processEntryLine(self, manifest, e):
out = e.output
src = e.source
# pick the right sourcedir -- l10n, topsrc or src
if e.is_locale:
src_base = self.localedirs
elif src.startswith('/'):
@@ -384,17 +385,17 @@ class JarMaker(object):
continue
emitted.add(reduced_path)
e = JarManifestEntry(
mozpath.join(out, reduced_path),
path,
is_locale=e.is_locale,
preprocess=e.preprocess,
)
- self._processEntryLine(e, outHelper)
+ self._processEntryLine(manifest, e)
return
# check if the source file exists
realsrc = None
for _srcdir in src_base:
if os.path.isfile(os.path.join(_srcdir, src)):
realsrc = os.path.join(_srcdir, src)
break
@@ -412,87 +413,23 @@ class JarMaker(object):
with open(realsrc) as fh:
pp = self.pp.clone()
if src[-4:] == '.css':
pp.setMarker('%')
pp.out = content
pp.do_include(fh)
pp.failUnused(realsrc)
- with outHelper.getOutput(out) as fh:
- fh.write(content.getvalue())
-
+ manifest.add_content(content.getvalue(), out)
return
- # copy or symlink if newer
-
- if getModTime(realsrc) > outHelper.getDestModTime(e.output):
- if self.outputFormat == 'symlink':
- outHelper.symlink(realsrc, out)
- return
-
- with outHelper.getOutput(out) as ofh:
- with open(realsrc, 'rb') as ifh:
- ofh.write(ifh.read())
-
- class OutputHelper_flat(object):
- '''Provide getDestModTime and getOutput for a given flat
- output directory. The helper method ensureDirFor is used by
- the symlink subclass.
- '''
-
- def __init__(self, basepath):
- self.basepath = basepath
-
- def getDestModTime(self, aPath):
- return getModTime(os.path.join(self.basepath, aPath))
-
- def getOutput(self, name):
- out = self.ensureDirFor(name)
-
- # remove previous link or file
- try:
- os.remove(out)
- except OSError, e:
- if e.errno != errno.ENOENT:
- raise
- return open(out, 'wb')
-
- def ensureDirFor(self, name):
- out = os.path.join(self.basepath, name)
- outdir = os.path.dirname(out)
- if not os.path.isdir(outdir):
- try:
- os.makedirs(outdir)
- except OSError, error:
- if error.errno != errno.EEXIST:
- raise
- return out
-
- class OutputHelper_symlink(OutputHelper_flat):
- '''Subclass of OutputHelper_flat that provides a helper for
- creating a symlink including creating the parent directories.
- '''
-
- def symlink(self, src, dest):
- out = self.ensureDirFor(dest)
-
- # remove previous link or file
- try:
- os.remove(out)
- except OSError, e:
- if e.errno != errno.ENOENT:
- raise
- if sys.platform != 'win32':
- os.symlink(src, out)
- else:
- # On Win32, use ctypes to create a hardlink
- rv = CreateHardLink(out, src, None)
- if rv == 0:
- raise WinError()
+ if self.outputFormat == 'symlink':
+ manifest.add_symlink(realsrc, out)
+ else:
+ manifest.add_copy(realsrc, out)
def main(args=None):
args = args or sys.argv
jm = JarMaker()
p = jm.getCommandLineParser()
(options, args) = p.parse_args(args)
jm.outputFormat = options.f
--- a/python/mozbuild/mozbuild/test/test_jarmaker.py
+++ b/python/mozbuild/mozbuild/test/test_jarmaker.py
@@ -10,112 +10,28 @@ from filecmp import dircmp
from tempfile import mkdtemp
from shutil import rmtree, copy2
from StringIO import StringIO
import mozunit
from mozbuild.jar import JarMaker
-if sys.platform == "win32":
- import ctypes
- from ctypes import POINTER, WinError
- DWORD = ctypes.c_ulong
- LPDWORD = POINTER(DWORD)
- HANDLE = ctypes.c_void_p
- GENERIC_READ = 0x80000000
- FILE_SHARE_READ = 0x00000001
- OPEN_EXISTING = 3
- MAX_PATH = 260
-
- class FILETIME(ctypes.Structure):
- _fields_ = [("dwLowDateTime", DWORD),
- ("dwHighDateTime", DWORD)]
-
- class BY_HANDLE_FILE_INFORMATION(ctypes.Structure):
- _fields_ = [("dwFileAttributes", DWORD),
- ("ftCreationTime", FILETIME),
- ("ftLastAccessTime", FILETIME),
- ("ftLastWriteTime", FILETIME),
- ("dwVolumeSerialNumber", DWORD),
- ("nFileSizeHigh", DWORD),
- ("nFileSizeLow", DWORD),
- ("nNumberOfLinks", DWORD),
- ("nFileIndexHigh", DWORD),
- ("nFileIndexLow", DWORD)]
-
- # http://msdn.microsoft.com/en-us/library/aa363858
- CreateFile = ctypes.windll.kernel32.CreateFileA
- CreateFile.argtypes = [ctypes.c_char_p, DWORD, DWORD, ctypes.c_void_p,
- DWORD, DWORD, HANDLE]
- CreateFile.restype = HANDLE
-
- # http://msdn.microsoft.com/en-us/library/aa364952
- GetFileInformationByHandle = ctypes.windll.kernel32.GetFileInformationByHandle
- GetFileInformationByHandle.argtypes = [HANDLE, POINTER(BY_HANDLE_FILE_INFORMATION)]
- GetFileInformationByHandle.restype = ctypes.c_int
+def symlinks_supported():
+ return hasattr(os, 'symlink')
- # http://msdn.microsoft.com/en-us/library/aa364996
- GetVolumePathName = ctypes.windll.kernel32.GetVolumePathNameA
- GetVolumePathName.argtypes = [ctypes.c_char_p, ctypes.c_char_p, DWORD]
- GetVolumePathName.restype = ctypes.c_int
-
- # http://msdn.microsoft.com/en-us/library/aa364993
- GetVolumeInformation = ctypes.windll.kernel32.GetVolumeInformationA
- GetVolumeInformation.argtypes = [ctypes.c_char_p, ctypes.c_char_p, DWORD,
- LPDWORD, LPDWORD, LPDWORD, ctypes.c_char_p,
- DWORD]
- GetVolumeInformation.restype = ctypes.c_int
-
-def symlinks_supported(path):
- if sys.platform == "win32":
- # Add 1 for a trailing backslash if necessary, and 1 for the terminating
- # null character.
- volpath = ctypes.create_string_buffer(len(path) + 2)
- rv = GetVolumePathName(path, volpath, len(volpath))
- if rv == 0:
- raise WinError()
-
- fsname = ctypes.create_string_buffer(MAX_PATH + 1)
- rv = GetVolumeInformation(volpath, None, 0, None, None, None, fsname,
- len(fsname))
- if rv == 0:
- raise WinError()
-
- # Return true only if the fsname is NTFS
- return fsname.value == "NTFS"
- else:
- return True
-
-def _getfileinfo(path):
- """Return information for the given file. This only works on Windows."""
- fh = CreateFile(path, GENERIC_READ, FILE_SHARE_READ, None, OPEN_EXISTING, 0, None)
- if fh is None:
- raise WinError()
- info = BY_HANDLE_FILE_INFORMATION()
- rv = GetFileInformationByHandle(fh, info)
- if rv == 0:
- raise WinError()
- return info
def is_symlink_to(dest, src):
- if sys.platform == "win32":
- # Check if both are on the same volume and have the same file ID
- destinfo = _getfileinfo(dest)
- srcinfo = _getfileinfo(src)
- return (destinfo.dwVolumeSerialNumber == srcinfo.dwVolumeSerialNumber and
- destinfo.nFileIndexHigh == srcinfo.nFileIndexHigh and
- destinfo.nFileIndexLow == srcinfo.nFileIndexLow)
- else:
- # Read the link and check if it is correct
- if not os.path.islink(dest):
- return False
- target = os.path.abspath(os.readlink(dest))
- abssrc = os.path.abspath(src)
- return target == abssrc
+ # Read the link and check if it is correct
+ if not os.path.islink(dest):
+ return False
+ target = os.path.abspath(os.readlink(dest))
+ abssrc = os.path.abspath(src)
+ return target == abssrc
+
class _TreeDiff(dircmp):
"""Helper to report rich results on difference between two directories.
"""
def _fillDiff(self, dc, rv, basepath="{0}"):
rv['right_only'] += map(lambda l: basepath.format(l), dc.right_only)
rv['left_only'] += map(lambda l: basepath.format(l), dc.left_only)
rv['diff_files'] += map(lambda l: basepath.format(l), dc.diff_files)
@@ -206,17 +122,17 @@ class TestJarMaker(unittest.TestCase):
self._create_simple_setup()
# call JarMaker
rv = self._jar_and_compare(os.path.join(self.srcdir,'jar.mn'),
sourcedirs = [self.srcdir])
self.assertTrue(not rv, rv)
def test_a_simple_symlink(self):
'''Test a simple jar.mn with a symlink'''
- if not symlinks_supported(self.srcdir):
+ if not symlinks_supported():
raise unittest.SkipTest('symlinks not supported')
self._create_simple_setup()
jm = JarMaker(outputFormat='symlink')
jm.sourcedirs = [self.srcdir]
jm.topsourcedir = self.srcdir
jm.makeJar(os.path.join(self.srcdir,'jar.mn'), self.builddir)
# All we do is check that srcdir/bar points to builddir/chrome/test/dir/foo
@@ -254,17 +170,17 @@ class TestJarMaker(unittest.TestCase):
self._create_wildcard_setup()
# call JarMaker
rv = self._jar_and_compare(os.path.join(self.srcdir,'jar.mn'),
sourcedirs = [self.srcdir])
self.assertTrue(not rv, rv)
def test_a_wildcard_symlink(self):
'''Test a wildcard in jar.mn with symlinks'''
- if not symlinks_supported(self.srcdir):
+ if not symlinks_supported():
raise unittest.SkipTest('symlinks not supported')
self._create_wildcard_setup()
jm = JarMaker(outputFormat='symlink')
jm.sourcedirs = [self.srcdir]
jm.topsourcedir = self.srcdir
jm.makeJar(os.path.join(self.srcdir,'jar.mn'), self.builddir)