Bug 1382697 - Use mozpack for I/O when processing jar.mn files; r?glandium draft
authorGregory Szorc <gps@mozilla.com>
Fri, 21 Jul 2017 22:40:01 -0700
changeset 613671 2544a1304096c9e7baa4ab04f594e1afb6a67244
parent 613655 55ce1dd5110e6709d9f2e0f30ace308eb4bf5248
child 613672 64f2a9a404205690fa6bb53dacf8471b0e4810c7
push id69832
push usergszorc@mozilla.com
push dateSat, 22 Jul 2017 06:27:51 +0000
reviewersglandium
bugs1382697
milestone56.0a1
Bug 1382697 - Use mozpack for I/O when processing jar.mn files; r?glandium jar.py predates mozpack. mozpack does the file management part of what jar.py does only better. (For the most part.) This commits converts jar.py to use mozpack for I/O. Essentially, we populate an InstallManifest instead of doing I/O at each entry time. Then once all entries are processed, we instantiate a FileCopier and do all I/O in bulk. A side-effect of this change is that we no longer create links on Windows. This is slightly unfortunate. But mozpack should soon gain hard link support. So this regression won't last for long. There is still room to optimize this code: we currently process an InstallManifest per jar.mn section. It should be possible to consolidate down to a single InstallManifest. This optimization is deferred to another time. A lot of test code to support links on Windows was deleted. It will likely be reborn as part of mozpack some day. MozReview-Commit-ID: 4ed4SKeJ8VW
python/mozbuild/mozbuild/jar.py
python/mozbuild/mozbuild/test/test_jarmaker.py
--- a/python/mozbuild/mozbuild/jar.py
+++ b/python/mozbuild/mozbuild/jar.py
@@ -8,39 +8,35 @@ processing jar.mn files.
 See the documentation for jar.mn on MDC for further details on the format.
 '''
 
 from __future__ import absolute_import
 
 import io
 import sys
 import os
-import errno
 import re
 import logging
-from time import localtime
 
 from mozbuild.preprocessor import Preprocessor
 from mozbuild.action.buildlist import addEntriesToListFile
-from mozpack.files import FileFinder
+from mozpack.copier import (
+    FileCopier,
+)
+from mozpack.files import (
+    FileFinder,
+)
+from mozpack.manifests import (
+    InstallManifest,
+)
 import mozpack.path as mozpath
-if sys.platform == 'win32':
-    from ctypes import windll, WinError
-    CreateHardLink = windll.kernel32.CreateHardLinkA
 
 __all__ = ['JarMaker']
 
 
-def getModTime(aPath):
-    if not os.path.isfile(aPath):
-        return 0
-    mtime = os.stat(aPath).st_mtime
-    return localtime(mtime)
-
-
 class JarManifestEntry(object):
     def __init__(self, output, source, is_locale=False, preprocess=False):
         self.output = output
         self.source = source
         self.is_locale = is_locale
         self.preprocess = preprocess
 
 
@@ -330,30 +326,35 @@ class JarMaker(object):
         '''
 
         # chromebasepath is used for chrome registration manifests
         # {0} is getting replaced with chrome/ for chrome.manifest, and with
         # an empty string for jarfile.manifest
 
         chromebasepath = '{0}%s/' % os.path.basename(jarinfo.name)
 
-        jarfile = os.path.join(jardir, jarinfo.base, jarinfo.name)
-        outHelper = getattr(self, 'OutputHelper_'
-                            + self.outputFormat)(jarfile)
+        dest_path = os.path.join(jardir, jarinfo.base, jarinfo.name)
 
         if jarinfo.relativesrcdir:
             self.localedirs = self.generateLocaleDirs(jarinfo.relativesrcdir)
 
+        manifest = InstallManifest()
+
         for e in jarinfo.entries:
-            self._processEntryLine(e, outHelper)
+            self._processEntryLine(manifest, e)
+
+        if len(manifest):
+            copier = FileCopier()
+            manifest.populate_registry(copier)
+            copier.copy(dest_path, remove_unaccounted=False)
 
         self.finalizeJar(jardir, jarinfo.base, jarinfo.name, chromebasepath,
                          jarinfo.chrome_manifests)
 
-    def _processEntryLine(self, e, outHelper):
+    def _processEntryLine(self, manifest, e):
         out = e.output
         src = e.source
 
         # pick the right sourcedir -- l10n, topsrc or src
 
         if e.is_locale:
             src_base = self.localedirs
         elif src.startswith('/'):
@@ -384,17 +385,17 @@ class JarMaker(object):
                         continue
                     emitted.add(reduced_path)
                     e = JarManifestEntry(
                         mozpath.join(out, reduced_path),
                         path,
                         is_locale=e.is_locale,
                         preprocess=e.preprocess,
                     )
-                    self._processEntryLine(e, outHelper)
+                    self._processEntryLine(manifest, e)
             return
 
         # check if the source file exists
         realsrc = None
         for _srcdir in src_base:
             if os.path.isfile(os.path.join(_srcdir, src)):
                 realsrc = os.path.join(_srcdir, src)
                 break
@@ -412,87 +413,23 @@ class JarMaker(object):
             with open(realsrc) as fh:
                 pp = self.pp.clone()
                 if src[-4:] == '.css':
                     pp.setMarker('%')
                 pp.out = content
                 pp.do_include(fh)
                 pp.failUnused(realsrc)
 
-            with outHelper.getOutput(out) as fh:
-                fh.write(content.getvalue())
-
+            manifest.add_content(content.getvalue(), out)
             return
 
-        # copy or symlink if newer
-
-        if getModTime(realsrc) > outHelper.getDestModTime(e.output):
-            if self.outputFormat == 'symlink':
-                outHelper.symlink(realsrc, out)
-                return
-
-            with outHelper.getOutput(out) as ofh:
-                with open(realsrc, 'rb') as ifh:
-                    ofh.write(ifh.read())
-
-    class OutputHelper_flat(object):
-        '''Provide getDestModTime and getOutput for a given flat
-        output directory. The helper method ensureDirFor is used by
-        the symlink subclass.
-        '''
-
-        def __init__(self, basepath):
-            self.basepath = basepath
-
-        def getDestModTime(self, aPath):
-            return getModTime(os.path.join(self.basepath, aPath))
-
-        def getOutput(self, name):
-            out = self.ensureDirFor(name)
-
-            # remove previous link or file
-            try:
-                os.remove(out)
-            except OSError, e:
-                if e.errno != errno.ENOENT:
-                    raise
-            return open(out, 'wb')
-
-        def ensureDirFor(self, name):
-            out = os.path.join(self.basepath, name)
-            outdir = os.path.dirname(out)
-            if not os.path.isdir(outdir):
-                try:
-                    os.makedirs(outdir)
-                except OSError, error:
-                    if error.errno != errno.EEXIST:
-                        raise
-            return out
-
-    class OutputHelper_symlink(OutputHelper_flat):
-        '''Subclass of OutputHelper_flat that provides a helper for
-        creating a symlink including creating the parent directories.
-        '''
-
-        def symlink(self, src, dest):
-            out = self.ensureDirFor(dest)
-
-            # remove previous link or file
-            try:
-                os.remove(out)
-            except OSError, e:
-                if e.errno != errno.ENOENT:
-                    raise
-            if sys.platform != 'win32':
-                os.symlink(src, out)
-            else:
-                # On Win32, use ctypes to create a hardlink
-                rv = CreateHardLink(out, src, None)
-                if rv == 0:
-                    raise WinError()
+        if self.outputFormat == 'symlink':
+            manifest.add_symlink(realsrc, out)
+        else:
+            manifest.add_copy(realsrc, out)
 
 
 def main(args=None):
     args = args or sys.argv
     jm = JarMaker()
     p = jm.getCommandLineParser()
     (options, args) = p.parse_args(args)
     jm.outputFormat = options.f
--- a/python/mozbuild/mozbuild/test/test_jarmaker.py
+++ b/python/mozbuild/mozbuild/test/test_jarmaker.py
@@ -10,112 +10,28 @@ from filecmp import dircmp
 from tempfile import mkdtemp
 from shutil import rmtree, copy2
 from StringIO import StringIO
 import mozunit
 
 from mozbuild.jar import JarMaker
 
 
-if sys.platform == "win32":
-    import ctypes
-    from ctypes import POINTER, WinError
-    DWORD = ctypes.c_ulong
-    LPDWORD = POINTER(DWORD)
-    HANDLE = ctypes.c_void_p
-    GENERIC_READ = 0x80000000
-    FILE_SHARE_READ = 0x00000001
-    OPEN_EXISTING = 3
-    MAX_PATH = 260
-
-    class FILETIME(ctypes.Structure):
-        _fields_ = [("dwLowDateTime", DWORD),
-                    ("dwHighDateTime", DWORD)]
-
-    class BY_HANDLE_FILE_INFORMATION(ctypes.Structure):
-        _fields_ = [("dwFileAttributes", DWORD),
-                    ("ftCreationTime", FILETIME),
-                    ("ftLastAccessTime", FILETIME),
-                    ("ftLastWriteTime", FILETIME),
-                    ("dwVolumeSerialNumber", DWORD),
-                    ("nFileSizeHigh", DWORD),
-                    ("nFileSizeLow", DWORD),
-                    ("nNumberOfLinks", DWORD),
-                    ("nFileIndexHigh", DWORD),
-                    ("nFileIndexLow", DWORD)]
-
-    # http://msdn.microsoft.com/en-us/library/aa363858
-    CreateFile = ctypes.windll.kernel32.CreateFileA
-    CreateFile.argtypes = [ctypes.c_char_p, DWORD, DWORD, ctypes.c_void_p,
-                           DWORD, DWORD, HANDLE]
-    CreateFile.restype = HANDLE
-
-    # http://msdn.microsoft.com/en-us/library/aa364952
-    GetFileInformationByHandle = ctypes.windll.kernel32.GetFileInformationByHandle
-    GetFileInformationByHandle.argtypes = [HANDLE, POINTER(BY_HANDLE_FILE_INFORMATION)]
-    GetFileInformationByHandle.restype = ctypes.c_int
+def symlinks_supported():
+    return hasattr(os, 'symlink')
 
-    # http://msdn.microsoft.com/en-us/library/aa364996
-    GetVolumePathName = ctypes.windll.kernel32.GetVolumePathNameA
-    GetVolumePathName.argtypes = [ctypes.c_char_p, ctypes.c_char_p, DWORD]
-    GetVolumePathName.restype = ctypes.c_int
-
-    # http://msdn.microsoft.com/en-us/library/aa364993
-    GetVolumeInformation = ctypes.windll.kernel32.GetVolumeInformationA
-    GetVolumeInformation.argtypes = [ctypes.c_char_p, ctypes.c_char_p, DWORD,
-                                     LPDWORD, LPDWORD, LPDWORD, ctypes.c_char_p,
-                                     DWORD]
-    GetVolumeInformation.restype = ctypes.c_int
-
-def symlinks_supported(path):
-    if sys.platform == "win32":
-        # Add 1 for a trailing backslash if necessary, and 1 for the terminating
-        # null character.
-        volpath = ctypes.create_string_buffer(len(path) + 2)
-        rv = GetVolumePathName(path, volpath, len(volpath))
-        if rv == 0:
-            raise WinError()
-
-        fsname = ctypes.create_string_buffer(MAX_PATH + 1)
-        rv = GetVolumeInformation(volpath, None, 0, None, None, None, fsname,
-                                  len(fsname))
-        if rv == 0:
-            raise WinError()
-
-        # Return true only if the fsname is NTFS
-        return fsname.value == "NTFS"
-    else:
-        return True
-
-def _getfileinfo(path):
-    """Return information for the given file. This only works on Windows."""
-    fh = CreateFile(path, GENERIC_READ, FILE_SHARE_READ, None, OPEN_EXISTING, 0, None)
-    if fh is None:
-        raise WinError()
-    info = BY_HANDLE_FILE_INFORMATION()
-    rv = GetFileInformationByHandle(fh, info)
-    if rv == 0:
-        raise WinError()
-    return info
 
 def is_symlink_to(dest, src):
-    if sys.platform == "win32":
-        # Check if both are on the same volume and have the same file ID
-        destinfo = _getfileinfo(dest)
-        srcinfo = _getfileinfo(src)
-        return (destinfo.dwVolumeSerialNumber == srcinfo.dwVolumeSerialNumber and
-                destinfo.nFileIndexHigh == srcinfo.nFileIndexHigh and
-                destinfo.nFileIndexLow == srcinfo.nFileIndexLow)
-    else:
-        # Read the link and check if it is correct
-        if not os.path.islink(dest):
-            return False
-        target = os.path.abspath(os.readlink(dest))
-        abssrc = os.path.abspath(src)
-        return target == abssrc
+    # Read the link and check if it is correct
+    if not os.path.islink(dest):
+        return False
+    target = os.path.abspath(os.readlink(dest))
+    abssrc = os.path.abspath(src)
+    return target == abssrc
+
 
 class _TreeDiff(dircmp):
     """Helper to report rich results on difference between two directories.
     """
     def _fillDiff(self, dc, rv, basepath="{0}"):
         rv['right_only'] += map(lambda l: basepath.format(l), dc.right_only)
         rv['left_only'] += map(lambda l: basepath.format(l), dc.left_only)
         rv['diff_files'] += map(lambda l: basepath.format(l), dc.diff_files)
@@ -206,17 +122,17 @@ class TestJarMaker(unittest.TestCase):
         self._create_simple_setup()
         # call JarMaker
         rv = self._jar_and_compare(os.path.join(self.srcdir,'jar.mn'),
                                    sourcedirs = [self.srcdir])
         self.assertTrue(not rv, rv)
 
     def test_a_simple_symlink(self):
         '''Test a simple jar.mn with a symlink'''
-        if not symlinks_supported(self.srcdir):
+        if not symlinks_supported():
             raise unittest.SkipTest('symlinks not supported')
 
         self._create_simple_setup()
         jm = JarMaker(outputFormat='symlink')
         jm.sourcedirs = [self.srcdir]
         jm.topsourcedir = self.srcdir
         jm.makeJar(os.path.join(self.srcdir,'jar.mn'), self.builddir)
         # All we do is check that srcdir/bar points to builddir/chrome/test/dir/foo
@@ -254,17 +170,17 @@ class TestJarMaker(unittest.TestCase):
         self._create_wildcard_setup()
         # call JarMaker
         rv = self._jar_and_compare(os.path.join(self.srcdir,'jar.mn'),
                                    sourcedirs = [self.srcdir])
         self.assertTrue(not rv, rv)
 
     def test_a_wildcard_symlink(self):
         '''Test a wildcard in jar.mn with symlinks'''
-        if not symlinks_supported(self.srcdir):
+        if not symlinks_supported():
             raise unittest.SkipTest('symlinks not supported')
 
         self._create_wildcard_setup()
         jm = JarMaker(outputFormat='symlink')
         jm.sourcedirs = [self.srcdir]
         jm.topsourcedir = self.srcdir
         jm.makeJar(os.path.join(self.srcdir,'jar.mn'), self.builddir)