--- a/config/Makefile.in
+++ b/config/Makefile.in
@@ -86,14 +86,13 @@ FORCE:
ifndef JS_STANDALONE
check-preqs += check-jar-mn
endif
check:: $(check-preqs)
check-jar-mn::
- $(MAKE) -C tests/src-simple check-jar
$(MAKE) -C tests/src-simple check-flat
$(MAKE) -C tests/src-simple check-flat USE_EXTENSION_MANIFEST=1
ifneq (,$(filter-out WINNT,$(OS_ARCH)))
$(MAKE) -C tests/src-simple check-symlink
endif
deleted file mode 100644
--- a/config/MozZipFile.py
+++ /dev/null
@@ -1,137 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-import os
-import time
-import zipfile
-
-from mozbuild.util import lock_file
-
-
-class ZipFile(zipfile.ZipFile):
- """ Class with methods to open, read, write, close, list zip files.
-
- Subclassing zipfile.ZipFile to allow for overwriting of existing
- entries, though only for writestr, not for write.
- """
- def __init__(self, file, mode="r", compression=zipfile.ZIP_STORED,
- lock = False):
- if lock:
- assert isinstance(file, basestring)
- self.lockfile = lock_file(file + '.lck')
- else:
- self.lockfile = None
-
- if mode == 'a' and lock:
- # appending to a file which doesn't exist fails, but we can't check
- # existence util we hold the lock
- if (not os.path.isfile(file)) or os.path.getsize(file) == 0:
- mode = 'w'
-
- zipfile.ZipFile.__init__(self, file, mode, compression)
- self._remove = []
- self.end = self.fp.tell()
- self.debug = 0
-
- def writestr(self, zinfo_or_arcname, bytes):
- """Write contents into the archive.
-
- The contents is the argument 'bytes', 'zinfo_or_arcname' is either
- a ZipInfo instance or the name of the file in the archive.
- This method is overloaded to allow overwriting existing entries.
- """
- if not isinstance(zinfo_or_arcname, zipfile.ZipInfo):
- zinfo = zipfile.ZipInfo(filename=zinfo_or_arcname,
- date_time=time.localtime(time.time()))
- zinfo.compress_type = self.compression
- # Add some standard UNIX file access permissions (-rw-r--r--).
- zinfo.external_attr = (0x81a4 & 0xFFFF) << 16L
- else:
- zinfo = zinfo_or_arcname
-
- # Now to the point why we overwrote this in the first place,
- # remember the entry numbers if we already had this entry.
- # Optimizations:
- # If the entry to overwrite is the last one, just reuse that.
- # If we store uncompressed and the new content has the same size
- # as the old, reuse the existing entry.
-
- doSeek = False # store if we need to seek to the eof after overwriting
- if self.NameToInfo.has_key(zinfo.filename):
- # Find the last ZipInfo with our name.
- # Last, because that's catching multiple overwrites
- i = len(self.filelist)
- while i > 0:
- i -= 1
- if self.filelist[i].filename == zinfo.filename:
- break
- zi = self.filelist[i]
- if ((zinfo.compress_type == zipfile.ZIP_STORED
- and zi.compress_size == len(bytes))
- or (i + 1) == len(self.filelist)):
- # make sure we're allowed to write, otherwise done by writestr below
- self._writecheck(zi)
- # overwrite existing entry
- self.fp.seek(zi.header_offset)
- if (i + 1) == len(self.filelist):
- # this is the last item in the file, just truncate
- self.fp.truncate()
- else:
- # we need to move to the end of the file afterwards again
- doSeek = True
- # unhook the current zipinfo, the writestr of our superclass
- # will add a new one
- self.filelist.pop(i)
- self.NameToInfo.pop(zinfo.filename)
- else:
- # Couldn't optimize, sadly, just remember the old entry for removal
- self._remove.append(self.filelist.pop(i))
- zipfile.ZipFile.writestr(self, zinfo, bytes)
- self.filelist.sort(lambda l, r: cmp(l.header_offset, r.header_offset))
- if doSeek:
- self.fp.seek(self.end)
- self.end = self.fp.tell()
-
- def close(self):
- """Close the file, and for mode "w" and "a" write the ending
- records.
-
- Overwritten to compact overwritten entries.
- """
- if not self._remove:
- # we don't have anything special to do, let's just call base
- r = zipfile.ZipFile.close(self)
- self.lockfile = None
- return r
-
- if self.fp.mode != 'r+b':
- # adjust file mode if we originally just wrote, now we rewrite
- self.fp.close()
- self.fp = open(self.filename, 'r+b')
- all = map(lambda zi: (zi, True), self.filelist) + \
- map(lambda zi: (zi, False), self._remove)
- all.sort(lambda l, r: cmp(l[0].header_offset, r[0].header_offset))
- # empty _remove for multiple closes
- self._remove = []
-
- lengths = [all[i+1][0].header_offset - all[i][0].header_offset
- for i in xrange(len(all)-1)]
- lengths.append(self.end - all[-1][0].header_offset)
- to_pos = 0
- for (zi, keep), length in zip(all, lengths):
- if not keep:
- continue
- oldoff = zi.header_offset
- # python <= 2.4 has file_offset
- if hasattr(zi, 'file_offset'):
- zi.file_offset = zi.file_offset + to_pos - oldoff
- zi.header_offset = to_pos
- self.fp.seek(oldoff)
- content = self.fp.read(length)
- self.fp.seek(to_pos)
- self.fp.write(content)
- to_pos += length
- self.fp.truncate()
- zipfile.ZipFile.close(self)
- self.lockfile = None
--- a/config/tests/src-simple/Makefile.in
+++ b/config/tests/src-simple/Makefile.in
@@ -23,16 +23,13 @@ REF_MANIFEST = $(if $(USE_EXTENSION_MANI
check-%::
if test -d $(FINAL_TARGET); then rm -rf $(FINAL_TARGET); fi;
$(MAKE) realchrome MOZ_JAR_MAKER_FILE_FORMAT=$*
@echo 'Comparing manifests...'
@if ! sort $(MY_MANIFEST) | diff --text -U 0 $(srcdir)/../$(REF_MANIFEST).$* - ; then \
echo 'TEST-UNEXPECTED-FAIL | config/tests/$(REF_MANIFEST).$* | differing content in manifest!' ; \
false; \
fi
- @if [ $* = 'jar' ]; then \
- $(UNZIP) -d $(FINAL_TARGET)/chrome/test $(FINAL_TARGET)/chrome/test.jar; \
- fi
@echo 'Comparing packages...'
@if ! diff -ur $(srcdir)/../ref-simple $(FINAL_TARGET)/chrome/test ; then\
echo 'TEST-UNEXPECTED-FAIL | config/tests/ref-simple | different content in jar' ; \
false; \
fi
deleted file mode 100644
--- a/config/tests/unitMozZipFile.py
+++ /dev/null
@@ -1,201 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-import unittest
-
-import shutil
-import os
-import re
-import sys
-import random
-import copy
-from string import letters
-
-'''
-Test case infrastructure for MozZipFile.
-
-This isn't really a unit test, but a test case generator and runner.
-For a given set of files, lengths, and number of writes, we create
-a testcase for every combination of the three. There are some
-symmetries used to reduce the number of test cases, the first file
-written is always the first file, the second is either the first or
-the second, the third is one of the first three. That is, if we
-had 4 files, but only three writes, the fourth file would never even
-get tried.
-
-The content written to the jars is pseudorandom with a fixed seed.
-'''
-
-if not __file__:
- __file__ = sys.argv[0]
-sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
-
-from MozZipFile import ZipFile
-import zipfile
-
-leafs = (
- 'firstdir/oneleaf',
- 'seconddir/twoleaf',
- 'thirddir/with/sub/threeleaf')
-_lengths = map(lambda n: n * 64, [16, 64, 80])
-lengths = 3
-writes = 5
-
-def givenlength(i):
- '''Return a length given in the _lengths array to allow manual
- tuning of which lengths of zip entries to use.
- '''
- return _lengths[i]
-
-
-def prod(*iterables):
- ''''Tensor product of a list of iterables.
-
- This generator returns lists of items, one of each given
- iterable. It iterates over all possible combinations.
- '''
- for item in iterables[0]:
- if len(iterables) == 1:
- yield [item]
- else:
- for others in prod(*iterables[1:]):
- yield [item] + others
-
-
-def getid(descs):
- 'Convert a list of ints to a string.'
- return reduce(lambda x,y: x+'{0}{1}'.format(*tuple(y)), descs,'')
-
-
-def getContent(length):
- 'Get pseudo random content of given length.'
- rv = [None] * length
- for i in xrange(length):
- rv[i] = random.choice(letters)
- return ''.join(rv)
-
-
-def createWriter(sizer, *items):
- 'Helper method to fill in tests, one set of writes, one for each item'
- locitems = copy.deepcopy(items)
- for item in locitems:
- item['length'] = sizer(item.pop('length', 0))
- def helper(self):
- mode = 'w'
- if os.path.isfile(self.f):
- mode = 'a'
- zf = ZipFile(self.f, mode, self.compression)
- for item in locitems:
- self._write(zf, **item)
- zf = None
- pass
- return helper
-
-def createTester(name, *writes):
- '''Helper method to fill in tests, calls into a list of write
- helper methods.
- '''
- _writes = copy.copy(writes)
- def tester(self):
- for w in _writes:
- getattr(self, w)()
- self._verifyZip()
- pass
- # unit tests get confused if the method name isn't test...
- tester.__name__ = name
- return tester
-
-class TestExtensiveStored(unittest.TestCase):
- '''Unit tests for MozZipFile
-
- The testcase are actually populated by code following the class
- definition.
- '''
-
- stage = "mozzipfilestage"
- compression = zipfile.ZIP_STORED
-
- def leaf(self, *leafs):
- return os.path.join(self.stage, *leafs)
- def setUp(self):
- if os.path.exists(self.stage):
- shutil.rmtree(self.stage)
- os.mkdir(self.stage)
- self.f = self.leaf('test.jar')
- self.ref = {}
- self.seed = 0
-
- def tearDown(self):
- self.f = None
- self.ref = None
-
- def _verifyZip(self):
- zf = zipfile.ZipFile(self.f)
- badEntry = zf.testzip()
- self.failIf(badEntry, badEntry)
- zlist = zf.namelist()
- zlist.sort()
- vlist = self.ref.keys()
- vlist.sort()
- self.assertEqual(zlist, vlist)
- for leaf, content in self.ref.iteritems():
- zcontent = zf.read(leaf)
- self.assertEqual(content, zcontent)
-
- def _write(self, zf, seed=None, leaf=0, length=0):
- if seed is None:
- seed = self.seed
- self.seed += 1
- random.seed(seed)
- leaf = leafs[leaf]
- content = getContent(length)
- self.ref[leaf] = content
- zf.writestr(leaf, content)
- dir = os.path.dirname(self.leaf('stage', leaf))
- if not os.path.isdir(dir):
- os.makedirs(dir)
- open(self.leaf('stage', leaf), 'w').write(content)
-
-# all leafs in all lengths
-atomics = list(prod(xrange(len(leafs)), xrange(lengths)))
-
-# populate TestExtensiveStore with testcases
-for w in xrange(writes):
- # Don't iterate over all files for the the first n passes,
- # those are redundant as long as w < lengths.
- # There are symmetries in the trailing end, too, but I don't know
- # how to reduce those out right now.
- nonatomics = [list(prod(range(min(i,len(leafs))), xrange(lengths)))
- for i in xrange(1, w+1)] + [atomics]
- for descs in prod(*nonatomics):
- suffix = getid(descs)
- dicts = [dict(leaf=leaf, length=length) for leaf, length in descs]
- setattr(TestExtensiveStored, '_write' + suffix,
- createWriter(givenlength, *dicts))
- setattr(TestExtensiveStored, 'test' + suffix,
- createTester('test' + suffix, '_write' + suffix))
-
-# now create another round of tests, with two writing passes
-# first, write all file combinations into the jar, close it,
-# and then write all atomics again.
-# This should catch more or less all artifacts generated
-# by the final ordering step when closing the jar.
-files = [list(prod([i], xrange(lengths))) for i in xrange(len(leafs))]
-allfiles = reduce(lambda l,r:l+r,
- [list(prod(*files[:(i+1)])) for i in xrange(len(leafs))])
-
-for first in allfiles:
- testbasename = 'test{0}_'.format(getid(first))
- test = [None, '_write' + getid(first), None]
- for second in atomics:
- test[0] = testbasename + getid([second])
- test[2] = '_write' + getid([second])
- setattr(TestExtensiveStored, test[0], createTester(*test))
-
-class TestExtensiveDeflated(TestExtensiveStored):
- 'Test all that has been tested with ZIP_STORED with DEFLATED, too.'
- compression = zipfile.ZIP_DEFLATED
-
-if __name__ == '__main__':
- unittest.main()
--- a/python/mozbuild/mozbuild/jar.py
+++ b/python/mozbuild/mozbuild/jar.py
@@ -11,56 +11,28 @@ See the documentation for jar.mn on MDC
from __future__ import absolute_import
import sys
import os
import errno
import re
import logging
from time import localtime
-from MozZipFile import ZipFile
-from cStringIO import StringIO
-from collections import defaultdict
from mozbuild.preprocessor import Preprocessor
from mozbuild.action.buildlist import addEntriesToListFile
from mozpack.files import FileFinder
import mozpack.path as mozpath
if sys.platform == 'win32':
from ctypes import windll, WinError
CreateHardLink = windll.kernel32.CreateHardLinkA
__all__ = ['JarMaker']
-class ZipEntry(object):
- '''Helper class for jar output.
-
- This class defines a simple file-like object for a zipfile.ZipEntry
- so that we can consecutively write to it and then close it.
- This methods hooks into ZipFile.writestr on close().
- '''
-
- def __init__(self, name, zipfile):
- self._zipfile = zipfile
- self._name = name
- self._inner = StringIO()
-
- def write(self, content):
- '''Append the given content to this zip entry'''
-
- self._inner.write(content)
- return
-
- def close(self):
- '''The close method writes the content back to the zip file.'''
-
- self._zipfile.writestr(self._name, self._inner.getvalue())
-
-
def getModTime(aPath):
if not os.path.isfile(aPath):
return 0
mtime = os.stat(aPath).st_mtime
return localtime(mtime)
class JarManifestEntry(object):
@@ -218,20 +190,20 @@ class JarMaker(object):
This OptionParser has the options for jarmaker as well as
the options for the inner PreProcessor.
'''
# HACK, we need to unescape the string variables we get,
# the perl versions didn't grok strings right
p = self.pp.getCommandLineParser(unescapeDefines=True)
- p.add_option('-f', type='choice', default='jar',
- choices=('jar', 'flat', 'symlink'),
+ p.add_option('-f', type='choice', default='flat',
+ choices=('flat', 'symlink'),
help='fileformat used for output',
- metavar='[jar, flat, symlink]',
+ metavar='[flat, symlink]',
)
p.add_option('-v', action='store_true', dest='verbose',
help='verbose output')
p.add_option('-q', action='store_false', dest='verbose',
help='verbose output')
p.add_option('-e', action='store_true',
help='create chrome.manifest instead of jarfile.manifest'
)
@@ -355,49 +327,32 @@ class JarMaker(object):
'''Internal method called by makeJar to actually process a section
of a jar.mn file.
'''
# chromebasepath is used for chrome registration manifests
# {0} is getting replaced with chrome/ for chrome.manifest, and with
# an empty string for jarfile.manifest
- chromebasepath = '{0}' + os.path.basename(jarinfo.name)
- if self.outputFormat == 'jar':
- chromebasepath = 'jar:' + chromebasepath + '.jar!'
- chromebasepath += '/'
+ chromebasepath = '{0}%s/' % os.path.basename(jarinfo.name)
jarfile = os.path.join(jardir, jarinfo.base, jarinfo.name)
- jf = None
- if self.outputFormat == 'jar':
- # jar
- jarfilepath = jarfile + '.jar'
- try:
- os.makedirs(os.path.dirname(jarfilepath))
- except OSError, error:
- if error.errno != errno.EEXIST:
- raise
- jf = ZipFile(jarfilepath, 'a', lock=True)
- outHelper = self.OutputHelper_jar(jf)
- else:
- outHelper = getattr(self, 'OutputHelper_'
- + self.outputFormat)(jarfile)
+ outHelper = getattr(self, 'OutputHelper_'
+ + self.outputFormat)(jarfile)
if jarinfo.relativesrcdir:
self.localedirs = self.generateLocaleDirs(jarinfo.relativesrcdir)
for e in jarinfo.entries:
- self._processEntryLine(e, outHelper, jf)
+ self._processEntryLine(e, outHelper)
self.finalizeJar(jardir, jarinfo.base, jarinfo.name, chromebasepath,
jarinfo.chrome_manifests)
- if jf is not None:
- jf.close()
- def _processEntryLine(self, e, outHelper, jf):
+ def _processEntryLine(self, e, outHelper):
out = e.output
src = e.source
# pick the right sourcedir -- l10n, topsrc or src
if e.is_locale:
src_base = self.localedirs
elif src.startswith('/'):
@@ -428,28 +383,26 @@ class JarMaker(object):
continue
emitted.add(reduced_path)
e = JarManifestEntry(
mozpath.join(out, reduced_path),
path,
is_locale=e.is_locale,
preprocess=e.preprocess,
)
- self._processEntryLine(e, outHelper, jf)
+ self._processEntryLine(e, outHelper)
return
# check if the source file exists
realsrc = None
for _srcdir in src_base:
if os.path.isfile(os.path.join(_srcdir, src)):
realsrc = os.path.join(_srcdir, src)
break
if realsrc is None:
- if jf is not None:
- jf.close()
raise RuntimeError('File "{0}" not found in {1}'.format(src,
', '.join(src_base)))
if out in self._seen_output:
raise RuntimeError('%s already added' % out)
self._seen_output.add(out)
if e.preprocess:
@@ -475,32 +428,16 @@ class JarMaker(object):
# open in binary mode, this can be images etc
inf = open(realsrc, 'rb')
outf.write(inf.read())
outf.close()
inf.close()
- class OutputHelper_jar(object):
- '''Provide getDestModTime and getOutput for a given jarfile.'''
-
- def __init__(self, jarfile):
- self.jarfile = jarfile
-
- def getDestModTime(self, aPath):
- try:
- info = self.jarfile.getinfo(aPath)
- return info.date_time
- except:
- return 0
-
- def getOutput(self, name):
- return ZipEntry(name, self.jarfile)
-
class OutputHelper_flat(object):
'''Provide getDestModTime and getOutput for a given flat
output directory. The helper method ensureDirFor is used by
the symlink subclass.
'''
def __init__(self, basepath):
self.basepath = basepath
--- a/python/mozbuild/mozbuild/test/test_jarmaker.py
+++ b/python/mozbuild/mozbuild/test/test_jarmaker.py
@@ -1,21 +1,20 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import print_function
import unittest
-import os, sys, os.path, time, inspect
+import os, sys
from filecmp import dircmp
from tempfile import mkdtemp
from shutil import rmtree, copy2
from StringIO import StringIO
-from zipfile import ZipFile
import mozunit
from mozbuild.jar import JarMaker
if sys.platform == "win32":
import ctypes
from ctypes import POINTER, WinError
@@ -160,69 +159,50 @@ class TestJarMaker(unittest.TestCase):
def tearDown(self):
if self.debug:
print(self.tmpdir)
elif sys.platform != "win32":
# can't clean up on windows
rmtree(self.tmpdir)
def _jar_and_compare(self, infile, **kwargs):
- jm = JarMaker(outputFormat='jar')
+ jm = JarMaker(outputFormat='flat')
if 'topsourcedir' not in kwargs:
kwargs['topsourcedir'] = self.srcdir
for attr in ('topsourcedir', 'sourcedirs'):
if attr in kwargs:
setattr(jm, attr, kwargs[attr])
jm.makeJar(infile, self.builddir)
cwd = os.getcwd()
os.chdir(self.builddir)
try:
# expand build to stage
for path, dirs, files in os.walk('.'):
stagedir = os.path.join(self.stagedir, path)
if not os.path.isdir(stagedir):
os.mkdir(stagedir)
for file in files:
- if file.endswith('.jar'):
- # expand jar
- stagepath = os.path.join(stagedir, file)
- os.mkdir(stagepath)
- zf = ZipFile(os.path.join(path, file))
- # extractall is only in 2.6, do this manually :-(
- for entry_name in zf.namelist():
- segs = entry_name.split('/')
- fname = segs.pop()
- dname = os.path.join(stagepath, *segs)
- if not os.path.isdir(dname):
- os.makedirs(dname)
- if not fname:
- # directory, we're done
- continue
- _c = zf.read(entry_name)
- open(os.path.join(dname, fname), 'wb').write(_c)
- zf.close()
- else:
- copy2(os.path.join(path, file), stagedir)
+ copy2(os.path.join(path, file), stagedir)
# compare both dirs
os.chdir('..')
td = _TreeDiff('ref', 'stage')
return td.allResults('reference', 'build')
finally:
os.chdir(cwd)
def _create_simple_setup(self):
# create src content
jarf = open(os.path.join(self.srcdir, 'jar.mn'), 'w')
jarf.write('''test.jar:
dir/foo (bar)
''')
jarf.close()
open(os.path.join(self.srcdir,'bar'),'w').write('content\n')
# create reference
- refpath = os.path.join(self.refdir, 'chrome', 'test.jar', 'dir')
+ refpath = os.path.join(self.refdir, 'chrome', 'test', 'dir')
os.makedirs(refpath)
open(os.path.join(refpath, 'foo'), 'w').write('content\n')
def test_a_simple_jar(self):
'''Test a simple jar.mn'''
self._create_simple_setup()
# call JarMaker
rv = self._jar_and_compare(os.path.join(self.srcdir,'jar.mn'),
@@ -255,17 +235,17 @@ class TestJarMaker(unittest.TestCase):
jarf.close()
open(os.path.join(self.srcdir,'foo.js'),'w').write('foo.js\n')
open(os.path.join(self.srcdir,'bar.js'),'w').write('bar.js\n')
os.makedirs(os.path.join(self.srcdir, 'qux', 'foo'))
open(os.path.join(self.srcdir,'qux', 'foo', '1'),'w').write('1\n')
open(os.path.join(self.srcdir,'qux', 'foo', '2'),'w').write('2\n')
open(os.path.join(self.srcdir,'qux', 'baz'),'w').write('baz\n')
# create reference
- refpath = os.path.join(self.refdir, 'chrome', 'test.jar', 'dir')
+ refpath = os.path.join(self.refdir, 'chrome', 'test', 'dir')
os.makedirs(os.path.join(refpath, 'bar'))
os.makedirs(os.path.join(refpath, 'hoge', 'foo'))
open(os.path.join(refpath, 'bar', 'foo.js'), 'w').write('foo.js\n')
open(os.path.join(refpath, 'bar', 'bar.js'), 'w').write('bar.js\n')
open(os.path.join(refpath, 'hoge', 'foo', '1'), 'w').write('1\n')
open(os.path.join(refpath, 'hoge', 'foo', '2'), 'w').write('2\n')
open(os.path.join(refpath, 'hoge', 'baz'), 'w').write('baz\n')