Bug 1359965 - Support mozpack.file.BaseFile in create_tar_from_files; r?glandium
This allows us to write files coming from a finder or other source
that isn't directly the filesystem.
MozReview-Commit-ID: KhPSD0JYzsQ
--- a/python/mozbuild/mozpack/archive.py
+++ b/python/mozbuild/mozpack/archive.py
@@ -1,64 +1,78 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import
import bz2
import gzip
+import io
import stat
import tarfile
+from .files import (
+ BaseFile,
+)
# 2016-01-01T00:00:00+0000
DEFAULT_MTIME = 1451606400
def create_tar_from_files(fp, files):
"""Create a tar file deterministically.
Receives a dict mapping names of files in the archive to local filesystem
- paths.
+ paths or ``mozpack.files.BaseFile`` instances.
The files will be archived and written to the passed file handle opened
for writing.
Only regular files can be written.
- FUTURE accept mozpack.files classes for writing
FUTURE accept a filename argument (or create APIs to write files)
"""
with tarfile.open(name='', mode='w', fileobj=fp, dereference=True) as tf:
- for archive_path, fs_path in sorted(files.items()):
- ti = tf.gettarinfo(fs_path, archive_path)
+ for archive_path, f in sorted(files.items()):
+ if isinstance(f, BaseFile):
+ ti = tarfile.TarInfo(archive_path)
+ ti.mode = f.mode or 0644
+ ti.type = tarfile.REGTYPE
+ else:
+ ti = tf.gettarinfo(f, archive_path)
if not ti.isreg():
- raise ValueError('not a regular file: %s' % fs_path)
+ raise ValueError('not a regular file: %s' % f)
# Disallow setuid and setgid bits. This is an arbitrary restriction.
# However, since we set uid/gid to root:root, setuid and setgid
# would be a glaring security hole if the archive were
# uncompressed as root.
if ti.mode & (stat.S_ISUID | stat.S_ISGID):
raise ValueError('cannot add file with setuid or setgid set: '
- '%s' % fs_path)
+ '%s' % f)
# Set uid, gid, username, and group as deterministic values.
ti.uid = 0
ti.gid = 0
ti.uname = ''
ti.gname = ''
# Set mtime to a constant value.
ti.mtime = DEFAULT_MTIME
- with open(fs_path, 'rb') as fh:
- tf.addfile(ti, fh)
+ if isinstance(f, BaseFile):
+ ti.size = f.size()
+ # tarfile wants to pass a size argument to read(). So just
+ # wrap/buffer in a proper file object interface.
+ tf.addfile(ti, f.open())
+ else:
+ with open(f, 'rb') as fh:
+ tf.addfile(ti, fh)
def create_tar_gz_from_files(fp, files, filename=None, compresslevel=9):
"""Create a tar.gz file deterministically from files.
This is a glorified wrapper around ``create_tar_from_files`` that
adds gzip compression.
--- a/python/mozbuild/mozpack/files.py
+++ b/python/mozbuild/mozpack/files.py
@@ -214,16 +214,24 @@ class BaseFile(object):
a custom file-like object.
'''
assert self.path is not None
return open(self.path, 'rb')
def read(self):
raise NotImplementedError('BaseFile.read() not implemented. Bug 1170329.')
+ def size(self):
+ """Returns size of the entry.
+
+ Derived classes are highly encouraged to override this with a more
+ optimal implementation.
+ """
+ return len(self.read())
+
@property
def mode(self):
'''
Return the file's unix mode, or None if it has no meaning.
'''
return None
@@ -245,16 +253,19 @@ class File(BaseFile):
mode = os.stat(self.path).st_mode
return self.normalize_mode(mode)
def read(self):
'''Return the contents of the file.'''
with open(self.path, 'rb') as fh:
return fh.read()
+ def size(self):
+ return os.stat(self.path).st_size
+
class ExecutableFile(File):
'''
File class for executable and library files on OS/2, OS/X and ELF systems.
(see mozpack.executables.is_executable documentation).
'''
def copy(self, dest, skip_if_older=True):
real_dest = dest
@@ -492,16 +503,22 @@ class GeneratedFile(BaseFile):
File class for content with no previous existence on the filesystem.
'''
def __init__(self, content):
self.content = content
def open(self):
return BytesIO(self.content)
+ def read(self):
+ return self.content
+
+ def size(self):
+ return len(self.content)
+
class DeflatedFile(BaseFile):
'''
File class for members of a jar archive. DeflatedFile.copy() effectively
extracts the file from the jar archive.
'''
def __init__(self, file):
from mozpack.mozjar import JarFileReader
--- a/python/mozbuild/mozpack/test/test_archive.py
+++ b/python/mozbuild/mozpack/test/test_archive.py
@@ -13,16 +13,19 @@ import tempfile
import unittest
from mozpack.archive import (
DEFAULT_MTIME,
create_tar_from_files,
create_tar_gz_from_files,
create_tar_bz2_from_files,
)
+from mozpack.files import (
+ GeneratedFile,
+)
from mozunit import main
MODE_STANDARD = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH
def file_hash(path):
@@ -36,29 +39,32 @@ def file_hash(path):
return h.hexdigest()
class TestArchive(unittest.TestCase):
def _create_files(self, root):
files = {}
for i in range(10):
- p = os.path.join(root, b'file%d' % i)
+ p = os.path.join(root, b'file%02d' % i)
with open(p, 'wb') as fh:
- fh.write(b'file%d' % i)
+ fh.write(b'file%02d' % i)
# Need to set permissions or umask may influence testing.
os.chmod(p, MODE_STANDARD)
- files[b'file%d' % i] = p
+ files[b'file%02d' % i] = p
+
+ for i in range(10):
+ files[b'file%02d' % (i + 10)] = GeneratedFile('file%02d' % (i + 10))
return files
def _verify_basic_tarfile(self, tf):
- self.assertEqual(len(tf.getmembers()), 10)
+ self.assertEqual(len(tf.getmembers()), 20)
- names = ['file%d' % i for i in range(10)]
+ names = ['file%02d' % i for i in range(20)]
self.assertEqual(tf.getnames(), names)
for ti in tf.getmembers():
self.assertEqual(ti.uid, 0)
self.assertEqual(ti.gid, 0)
self.assertEqual(ti.uname, '')
self.assertEqual(ti.gname, '')
self.assertEqual(ti.mode, MODE_STANDARD)
@@ -101,17 +107,17 @@ class TestArchive(unittest.TestCase):
try:
files = self._create_files(d)
tp = os.path.join(d, 'test.tar')
with open(tp, 'wb') as fh:
create_tar_from_files(fh, files)
# Output should be deterministic.
- self.assertEqual(file_hash(tp), 'cd16cee6f13391abd94dfa435d2633b61ed727f1')
+ self.assertEqual(file_hash(tp), '01cd314e277f060e98c7de6c8ea57f96b3a2065c')
with tarfile.open(tp, 'r') as tf:
self._verify_basic_tarfile(tf)
finally:
shutil.rmtree(d)
def test_executable_preserved(self):
@@ -139,51 +145,51 @@ class TestArchive(unittest.TestCase):
d = tempfile.mkdtemp()
try:
files = self._create_files(d)
gp = os.path.join(d, 'test.tar.gz')
with open(gp, 'wb') as fh:
create_tar_gz_from_files(fh, files)
- self.assertEqual(file_hash(gp), 'acb602239c1aeb625da5e69336775609516d60f5')
+ self.assertEqual(file_hash(gp), '7c4da5adc5088cdf00911d5daf9a67b15de714b7')
with tarfile.open(gp, 'r:gz') as tf:
self._verify_basic_tarfile(tf)
finally:
shutil.rmtree(d)
def test_tar_gz_name(self):
d = tempfile.mkdtemp()
try:
files = self._create_files(d)
gp = os.path.join(d, 'test.tar.gz')
with open(gp, 'wb') as fh:
create_tar_gz_from_files(fh, files, filename='foobar', compresslevel=1)
- self.assertEqual(file_hash(gp), 'fd099f96480cc1100f37baa8e89a6b820dbbcbd3')
+ self.assertEqual(file_hash(gp), '1cc8b96f0262350977c2e9d61f40a1fa76f35c52')
with tarfile.open(gp, 'r:gz') as tf:
self._verify_basic_tarfile(tf)
finally:
shutil.rmtree(d)
def test_create_tar_bz2_basic(self):
d = tempfile.mkdtemp()
try:
files = self._create_files(d)
bp = os.path.join(d, 'test.tar.bz2')
with open(bp, 'wb') as fh:
create_tar_bz2_from_files(fh, files)
- self.assertEqual(file_hash(bp), '1827ad00dfe7acf857b7a1c95ce100361e3f6eea')
+ self.assertEqual(file_hash(bp), 'eb5096d2fbb71df7b3d690001a6f2e82a5aad6a7')
with tarfile.open(bp, 'r:bz2') as tf:
self._verify_basic_tarfile(tf)
finally:
shutil.rmtree(d)
if __name__ == '__main__':