--- a/toolkit/crashreporter/tools/symbolstore.py
+++ b/toolkit/crashreporter/tools/symbolstore.py
@@ -344,75 +344,34 @@ def SourceIndex(fileStream, outputPath,
pdbStreamFile.write('''SRCSRV: ini ------------------------------------------------\r\nVERSION=2\r\nINDEXVERSION=2\r\nVERCTRL=http\r\nSRCSRV: variables ------------------------------------------\r\nHGSERVER=''')
pdbStreamFile.write(vcs_root)
pdbStreamFile.write('''\r\nSRCSRVVERCTRL=http\r\nHTTP_EXTRACT_TARGET=%hgserver%/raw-file/%var3%/%var2%\r\nSRCSRVTRG=%http_extract_target%\r\nSRCSRV: source files ---------------------------------------\r\n''')
pdbStreamFile.write(fileStream) # can't do string interpolation because the source server also uses this and so there are % in the above
pdbStreamFile.write("SRCSRV: end ------------------------------------------------\r\n\n")
pdbStreamFile.close()
return result
-def StartJob(dumper, lock, srcdirRepoInfo, func_name, args):
- # Windows worker processes won't have run GlobalInit,
- # and due to a lack of fork(), won't inherit the class
- # variables from the parent, so set them here.
- Dumper.lock = lock
- Dumper.srcdirRepoInfo = srcdirRepoInfo
- return getattr(dumper, func_name)(*args)
-
-class JobPool(object):
- jobs = {}
- executor = None
-
- @classmethod
- def init(cls, executor):
- cls.executor = executor
-
- @classmethod
- def shutdown(cls):
- cls.executor.shutdown()
-
- @classmethod
- def submit(cls, args, callback):
- cls.jobs[cls.executor.submit(StartJob, *args)] = callback
-
- @classmethod
- def as_completed(cls):
- '''Like concurrent.futures.as_completed, but allows adding new futures
- between generator steps. Iteration will end when the generator has
- yielded all completed futures and JobQueue.jobs is empty.
- Yields (future, callback) pairs.
- '''
- while cls.jobs:
- completed, _ = concurrent.futures.wait(cls.jobs.keys(), return_when=concurrent.futures.FIRST_COMPLETED)
- for f in completed:
- callback = cls.jobs[f]
- del cls.jobs[f]
- yield f, callback
class Dumper:
"""This class can dump symbols from a file with debug info, and
store the output in a directory structure that is valid for use as
a Breakpad symbol server. Requires a path to a dump_syms binary--
|dump_syms| and a directory to store symbols in--|symbol_path|.
Optionally takes a list of processor architectures to process from
each debug file--|archs|, the full path to the top source
directory--|srcdir|, for generating relative source file names,
and an option to copy debug info files alongside the dumped
symbol files--|copy_debug|, mostly useful for creating a
Microsoft Symbol Server from the resulting output.
You don't want to use this directly if you intend to process files.
Instead, call GetPlatformSpecificDumper to get an instance of a
- subclass.
+ subclass."""
+ srcdirRepoInfo = {}
- Processing is performed asynchronously via worker processes; in
- order to wait for processing to finish and cleanup correctly, you
- must call Finish after all ProcessFiles calls have been made.
- You must also call Dumper.GlobalInit before creating or using any
- instances."""
def __init__(self, dump_syms, symbol_path,
archs=None,
srcdirs=[],
copy_debug=False,
vcsinfo=False,
srcsrv=False,
exclude=[],
repo_manifest=None,
@@ -432,40 +391,20 @@ class Dumper:
self.exclude = exclude[:]
if repo_manifest:
self.parse_repo_manifest(repo_manifest)
self.file_mapping = file_mapping or {}
# book-keeping to keep track of the cleanup work per file tuple
self.files_record = {}
- @classmethod
- def GlobalInit(cls, executor=concurrent.futures.ProcessPoolExecutor):
- """Initialize the class globals for the multiprocessing setup; must
- be called before any Dumper instances are created and used. Test cases
- may pass in a different executor to use, usually
- concurrent.futures.ThreadPoolExecutor."""
- num_cpus = multiprocessing.cpu_count()
- if num_cpus is None:
- # assume a dual core machine if we can't find out for some reason
- # probably better on single core anyway due to I/O constraints
- num_cpus = 2
-
- # have to create any locks etc before the pool
- manager = multiprocessing.Manager()
- cls.lock = manager.RLock()
- cls.srcdirRepoInfo = manager.dict()
- JobPool.init(executor(max_workers=num_cpus))
-
def output(self, dest, output_str):
- """Writes |output_str| to |dest|, holding |lock|;
- terminates with a newline."""
- with Dumper.lock:
- dest.write(output_str + "\n")
- dest.flush()
+ """Writes |output_str| to |dest|, holding, terminates with a newline."""
+ dest.write(output_str + "\n")
+ dest.flush()
def output_pid(self, dest, output_str):
"""Debugging output; prepends the pid to the string."""
self.output(dest, "%d: %s" % (os.getpid(), output_str))
def parse_repo_manifest(self, repo_manifest):
"""
Parse an XML manifest of repository info as produced
@@ -540,30 +479,16 @@ class Dumper:
# This is a no-op except on Win32
def SourceServerIndexing(self, debug_file, guid, sourceFileStream, vcs_root):
return ""
# subclasses override this if they want to support this
def CopyDebug(self, file, debug_file, guid, code_file, code_id):
pass
- def Finish(self, stop_pool=True):
- '''Process all pending jobs and any jobs their callbacks submit.
- By default, will shutdown the executor, but for testcases that
- need multiple runs, pass stop_pool = False.'''
- for job, callback in JobPool.as_completed():
- try:
- res = job.result()
- except Exception as e:
- self.output(sys.stderr, 'Job raised exception: %s' % e)
- continue
- callback(res)
- if stop_pool:
- JobPool.shutdown()
-
def Process(self, *args):
"""Process files recursively in args."""
# We collect all files to process first then sort by size to schedule
# larger files first because larger files tend to take longer and we
# don't like long pole stragglers.
files = set()
for arg in args:
for f in self.get_files_to_process(arg):
@@ -589,55 +514,52 @@ class Dumper:
for d in dirs[:]:
if self.ShouldSkipDir(d):
dirs.remove(d)
for f in files:
fullpath = os.path.join(root, f)
if self.ShouldProcess(fullpath):
yield fullpath
- def SubmitJob(self, file_key, func_name, args, callback):
- """Submits a job to the pool of workers"""
- JobPool.submit((self, Dumper.lock, Dumper.srcdirRepoInfo, func_name, args), callback)
-
def ProcessFilesFinished(self, res):
"""Callback from multiprocesing when ProcessFilesWork finishes;
run the cleanup work, if any"""
# only run the cleanup function once per tuple of files
self.files_record[res['files']] += 1
if self.files_record[res['files']] == len(self.archs):
del self.files_record[res['files']]
if res['after']:
res['after'](res['status'], res['after_arg'])
def ProcessFiles(self, files, after=None, after_arg=None):
"""Dump symbols from these files into a symbol file, stored
in the proper directory structure in |symbol_path|; processing is performed
asynchronously, and Finish must be called to wait for it complete and cleanup.
All files after the first are fallbacks in case the first file does not process
successfully; if it does, no other files will be touched."""
- self.output_pid(sys.stderr, "Submitting jobs for files: %s" % str(files))
+ self.output_pid(sys.stderr, "Beginning work for files: %s" % str(files))
# tries to get the vcs root from the .mozconfig first - if it's not set
# the tinderbox vcs path will be assigned further down
vcs_root = os.environ.get('MOZ_SOURCE_REPO')
for arch_num, arch in enumerate(self.archs):
self.files_record[files] = 0 # record that we submitted jobs for this tuple of files
- self.SubmitJob(files[-1], 'ProcessFilesWork', args=(files, arch_num, arch, vcs_root, after, after_arg), callback=self.ProcessFilesFinished)
+ res = self.ProcessFilesWork(files, arch_num, arch, vcs_root, after, after_arg)
+ self.ProcessFilesFinished(res)
def dump_syms_cmdline(self, file, arch, files):
'''
Get the commandline used to invoke dump_syms.
'''
# The Mac dumper overrides this.
return [self.dump_syms, file]
def ProcessFilesWork(self, files, arch_num, arch, vcs_root, after, after_arg):
t_start = time.time()
- self.output_pid(sys.stderr, "Worker processing files: %s" % (files,))
+ self.output_pid(sys.stderr, "Processing files: %s" % (files,))
# our result is a status, a cleanup function, an argument to that function, and the tuple of files we were called on
result = { 'status' : False, 'after' : after, 'after_arg' : after_arg, 'files' : files }
sourceFileStream = ''
code_id, code_file = None, None
for file in files:
# files is a tuple of files, containing fallbacks in case the first file doesn't process successfully
@@ -715,17 +637,17 @@ class Dumper:
except Exception as e:
self.output(sys.stderr, "Unexpected error: %s" % (str(e),))
raise
if result['status']:
# we only need 1 file to work
break
elapsed = time.time() - t_start
- self.output_pid(sys.stderr, 'Worker finished processing %s in %.2fs' %
+ self.output_pid(sys.stderr, 'Finished processing %s in %.2fs' %
(files, elapsed))
return result
# Platform-specific subclasses. For the most part, these just have
# logic to determine what files to extract symbols from.
class Dumper_Win32(Dumper):
fixedFilenameCaseCache = {}
@@ -928,18 +850,19 @@ class Dumper_Mac(Dumper):
dump the inner bits again."""
if dir.endswith(".dSYM"):
return True
return False
def ProcessFiles(self, files, after=None, after_arg=None):
# also note, files must be len 1 here, since we're the only ones
# that ever add more than one file to the list
- self.output_pid(sys.stderr, "Submitting job for Mac pre-processing on file: %s" % (files[0]))
- self.SubmitJob(files[0], 'ProcessFilesWorkMac', args=(files[0],), callback=self.ProcessFilesMacFinished)
+ self.output_pid(sys.stderr, "Starting Mac pre-processing on file: %s" % (files[0]))
+ res = self.ProcessFilesWorkMac(files[0])
+ self.ProcessFilesMacFinished(res)
def ProcessFilesMacFinished(self, result):
if result['status']:
# kick off new jobs per-arch with our new list of files
Dumper.ProcessFiles(self, result['files'], after=AfterMac, after_arg=result['files'][0])
def dump_syms_cmdline(self, file, arch, files):
'''
@@ -952,17 +875,17 @@ class Dumper_Mac(Dumper):
return [self.dump_syms] + arch.split() + ['-g', file, files[1]]
return Dumper.dump_syms_cmdline(self, file, arch, files)
def ProcessFilesWorkMac(self, file):
"""dump_syms on Mac needs to be run on a dSYM bundle produced
by dsymutil(1), so run dsymutil here and pass the bundle name
down to the superclass method instead."""
t_start = time.time()
- self.output_pid(sys.stderr, "Worker running Mac pre-processing on file: %s" % (file,))
+ self.output_pid(sys.stderr, "Running Mac pre-processing on file: %s" % (file,))
# our return is a status and a tuple of files to dump symbols for
# the extra files are fallbacks; as soon as one is dumped successfully, we stop
result = { 'status' : False, 'files' : None, 'file_key' : file }
dsymbundle = file + ".dSYM"
if os.path.exists(dsymbundle):
shutil.rmtree(dsymbundle)
dsymutil = buildconfig.substs['DSYMUTIL']
@@ -981,17 +904,17 @@ class Dumper_Mac(Dumper):
self.output_pid(sys.stderr, "No symbols found in file: %s" % (file,))
result['status'] = False
result['files'] = (file, )
return result
result['status'] = True
result['files'] = (dsymbundle, file)
elapsed = time.time() - t_start
- self.output_pid(sys.stderr, 'Worker finished processing %s in %.2fs' %
+ self.output_pid(sys.stderr, 'Finished processing %s in %.2fs' %
(file, elapsed))
return result
def CopyDebug(self, file, debug_file, guid, code_file, code_id):
"""ProcessFiles has already produced a dSYM bundle, so we should just
copy that to the destination directory. However, we'll package it
into a .tar.bz2 because the debug symbols are pretty huge, and
also because it's a bundle, so it's a directory. |file| here is the
@@ -1066,17 +989,12 @@ to canonical locations in the source rep
srcdirs=options.srcdir,
vcsinfo=options.vcsinfo,
srcsrv=options.srcsrv,
exclude=options.exclude,
repo_manifest=options.repo_manifest,
file_mapping=file_mapping)
dumper.Process(*args[2:])
- dumper.Finish()
# run main if run directly
if __name__ == "__main__":
- # set up the multiprocessing infrastructure before we start;
- # note that this needs to be in the __main__ guard, or else Windows will choke
- Dumper.GlobalInit()
-
main()
--- a/toolkit/crashreporter/tools/unit-symbolstore.py
+++ b/toolkit/crashreporter/tools/unit-symbolstore.py
@@ -101,17 +101,16 @@ class TestSizeOrder(HelperMixin, unittes
if d and not os.path.exists(d):
os.makedirs(d)
open(f, 'wb').write('x' * size)
d = symbolstore.GetPlatformSpecificDumper(dump_syms="dump_syms",
symbol_path="symbol_path")
d.ShouldProcess = lambda f: True
d.ProcessFiles = mock_process_file
d.Process(self.test_dir)
- d.Finish(stop_pool=False)
self.assertEqual(processed, ['b/c/two', 'c/three', 'a/one'])
class TestExclude(HelperMixin, unittest.TestCase):
def test_exclude_wildcard(self):
"""
Test that using an exclude list with a wildcard pattern works.
"""
@@ -121,17 +120,16 @@ class TestExclude(HelperMixin, unittest.
processed.append((filename[len(self.test_dir):] if filename.startswith(self.test_dir) else filename).replace('\\', '/'))
return True
self.add_test_files(add_extension(["foo", "bar", "abc/xyz", "abc/fooxyz", "def/asdf", "def/xyzfoo"]))
d = symbolstore.GetPlatformSpecificDumper(dump_syms="dump_syms",
symbol_path="symbol_path",
exclude=["*foo*"])
d.ProcessFiles = mock_process_file
d.Process(self.test_dir)
- d.Finish(stop_pool=False)
processed.sort()
expected = add_extension(["bar", "abc/xyz", "def/asdf"])
expected.sort()
self.assertEqual(processed, expected)
def test_exclude_filenames(self):
"""
@@ -143,17 +141,16 @@ class TestExclude(HelperMixin, unittest.
processed.append((filename[len(self.test_dir):] if filename.startswith(self.test_dir) else filename).replace('\\', '/'))
return True
self.add_test_files(add_extension(["foo", "bar", "abc/foo", "abc/bar", "def/foo", "def/bar"]))
d = symbolstore.GetPlatformSpecificDumper(dump_syms="dump_syms",
symbol_path="symbol_path",
exclude=add_extension(["foo"]))
d.ProcessFiles = mock_process_file
d.Process(self.test_dir)
- d.Finish(stop_pool=False)
processed.sort()
expected = add_extension(["bar", "abc/bar", "def/bar"])
expected.sort()
self.assertEqual(processed, expected)
def mock_dump_syms(module_id, filename, extra=[]):
return ["MODULE os x86 %s %s" % (module_id, filename)
@@ -206,17 +203,16 @@ class TestCopyDebug(HelperMixin, unittes
return 0
self.mock_call.side_effect = mock_dsymutil
d = symbolstore.GetPlatformSpecificDumper(dump_syms="dump_syms",
symbol_path=self.symbol_dir,
copy_debug=True,
archs="abc xyz")
d.CopyDebug = mock_copy_debug
d.Process(self.test_dir)
- d.Finish(stop_pool=False)
self.assertEqual(1, len(copied))
@patch.dict('buildconfig.substs', {'MAKECAB': 'makecab'})
def test_copy_debug_copies_binaries(self):
"""
Test that CopyDebug copies binaries as well on Windows.
"""
test_file = os.path.join(self.test_dir, 'foo.dll')
@@ -230,17 +226,16 @@ class TestCopyDebug(HelperMixin, unittes
open(filename, 'w').write('stuff')
return 0
self.mock_call.side_effect = mock_compress
d = symbolstore.Dumper_Win32(dump_syms='dump_syms',
symbol_path=self.symbol_dir,
copy_debug=True)
d.FixFilenameCase = lambda f: f
d.Process(self.test_dir)
- d.Finish(stop_pool=False)
self.assertTrue(os.path.isfile(os.path.join(self.symbol_dir, code_file, code_id, code_file[:-1] + '_')))
class TestGetVCSFilename(HelperMixin, unittest.TestCase):
def setUp(self):
HelperMixin.setUp(self)
def tearDown(self):
HelperMixin.tearDown(self)
@@ -378,17 +373,16 @@ if target_platform() == 'WINNT':
symbol_path=symbolpath,
srcdirs=[srcdir],
vcsinfo=True,
srcsrv=True,
copy_debug=True)
# stub out CopyDebug
d.CopyDebug = lambda *args: True
d.Process(self.test_dir)
- d.Finish(stop_pool=False)
self.assertNotEqual(srcsrv_stream, None)
hgserver = [x.rstrip() for x in srcsrv_stream.splitlines() if x.startswith("HGSERVER=")]
self.assertEqual(len(hgserver), 1)
self.assertEqual(hgserver[0].split("=")[1], "http://example.com/repo")
class TestInstallManifest(HelperMixin, unittest.TestCase):
def setUp(self):
HelperMixin.setUp(self)
@@ -498,17 +492,16 @@ class TestFileMapping(HelperMixin, unitt
)
mock_Popen.return_value.stdout = mk_output(dumped_files)
d = symbolstore.Dumper('dump_syms', self.symboldir,
file_mapping=file_mapping)
f = os.path.join(self.objdir, 'somefile')
open(f, 'wb').write('blah')
d.Process(f)
- d.Finish(stop_pool=False)
expected_output = ''.join(mk_output(expected_files))
symbol_file = os.path.join(self.symboldir,
file_id[1], file_id[0], file_id[1] + '.sym')
self.assertEqual(open(symbol_file, 'r').read(), expected_output)
class TestFunctional(HelperMixin, unittest.TestCase):
'''Functional tests of symbolstore.py, calling it with a real
dump_syms binary and passing in a real binary to dump symbols from.
@@ -576,14 +569,9 @@ class TestFunctional(HelperMixin, unitte
filename = file_lines[0].split(None, 2)[2]
# Skip this check for local git repositories.
if os.path.isdir(mozpath.join(self.topsrcdir, '.hg')):
self.assertEqual('hg:', filename[:3])
if __name__ == '__main__':
- # use ThreadPoolExecutor to use threading instead of processes so
- # that our mocking/module-patching works.
- symbolstore.Dumper.GlobalInit(concurrent.futures.ThreadPoolExecutor)
-
mozunit.main()
-