--- a/memory/replace/dmd/block_analyzer.py
+++ b/memory/replace/dmd/block_analyzer.py
@@ -56,23 +56,25 @@ allocatorFns = [
# functions won't be stripped, as explained above.
'???',
]
####
# Command line arguments
+
def range_1_24(string):
value = int(string)
if value < 1 or value > 24:
msg = '{:s} is not in the range 1..24'.format(string)
raise argparse.ArgumentTypeError(msg)
return value
+
parser = argparse.ArgumentParser(description='Analyze the heap graph to find out things about an object. \
By default this prints out information about blocks that point to the given block.')
parser.add_argument('dmd_log_file_name',
help='clamped DMD log file name')
parser.add_argument('block',
help='address of the block of interest')
@@ -139,17 +141,18 @@ def show_referrers(args, blocks, stacks,
# XXX This means that this output will be wrong for logs from 32-bit systems!
referrers.setdefault(b, []).append(8 * which_edge)
anyFound = True
which_edge += 1
for r in referrers:
sys.stdout.write('0x{} size = {} bytes'.format(blocks[r].addr, blocks[r].req_size))
plural = 's' if len(referrers[r]) > 1 else ''
- sys.stdout.write(' at byte offset' + plural + ' ' + (', '.join(str(x) for x in referrers[r])))
+ sys.stdout.write(' at byte offset' + plural + ' ' +
+ (', '.join(str(x) for x in referrers[r])))
print
print_trace_segment(args, stacks, blocks[r])
print
if args.chain_reports:
if len(referrers) == 0:
sys.stdout.write('Found no more referrers.\n')
break
--- a/memory/replace/dmd/dmd.py
+++ b/memory/replace/dmd/dmd.py
@@ -40,36 +40,37 @@ allocatorFns = [
'operator new[](',
'g_slice_alloc',
# This one necessary to fully filter some sequences of allocation functions
# that happen in practice. Note that ??? entries that follow non-allocation
# functions won't be stripped, as explained above.
'???',
]
+
class Record(object):
'''A record is an aggregation of heap blocks that have identical stack
traces. It can also be used to represent the difference between two
records.'''
def __init__(self):
self.numBlocks = 0
self.reqSize = 0
self.slopSize = 0
self.usableSize = 0
self.allocatedAtDesc = None
self.reportedAtDescs = []
self.usableSizes = collections.defaultdict(int)
def isZero(self, args):
return self.numBlocks == 0 and \
- self.reqSize == 0 and \
- self.slopSize == 0 and \
- self.usableSize == 0 and \
- len(self.usableSizes) == 0
+ self.reqSize == 0 and \
+ self.slopSize == 0 and \
+ self.usableSize == 0 and \
+ len(self.usableSizes) == 0
def negate(self):
self.numBlocks = -self.numBlocks
self.reqSize = -self.reqSize
self.slopSize = -self.slopSize
self.usableSize = -self.usableSize
negatedUsableSizes = collections.defaultdict(int)
@@ -109,33 +110,33 @@ class Record(object):
usableSizes3[-usableSize] = usableSizes2[usableSize]
self.usableSizes = usableSizes3
@staticmethod
def cmpByUsableSize(r1, r2):
# Sort by usable size, then by req size.
return cmp(abs(r1.usableSize), abs(r2.usableSize)) or \
- Record.cmpByReqSize(r1, r2)
+ Record.cmpByReqSize(r1, r2)
@staticmethod
def cmpByReqSize(r1, r2):
# Sort by req size.
return cmp(abs(r1.reqSize), abs(r2.reqSize))
@staticmethod
def cmpBySlopSize(r1, r2):
# Sort by slop size.
return cmp(abs(r1.slopSize), abs(r2.slopSize))
@staticmethod
def cmpByNumBlocks(r1, r2):
# Sort by block counts, then by usable size.
return cmp(abs(r1.numBlocks), abs(r2.numBlocks)) or \
- Record.cmpByUsableSize(r1, r2)
+ Record.cmpByUsableSize(r1, r2)
sortByChoices = {
'usable': Record.cmpByUsableSize, # the default
'req': Record.cmpByReqSize,
'slop': Record.cmpBySlopSize,
'num-blocks': Record.cmpByNumBlocks,
}
@@ -203,23 +204,26 @@ def fixStackTraces(inputFilename, isZipp
# This append() call is needed to make the import statements work when this
# script is installed as a symlink.
sys.path.append(os.path.dirname(__file__))
bpsyms = os.environ.get('BREAKPAD_SYMBOLS_PATH', None)
sysname = platform.system()
if bpsyms and os.path.exists(bpsyms):
import fix_stack_using_bpsyms as fixModule
- fix = lambda line: fixModule.fixSymbols(line, bpsyms)
+
+ def fix(line): return fixModule.fixSymbols(line, bpsyms)
elif sysname == 'Linux':
import fix_linux_stack as fixModule
- fix = lambda line: fixModule.fixSymbols(line)
+
+ def fix(line): return fixModule.fixSymbols(line)
elif sysname == 'Darwin':
import fix_macosx_stack as fixModule
- fix = lambda line: fixModule.fixSymbols(line)
+
+ def fix(line): return fixModule.fixSymbols(line)
else:
fix = None # there is no fix script for Windows
if fix:
# Fix stacks, writing output to a temporary file, and then
# overwrite the original file.
tmpFile = tempfile.NamedTemporaryFile(delete=False)
@@ -338,18 +342,18 @@ def getDigestFromFile(args, inputFile):
return desc
# Aggregate blocks into records. All sufficiently similar blocks go into a
# single record.
if mode in ['live', 'cumulative']:
liveOrCumulativeRecords = collections.defaultdict(Record)
elif mode == 'dark-matter':
- unreportedRecords = collections.defaultdict(Record)
- onceReportedRecords = collections.defaultdict(Record)
+ unreportedRecords = collections.defaultdict(Record)
+ onceReportedRecords = collections.defaultdict(Record)
twiceReportedRecords = collections.defaultdict(Record)
heapUsableSize = 0
heapBlocks = 0
recordKeyPartCache = {}
for block in blockList:
@@ -413,30 +417,30 @@ def getDigestFromFile(args, inputFile):
num = block['num']
else:
num = 1
usableSize = reqSize + slopSize
heapUsableSize += num * usableSize
heapBlocks += num
- record.numBlocks += num
- record.reqSize += num * reqSize
- record.slopSize += num * slopSize
+ record.numBlocks += num
+ record.reqSize += num * reqSize
+ record.slopSize += num * slopSize
record.usableSize += num * usableSize
if record.allocatedAtDesc == None:
record.allocatedAtDesc = \
buildTraceDescription(traceTable, frameTable,
allocatedAtTraceKey)
if mode in ['live', 'cumulative']:
pass
elif mode == 'dark-matter':
if 'reps' in block and record.reportedAtDescs == []:
- f = lambda k: buildTraceDescription(traceTable, frameTable, k)
+ def f(k): return buildTraceDescription(traceTable, frameTable, k)
record.reportedAtDescs = map(f, reportedAtTraceKeys)
record.usableSizes[usableSize] += num
# All the processed data for a single DMD file is called a "digest".
digest = {}
digest['dmdEnvVar'] = dmdEnvVar
digest['mode'] = mode
digest['heapUsableSize'] = heapUsableSize
@@ -478,41 +482,41 @@ def diffRecords(args, records1, records2
def diffDigests(args, d1, d2):
if (d1['mode'] != d2['mode']):
raise Exception("the input files have different 'mode' properties")
d3 = {}
d3['dmdEnvVar'] = (d1['dmdEnvVar'], d2['dmdEnvVar'])
d3['mode'] = d1['mode']
d3['heapUsableSize'] = d2['heapUsableSize'] - d1['heapUsableSize']
- d3['heapBlocks'] = d2['heapBlocks'] - d1['heapBlocks']
+ d3['heapBlocks'] = d2['heapBlocks'] - d1['heapBlocks']
if d1['mode'] in ['live', 'cumulative']:
d3['liveOrCumulativeRecords'] = \
diffRecords(args, d1['liveOrCumulativeRecords'],
- d2['liveOrCumulativeRecords'])
+ d2['liveOrCumulativeRecords'])
elif d1['mode'] == 'dark-matter':
- d3['unreportedRecords'] = diffRecords(args, d1['unreportedRecords'],
- d2['unreportedRecords'])
- d3['onceReportedRecords'] = diffRecords(args, d1['onceReportedRecords'],
- d2['onceReportedRecords'])
+ d3['unreportedRecords'] = diffRecords(args, d1['unreportedRecords'],
+ d2['unreportedRecords'])
+ d3['onceReportedRecords'] = diffRecords(args, d1['onceReportedRecords'],
+ d2['onceReportedRecords'])
d3['twiceReportedRecords'] = diffRecords(args, d1['twiceReportedRecords'],
- d2['twiceReportedRecords'])
+ d2['twiceReportedRecords'])
return d3
def printDigest(args, digest):
- dmdEnvVar = digest['dmdEnvVar']
- mode = digest['mode']
- heapUsableSize = digest['heapUsableSize']
- heapBlocks = digest['heapBlocks']
+ dmdEnvVar = digest['dmdEnvVar']
+ mode = digest['mode']
+ heapUsableSize = digest['heapUsableSize']
+ heapBlocks = digest['heapBlocks']
if mode in ['live', 'cumulative']:
liveOrCumulativeRecords = digest['liveOrCumulativeRecords']
elif mode == 'dark-matter':
- unreportedRecords = digest['unreportedRecords']
- onceReportedRecords = digest['onceReportedRecords']
+ unreportedRecords = digest['unreportedRecords']
+ onceReportedRecords = digest['onceReportedRecords']
twiceReportedRecords = digest['twiceReportedRecords']
separator = '#' + '-' * 65 + '\n'
def number(n):
'''Format a number with comma as a separator.'''
return '{:,d}'.format(n)
@@ -537,17 +541,17 @@ def printDigest(args, digest):
cmpRecords = sortByChoices[args.sort_by]
sortedRecords = sorted(records.values(), cmp=cmpRecords, reverse=True)
kindBlocks = 0
kindUsableSize = 0
maxRecord = 1000
# First iteration: get totals, etc.
for record in sortedRecords:
- kindBlocks += record.numBlocks
+ kindBlocks += record.numBlocks
kindUsableSize += record.usableSize
# Second iteration: print.
if numRecords == 0:
out('# no {:} heap blocks\n'.format(recordKind))
kindCumulativeUsableSize = 0
for i, record in enumerate(sortedRecords, start=1):
@@ -563,18 +567,18 @@ def printDigest(args, digest):
out(' {:} block{:} in heap block record {:,d} of {:,d}'.
format(number(record.numBlocks),
plural(record.numBlocks), i, numRecords))
out(' {:} bytes ({:} requested / {:} slop)'.
format(number(record.usableSize),
number(record.reqSize),
number(record.slopSize)))
- abscmp = lambda (usableSize1, _1), (usableSize2, _2): \
- cmp(abs(usableSize1), abs(usableSize2))
+ def abscmp((usableSize1, _1), (usableSize2, _2)): return \
+ cmp(abs(usableSize1), abs(usableSize2))
usableSizes = sorted(record.usableSizes.items(), cmp=abscmp,
reverse=True)
hasSingleBlock = len(usableSizes) == 1 and usableSizes[0][1] == 1
if not hasSingleBlock:
out(' Individual block sizes: ', end='')
if len(usableSizes) == 0:
@@ -610,17 +614,16 @@ def printDigest(args, digest):
again = 'again ' if n > 0 else ''
out(' Reported {:}at {{'.format(again))
printStack(reportedAtDesc)
out(' }')
out('}\n')
return (kindUsableSize, kindBlocks)
-
def printInvocation(n, dmdEnvVar, mode):
out('Invocation{:} {{'.format(n))
if dmdEnvVar == None:
out(' $DMD is undefined')
else:
out(' $DMD = \'' + dmdEnvVar + '\'')
out(' Mode = \'' + mode + '\'')
out('}\n')
@@ -755,34 +758,36 @@ class ClampStats:
# Number of null pointers.
self.nullPtr = 0
# Number of non-null pointers that didn't point into the middle
# of any blocks. These are clamped to null.
self.nonNullNonBlockPtr = 0
-
def clampedBlockAddr(self, sameAddress):
if sameAddress:
self.startBlockPtr += 1
else:
self.midBlockPtr += 1
def nullAddr(self):
self.nullPtr += 1
def clampedNonBlockAddr(self):
self.nonNullNonBlockPtr += 1
def log(self):
sys.stderr.write('Results:\n')
- sys.stderr.write(' Number of pointers already pointing to start of blocks: ' + str(self.startBlockPtr) + '\n')
- sys.stderr.write(' Number of pointers clamped to start of blocks: ' + str(self.midBlockPtr) + '\n')
- sys.stderr.write(' Number of non-null pointers not pointing into blocks clamped to null: ' + str(self.nonNullNonBlockPtr) + '\n')
+ sys.stderr.write(
+ ' Number of pointers already pointing to start of blocks: ' + str(self.startBlockPtr) + '\n')
+ sys.stderr.write(' Number of pointers clamped to start of blocks: ' +
+ str(self.midBlockPtr) + '\n')
+ sys.stderr.write(' Number of non-null pointers not pointing into blocks clamped to null: ' +
+ str(self.nonNullNonBlockPtr) + '\n')
sys.stderr.write(' Number of null pointers: ' + str(self.nullPtr) + '\n')
# Search the block ranges array for a block that address points into.
# The search is carried out in an array of starting addresses for each blocks
# because it is faster.
def clampAddress(blockRanges, blockStarts, clampStats, address):
i = bisect_right(blockStarts, address)
@@ -882,9 +887,8 @@ def main():
if args.input_file2:
digest2 = getDigestFromFile(args, args.input_file2)
digest = diffDigests(args, digest, digest2)
printDigest(args, digest)
if __name__ == '__main__':
main()
-
--- a/memory/replace/logalloc/replay/logalloc_munge.py
+++ b/memory/replace/logalloc/replay/logalloc_munge.py
@@ -20,23 +20,25 @@ See README for more details.
from __future__ import print_function
import sys
from collections import (
defaultdict,
deque,
)
+
class IdMapping(object):
"""Class to map values to ids.
Each value is associated to an increasing id, starting from 1.
When a value is removed, its id is recycled and will be reused for
subsequent values.
"""
+
def __init__(self):
self.id = 1
self._values = {}
self._recycle = deque()
def __getitem__(self, value):
if value not in self._values:
if self._recycle:
@@ -51,17 +53,18 @@ class IdMapping(object):
return
self._recycle.append(self._values[value])
del self._values[value]
def __contains__(self, value):
return value == 0 or value in self._values
-class Ignored(Exception): pass
+class Ignored(Exception):
+ pass
def split_log_line(line):
try:
# The format for each line is:
# <pid> [<tid>] <function>([<args>])[=<result>]
#
# The original format didn't include the tid, so we try to parse
@@ -93,18 +96,18 @@ NUM_ARGUMENTS = {
'realloc': 2,
'memalign': 2,
'valloc': 1,
}
def main():
pids = IdMapping()
- processes = defaultdict(lambda: { 'pointers': IdMapping(),
- 'tids': IdMapping() })
+ processes = defaultdict(lambda: {'pointers': IdMapping(),
+ 'tids': IdMapping()})
for line in sys.stdin:
line = line.strip()
try:
pid, tid, func, args, result = split_log_line(line)
# Replace pid with an id.
pid = pids[int(pid)]
@@ -132,16 +135,16 @@ def main():
if result:
result = int(result, 16)
if not result:
raise Ignored('Result is NULL')
result = "#%d" % pointers[result]
print('%d %d %s(%s)%s' % (pid, tid, func, ','.join(args),
- '=%s' % result if result else ''))
+ '=%s' % result if result else ''))
except Exception as e:
print('Ignored "%s": %s' % (line, e.message), file=sys.stderr)
if __name__ == '__main__':
main()