[pypy-svn] r58737 - pypy/dist/pypy/translator/tool

[xoraxax at codespeak.net]

Author: xoraxax
Date: Tue Oct  7 14:30:19 2008
New Revision: 58737

Modified:
   pypy/dist/pypy/translator/tool/staticsizereport.py
Log:
(cfbolz, antocuni, xoraxax) Add reporting for static data in RPython programs by module. Can be used e.g. on the pdb prompt after source generation.

Modified: pypy/dist/pypy/translator/tool/staticsizereport.py
==============================================================================
--- pypy/dist/pypy/translator/tool/staticsizereport.py	(original)
+++ pypy/dist/pypy/translator/tool/staticsizereport.py	Tue Oct  7 14:30:19 2008
@@ -1,10 +1,34 @@
 from __future__ import division
 
-from pypy.rpython.lltypesystem.lltype import typeOf, _ptr, Ptr
+from pypy.rpython.lltypesystem.lltype import typeOf, _ptr, Ptr, ContainerType
 from pypy.rpython.lltypesystem import llmemory
 from pypy.rpython.memory.lltypelayout import convert_offset_to_int
 
-def guess_size(obj):
+
+def guess_module(graph):
+    func = getattr(graph, 'func', None)
+    name = '<unknown>'
+    if func is not None:
+        newname = func.func_globals.get('__name__',  None)
+        if newname is not None:
+            name = newname
+        else:
+            if func.__module__:
+                name = func.__module__
+    return name
+
+
+def values_to_nodes(database, values):
+    nodes = []
+    for value in values:
+        if isinstance(typeOf(value), Ptr) and isinstance(typeOf(value._obj), ContainerType):
+            node = database.getcontainernode(value._obj)
+            if node.nodekind != 'func':
+                nodes.append(node)
+    return nodes
+
+
+def guess_size_obj(obj):
     TYPE = typeOf(obj)
     ptr = _ptr(Ptr(TYPE), obj)
     if TYPE._is_varsize():
@@ -22,24 +46,94 @@
     return convert_offset_to_int(llmemory.sizeof(TYPE, length))
 
 
+def guess_size(database, node, recursive=None):
+    obj = node.obj
+    size = guess_size_obj(obj)
+    if recursive is None:
+        return size
+    if node in recursive:
+        return 0
+    recursive.add(node)
+    for dep in values_to_nodes(database, node.enum_dependencies()):
+        recursive.add(dep)
+        size += guess_size(database, dep, recursive)
+    return size
+
+
 def by_lltype(obj):
     return typeOf(obj)
 
-def group_static_size(database, grouper=by_lltype):
+def group_static_size(database, nodes, grouper=by_lltype, recursive=None):
     totalsize = {}
     numobjects = {}
-    for node in database.globalcontainers():
+    for node in nodes:
         obj = node.obj
         group = grouper(obj)
-        totalsize[group] = totalsize.get(group, 0) + guess_size(obj)
+        totalsize[group] = totalsize.get(group, 0) + guess_size(database, node, recursive)
         numobjects[group] = numobjects.get(group, 0) + 1
     return totalsize, numobjects
 
-def print_static_size(database, grouper=by_lltype):
-    totalsize, numobjects = group_static_size(database, grouper)
+def make_report_static_size(database, nodes, grouper, recursive=None):
+    totalsize, numobjects = group_static_size(database, nodes, grouper, recursive)
     l = [(size, key) for key, size in totalsize.iteritems()]
     l.sort()
     l.reverse()
+    sizesum = 0
+    report = []
     for size, key in l:
-        print key, size, numobjects[key], size / numobjects[key]
+        sizesum += size
+        report.append((key, size, numobjects[key], size / numobjects[key]))
+    return sizesum, report
+
+def format_report_line(line):
+    return str(line[0])[:50] + " " + " ".join([str(x) for x in line[1:]])
+
+
+def print_report_static_size(database, grouper=by_lltype):
+    " Reports all objects with a specified grouper. "
+    _, report = report_static_size(database.globalcontainers(), grouper)
+    for line in report:
+        print format_report_line(line)
+
+
+def print_aggregated_values_by_module_and_type(database, count_modules_separately=False):
+    " Reports all objects by module and by lltype. "
+    modules = {}
+    reports = []
+    funcnodes = [node for node in database.globalcontainers()
+                     if node.nodekind == "func"]
+    # extract all prebuilt nodes per module
+    for node in funcnodes:
+        graph = getattr(node.obj, 'graph', None)
+        if not graph:
+            continue
+        nodes_set = modules.setdefault(guess_module(graph), set())
+        assert len(node.funcgens) == 1
+        nodes_set.update(values_to_nodes(database, node.funcgens[0].all_cached_consts))
+    modules = modules.items()
+    # make sure that gc modules are reported latest to avoid them eating all objects
+    def gc_module_key(tup):
+        if "module.gc" in tup[0]:
+            return ("\xff", ) + tup
+        return tup
+    modules.sort(key=gc_module_key)
+
+    # report sizes per module
+    seen = set()
+    for modulename, nodes in modules:
+        if count_modules_separately:
+            seen = set()
+        if not nodes:
+            continue
+        size, report = make_report_static_size(database, nodes, by_lltype, seen)
+        reports.append((size, modulename, report))
+    reports.sort()
+    reports.reverse()
+    for size, modulename, report in reports:
+        if not size:
+            continue
+        print "########### %i %s ####################################" % (size, modulename)
+        for line in report:
+            print " " * 4 + format_report_line(line)
+        print