[pypy-commit] lang-smalltalk storage: Removed storage_statistics module, replaced with very simple storage_logger module.
anton_gulenko
noreply at buildbot.pypy.org
Mon Jul 7 13:16:22 CEST 2014
Author: Anton Gulenko <anton.gulenko at googlemail.com>
Branch: storage
Changeset: r855:9468db2cf599
Date: 2014-06-30 17:57 +0200
http://bitbucket.org/pypy/lang-smalltalk/changeset/9468db2cf599/
Log: Removed storage_statistics module, replaced with very simple
storage_logger module.
diff --git a/spyvm/model.py b/spyvm/model.py
--- a/spyvm/model.py
+++ b/spyvm/model.py
@@ -15,7 +15,7 @@
that create W_PointersObjects of correct size with attached shadows.
"""
import sys, weakref
-from spyvm import constants, error, version, storage_statistics
+from spyvm import constants, error, version, storage_logger
from spyvm.version import elidable_for_version, constant_for_version, constant_for_version_arg
from rpython.rlib import rrandom, objectmodel, jit, signature
@@ -566,7 +566,7 @@
_attrs_ = ['shadow']
shadow = None
repr_classname = "W_PointersObject"
- log_storage = storage_statistics.log
+ log_storage = storage_logger.log
@jit.unroll_safe
def __init__(self, space, w_class, size, weak=False):
diff --git a/spyvm/storage_logger.py b/spyvm/storage_logger.py
new file mode 100644
--- /dev/null
+++ b/spyvm/storage_logger.py
@@ -0,0 +1,27 @@
+
+# Put flag in a list to make it modifyable after compile time.
+_active = [False]
+
+def activate():
+ _active[0] = True
+
+def log(w_obj, operation, old_storage_object=None, log_classname=True):
+ if not _active[0]:
+ return
+
+ # Gather information to be logged
+ new_storage = w_obj.shadow.repr_classname
+ if old_storage_object:
+ old_storage = old_storage_object.repr_classname
+ else:
+ old_storage = None
+ size = w_obj.size()
+ if log_classname:
+ classname = w_obj.guess_classname()
+ else:
+ classname = None
+
+ # Construct and print the logstring
+ old_storage_string = "%s -> " % old_storage if old_storage else ""
+ classname_string = " of %s" % classname if classname else ""
+ print "%s (%s%s)%s size %d" % (operation, old_storage_string, new_storage, classname_string, size)
diff --git a/spyvm/storage_statistics.py b/spyvm/storage_statistics.py
deleted file mode 100644
--- a/spyvm/storage_statistics.py
+++ /dev/null
@@ -1,255 +0,0 @@
-
-from rpython.rlib.listsort import TimSort
-from rpython.rlib.objectmodel import import_from_mixin
-
-class StatsSorter(TimSort):
- """Sort a tuple of 3 strings"""
- def lt(self, a, b):
- if a[0] == b[0]:
- if a[1] == b[1]:
- return a[2] < b[2]
- else:
- return a[1] < b[1]
- else:
- return a[0] < b[0]
-
-class StorageStatistics(object):
-
- def __init__(self):
- self.modules = []
- self.using_classname = False
-
- def add_module(self, module):
- if module not in self.modules:
- self.modules.append(module)
- self.using_classname = self.using_classname or module.uses_classname
-
- def log(self, w_obj, operation, old_storage_object, log_classname):
- if len(self.modules) > 0:
- new_storage = w_obj.shadow.repr_classname
- if old_storage_object:
- old_storage = old_storage_object.repr_classname
- else:
- old_storage = None
- size = w_obj.size()
-
- if self.using_classname and log_classname:
- classname = w_obj.guess_classname()
- else:
- classname = None
- for module in self.modules:
- key = module.make_key(operation, old_storage, new_storage)
- module.storage_operation(key, size, classname)
-
- def print_results(self):
- for module in self.modules:
- module.print_results()
- print "\n\n"
-
-class StatisticsModule(object):
- uses_classname = False
- def storage_operation(self, operation_key, storage_size, container_classname):
- raise NotImplementedError("Abstract class")
- def print_results(self):
- raise NotImplementedError("Abstract class")
- def make_key(self, operation, old_storage, new_storage):
- return (operation, old_storage, new_storage)
- def key_string(self, key):
- if key[1]:
- return "%s (%s -> %s)" % (key[0], key[1], key[2])
- else:
- return "%s (%s)" % (key[0], key[2])
-
-class StatisticsLogger(StatisticsModule):
- uses_classname = True
- def storage_operation(self, operation_key, storage_size, container_classname):
- print self.log_string(operation_key, storage_size, container_classname)
-
- def log_string(self, operation_key, storage_size, container_classname):
- if container_classname:
- return "%s of %s size %d" % (self.key_string(operation_key), container_classname, storage_size)
- else:
- return "%s size %d" % (self.key_string(operation_key), storage_size)
-
- def print_results(self):
- # Nothing to do, this is just for logging during runtime.
- pass
-
-class StatisticsCollectorMixin(StatisticsModule):
-
- def __init__(self):
- self.stats = {}
-
- def storage_operation(self, operation_key, storage_size, container_classname):
- if not operation_key in self.stats:
- self.stats[operation_key] = self.initial_value()
- self.increment_value(self.stats[operation_key], storage_size, container_classname)
-
- def sorted_keys(self):
- keys = [ x for x in self.stats ]
- StatsSorter(keys).sort()
- return keys
-
-class HistogramStatisticsCollector(StatisticsModule):
- # Stores classnames with sizes
- # Value: map <classname, (count, elements)>
- import_from_mixin(StatisticsCollectorMixin)
-
- uses_classname = True
- def initial_value(self): return {}
- def increment_value(self, value_object, storage_size, container_classname):
- if not container_classname in value_object:
- value_object[container_classname] = [0, 0]
- m = value_object[container_classname]
- m[0] = m[0] + storage_size
- m[1] = m[1] + 1
-
- def make_key(self, operation, old_storage, new_storage):
- return (new_storage, "", "")
-
- def print_results(self):
- print "## Histogram statistics:"
- for key_tuple in self.sorted_keys():
- key = key_tuple[0]
- if not "Storage" in key:
- continue
- print "\n# %s" % key
- print "Data Objects Elements"
- classes = self.stats[key_tuple]
- for cls in classes:
- tuple = classes[cls]
- sum = tuple[0]
- num = tuple[1]
- print "%d slots in %d objects: %s" % (sum, num, cls)
-
-class StatisticsCollector(StatisticsModule):
- # Value: [total_size, num_operations]
- import_from_mixin(StatisticsCollectorMixin)
-
- def initial_value(self): return [0, 0]
- def increment_value(self, value_object, storage_size, container_classname):
- value_object[0] = value_object[0] + storage_size
- value_object[1] = value_object[1] + 1
- def print_results(self):
- print "Storage Statistics:"
- for key in self.sorted_keys():
- tuple = self.stats[key]
- sum = tuple[0]
- num = tuple[1]
- print "\t%s: %d times, avg size: %f" % (self.key_string(key), num, float(sum)/num)
-
-class DotStatisticsCollector(StatisticsCollector):
-
- def __init__(self):
- StatisticsCollector.__init__(self)
- self.incoming_operations = {}
- self.incoming_elements = {}
- self.outgoing_operations = {}
- self.outgoing_elements = {}
-
- def storage_operation(self, key, storage_size, container_classname):
- StatisticsCollector.storage_operation(self, key, storage_size, container_classname)
- source_type = key[1]
- target_type = key[2]
- self.fill_maps(self.incoming_operations, self.incoming_elements, target_type, storage_size)
- if source_type:
- self.fill_maps(self.outgoing_operations, self.outgoing_elements, source_type, storage_size)
-
- def fill_maps(self, operations_map, elements_map, key_type, size):
- if key_type not in operations_map:
- operations_map[key_type] = 0
- elements_map[key_type] = 0
- operations_map[key_type] = operations_map[key_type] + 1
- elements_map[key_type] = elements_map[key_type] + size
-
- def print_results(self):
- print "Storage Statistics (dot format):"
- print "================================"
- print "*/" # End the commend started in activate_statistics()
- print self.dot_string()
-
- def dot_string(self):
- # Return a string that is valid dot code and can be parsed by the graphviz dot utility.
- # Unfortunately, this is pretty complicated and messy... Sorry.
-
- result = "digraph G {"
- result += "loading_image [label=\"Image Loading\",shape=box];"
- result += "created_object [label=\"Object Creation\",shape=box];"
- for key in self.stats:
- operation_type = key[0]
- target_node = key[2]
- elements = self.stats[key][0]
- operations = self.stats[key][1]
- label_suffix = ""
- if operation_type == "Switched":
- source_node = key[1]
- percent_ops = float(operations) / float(self.incoming_operations[source_node]) * 100
- percent_elements = float(elements) / float(self.incoming_elements[source_node]) * 100
- label_suffix = "\n%d%% objects\n%d%% elements" % (int(percent_ops), int(percent_elements))
- elif operation_type == "Initialized":
- source_node = "created_object"
- elif operation_type == "Filledin":
- source_node = "loading_image"
- else:
- print "Could not handle storage operation %s" % operation_type
- continue
- result += "%s -> %s [label=\"%d objects\n%d elements per object%s\"];" % (source_node, target_node, operations, elements/operations, label_suffix)
- for type in self.incoming_operations:
- incoming_ops = self.incoming_operations[type]
- incoming_els = self.incoming_elements[type]
- label = "\nIncoming objects: %d" % incoming_ops
- label += "\nIncoming elements: %d" % incoming_els
- if type in self.outgoing_operations:
- remaining_ops = incoming_ops - self.outgoing_operations[type]
- remaining_els = incoming_els - self.outgoing_elements[type]
- else:
- remaining_ops = incoming_ops
- remaining_els = incoming_els
- percent_remaining_ops = float(remaining_ops) / incoming_ops * 100
- percent_remaining_els = float(remaining_els) / incoming_els * 100
- label += "\nRemaining objects: %d (%d%%)" % (remaining_ops, int(percent_remaining_ops))
- label += "\nRemaining elements: %d (%d%%)" % (remaining_els, int(percent_remaining_els))
- result += "%s [label=\"%s%s\"];" % (type, type, label)
- result += "}"
- return result
-
-class DetailedStatisticsCollector(StatisticsModule):
- # Value: list of numbers (sizes)
- import_from_mixin(StatisticsCollectorMixin)
-
- def initial_value(self): return []
- def increment_value(self, value_object, storage_size, container_classname):
- value_object.append(storage_size)
- def print_results(self):
- print "Detailed Storage Statistics:"
- for key in self.sorted_keys():
- print "\t%s: s" % (self.key_string(key), self.stats[key])
-
-# Static & global access to a StorageStatistics instance.
-
-_stats = StorageStatistics()
-_logger = StatisticsLogger()
-_collector = StatisticsCollector()
-_detailedcollector = DetailedStatisticsCollector()
-_dotcollector = DotStatisticsCollector()
-_histogramcollector = HistogramStatisticsCollector()
-
-def activate_statistics(log=False, statistics=False, detailed_statistics=False, dot=False, histogram=False):
- if log:
- _stats.add_module(_logger)
- if statistics:
- _stats.add_module(_collector)
- if detailed_statistics:
- _stats.add_module(_detailedcollector)
- if dot:
- _stats.add_module(_dotcollector)
- # Start a comment in order to make the entire output valid dot code. Hack.
- print "/*"
- if histogram:
- _stats.add_module(_histogramcollector)
-
-def print_statistics():
- _stats.print_results()
-
-def log(w_obj, operation, old_storage=None, log_classname=True):
- _stats.log(w_obj, operation, old_storage, log_classname)
diff --git a/spyvm/test/test_strategies.py b/spyvm/test/test_strategies.py
--- a/spyvm/test/test_strategies.py
+++ b/spyvm/test/test_strategies.py
@@ -1,5 +1,5 @@
import py
-from spyvm import wrapper, model, interpreter, shadow, storage_statistics
+from spyvm import wrapper, model, interpreter, shadow
from spyvm.error import WrapperException, FatalError
from .util import read_image, copy_to_module, cleanup_module
@@ -175,50 +175,3 @@
a.store(space, 1, space.wrap_int(2))
assert isinstance(a.shadow, shadow.ListStorageShadow)
check_arr(a, [1.2, 2, w_nil, w_nil, w_nil])
-
-def test_statistics_stats():
- col = storage_statistics.DetailedStatisticsCollector()
- stats = storage_statistics.StorageStatistics()
- col.storage_operation(stats.make_key("B", "old", "new"), 3, None)
- col.storage_operation(stats.make_key("B", "old", "new"), 4, None)
- col.storage_operation(stats.make_key("B", "old2", "new2"), 20, None)
- col.storage_operation(stats.make_key("B", "old", "new"), 5, None)
- col.storage_operation(stats.make_key("A", "old", "new"), 1, None)
- col.storage_operation(stats.make_key("A", "old", "new"), 2, None)
- col.storage_operation(stats.make_key("C", "old", "new"), 10, None)
- col.storage_operation(stats.make_key("C", "old", "new"), 11, None)
- keys = col.sorted_keys()
- assert keys == [ ("A", "old", "new"), ("B", "old", "new"), ("B", "old2", "new2"), ("C", "old", "new") ]
- assert col.stats[keys[0]] == [1, 2]
- assert col.stats[keys[1]] == [3, 4, 5]
- assert col.stats[keys[2]] == [20]
- assert col.stats[keys[3]] == [10, 11]
-
-def test_statistics_log():
- stats = storage_statistics.StorageStatistics()
- log = storage_statistics.StatisticsLogger()
- s = log.log_string(stats.make_key("Operation", "old_storage", "new_storage"), 22, "classname")
- assert s == "Operation (old_storage -> new_storage) of classname size 22"
- s = log.log_string(stats.make_key("InitialOperation", None, "some_new_storage"), 40, "a_classname")
- assert s == "InitialOperation (some_new_storage) of a_classname size 40"
-
-def test_statistics_stats_dot():
- col = storage_statistics.DotStatisticsCollector()
- stats = storage_statistics.StorageStatistics()
-
- col.storage_operation(stats.make_key("Switched", "old", "new"), 10, None)
- col.storage_operation(stats.make_key("Switched", "old", "new"), 10, None)
- col.storage_operation(stats.make_key("Switched", "new", "new2"), 10, None)
- col.storage_operation(stats.make_key("Switched", "old2", "new"), 5, None)
- col.storage_operation(stats.make_key("Initialized", None, "old"), 13, None)
- col.storage_operation(stats.make_key("Initialized", None, "old"), 10, None)
- col.storage_operation(stats.make_key("Initialized", None, "old"), 10, None)
- col.storage_operation(stats.make_key("Initialized", None, "old2"), 15, None)
- col.storage_operation(stats.make_key("Filledin", None, "old2"), 20, None)
- col.storage_operation(stats.make_key("Filledin", None, "new"), 10, None)
- col.storage_operation(stats.make_key("Filledin", None, "new"), 11, None)
-
- # The dot-code is correct, I checked ;)
- assert col.dot_string() == \
- 'digraph G {loading_image [label="Image Loading",shape=box];created_object [label="Object Creation",shape=box];created_object -> old2 [label="1 objects\n15 elements per object"];loading_image -> new [label="2 objects\n10 elements per object"];old -> new [label="2 objects\n10 elements per object\n66% objects\n60% elements"];loading_image -> old2 [label="1 objects\n20 elements per object"];created_object -> old [label="3 objects\n11 elements per object"];old2 -> new [label="1 objects\n5 elements per object\n50% objects\n14% elements"];new -> new2 [label="1 objects\n10 elements per object\n20% objects\n21% elements"];new2 [label="new2\nIncoming objects: 1\nIncoming elements: 10\nRemaining objects: 1 (100%)\nRemaining elements: 10 (100%)"];new [label="new\nIncoming objects: 5\nIncoming elements: 46\nRemaining objects: 4 (80%)\nRemaining elements: 36 (78%)"];old2 [label="old2\nIncoming objects: 2\nIncoming elements: 35\nRemaining objects: 1 (50%)\nRemaining elements: 30 (85%)"];old [label="old\nIncoming objects: 3\nIncoming elements: 33\nRemaining objects: 1 (33%)\nRemaining elements: 13 (39%)"];}'
-
\ No newline at end of file
diff --git a/targetimageloadingsmalltalk.py b/targetimageloadingsmalltalk.py
--- a/targetimageloadingsmalltalk.py
+++ b/targetimageloadingsmalltalk.py
@@ -6,7 +6,7 @@
from rpython.rlib import jit, rpath
from spyvm import model, interpreter, squeakimage, objspace, wrapper,\
- error, shadow, storage_statistics, constants
+ error, shadow, storage_logger, constants
from spyvm.tool.analyseimage import create_image
from spyvm.interpreter_proxy import VirtualMachine
@@ -130,11 +130,7 @@
-p|--poll_events
-ni|--no-interrupts
-d|--max-stack-depth [number, default %d, <= 0 disables stack protection]
- --strategy-log
- --strategy-stats
- --strategy-stats-dot
- --strategy-stats-details
- --strategy-stats-histogram
+ -l|--storage-log
[image path, default: Squeak.image]
""" % (argv[0], constants.MAX_LOOP_DEPTH)
@@ -199,16 +195,8 @@
_arg_missing(argv, idx, arg)
max_stack_depth = int(argv[idx + 1])
idx += 1
- elif arg == "--strategy-log":
- storage_statistics.activate_statistics(log=True)
- elif arg == "--strategy-stats":
- storage_statistics.activate_statistics(statistics=True)
- elif arg == "--strategy-stats-dot":
- storage_statistics.activate_statistics(dot=True)
- elif arg == "--strategy-stats-histogram":
- storage_statistics.activate_statistics(histogram=True)
- elif arg == "--strategy-stats-details":
- storage_statistics.activate_statistics(statistics=True, detailed_statistics=True)
+ elif arg in ["-l", "--storage-log"]:
+ storage_logger.activate()
elif path is None:
path = argv[idx]
else:
@@ -245,7 +233,6 @@
else:
_run_image(interp)
result = 0
- storage_statistics.print_statistics()
return result
More information about the pypy-commit
mailing list