[pypy-commit] lang-smalltalk storage: Refactored storage statistics to save memory when details are not required.
anton_gulenko
noreply at buildbot.pypy.org
Wed May 7 21:16:12 CEST 2014
Author: Anton Gulenko <anton.gulenko at googlemail.com>
Branch: storage
Changeset: r800:4e5d3401cdf2
Date: 2014-05-06 10:52 +0200
http://bitbucket.org/pypy/lang-smalltalk/changeset/4e5d3401cdf2/
Log: Refactored storage statistics to save memory when details are not
required.
diff --git a/spyvm/storage_statistics.py b/spyvm/storage_statistics.py
--- a/spyvm/storage_statistics.py
+++ b/spyvm/storage_statistics.py
@@ -13,16 +13,16 @@
return a[0] < b[0]
class StorageStatistics(object):
- # Key: (operation_name, old_storage, new_storage)
- # Value: [sizes]
- stats = {}
+ modules = []
+ using_classname = False
- do_log = False
- do_stats = False
- do_stats_sizes = False
+ def add_module(self, module):
+ if module not in self.modules:
+ self.modules.append(module)
+ self.using_classname = self.using_classname or module.uses_classname
def log(self, w_obj, operation, old_storage_object, log_classname):
- if self.do_log or self.do_stats:
+ if len(self.modules) > 0:
new_storage = w_obj.shadow.repr_classname
if old_storage_object:
old_storage = old_storage_object.repr_classname
@@ -31,62 +31,101 @@
size = w_obj.size()
key = self.make_key(operation, old_storage, new_storage)
- if self.do_stats:
- self.stat_operation(key, size)
- if self.do_log:
- if log_classname:
- classname = w_obj.guess_classname()
- else:
- classname = None
- self.log_operation(key, size, classname)
+ if self.using_classname and log_classname:
+ classname = w_obj.guess_classname()
+ else:
+ classname = None
+ for module in self.modules:
+ module.storage_operation(key, size, classname)
def make_key(self, operation, old_storage, new_storage):
return (operation, old_storage, new_storage)
-
- def stat_operation(self, key, size):
- if not key in self.stats:
- self.stats[key] = []
- self.stats[key].append(size)
+
+ def print_results(self):
+ for module in self.modules:
+ module.print_results()
- def log_operation(self, key, size, classname):
- print self.log_operation_string(key, size, classname)
-
+class StatisticsModule(object):
+ uses_classname = False
+ def storage_operation(self, operation_key, storage_size, element_classname):
+ raise NotImplementedError("Abstract class")
+ def print_results(self):
+ raise NotImplementedError("Abstract class")
def key_string(self, key):
if key[1]:
return "%s (%s -> %s)" % (key[0], key[1], key[2])
else:
return "%s (%s)" % (key[0], key[2])
-
- def log_operation_string(self, key, size, classname):
- if classname:
- return "%s of %s size %d" % (self.key_string(key), classname, size)
+
+class StatisticsLogger(StatisticsModule):
+ uses_classname = True
+ def storage_operation(self, operation_key, storage_size, element_classname):
+ print self.log_string(operation_key, storage_size, element_classname)
+
+ def log_string(self, operation_key, storage_size, element_classname):
+ if element_classname:
+ return "%s of %s size %d" % (self.key_string(operation_key), element_classname, storage_size)
else:
- return "%s size %d" % (self.key_string(key), size)
-
+ return "%s size %d" % (self.key_string(operation_key), storage_size)
+
+ def print_results(self):
+ # Nothing to do, this is just for logging during runtime.
+ pass
+
+class AbstractStatisticsCollector(StatisticsModule):
+ stats = {}
+
+ def storage_operation(self, operation_key, storage_size, element_classname):
+ if not operation_key in self.stats:
+ self.stats[operation_key] = self.initial_value()
+ self.increment_value(self.stats[operation_key], storage_size)
+
def sorted_keys(self):
keys = [ x for x in self.stats ]
StatsSorter(keys).sort()
return keys
-
- def print_stats(self):
+
+class StatisticsCollector(AbstractStatisticsCollector):
+ # Value: [total_size, num_operations]
+ def initial_value(self): return [0, 0]
+ def increment_value(self, value_object, storage_size):
+ value_object[0] = value_object[0] + storage_size
+ value_object[1] = value_object[1] + 1
+ def print_results(self):
+ print "Storage Statistics:"
for key in self.sorted_keys():
- sizes = self.stats[key]
- sum = 0
- for s in sizes: sum += s
- print "%s: %d times, avg size: %f" % (self.key_string(key), len(sizes), float(sum)/len(sizes))
- if self.do_stats_sizes:
- print " All sizes: %s" % sizes
+ tuple = self.stats[key]
+ sum = tuple[0]
+ num = tuple[1]
+ print "\t%s: %d times, avg size: %f" % (self.key_string(key), num, float(sum)/num)
+
+class DetailedStatisticsCollector(AbstractStatisticsCollector):
+ # Value: list of numbers (sizes)
+ def initial_value(self): return []
+ def increment_value(self, value_object, storage_size):
+ value_object.append(storage_size)
+ def print_results(self):
+ print "Detailed Storage Statistics:"
+ for key in self.sorted_keys():
+ print "\t%s: s" % (self.key_string(key), self.stats[key])
+
+# Static & global access to a StorageStatistics instance.
_stats = StorageStatistics()
+_logger = StatisticsLogger()
+_collector = StatisticsCollector()
+_detailedcollector = DetailedStatisticsCollector()
-def activate_statistics(log=False, statistics=False, statstics_sizes=False):
- _stats.do_log = _stats.do_log or log
- _stats.do_stats = _stats.do_stats or statistics
- _stats.do_stats_sizes = _stats.do_stats_sizes or statstics_sizes
+def activate_statistics(log=False, statistics=False, detailed_statistics=False):
+ if log:
+ _stats.add_module(_logger)
+ if statistics:
+ _stats.add_module(_collector)
+ if detailed_statistics:
+ _stats.add_module(_detailedcollector)
def print_statistics():
- if _stats.do_stats:
- _stats.print_stats()
+ _stats.print_results()
def log(w_obj, operation, old_storage=None, log_classname=True):
_stats.log(w_obj, operation, old_storage, log_classname)
diff --git a/spyvm/test/test_strategies.py b/spyvm/test/test_strategies.py
--- a/spyvm/test/test_strategies.py
+++ b/spyvm/test/test_strategies.py
@@ -178,25 +178,27 @@
def test_statistics_stats():
stats = storage_statistics.StorageStatistics()
- stats.stat_operation(stats.make_key("B", "old", "new"), 3)
- stats.stat_operation(stats.make_key("B", "old", "new"), 4)
- stats.stat_operation(stats.make_key("B", "old2", "new2"), 20)
- stats.stat_operation(stats.make_key("B", "old", "new"), 5)
- stats.stat_operation(stats.make_key("A", "old", "new"), 1)
- stats.stat_operation(stats.make_key("A", "old", "new"), 2)
- stats.stat_operation(stats.make_key("C", "old", "new"), 10)
- stats.stat_operation(stats.make_key("C", "old", "new"), 11)
- keys = stats.sorted_keys()
+ col = storage_statistics.DetailedStatisticsCollector()
+ col.storage_operation(stats.make_key("B", "old", "new"), 3, None)
+ col.storage_operation(stats.make_key("B", "old", "new"), 4, None)
+ col.storage_operation(stats.make_key("B", "old2", "new2"), 20, None)
+ col.storage_operation(stats.make_key("B", "old", "new"), 5, None)
+ col.storage_operation(stats.make_key("A", "old", "new"), 1, None)
+ col.storage_operation(stats.make_key("A", "old", "new"), 2, None)
+ col.storage_operation(stats.make_key("C", "old", "new"), 10, None)
+ col.storage_operation(stats.make_key("C", "old", "new"), 11, None)
+ keys = col.sorted_keys()
assert keys == [ ("A", "old", "new"), ("B", "old", "new"), ("B", "old2", "new2"), ("C", "old", "new") ]
- assert stats.stats[keys[0]] == [1, 2]
- assert stats.stats[keys[1]] == [3, 4, 5]
- assert stats.stats[keys[2]] == [20]
- assert stats.stats[keys[3]] == [10, 11]
+ assert col.stats[keys[0]] == [1, 2]
+ assert col.stats[keys[1]] == [3, 4, 5]
+ assert col.stats[keys[2]] == [20]
+ assert col.stats[keys[3]] == [10, 11]
def test_statistics_log():
stats = storage_statistics.StorageStatistics()
- s = stats.log_operation_string(stats.make_key("Operation", "old_storage", "new_storage"), 22, "classname")
+ log = storage_statistics.StatisticsLogger()
+ s = log.log_string(stats.make_key("Operation", "old_storage", "new_storage"), 22, "classname")
assert s == "Operation (old_storage -> new_storage) of classname size 22"
- s = stats.log_operation_string(stats.make_key("InitialOperation", None, "some_new_storage"), 40, "a_classname")
+ s = log.log_string(stats.make_key("InitialOperation", None, "some_new_storage"), 40, "a_classname")
assert s == "InitialOperation (some_new_storage) of a_classname size 40"
\ No newline at end of file
diff --git a/targetimageloadingsmalltalk.py b/targetimageloadingsmalltalk.py
--- a/targetimageloadingsmalltalk.py
+++ b/targetimageloadingsmalltalk.py
@@ -127,7 +127,7 @@
-p|--poll_events
--strategy-log
--strategy-stats
- --strategy-stats-with-sizes
+ --strategy-stats-details
[image path, default: Squeak.image]
""" % argv[0]
@@ -189,8 +189,8 @@
storage_statistics.activate_statistics(log=True)
elif arg == "--strategy-stats":
storage_statistics.activate_statistics(statistics=True)
- elif arg == "--strategy-stats-with-sizes":
- storage_statistics.activate_statistics(statistics=True, statstics_sizes=True)
+ elif arg == "--strategy-stats-details":
+ storage_statistics.activate_statistics(statistics=True, detailed_statistics=True)
elif path is None:
path = argv[idx]
else:
More information about the pypy-commit
mailing list