[pypy-svn] r58423 - in pypy/build/benchmem: . benchmark testing
hpk at codespeak.net
hpk at codespeak.net
Thu Sep 25 11:41:46 CEST 2008
Author: hpk
Date: Thu Sep 25 11:41:42 2008
New Revision: 58423
Added:
pypy/build/benchmem/benchmark/sizes.py
pypy/build/benchmem/readme.txt
Removed:
pypy/build/benchmem/benchmark/create_recursive_tuples.py
pypy/build/benchmem/benchmark/empty_instances.py
pypy/build/benchmem/benchmark/linked_list_with_floats.py
pypy/build/benchmem/benchmark/list_of_instances_with_ints.py
pypy/build/benchmem/benchmark/lists.py
pypy/build/benchmem/benchmark/simple_linked_instances.py
Modified:
pypy/build/benchmem/report.py
pypy/build/benchmem/runbench.py
pypy/build/benchmem/testing/test_benchtool.py
Log:
* put all "size" benchmarks into sizes.py
* add a readme.txt with some XXX
* add functional tests for runbench and reporting
* add option parsing to reporting
Added: pypy/build/benchmem/benchmark/sizes.py
==============================================================================
--- (empty file)
+++ pypy/build/benchmem/benchmark/sizes.py Thu Sep 25 11:41:42 2008
@@ -0,0 +1,60 @@
+
+def bench_list_of_None(iter1):
+ l = [None for i in range(iter1)]
+ checkpoint(collect=True)
+
+def bench_list_of_int(iter1):
+ l = [int(i+1000) for i in range(iter1)]
+ checkpoint(collect=True)
+
+def bench_list_of_emptylist(iter1):
+ l = [[] for i in range(iter1)]
+ checkpoint(collect=True)
+
+def bench_list_of_list_with_int(iter1):
+ l = [[i] for i in range(iter1)]
+ checkpoint(collect=True)
+
+def bench_list_of_list_with_float(iter1):
+ l = [[float(i)] for i in range(iter1)]
+ checkpoint(collect=True)
+
+def bench_list_of_emptydict(iter1):
+ l = [{} for i in range(iter1)]
+ checkpoint(collect=True)
+
+def bench_list_of_dict_intkey(iter1):
+ l = [{i:None} for i in range(iter1)]
+ checkpoint(collect=True)
+
+def bench_list_of_empty_instances_old(iter1):
+ class A:
+ pass
+ x = [A() for i in range(iter1)]
+ checkpoint(collect=True)
+
+def bench_list_of_empty_instances_new(iter1):
+ class A(object):
+ pass
+ x = [A() for i in range(iter1)]
+ checkpoint(collect=True)
+
+def bench_linked_instances(iter1):
+ class A(object):
+ def __init__(self, prev):
+ self.prev = prev
+ l = []
+ prev = None
+ for i in range(iter1):
+ x = A(prev)
+ l.append(prev)
+ prev = x
+
+ checkpoint(collect=True)
+
+def bench_recursive_list(iter1):
+ x = []
+ for i in range(iter1):
+ x = [x]
+ checkpoint(collect=True)
+
Added: pypy/build/benchmem/readme.txt
==============================================================================
--- (empty file)
+++ pypy/build/benchmem/readme.txt Thu Sep 25 11:41:42 2008
@@ -0,0 +1,26 @@
+memory benchmarks running/reporting
+-----------------------------------------
+
+``runbench.py`` runs all available and stores resulting information in a log file.
+
+``report.py`` takes a logfile and produces reporting information.
+
+In order to run benchmarks with multiple python interpreters::
+
+ python runbench.py -e python2.5,pypy-c-opt3
+
+
+Benchmark types / files
+-------------------------
+
+sizes.py contains explicitely checkpointed benchmarks used to measure sizes of several object types and constellations.
+ all 'bench_' functions take an NUMITER parameter which indicates how often an object
+ structure should be put in a list. This parameter can be set from ``runbench.py`` command line.
+
+XXX In order to run interpreter size benchmarks::
+
+ XXX python runbench.py -e python2.5,pypy-c-opt3 --interpsize
+
+XXX todo:
+app*.py contains time-checkpointed benchmarks used to measure memory usage of an application scenario.
+ --runwithparams=500,1000,1500 will run application benchmarks with with several paramat
Modified: pypy/build/benchmem/report.py
==============================================================================
--- pypy/build/benchmem/report.py (original)
+++ pypy/build/benchmem/report.py Thu Sep 25 11:41:42 2008
@@ -9,6 +9,10 @@
import py, os
import smaps, runbench
+parser = py.compat.optparse.OptionParser(usage=__doc__)
+parser.add_option("-l", "--benchlog", action="store", dest="benchlog", default="bench.log",
+ help="logfile to read results from")
+
def asciitable(table):
colmax = []
for row in table:
@@ -88,9 +92,11 @@
os.system("gnuplot gnuplotcmd")
if __name__ == "__main__":
- benchlog = py.path.local("bench.log")
+ options, args = parser.parse_args()
+
+ benchlog = py.path.local(options.benchlog)
reader = runbench.LogReader()
- reader.parse_logfile(benchlog)
+ reader.parse(benchlog)
#maxtable_overview(reader)
checkpointdetails(reader)
Modified: pypy/build/benchmem/runbench.py
==============================================================================
--- pypy/build/benchmem/runbench.py (original)
+++ pypy/build/benchmem/runbench.py Thu Sep 25 11:41:42 2008
@@ -13,31 +13,26 @@
mydir = py.magic.autopath().dirpath()
benchmarkdir = mydir.join("benchmark")
-
-class BenchRunner(object):
+class BenchRunner:
SEPBENCH = "=" * 80
-
- def __init__(self, executable, benchlog):
- self.benchlogpath = py.path.local(benchlog)
- self.logstream = self.benchlogpath.open("a")
+
+class BenchRunnerSizes(BenchRunner):
+ def __init__(self, executable, fsname, logpath, options):
self.executable = executable
+ self.benchpath = benchmarkdir.join(fsname)
+ assert self.benchpath.check()
+ self.logpath = py.path.local(logpath)
+ self.logstream = self.logpath.open("a")
self.tmpdir = py.path.local.make_numbered_dir(prefix="bench")
+ self.options = options
def log(self, *args):
print " ".join(map(str, args))
- def _preparebench(self, path, args):
- path = py.path.local(path)
- for name, obj in vars(path.pyimport()).items():
- if name.startswith("bench") and callable(obj):
- break # xxx only first bench function is considered
- else:
- raise LookupError("no benchmark found in %s" %(path,))
-
- arglist = ",".join(map(str, args))
- source = py.code.Source(path.read(), """
+ def makebench(self, name):
+ arglist = str(self.options.numiter)
+ source = py.code.Source(self.benchpath.read(), """
import gc
-
def write(c):
sys.stdout.write(c)
sys.stdout.flush()
@@ -47,35 +42,47 @@
gc.collect()
write("c")
sys.stdin.read(1)
+
if __name__ == "__main__":
import os, sys, gc
pid = os.getpid()
write(str(pid) + "\\n")
checkpoint(collect=True)
- %s(checkpoint, %s)
+ %s(%s)
checkpoint(collect=True)
write("F")
sys.stdin.close()
""" %(name, arglist))
- p = self.tmpdir.join(path.basename)
+ p = self.tmpdir.join(self.benchpath.basename)
p.write(source)
return p
def write_benchheader(self, benchname, args):
print >>self.logstream, self.SEPBENCH
print >>self.logstream, "#executable=%r" %(str(self.executable ),)
+ print >>self.logstream, "#benchpath=%r" %(self.benchpath.basename,)
print >>self.logstream, "#benchname=%r" %(benchname,)
print >>self.logstream, "#benchargs=%r" %(args,)
- def run_checkpointed_bench(self, filename, args):
- benchpyfile = self._preparebench(filename, args)
- self.log("created", benchpyfile)
+ def getnames(self):
+ l = []
+ for name, obj in vars(self.benchpath.pyimport()).items():
+ if name.startswith("bench") and callable(obj):
+ l.append(name)
+ return l
+
+ def run(self):
+ for name in self.getnames():
+ self.run_checkpointed(name)
+
+ def run_checkpointed(self, name):
+ benchpyfile = self.makebench(name)
+ #self.log("created", benchpyfile)
cmd = "%s -u %s" %(self.executable, benchpyfile)
- self.log("exec", cmd)
+ self.log("running %s(%s)" %(name, self.options.numiter))
stdout, stdin = os.popen2(cmd)
pid = int(stdin.readline())
-
- self.write_benchheader(benchpyfile.basename, args)
+ self.write_benchheader(name, self.options.numiter)
rec = smaps.SmapsRecorder(pid, self.logstream)
self.interact_with_child_checkpoints(rec, stdout, stdin)
@@ -103,7 +110,7 @@
def __init__(self):
self.name2results = {}
- def parse_logfile(self, logpath):
+ def parse(self, logpath):
f = logpath.open()
for result in BenchmarkResult.parse(f):
#print "parsed", result
@@ -132,9 +139,12 @@
if not line.strip():
break
if not line.startswith("#executable"):
- print "ignoring", line
+ if line != BenchRunner.SEPBENCH:
+ print "ignoring", line
continue
+ # see write_benchheader
executable = eval(line.split("=", 1)[1])
+ benchbasename = eval(f.readline().split("=", 1)[1])
benchname = eval(f.readline().split("=", 1)[1])
benchargs = eval(f.readline().split("=", 1)[1])
@@ -184,35 +194,20 @@
help="logfile for recording benchmark measurements")
parser.add_option("-a", "--append", action="store_true", dest="append", default=False,
help="append to logfile")
-parser.add_option("-c", "--coefficient", action="store", dest="coeff",
- default=1, help="Coefficient of number of iterations")
-
-def getbenchfiles(options, args):
- if args:
- benchfiles = [py.path.local(x) for x in args]
- else:
- benchfiles = [x for x in benchmarkdir.listdir('*.py')
- if x.basename[0] != "_"]
- for x in benchfiles:
- if not x.check():
- raise LookupError("%s does not exist" %(x,))
- assert benchfiles
- return benchfiles
+parser.add_option("-n", "--numiter", action="store", dest="numiter",
+ default=100000, help="number of iterations")
def getbenchlog(options):
benchlog = options.benchlog
if benchlog is None:
benchlog = "bench.log"
- return py.path.local(benchlog)
-
-
-if __name__ == '__main__':
- (options, args) = parser.parse_args()
-
- names = getbenchfiles(options, args)
- benchlog = getbenchlog(options)
+ benchlog = py.path.local(benchlog)
if not options.append and benchlog.check():
benchlog.remove()
+ return benchlog
+
+def getexecutables(options):
+ l = []
for executable in options.executable.split(","):
if not executable:
continue
@@ -221,8 +216,22 @@
p = py.path.local.sysfind(executable)
if not p.check():
raise SystemExit("could not find %r"% (executable))
- runner = BenchRunner(executable, benchlog)
- for name in names:
- iter1 = int(100000*float(options.coeff))
- runner.run_checkpointed_bench(name, (iter1,))
+ l.append(executable)
+ return l
+
+def getrunnerclass(fsname):
+ if fsname == "sizes.py":
+ return BenchRunnerSizes
+
+if __name__ == '__main__':
+ (options, args) = parser.parse_args()
+
+ benchlog = getbenchlog(options)
+ if not args:
+ args = ("sizes.py",)
+ for executable in getexecutables(options):
+ for fsname in args:
+ Runner = getrunnerclass(fsname)
+ runner = Runner(executable, fsname, benchlog, options)
+ runner.run()
print "bench results append to -->>>", benchlog
Modified: pypy/build/benchmem/testing/test_benchtool.py
==============================================================================
--- pypy/build/benchmem/testing/test_benchtool.py (original)
+++ pypy/build/benchmem/testing/test_benchtool.py Thu Sep 25 11:41:42 2008
@@ -10,7 +10,7 @@
def test_smapsrecorder():
tmpdir = py.test.ensuretemp("smapsrecorder")
- logpath = tmpdir.join("logfile")
+ logpath = tmpdir.join("benchlog")
pid = os.getpid()
f = logpath.open("w")
rec = smaps.SmapsRecorder(pid=pid, stream=f)
@@ -22,27 +22,38 @@
s = logpath.read()
assert s.count(smaps.SmapsRecorder.SEPSNAPSHOT) == 2
-def test_benchrunner_functional():
+def test_sizes_runbench_and_read_results():
tmpdir = py.test.ensuretemp("benchrunner")
- log=tmpdir.join("log")
- def checker(path, *args):
- if log.check():
- log.remove()
- runner = runbench.BenchRunner(executable="python2.5", benchlog=log)
- runner.run_checkpointed_bench(path, args)
- assert log.check()
- benchresult = runbench.LogReader()
- benchresult.parse_logfile(log)
- #assert reader.executable
- #assert reader.executable
- assert len(benchresult.name2results) == 1
- results = benchresult.name2results.values()[0]
- assert len(results) == 1
+ benchlog=tmpdir.join("log")
+ class options:
+ numiter = 10
+ runner = runbench.BenchRunnerSizes("python2.5", "sizes.py", benchlog, options)
+ assert runner.benchpath.basename == "sizes.py"
+ runner.run()
+ benchresult = runbench.LogReader()
+ benchresult.parse(benchlog)
+
+ names = runner.getnames()
+ assert len(benchresult.name2results) == len(names)
+ for results in benchresult.name2results.values():
+ assert len(results) ==1
assert len(results[0].snapshots) == 2 + 1
- for path in runbench.benchmarkdir.listdir("*.py"):
- if path.basename[0] != "_":
- yield checker, path, 10
+def test_runbench_functional():
+ script = py.path.local(runbench.__file__).dirpath("runbench.py")
+ output = py.process.cmdexec("python %s --numiter=10" %(script))
+
+def test_report_functional():
+ tmpdir = py.test.ensuretemp("test_report_functional")
+ script = py.path.local(runbench.__file__).dirpath("runbench.py")
+ benchlog = tmpdir.join("benchlog")
+ py.process.cmdexec("%s --benchlog=%s --numiter=10" %(script, benchlog))
+ report = script.dirpath("report.py")
+ old = tmpdir.chdir()
+ try:
+ py.process.cmdexec("%s --benchlog %s" %(report, benchlog,))
+ finally:
+ old.chdir()
def test_log_mapping():
s = py.std.textwrap.dedent("""\
@@ -114,3 +125,4 @@
val1 = getattr(snap.mappings[0], name)
val2 = getattr(snap.mappings[1], name)
assert sumval == val1 + val2
+
More information about the Pypy-commit
mailing list