[pypy-svn] r23526 - pypy/dist/pypy/translator/goal

ericvrp at codespeak.net ericvrp at codespeak.net
Mon Feb 20 21:09:59 CET 2006


Author: ericvrp
Date: Mon Feb 20 21:09:55 2006
New Revision: 23526

Modified:
   pypy/dist/pypy/translator/goal/bench-unix.py
Log:
Made nightly benchmark results more stable by caching the best of 10 runs.
This also speeds up the cronjob quite a bit as soon as the ~300 pypy's
are measured often enough.


Modified: pypy/dist/pypy/translator/goal/bench-unix.py
==============================================================================
--- pypy/dist/pypy/translator/goal/bench-unix.py	(original)
+++ pypy/dist/pypy/translator/goal/bench-unix.py	Mon Feb 20 21:09:55 2006
@@ -2,13 +2,55 @@
 # to be executed in the goal folder,
 # where a couple of pypy-* files is expected.
 
-import os, sys, time
+import os, sys, time, pickle
 
-MAX_BENCHMARKS = 40
 PYSTONE_CMD = 'from test import pystone;pystone.main(%s)'
 PYSTONE_PATTERN = 'This machine benchmarks at'
+PYSTONE_ASCENDING_GOOD = True 
+
 RICHARDS_CMD = 'from richards import *;main(iterations=%d)'
 RICHARDS_PATTERN = 'Average time per iteration:'
+RICHARDS_ASCENDING_GOOD = False 
+
+class BenchmarkResult(object):
+
+    def __init__(self, filename, max_results=10):
+        self.filename    = filename
+        self.max_results = max_results
+        try:
+            f = open(filename, 'r')
+            self.n_results   = pickle.load(f)
+            self.best_result = pickle.load(f)
+            f.close()
+        except:
+            self.n_results   = {}
+            self.best_result = {}
+
+    def is_stable(self, name):
+        try:
+            return self.n_results[name] >= self.max_results
+        except:
+            return False
+
+    def update(self, name, result, ascending_good):
+        try:
+            if ascending_good:
+                self.best_result[name] = max(self.best_result[name], result)
+            else:
+                self.best_result[name] = min(self.best_result[name], result)
+        except KeyError:
+            self.n_results[name] = 0
+            self.best_result[name] = result
+        self.n_results[name] += 1
+
+        f = open(self.filename, 'w')
+        pickle.dump(self.n_results  , f)
+        pickle.dump(self.best_result, f)
+        f.close()
+
+    def get_best_result(self, name):
+        return self.best_result[name]
+
 
 def get_result(txt, pattern):
     for line in txt.split('\n'):
@@ -35,29 +77,53 @@
     return get_result(txt, RICHARDS_PATTERN)
 
 def get_executables():  #sorted by revision number (highest first)
-    exes = [os.path.join('.', name) for name in os.listdir('.') if name.startswith('pypy-')]
-    exes = [(s.split('-',2)[2], s) for s in exes]
+    exes = []
+    for exe in [os.path.join('.', name) for name in os.listdir('.') if name.startswith('pypy-')]:
+        try:
+            exes.append( (exe.split('-',2)[2], exe) )
+        except:
+            pass    #skip filenames without version number
     exes.sort()
     exes.reverse()
     exes = [s[1] for s in exes]
     return exes
 
-HEADLINE = 'date                       executable                        richards             pystone'
-FMT      = '%-26s %-30s   %6dms (%6.1fx)   %6d (%6.1fx)'
-
 def main():
-    print HEADLINE
+    benchmark_result = BenchmarkResult('bench-unix.benchmark_result')
+
+    print 'date                       executable                        richards             pystone'
     sys.stdout.flush()
-    ref_rich = run_richards()
-    ref_stone = run_pystone()
-    print FMT % (time.ctime(), 'python %s' % sys.version.split()[0], ref_rich, 1.0, ref_stone, 1.0)
+
+    v = 'python ' + sys.version.split()[0]
+    r = v + '_richards'
+    if not benchmark_result.is_stable(r):
+        benchmark_result.update(r, run_richards(), RICHARDS_ASCENDING_GOOD)
+    ref_rich = benchmark_result.get_best_result(r)
+
+    p = v + '_pystone'
+    if not benchmark_result.is_stable(p):
+        benchmark_result.update(p, run_pystone(), PYSTONE_ASCENDING_GOOD)
+    ref_stone = benchmark_result.get_best_result(p)
+
+    fmt = '%-26s %-30s   %6dms (%6.1fx)   %6d (%6.1fx)'
+    print fmt % (time.ctime(), v, ref_rich, 1.0, ref_stone, 1.0)
     sys.stdout.flush()
-    for exe in get_executables()[:MAX_BENCHMARKS]:
+
+    for exe in get_executables():
         exename = os.path.splitext(exe)[0].lstrip('./')
         ctime   = time.ctime( os.path.getctime(exename) )
-        rich    = run_richards(exe, 1)
-        stone   = run_pystone(exe)
-        print FMT % (ctime, exename, rich, rich / ref_rich, stone, ref_stone / stone)
+
+        r = exe + '_richards'
+        if not benchmark_result.is_stable(r):
+            benchmark_result.update(r, run_richards(exe, 1), RICHARDS_ASCENDING_GOOD)
+        rich = benchmark_result.get_best_result(r)
+
+        p = exe + '_pystone'
+        if not benchmark_result.is_stable(p):
+            benchmark_result.update(p, run_pystone(exe), PYSTONE_ASCENDING_GOOD)
+        stone = benchmark_result.get_best_result(p)
+
+        print fmt % (ctime, exename, rich, rich / ref_rich, stone, ref_stone / stone)
         sys.stdout.flush()
 
 if __name__ == '__main__':



More information about the Pypy-commit mailing list