[pypy-commit] pypy even-more-jit-hooks: a bit of refactoring but now one can get to details about counters from the interpreter if he really feels like it
fijal
noreply at buildbot.pypy.org
Tue Jun 26 22:16:54 CEST 2012
Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: even-more-jit-hooks
Changeset: r55851:f09f03b9e5ff
Date: 2012-06-26 20:46 +0200
http://bitbucket.org/pypy/pypy/changeset/f09f03b9e5ff/
Log: a bit of refactoring but now one can get to details about counters
from the interpreter if he really feels like it
diff --git a/pypy/jit/metainterp/jitprof.py b/pypy/jit/metainterp/jitprof.py
--- a/pypy/jit/metainterp/jitprof.py
+++ b/pypy/jit/metainterp/jitprof.py
@@ -6,42 +6,11 @@
from pypy.rlib.debug import debug_print, debug_start, debug_stop
from pypy.rlib.debug import have_debug_prints
from pypy.jit.metainterp.jitexc import JitException
+from pypy.rlib.jit import Counters
-counters="""
-TRACING
-BACKEND
-OPS
-RECORDED_OPS
-GUARDS
-OPT_OPS
-OPT_GUARDS
-OPT_FORCINGS
-ABORT_TOO_LONG
-ABORT_BRIDGE
-ABORT_BAD_LOOP
-ABORT_ESCAPE
-ABORT_FORCE_QUASIIMMUT
-NVIRTUALS
-NVHOLES
-NVREUSED
-TOTAL_COMPILED_LOOPS
-TOTAL_COMPILED_BRIDGES
-TOTAL_FREED_LOOPS
-TOTAL_FREED_BRIDGES
-"""
-counter_names = []
-
-def _setup():
- names = counters.split()
- for i, name in enumerate(names):
- globals()[name] = i
- counter_names.append(name)
- global ncounters
- ncounters = len(names)
-_setup()
-
-JITPROF_LINES = ncounters + 1 + 1 # one for TOTAL, 1 for calls, update if needed
+JITPROF_LINES = Counters.ncounters + 1 + 1
+# one for TOTAL, 1 for calls, update if needed
_CPU_LINES = 4 # the last 4 lines are stored on the cpu
class BaseProfiler(object):
@@ -71,9 +40,12 @@
def count(self, kind, inc=1):
pass
- def count_ops(self, opnum, kind=OPS):
+ def count_ops(self, opnum, kind=Counters.OPS):
pass
+ def get_counter(self, num):
+ return -1.0
+
class Profiler(BaseProfiler):
initialized = False
timer = time.time
@@ -89,7 +61,7 @@
self.starttime = self.timer()
self.t1 = self.starttime
self.times = [0, 0]
- self.counters = [0] * (ncounters - _CPU_LINES)
+ self.counters = [0] * (Counters.ncounters - _CPU_LINES)
self.calls = 0
self.current = []
@@ -117,19 +89,30 @@
return
self.times[ev1] += self.t1 - t0
- def start_tracing(self): self._start(TRACING)
- def end_tracing(self): self._end (TRACING)
+ def start_tracing(self): self._start(Counters.TRACING)
+ def end_tracing(self): self._end (Counters.TRACING)
- def start_backend(self): self._start(BACKEND)
- def end_backend(self): self._end (BACKEND)
+ def start_backend(self): self._start(Counters.BACKEND)
+ def end_backend(self): self._end (Counters.BACKEND)
def count(self, kind, inc=1):
self.counters[kind] += inc
-
- def count_ops(self, opnum, kind=OPS):
+
+ def get_counter(self, num):
+ if num == Counters.TOTAL_COMPILED_LOOPS:
+ return float(self.cpu.total_compiled_loops)
+ elif num == Counters.TOTAL_COMPILED_BRIDGES:
+ return float(self.cpu.total_compiled_bridges)
+ elif num == Counters.TOTAL_FREED_LOOPS:
+ return float(self.cpu.total_freed_loops)
+ elif num == Counters.TOTAL_FREED_BRIDGES:
+ return float(self.cpu.total_freed_bridges)
+ return float(self.counters[num])
+
+ def count_ops(self, opnum, kind=Counters.OPS):
from pypy.jit.metainterp.resoperation import rop
self.counters[kind] += 1
- if opnum == rop.CALL and kind == RECORDED_OPS:# or opnum == rop.OOSEND:
+ if opnum == rop.CALL and kind == Counters.RECORDED_OPS:# or opnum == rop.OOSEND:
self.calls += 1
def print_stats(self):
@@ -142,26 +125,29 @@
cnt = self.counters
tim = self.times
calls = self.calls
- self._print_line_time("Tracing", cnt[TRACING], tim[TRACING])
- self._print_line_time("Backend", cnt[BACKEND], tim[BACKEND])
+ self._print_line_time("Tracing", cnt[Counters.TRACING],
+ tim[Counters.TRACING])
+ self._print_line_time("Backend", cnt[Counters.BACKEND],
+ tim[Counters.BACKEND])
line = "TOTAL: \t\t%f" % (self.tk - self.starttime, )
debug_print(line)
- self._print_intline("ops", cnt[OPS])
- self._print_intline("recorded ops", cnt[RECORDED_OPS])
+ self._print_intline("ops", cnt[Counters.OPS])
+ self._print_intline("recorded ops", cnt[Counters.RECORDED_OPS])
self._print_intline(" calls", calls)
- self._print_intline("guards", cnt[GUARDS])
- self._print_intline("opt ops", cnt[OPT_OPS])
- self._print_intline("opt guards", cnt[OPT_GUARDS])
- self._print_intline("forcings", cnt[OPT_FORCINGS])
- self._print_intline("abort: trace too long", cnt[ABORT_TOO_LONG])
- self._print_intline("abort: compiling", cnt[ABORT_BRIDGE])
- self._print_intline("abort: vable escape", cnt[ABORT_ESCAPE])
- self._print_intline("abort: bad loop", cnt[ABORT_BAD_LOOP])
+ self._print_intline("guards", cnt[Counters.GUARDS])
+ self._print_intline("opt ops", cnt[Counters.OPT_OPS])
+ self._print_intline("opt guards", cnt[Counters.OPT_GUARDS])
+ self._print_intline("forcings", cnt[Counters.OPT_FORCINGS])
+ self._print_intline("abort: trace too long",
+ cnt[Counters.ABORT_TOO_LONG])
+ self._print_intline("abort: compiling", cnt[Counters.ABORT_BRIDGE])
+ self._print_intline("abort: vable escape", cnt[Counters.ABORT_ESCAPE])
+ self._print_intline("abort: bad loop", cnt[Counters.ABORT_BAD_LOOP])
self._print_intline("abort: force quasi-immut",
- cnt[ABORT_FORCE_QUASIIMMUT])
- self._print_intline("nvirtuals", cnt[NVIRTUALS])
- self._print_intline("nvholes", cnt[NVHOLES])
- self._print_intline("nvreused", cnt[NVREUSED])
+ cnt[Counters.ABORT_FORCE_QUASIIMMUT])
+ self._print_intline("nvirtuals", cnt[Counters.NVIRTUALS])
+ self._print_intline("nvholes", cnt[Counters.NVHOLES])
+ self._print_intline("nvreused", cnt[Counters.NVREUSED])
cpu = self.cpu
if cpu is not None: # for some tests
self._print_intline("Total # of loops",
diff --git a/pypy/jit/metainterp/optimizeopt/optimizer.py b/pypy/jit/metainterp/optimizeopt/optimizer.py
--- a/pypy/jit/metainterp/optimizeopt/optimizer.py
+++ b/pypy/jit/metainterp/optimizeopt/optimizer.py
@@ -401,7 +401,7 @@
o.turned_constant(value)
def forget_numberings(self, virtualbox):
- self.metainterp_sd.profiler.count(jitprof.OPT_FORCINGS)
+ self.metainterp_sd.profiler.count(jitprof.Counters.OPT_FORCINGS)
self.resumedata_memo.forget_numberings(virtualbox)
def getinterned(self, box):
@@ -535,9 +535,9 @@
else:
self.ensure_imported(value)
op.setarg(i, value.force_box(self))
- self.metainterp_sd.profiler.count(jitprof.OPT_OPS)
+ self.metainterp_sd.profiler.count(jitprof.Counters.OPT_OPS)
if op.is_guard():
- self.metainterp_sd.profiler.count(jitprof.OPT_GUARDS)
+ self.metainterp_sd.profiler.count(jitprof.Counters.OPT_GUARDS)
if self.replaces_guard and op in self.replaces_guard:
self.replace_op(self.replaces_guard[op], op)
del self.replaces_guard[op]
diff --git a/pypy/jit/metainterp/pyjitpl.py b/pypy/jit/metainterp/pyjitpl.py
--- a/pypy/jit/metainterp/pyjitpl.py
+++ b/pypy/jit/metainterp/pyjitpl.py
@@ -13,9 +13,7 @@
from pypy.jit.metainterp import executor
from pypy.jit.metainterp.logger import Logger
from pypy.jit.metainterp.jitprof import EmptyProfiler
-from pypy.jit.metainterp.jitprof import GUARDS, RECORDED_OPS, ABORT_ESCAPE
-from pypy.jit.metainterp.jitprof import ABORT_TOO_LONG, ABORT_BRIDGE, \
- ABORT_FORCE_QUASIIMMUT, ABORT_BAD_LOOP
+from pypy.rlib.jit import Counters
from pypy.jit.metainterp.jitexc import JitException, get_llexception
from pypy.jit.metainterp.heapcache import HeapCache
from pypy.rlib.objectmodel import specialize
@@ -675,7 +673,7 @@
from pypy.jit.metainterp.quasiimmut import do_force_quasi_immutable
do_force_quasi_immutable(self.metainterp.cpu, box.getref_base(),
mutatefielddescr)
- raise SwitchToBlackhole(ABORT_FORCE_QUASIIMMUT)
+ raise SwitchToBlackhole(Counters.ABORT_FORCE_QUASIIMMUT)
self.generate_guard(rop.GUARD_ISNULL, mutatebox, resumepc=orgpc)
def _nonstandard_virtualizable(self, pc, box):
@@ -1255,7 +1253,7 @@
guard_op = metainterp.history.record(opnum, moreargs, None,
descr=resumedescr)
self.capture_resumedata(resumedescr, resumepc)
- self.metainterp.staticdata.profiler.count_ops(opnum, GUARDS)
+ self.metainterp.staticdata.profiler.count_ops(opnum, Counters.GUARDS)
# count
metainterp.attach_debug_info(guard_op)
return guard_op
@@ -1776,7 +1774,7 @@
return resbox.constbox()
# record the operation
profiler = self.staticdata.profiler
- profiler.count_ops(opnum, RECORDED_OPS)
+ profiler.count_ops(opnum, Counters.RECORDED_OPS)
self.heapcache.invalidate_caches(opnum, descr, argboxes)
op = self.history.record(opnum, argboxes, resbox, descr)
self.attach_debug_info(op)
@@ -1837,7 +1835,7 @@
if greenkey_of_huge_function is not None:
warmrunnerstate.disable_noninlinable_function(
greenkey_of_huge_function)
- raise SwitchToBlackhole(ABORT_TOO_LONG)
+ raise SwitchToBlackhole(Counters.ABORT_TOO_LONG)
def _interpret(self):
# Execute the frames forward until we raise a DoneWithThisFrame,
@@ -1921,7 +1919,7 @@
try:
self.prepare_resume_from_failure(key.guard_opnum, dont_change_position)
if self.resumekey_original_loop_token is None: # very rare case
- raise SwitchToBlackhole(ABORT_BRIDGE)
+ raise SwitchToBlackhole(Counters.ABORT_BRIDGE)
self.interpret()
except SwitchToBlackhole, stb:
self.run_blackhole_interp_to_cancel_tracing(stb)
@@ -1996,7 +1994,7 @@
# raises in case it works -- which is the common case
if self.partial_trace:
if start != self.retracing_from:
- raise SwitchToBlackhole(ABORT_BAD_LOOP) # For now
+ raise SwitchToBlackhole(Counters.ABORT_BAD_LOOP) # For now
self.compile_loop(original_boxes, live_arg_boxes, start, resumedescr)
# creation of the loop was cancelled!
self.cancel_count += 1
@@ -2005,7 +2003,7 @@
if memmgr:
if self.cancel_count > memmgr.max_unroll_loops:
self.staticdata.log('cancelled too many times!')
- raise SwitchToBlackhole(ABORT_BAD_LOOP)
+ raise SwitchToBlackhole(Counters.ABORT_BAD_LOOP)
self.staticdata.log('cancelled, tracing more...')
# Otherwise, no loop found so far, so continue tracing.
@@ -2299,7 +2297,8 @@
if vinfo.tracing_after_residual_call(virtualizable):
# the virtualizable escaped during CALL_MAY_FORCE.
self.load_fields_from_virtualizable()
- raise SwitchToBlackhole(ABORT_ESCAPE, raising_exception=True)
+ raise SwitchToBlackhole(Counters.ABORT_ESCAPE,
+ raising_exception=True)
# ^^^ we set 'raising_exception' to True because we must still
# have the eventual exception raised (this is normally done
# after the call to vable_after_residual_call()).
diff --git a/pypy/jit/metainterp/resume.py b/pypy/jit/metainterp/resume.py
--- a/pypy/jit/metainterp/resume.py
+++ b/pypy/jit/metainterp/resume.py
@@ -254,9 +254,9 @@
self.cached_virtuals.clear()
def update_counters(self, profiler):
- profiler.count(jitprof.NVIRTUALS, self.nvirtuals)
- profiler.count(jitprof.NVHOLES, self.nvholes)
- profiler.count(jitprof.NVREUSED, self.nvreused)
+ profiler.count(jitprof.Counters.NVIRTUALS, self.nvirtuals)
+ profiler.count(jitprof.Counters.NVHOLES, self.nvholes)
+ profiler.count(jitprof.Counters.NVREUSED, self.nvreused)
_frame_info_placeholder = (None, 0, 0)
diff --git a/pypy/jit/metainterp/test/test_jitiface.py b/pypy/jit/metainterp/test/test_jitiface.py
--- a/pypy/jit/metainterp/test/test_jitiface.py
+++ b/pypy/jit/metainterp/test/test_jitiface.py
@@ -1,11 +1,11 @@
-from pypy.rlib.jit import JitDriver, JitHookInterface
+from pypy.rlib.jit import JitDriver, JitHookInterface, Counters
from pypy.rlib import jit_hooks
from pypy.jit.metainterp.test.support import LLJitMixin
from pypy.jit.codewriter.policy import JitPolicy
-from pypy.jit.metainterp.jitprof import ABORT_FORCE_QUASIIMMUT
from pypy.jit.metainterp.resoperation import rop
from pypy.rpython.annlowlevel import hlstr
+from pypy.jit.metainterp.jitprof import Profiler
class TestJitHookInterface(LLJitMixin):
def test_abort_quasi_immut(self):
@@ -41,7 +41,7 @@
assert f(100, 7) == 721
res = self.meta_interp(f, [100, 7], policy=JitPolicy(iface))
assert res == 721
- assert reasons == [ABORT_FORCE_QUASIIMMUT] * 2
+ assert reasons == [Counters.ABORT_FORCE_QUASIIMMUT] * 2
def test_on_compile(self):
called = []
@@ -146,3 +146,29 @@
assert jit_hooks.resop_getresult(op) == box5
self.meta_interp(main, [])
+
+ def test_get_stats(self):
+ driver = JitDriver(greens = [], reds = ['i', 's'])
+
+ def loop(i):
+ s = 0
+ while i > 0:
+ driver.jit_merge_point(i=i, s=s)
+ if i % 2:
+ s += 1
+ i -= 1
+ s+= 2
+ return s
+
+ def main():
+ loop(30)
+ stats = jit_hooks.get_stats()
+ assert jit_hooks.stats_get_counter_value(stats,
+ Counters.TOTAL_COMPILED_LOOPS) == 1
+ assert jit_hooks.stats_get_counter_value(stats,
+ Counters.TOTAL_COMPILED_BRIDGES) == 1
+ assert jit_hooks.stats_get_counter_value(stats,
+ Counters.TRACING) >= 0
+
+
+ self.meta_interp(main, [], ProfilerClass=Profiler)
diff --git a/pypy/jit/metainterp/warmspot.py b/pypy/jit/metainterp/warmspot.py
--- a/pypy/jit/metainterp/warmspot.py
+++ b/pypy/jit/metainterp/warmspot.py
@@ -6,6 +6,7 @@
from pypy.annotation import model as annmodel
from pypy.rpython.llinterp import LLException
from pypy.rpython.test.test_llinterp import get_interpreter, clear_tcache
+from pypy.rpython.annlowlevel import cast_instance_to_base_ptr
from pypy.objspace.flow.model import SpaceOperation, Variable, Constant
from pypy.objspace.flow.model import checkgraph, Link, copygraph
from pypy.rlib.objectmodel import we_are_translated
@@ -151,6 +152,9 @@
def find_set_param(graphs):
return _find_jit_marker(graphs, 'set_param')
+def find_get_stats(graphs):
+ return _find_jit_marker(graphs, 'get_stats', False)
+
def find_force_quasi_immutable(graphs):
results = []
for graph in graphs:
@@ -221,7 +225,7 @@
self.rewrite_access_helpers()
self.codewriter.make_jitcodes(verbose=verbose)
self.rewrite_can_enter_jits()
- self.rewrite_set_param()
+ self.rewrite_set_param_and_get_stats()
self.rewrite_force_virtual(vrefinfo)
self.rewrite_force_quasi_immutable()
self.add_finish()
@@ -638,7 +642,14 @@
# make sure we make a copy of function so it no longer belongs
# to extregistry
func = op.args[1].value
- func = func_with_new_name(func, func.func_name + '_compiled')
+ if func.func_name == 'get_stats':
+ # get special treatment since we rewrite it to a call that accepts
+ # jit driver
+ def func():
+ return lltype.cast_opaque_ptr(llmemory.GCREF,
+ cast_instance_to_base_ptr(self))
+ else:
+ func = func_with_new_name(func, func.func_name + '_compiled')
ptr = self.helper_func(FUNCPTR, func)
op.opname = 'direct_call'
op.args = [Constant(ptr, FUNCPTR)] + op.args[2:]
@@ -859,7 +870,7 @@
call_final_function(self.translator, finish,
annhelper = self.annhelper)
- def rewrite_set_param(self):
+ def rewrite_set_param_and_get_stats(self):
from pypy.rpython.lltypesystem.rstr import STR
closures = {}
@@ -907,6 +918,9 @@
op.opname = 'direct_call'
op.args[:3] = [closures[key]]
+ for graph, block, i in find_get_stats(graphs):
+ xxx
+
def rewrite_force_virtual(self, vrefinfo):
if self.cpu.ts.name != 'lltype':
py.test.skip("rewrite_force_virtual: port it to ootype")
diff --git a/pypy/module/pypyjit/interp_resop.py b/pypy/module/pypyjit/interp_resop.py
--- a/pypy/module/pypyjit/interp_resop.py
+++ b/pypy/module/pypyjit/interp_resop.py
@@ -42,7 +42,7 @@
Set a compiling hook that will be called each time a loop is compiled.
The hook will be called with the following signature:
hook(jitdriver_name, loop_type, greenkey or guard_number, operations,
- assembler_addr, assembler_length)
+ assembler_addr, assembler_length, loopno)
jitdriver_name is the name of this particular jitdriver, 'pypyjit' is
the main interpreter loop
@@ -51,13 +51,15 @@
in case loop is not `bridge`, greenkey will be a tuple of constants
or a string describing it.
- for the interpreter loop` it'll be a tuple
+ for the main interpreter loop` it'll be a tuple
(code, offset, is_being_profiled)
assembler_addr is an integer describing where assembler starts,
can be accessed via ctypes, assembler_lenght is the lenght of compiled
asm
+ loopno is the unique loop identifier (int)
+
Note that jit hook is not reentrant. It means that if the code
inside the jit hook is itself jitted, it will get compiled, but the
jit hook won't be called for that.
@@ -74,7 +76,8 @@
optimizations on Python level.
The hook will be called with the following signature:
- hook(jitdriver_name, loop_type, greenkey or guard_number, operations)
+ hook(jitdriver_name, loop_type, greenkey or guard_number, operations,
+ loopno)
jitdriver_name is the name of this particular jitdriver, 'pypyjit' is
the main interpreter loop
@@ -90,6 +93,8 @@
inside the jit hook is itself jitted, it will get compiled, but the
jit hook won't be called for that.
+ loopno is the unique loop identifier (int)
+
Result value will be the resulting list of operations, or None
"""
cache = space.fromcache(Cache)
diff --git a/pypy/rlib/jit.py b/pypy/rlib/jit.py
--- a/pypy/rlib/jit.py
+++ b/pypy/rlib/jit.py
@@ -600,7 +600,6 @@
raise ValueError
set_user_param._annspecialcase_ = 'specialize:arg(0)'
-
# ____________________________________________________________
#
# Annotation and rtyping of some of the JitDriver methods
@@ -901,11 +900,6 @@
instance, overwrite for custom behavior
"""
- def get_stats(self):
- """ Returns various statistics
- """
- raise NotImplementedError
-
def record_known_class(value, cls):
"""
Assure the JIT that value is an instance of cls. This is not a precise
@@ -932,3 +926,39 @@
v_cls = hop.inputarg(classrepr, arg=1)
return hop.genop('jit_record_known_class', [v_inst, v_cls],
resulttype=lltype.Void)
+
+class Counters(object):
+ counters="""
+ TRACING
+ BACKEND
+ OPS
+ RECORDED_OPS
+ GUARDS
+ OPT_OPS
+ OPT_GUARDS
+ OPT_FORCINGS
+ ABORT_TOO_LONG
+ ABORT_BRIDGE
+ ABORT_BAD_LOOP
+ ABORT_ESCAPE
+ ABORT_FORCE_QUASIIMMUT
+ NVIRTUALS
+ NVHOLES
+ NVREUSED
+ TOTAL_COMPILED_LOOPS
+ TOTAL_COMPILED_BRIDGES
+ TOTAL_FREED_LOOPS
+ TOTAL_FREED_BRIDGES
+ """
+
+ counter_names = []
+
+ @staticmethod
+ def _setup():
+ names = Counters.counters.split()
+ for i, name in enumerate(names):
+ setattr(Counters, name, i)
+ Counters.counter_names.append(name)
+ Counters.ncounters = len(names)
+
+Counters._setup()
diff --git a/pypy/rlib/jit_hooks.py b/pypy/rlib/jit_hooks.py
--- a/pypy/rlib/jit_hooks.py
+++ b/pypy/rlib/jit_hooks.py
@@ -108,3 +108,27 @@
def box_isconst(llbox):
from pypy.jit.metainterp.history import Const
return isinstance(_cast_to_box(llbox), Const)
+
+ at register_helper(annmodel.SomePtr(llmemory.GCREF))
+def get_stats():
+ """ Returns various statistics, including how many times each loop
+ was run. Note that on this level the resulting instance is completely
+ opaque, you need to use the interface specified below.
+
+ Note that since we pass warmspot by closure, the actual implementation
+ is in warmspot.py, this is just a placeholder
+ """
+ from pypy.rpython.lltypesystem import lltype, llmemory
+ return lltype.nullptr(llmemory.GCREF.TO) # unusable without the actual JIT
+
+# ------------------------- stats interface ---------------------------
+
+def _cast_to_warmrunnerdesc(llref):
+ from pypy.jit.metainterp.warmspot import WarmRunnerDesc
+
+ ptr = lltype.cast_opaque_ptr(rclass.OBJECTPTR, llref)
+ return cast_base_ptr_to_instance(WarmRunnerDesc, ptr)
+
+ at register_helper(annmodel.SomeFloat())
+def stats_get_counter_value(llref, no):
+ return _cast_to_warmrunnerdesc(llref).metainterp_sd.profiler.get_counter(no)
diff --git a/pypy/translator/goal/richards.py b/pypy/translator/goal/richards.py
--- a/pypy/translator/goal/richards.py
+++ b/pypy/translator/goal/richards.py
@@ -343,8 +343,6 @@
import time
-
-
def schedule():
t = taskWorkArea.taskList
while t is not None:
More information about the pypy-commit
mailing list