[pypy-svn] r79449 - in pypy/trunk/pypy: interpreter jit/backend jit/backend/llgraph jit/backend/llsupport jit/backend/test jit/backend/x86 jit/backend/x86/test jit/metainterp jit/metainterp/test jit/tool jit/tool/test module/__pypy__ module/__pypy__/test module/_pickle_support rlib rpython/memory/gc tool translator translator/c translator/c/test translator/tool
arigo at codespeak.net
arigo at codespeak.net
Wed Nov 24 13:15:21 CET 2010
Author: arigo
Date: Wed Nov 24 13:15:18 2010
New Revision: 79449
Added:
pypy/trunk/pypy/jit/metainterp/memmgr.py
- copied unchanged from r79448, pypy/branch/jit-free/pypy/jit/metainterp/memmgr.py
pypy/trunk/pypy/jit/metainterp/test/test_memmgr.py
- copied unchanged from r79448, pypy/branch/jit-free/pypy/jit/metainterp/test/test_memmgr.py
pypy/trunk/pypy/module/__pypy__/interp_debug.py
- copied unchanged from r79448, pypy/branch/jit-free/pypy/module/__pypy__/interp_debug.py
pypy/trunk/pypy/module/__pypy__/test/test_debug.py
- copied unchanged from r79448, pypy/branch/jit-free/pypy/module/__pypy__/test/test_debug.py
pypy/trunk/pypy/tool/debug_print.py
- copied unchanged from r79448, pypy/branch/jit-free/pypy/tool/debug_print.py
Modified:
pypy/trunk/pypy/interpreter/baseobjspace.py
pypy/trunk/pypy/interpreter/generator.py
pypy/trunk/pypy/jit/backend/llgraph/llimpl.py
pypy/trunk/pypy/jit/backend/llgraph/runner.py
pypy/trunk/pypy/jit/backend/llsupport/gc.py
pypy/trunk/pypy/jit/backend/model.py
pypy/trunk/pypy/jit/backend/test/runner_test.py
pypy/trunk/pypy/jit/backend/test/test_random.py
pypy/trunk/pypy/jit/backend/x86/assembler.py
pypy/trunk/pypy/jit/backend/x86/regalloc.py
pypy/trunk/pypy/jit/backend/x86/runner.py
pypy/trunk/pypy/jit/backend/x86/test/test_regalloc.py
pypy/trunk/pypy/jit/backend/x86/test/test_runner.py
pypy/trunk/pypy/jit/metainterp/compile.py
pypy/trunk/pypy/jit/metainterp/history.py
pypy/trunk/pypy/jit/metainterp/jitprof.py
pypy/trunk/pypy/jit/metainterp/pyjitpl.py
pypy/trunk/pypy/jit/metainterp/test/test_basic.py
pypy/trunk/pypy/jit/metainterp/test/test_compile.py
pypy/trunk/pypy/jit/metainterp/test/test_logger.py
pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py
pypy/trunk/pypy/jit/metainterp/test/test_pyjitpl.py
pypy/trunk/pypy/jit/metainterp/test/test_warmspot.py
pypy/trunk/pypy/jit/metainterp/test/test_warmstate.py
pypy/trunk/pypy/jit/metainterp/warmspot.py
pypy/trunk/pypy/jit/metainterp/warmstate.py
pypy/trunk/pypy/jit/tool/jitoutput.py
pypy/trunk/pypy/jit/tool/test/test_jitoutput.py
pypy/trunk/pypy/module/__pypy__/__init__.py
pypy/trunk/pypy/module/_pickle_support/maker.py
pypy/trunk/pypy/rlib/debug.py
pypy/trunk/pypy/rlib/jit.py
pypy/trunk/pypy/rpython/memory/gc/base.py
pypy/trunk/pypy/rpython/memory/gc/inspector.py
pypy/trunk/pypy/rpython/memory/gc/minimark.py
pypy/trunk/pypy/rpython/memory/gc/semispace.py
pypy/trunk/pypy/translator/c/funcgen.py
pypy/trunk/pypy/translator/c/test/test_newgc.py
pypy/trunk/pypy/translator/c/test/test_standalone.py
pypy/trunk/pypy/translator/driver.py
pypy/trunk/pypy/translator/tool/reftracker.py
Log:
(antocuni, arigo)
Merge branch/jit-free.
The major point of the branch is to reduce memory usage of long-running
pypy's with the JIT (such as "pypy translate.py"). Done by playing with
weakrefs and having a __del__ that kills the ResumeGuardDescrs of old
loops. This is controlled by "--jit loop_longevity=N". (Actually
freeing the assembler code would be the next step, but is less important,
because the ResumeDescrs take more memory.)
In addition, this branch contains a number of semi-related things:
* Generators: set their 'frame' to None when they finish. This
helps a lot the GC. (The issue is that generators need an
interp-level __del__; this let the GC free the frame earlier.)
* rgc.dump_rpy_heap() now works with almost no need for
additional memory.
* pypy.rlib.debug.debug_print(r_longlong(...)).
* add debug_{start,stop,print} to the __pypy__ module. This lets
you write to the PYPYLOG file from app-level.
* use this new interface in translator/driver, to record the begin
and end of tasks (via pypy.tool.debug_print so that it does not
crash on CPython).
* improve a bit reftracker.py.
The net result is that you can now translate pypy with a pypy-jit on
32-bits, using roughly 2.5 GB of RAM. It is twice as big (for now) as
CPython, but twice as fast :-)
Modified: pypy/trunk/pypy/interpreter/baseobjspace.py
==============================================================================
--- pypy/trunk/pypy/interpreter/baseobjspace.py (original)
+++ pypy/trunk/pypy/interpreter/baseobjspace.py Wed Nov 24 13:15:18 2010
@@ -147,7 +147,7 @@
__already_enqueued_for_destruction = False
- def _enqueue_for_destruction(self, space):
+ def _enqueue_for_destruction(self, space, call_user_del=True):
"""Put the object in the destructor queue of the space.
At a later, safe point in time, UserDelAction will use
space.userdel() to call the object's app-level __del__ method.
@@ -160,7 +160,8 @@
return
self.__already_enqueued_for_destruction = True
self.clear_all_weakrefs()
- space.user_del_action.register_dying_object(self)
+ if call_user_del:
+ space.user_del_action.register_dying_object(self)
def _call_builtin_destructor(self):
pass # method overridden in typedef.py
Modified: pypy/trunk/pypy/interpreter/generator.py
==============================================================================
--- pypy/trunk/pypy/interpreter/generator.py (original)
+++ pypy/trunk/pypy/interpreter/generator.py Wed Nov 24 13:15:18 2010
@@ -10,7 +10,7 @@
def __init__(self, frame):
self.space = frame.space
- self.frame = frame
+ self.frame = frame # turned into None when frame_finished_execution
self.running = False
def descr__reduce__(self, space):
@@ -19,9 +19,13 @@
mod = space.interp_w(MixedModule, w_mod)
new_inst = mod.get('generator_new')
w = space.wrap
+ if self.frame:
+ w_frame = w(self.frame)
+ else:
+ w_frame = space.w_None
tup = [
- w(self.frame),
+ w_frame,
w(self.running),
]
@@ -41,7 +45,8 @@
if self.running:
raise OperationError(space.w_ValueError,
space.wrap('generator already executing'))
- if self.frame.frame_finished_execution:
+ frame = self.frame
+ if frame is None:
# xxx a bit ad-hoc, but we don't want to go inside
# execute_generator_frame() if the frame is actually finished
if operr is None:
@@ -49,7 +54,7 @@
raise operr
# XXX it's not clear that last_instr should be promoted at all
# but as long as it is necessary for call_assembler, let's do it early
- last_instr = jit.hint(self.frame.last_instr, promote=True)
+ last_instr = jit.hint(frame.last_instr, promote=True)
if last_instr == -1:
if w_arg and not space.is_w(w_arg, space.w_None):
msg = "can't send non-None value to a just-started generator"
@@ -60,18 +65,19 @@
self.running = True
try:
try:
- w_result = self.frame.execute_generator_frame(w_arg, operr)
+ w_result = frame.execute_generator_frame(w_arg, operr)
except OperationError:
# errors finish a frame
- self.frame.frame_finished_execution = True
+ self.frame = None
raise
# if the frame is now marked as finished, it was RETURNed from
- if self.frame.frame_finished_execution:
+ if frame.frame_finished_execution:
+ self.frame = None
raise OperationError(space.w_StopIteration, space.w_None)
else:
return w_result # YIELDed
finally:
- self.frame.f_backref = jit.vref_None
+ frame.f_backref = jit.vref_None
self.running = False
def descr_throw(self, w_type, w_val=None, w_tb=None):
@@ -115,7 +121,7 @@
raise OperationError(space.w_RuntimeError, space.wrap(msg))
def descr_gi_frame(space, self):
- if not self.frame.frame_finished_execution:
+ if self.frame is not None and not self.frame.frame_finished_execution:
return self.frame
else:
return space.w_None
@@ -125,15 +131,17 @@
applevel __del__, which is called at a safe point after the
interp-level __del__ enqueued the object for destruction
"""
- # Only bother raising an exception if the frame is still not
- # finished and finally or except blocks are present.
- if not self.frame.frame_finished_execution:
+ self.descr_close()
+
+ def __del__(self):
+ # Only bother enqueuing self to raise an exception if the frame is
+ # still not finished and finally or except blocks are present.
+ must_call_close = False
+ if self.frame is not None:
block = self.frame.lastblock
while block is not None:
if not isinstance(block, LoopBlock):
- self.descr_close()
- return
+ must_call_close = True
+ break
block = block.previous
-
- def __del__(self):
- self._enqueue_for_destruction(self.space)
+ self._enqueue_for_destruction(self.space, must_call_close)
Modified: pypy/trunk/pypy/jit/backend/llgraph/llimpl.py
==============================================================================
--- pypy/trunk/pypy/jit/backend/llgraph/llimpl.py (original)
+++ pypy/trunk/pypy/jit/backend/llgraph/llimpl.py Wed Nov 24 13:15:18 2010
@@ -4,6 +4,7 @@
when executing on top of the llinterpreter.
"""
+import weakref
from pypy.objspace.flow.model import Variable, Constant
from pypy.annotation import model as annmodel
from pypy.jit.metainterp.history import (ConstInt, ConstPtr,
@@ -161,6 +162,8 @@
# ____________________________________________________________
class CompiledLoop(object):
+ has_been_freed = False
+
def __init__(self):
self.inputargs = []
self.operations = []
@@ -285,6 +288,11 @@
del _variables[:]
return _to_opaque(CompiledLoop())
+def mark_as_free(loop):
+ loop = _from_opaque(loop)
+ assert not loop.has_been_freed
+ loop.has_been_freed = True
+
def compile_start_int_var(loop):
return compile_start_ref_var(loop, lltype.Signed)
@@ -317,7 +325,7 @@
raise ValueError("CALL_ASSEMBLER not supported")
loop = _from_opaque(loop)
op = loop.operations[-1]
- op.descr = descr
+ op.descr = weakref.ref(descr)
def compile_add_var(loop, intvar):
loop = _from_opaque(loop)
@@ -429,6 +437,7 @@
verbose = True
self.opindex = 0
while True:
+ assert not self.loop.has_been_freed
op = self.loop.operations[self.opindex]
args = [self.getenv(v) for v in op.args]
if not op.is_final():
@@ -440,7 +449,10 @@
_stats.exec_conditional_jumps += 1
if op.jump_target is not None:
# a patched guard, pointing to further code
- args = [self.getenv(v) for v in op.fail_args if v]
+ if op.fail_args:
+ args = [self.getenv(v) for v in op.fail_args if v]
+ else:
+ args = []
assert len(op.jump_target.inputargs) == len(args)
self.env = dict(zip(op.jump_target.inputargs, args))
self.loop = op.jump_target
@@ -839,14 +851,22 @@
finally:
self._may_force = -1
- def op_call_assembler(self, loop_token, *args):
+ def op_call_assembler(self, wref_loop_token, *args):
+ if we_are_translated():
+ raise ValueError("CALL_ASSEMBLER not supported")
+ return self._do_call_assembler(wref_loop_token, *args)
+
+ def _do_call_assembler(self, wref_loop_token, *args):
global _last_exception
+ loop_token = wref_loop_token()
+ assert loop_token, "CALL_ASSEMBLER to a target that already died"
+ ctl = loop_token.compiled_loop_token
+ if hasattr(ctl, 'redirected'):
+ return self._do_call_assembler(ctl.redirected, *args)
assert not self._forced
- loop_token = self.cpu._redirected_call_assembler.get(loop_token,
- loop_token)
self._may_force = self.opindex
try:
- inpargs = _from_opaque(loop_token._llgraph_compiled_version).inputargs
+ inpargs = _from_opaque(ctl.compiled_version).inputargs
for i, inparg in enumerate(inpargs):
TYPE = inparg.concretetype
if TYPE is lltype.Signed:
@@ -1539,10 +1559,13 @@
do_setfield_gc_int(vable, fielddescr.ofs, 0)
def redirect_call_assembler(cpu, oldlooptoken, newlooptoken):
- OLD = _from_opaque(oldlooptoken._llgraph_compiled_version).getargtypes()
- NEW = _from_opaque(newlooptoken._llgraph_compiled_version).getargtypes()
+ oldclt = oldlooptoken.compiled_loop_token
+ newclt = newlooptoken.compiled_loop_token
+ OLD = _from_opaque(oldclt.compiled_version).getargtypes()
+ NEW = _from_opaque(newclt.compiled_version).getargtypes()
assert OLD == NEW
- cpu._redirected_call_assembler[oldlooptoken] = newlooptoken
+ assert not hasattr(oldclt, 'redirected')
+ oldclt.redirected = weakref.ref(newlooptoken)
# ____________________________________________________________
@@ -1609,6 +1632,7 @@
setannotation(compile_add_fail, annmodel.SomeInteger())
setannotation(compile_add_fail_arg, annmodel.s_None)
setannotation(compile_redirect_fail, annmodel.s_None)
+setannotation(mark_as_free, annmodel.s_None)
setannotation(new_frame, s_Frame)
setannotation(frame_clear, annmodel.s_None)
Modified: pypy/trunk/pypy/jit/backend/llgraph/runner.py
==============================================================================
--- pypy/trunk/pypy/jit/backend/llgraph/runner.py (original)
+++ pypy/trunk/pypy/jit/backend/llgraph/runner.py Wed Nov 24 13:15:18 2010
@@ -102,7 +102,6 @@
llimpl._llinterp = LLInterpreter(self.rtyper)
self._future_values = []
self._descrs = {}
- self._redirected_call_assembler = {}
def _freeze_(self):
assert self.translate_support_code
@@ -118,22 +117,34 @@
self._descrs[key] = descr
return descr
- def compile_bridge(self, faildescr, inputargs, operations, log=True):
+ def compile_bridge(self, faildescr, inputargs, operations,
+ original_loop_token, log=True):
c = llimpl.compile_start()
+ clt = original_loop_token.compiled_loop_token
+ clt.loop_and_bridges.append(c)
+ clt.compiling_a_bridge()
self._compile_loop_or_bridge(c, inputargs, operations)
old, oldindex = faildescr._compiled_fail
llimpl.compile_redirect_fail(old, oldindex, c)
- def compile_loop(self, inputargs, operations, loopdescr, log=True):
+ def compile_loop(self, inputargs, operations, looptoken, log=True):
"""In a real assembler backend, this should assemble the given
list of operations. Here we just generate a similar CompiledLoop
instance. The code here is RPython, whereas the code in llimpl
is not.
"""
c = llimpl.compile_start()
- loopdescr._llgraph_compiled_version = c
+ clt = model.CompiledLoopToken(self, looptoken.number)
+ clt.loop_and_bridges = [c]
+ clt.compiled_version = c
+ looptoken.compiled_loop_token = clt
self._compile_loop_or_bridge(c, inputargs, operations)
+ def free_loop_and_bridges(self, compiled_loop_token):
+ for c in compiled_loop_token.loop_and_bridges:
+ llimpl.mark_as_free(c)
+ model.AbstractCPU.free_loop_and_bridges(self, compiled_loop_token)
+
def _compile_loop_or_bridge(self, c, inputargs, operations):
var2index = {}
for box in inputargs:
@@ -207,7 +218,7 @@
if op.getopnum() == rop.JUMP:
targettoken = op.getdescr()
assert isinstance(targettoken, history.LoopToken)
- compiled_version = targettoken._llgraph_compiled_version
+ compiled_version = targettoken.compiled_loop_token.compiled_version
llimpl.compile_add_jump_target(c, compiled_version)
elif op.getopnum() == rop.FINISH:
faildescr = op.getdescr()
@@ -217,7 +228,7 @@
assert False, "unknown operation"
def _execute_token(self, loop_token):
- compiled_version = loop_token._llgraph_compiled_version
+ compiled_version = loop_token.compiled_loop_token.compiled_version
frame = llimpl.new_frame(self.is_oo, self)
# setup the frame
llimpl.frame_clear(frame, compiled_version)
Modified: pypy/trunk/pypy/jit/backend/llsupport/gc.py
==============================================================================
--- pypy/trunk/pypy/jit/backend/llsupport/gc.py (original)
+++ pypy/trunk/pypy/jit/backend/llsupport/gc.py Wed Nov 24 13:15:18 2010
@@ -254,7 +254,7 @@
def _enlarge_gcmap(self):
newlength = 250 + self._gcmap_maxlength * 2
newgcmap = lltype.malloc(self.GCMAP_ARRAY, newlength, flavor='raw',
- track_allocation=False)
+ track_allocation=False) # YYY leak
oldgcmap = self._gcmap
for i in range(self._gcmap_curlength):
newgcmap[i] = oldgcmap[i]
@@ -311,7 +311,7 @@
length = len(shape)
compressed = lltype.malloc(self.CALLSHAPE_ARRAY, length,
flavor='raw',
- track_allocation=False) # memory leak
+ track_allocation=False) # YYY leak
for i in range(length):
compressed[length-1-i] = rffi.cast(rffi.UCHAR, shape[i])
return llmemory.cast_ptr_to_adr(compressed)
Modified: pypy/trunk/pypy/jit/backend/model.py
==============================================================================
--- pypy/trunk/pypy/jit/backend/model.py (original)
+++ pypy/trunk/pypy/jit/backend/model.py Wed Nov 24 13:15:18 2010
@@ -1,3 +1,4 @@
+from pypy.rlib.debug import debug_start, debug_print, debug_stop
from pypy.jit.metainterp import history, compile
@@ -7,17 +8,27 @@
done_with_this_frame_int_v = -1
done_with_this_frame_ref_v = -1
done_with_this_frame_float_v = -1
+ total_compiled_loops = 0
+ total_compiled_bridges = 0
+ total_freed_loops = 0
+ total_freed_bridges = 0
def __init__(self):
self.fail_descr_list = []
+ self.fail_descr_free_list = []
def get_fail_descr_number(self, descr):
assert isinstance(descr, history.AbstractFailDescr)
n = descr.index
if n < 0:
lst = self.fail_descr_list
- n = len(lst)
- lst.append(descr)
+ if len(self.fail_descr_free_list) > 0:
+ n = self.fail_descr_free_list.pop()
+ assert lst[n] is None
+ lst[n] = descr
+ else:
+ n = len(lst)
+ lst.append(descr)
descr.index = n
return n
@@ -35,12 +46,14 @@
def compile_loop(self, inputargs, operations, looptoken, log=True):
"""Assemble the given loop.
- Extra attributes should be put in the LoopToken to
- point to the compiled loop in assembler.
+ Should create and attach a fresh CompiledLoopToken to
+ looptoken.compiled_loop_token and stick extra attributes
+ on it to point to the compiled loop in assembler.
"""
raise NotImplementedError
- def compile_bridge(self, faildescr, inputargs, operations, log=True):
+ def compile_bridge(self, faildescr, inputargs, operations,
+ original_loop_token, log=True):
"""Assemble the bridge.
The FailDescr is the descr of the original guard that failed.
"""
@@ -113,6 +126,24 @@
oldlooptoken so that from now own they will call newlooptoken."""
raise NotImplementedError
+ def free_loop_and_bridges(self, compiled_loop_token):
+ """This method is called to free resources (machine code,
+ references to resume guards, etc.) allocated by the compilation
+ of a loop and all bridges attached to it. After this call, the
+ frontend cannot use this compiled loop any more; in fact, it
+ guarantees that at the point of the call to free_code_group(),
+ none of the corresponding assembler is currently running.
+ """
+ # The base class provides a limited implementation: freeing the
+ # resume descrs. This is already quite helpful, because the
+ # resume descrs are the largest consumers of memory (about 3x
+ # more than the assembler, in the case of the x86 backend).
+ lst = self.fail_descr_list
+ for n in compiled_loop_token.faildescr_indices:
+ lst[n] = None
+ self.fail_descr_free_list.extend(compiled_loop_token.faildescr_indices)
+ # We expect 'compiled_loop_token' to be itself garbage-collected soon.
+
@staticmethod
def sizeof(S):
raise NotImplementedError
@@ -237,3 +268,37 @@
def force(self, force_token):
raise NotImplementedError
+
+
+class CompiledLoopToken(object):
+ def __init__(self, cpu, number):
+ cpu.total_compiled_loops += 1
+ self.cpu = cpu
+ self.number = number
+ self.bridges_count = 0
+ # This growing list gives the 'descr_number' of all fail descrs
+ # that belong to this loop or to a bridge attached to it.
+ # Filled by the frontend calling record_faildescr_index().
+ self.faildescr_indices = []
+ debug_start("jit-mem-looptoken-alloc")
+ debug_print("allocating Loop #", self.number)
+ debug_stop("jit-mem-looptoken-alloc")
+
+ def record_faildescr_index(self, n):
+ self.faildescr_indices.append(n)
+
+ def compiling_a_bridge(self):
+ self.cpu.total_compiled_bridges += 1
+ self.bridges_count += 1
+ debug_start("jit-mem-looptoken-alloc")
+ debug_print("allocating Bridge #", self.bridges_count, "of Loop #", self.number)
+ debug_stop("jit-mem-looptoken-alloc")
+
+ def __del__(self):
+ debug_start("jit-mem-looptoken-free")
+ debug_print("freeing Loop #", self.number, 'with',
+ self.bridges_count, 'attached bridges')
+ self.cpu.free_loop_and_bridges(self)
+ self.cpu.total_freed_loops += 1
+ self.cpu.total_freed_bridges += self.bridges_count
+ debug_stop("jit-mem-looptoken-free")
Modified: pypy/trunk/pypy/jit/backend/test/runner_test.py
==============================================================================
--- pypy/trunk/pypy/jit/backend/test/runner_test.py (original)
+++ pypy/trunk/pypy/jit/backend/test/runner_test.py Wed Nov 24 13:15:18 2010
@@ -174,6 +174,8 @@
assert not wr_i1() and not wr_guard()
def test_compile_bridge(self):
+ self.cpu.total_compiled_loops = 0
+ self.cpu.total_compiled_bridges = 0
i0 = BoxInt()
i1 = BoxInt()
i2 = BoxInt()
@@ -199,7 +201,7 @@
]
bridge[1].setfailargs([i1b])
- self.cpu.compile_bridge(faildescr1, [i1b], bridge)
+ self.cpu.compile_bridge(faildescr1, [i1b], bridge, looptoken)
self.cpu.set_future_value_int(0, 2)
fail = self.cpu.execute_token(looptoken)
@@ -207,6 +209,9 @@
res = self.cpu.get_latest_value_int(0)
assert res == 20
+ assert self.cpu.total_compiled_loops == 1
+ assert self.cpu.total_compiled_bridges == 1
+
def test_compile_bridge_with_holes(self):
i0 = BoxInt()
i1 = BoxInt()
@@ -233,7 +238,7 @@
]
bridge[1].setfailargs([i1b])
- self.cpu.compile_bridge(faildescr1, [i1b], bridge)
+ self.cpu.compile_bridge(faildescr1, [i1b], bridge, looptoken)
self.cpu.set_future_value_int(0, 2)
fail = self.cpu.execute_token(looptoken)
@@ -1050,7 +1055,7 @@
ResOperation(rop.JUMP, [f3] + fboxes2[1:], None, descr=looptoken),
]
- self.cpu.compile_bridge(faildescr1, fboxes2, bridge)
+ self.cpu.compile_bridge(faildescr1, fboxes2, bridge, looptoken)
for i in range(len(fboxes)):
self.cpu.set_future_value_float(i, 13.5 + 6.73 * i)
Modified: pypy/trunk/pypy/jit/backend/test/test_random.py
==============================================================================
--- pypy/trunk/pypy/jit/backend/test/test_random.py (original)
+++ pypy/trunk/pypy/jit/backend/test/test_random.py Wed Nov 24 13:15:18 2010
@@ -524,7 +524,7 @@
self.prebuilt_ptr_consts = []
self.r = r
self.build_random_loop(cpu, builder_factory, r, startvars)
-
+
def build_random_loop(self, cpu, builder_factory, r, startvars):
loop = TreeLoop('test_random_function')
@@ -685,11 +685,12 @@
subloop.operations[-1] = jump_op
self.guard_op = rl.guard_op
self.prebuilt_ptr_consts += rl.prebuilt_ptr_consts
+ self.loop.token.record_jump_to(rl.loop.token)
self.dont_generate_more = True
if r.random() < .05:
return False
self.builder.cpu.compile_bridge(fail_descr, fail_args,
- subloop.operations)
+ subloop.operations, self.loop.token)
return True
def check_random_function(cpu, BuilderClass, r, num=None, max=None):
Modified: pypy/trunk/pypy/jit/backend/x86/assembler.py
==============================================================================
--- pypy/trunk/pypy/jit/backend/x86/assembler.py (original)
+++ pypy/trunk/pypy/jit/backend/x86/assembler.py Wed Nov 24 13:15:18 2010
@@ -7,6 +7,7 @@
from pypy.rpython.lltypesystem.lloperation import llop
from pypy.rpython.annlowlevel import llhelper
from pypy.tool.uid import fixid
+from pypy.jit.backend.model import CompiledLoopToken
from pypy.jit.backend.x86.regalloc import (RegAlloc, X86RegisterManager,
X86XMMRegisterManager, get_ebp_ofs,
_get_scale)
@@ -33,7 +34,6 @@
from pypy.rlib.debug import debug_print, debug_start, debug_stop
from pypy.rlib import rgc
from pypy.jit.backend.x86.jump import remap_frame_layout
-from pypy.rlib.streamio import open_file_as_stream
from pypy.jit.metainterp.history import ConstInt, BoxInt
# darwin requires the stack to be 16 bytes aligned on calls. Same for gcc 4.5.0,
@@ -302,6 +302,8 @@
_x86_arglocs
_x86_debug_checksum
"""
+ looptoken.compiled_loop_token = CompiledLoopToken(self.cpu,
+ looptoken.number)
if not we_are_translated():
# Arguments should be unique
assert len(set(inputargs)) == len(inputargs)
@@ -406,8 +408,9 @@
def _register_counter(self):
if self._debug:
+ # YYY leak -- just put it in self.mc instead
struct = lltype.malloc(DEBUG_COUNTER, flavor='raw',
- track_allocation=False) # known to leak
+ track_allocation=False)
struct.i = 0
self.loop_run_counters.append((len(self.loop_run_counters), struct))
Modified: pypy/trunk/pypy/jit/backend/x86/regalloc.py
==============================================================================
--- pypy/trunk/pypy/jit/backend/x86/regalloc.py (original)
+++ pypy/trunk/pypy/jit/backend/x86/regalloc.py Wed Nov 24 13:15:18 2010
@@ -70,7 +70,7 @@
def _get_new_array(self):
n = self.BASE_CONSTANT_SIZE
# known to leak
- self.cur_array = lltype.malloc(rffi.CArray(lltype.Float), n,
+ self.cur_array = lltype.malloc(rffi.CArray(lltype.Float), n, # YYY leak
flavor='raw', track_allocation=False)
self.cur_array_free = n
_get_new_array._dont_inline_ = True
Modified: pypy/trunk/pypy/jit/backend/x86/runner.py
==============================================================================
--- pypy/trunk/pypy/jit/backend/x86/runner.py (original)
+++ pypy/trunk/pypy/jit/backend/x86/runner.py Wed Nov 24 13:15:18 2010
@@ -53,7 +53,10 @@
self.assembler.assemble_loop(inputargs, operations, looptoken,
log=log)
- def compile_bridge(self, faildescr, inputargs, operations, log=True):
+ def compile_bridge(self, faildescr, inputargs, operations,
+ original_loop_token, log=True):
+ clt = original_loop_token.compiled_loop_token
+ clt.compiling_a_bridge()
self.assembler.assemble_bridge(faildescr, inputargs, operations,
log=log)
Modified: pypy/trunk/pypy/jit/backend/x86/test/test_regalloc.py
==============================================================================
--- pypy/trunk/pypy/jit/backend/x86/test/test_regalloc.py (original)
+++ pypy/trunk/pypy/jit/backend/x86/test/test_regalloc.py Wed Nov 24 13:15:18 2010
@@ -166,7 +166,8 @@
assert ([box.type for box in bridge.inputargs] ==
[box.type for box in guard_op.getfailargs()])
faildescr = guard_op.getdescr()
- self.cpu.compile_bridge(faildescr, bridge.inputargs, bridge.operations)
+ self.cpu.compile_bridge(faildescr, bridge.inputargs, bridge.operations,
+ loop.token)
return bridge
def run(self, loop):
Modified: pypy/trunk/pypy/jit/backend/x86/test/test_runner.py
==============================================================================
--- pypy/trunk/pypy/jit/backend/x86/test/test_runner.py (original)
+++ pypy/trunk/pypy/jit/backend/x86/test/test_runner.py Wed Nov 24 13:15:18 2010
@@ -338,6 +338,7 @@
faildescr1 = BasicFailDescr(1)
faildescr2 = BasicFailDescr(2)
looptoken = LoopToken()
+ looptoken.number = 17
class FakeString(object):
def __init__(self, val):
self.val = val
@@ -356,7 +357,7 @@
operations[3].setfailargs([i1])
self.cpu.compile_loop(inputargs, operations, looptoken)
name, loopaddress, loopsize = agent.functions[0]
- assert name == "Loop # 0: hello"
+ assert name == "Loop # 17: hello"
assert loopaddress <= looptoken._x86_loop_code
assert loopsize >= 40 # randomish number
@@ -370,7 +371,7 @@
]
bridge[1].setfailargs([i1b])
- self.cpu.compile_bridge(faildescr1, [i1b], bridge)
+ self.cpu.compile_bridge(faildescr1, [i1b], bridge, looptoken)
name, address, size = agent.functions[1]
assert name == "Bridge # 0: bye"
# Would be exactly ==, but there are some guard failure recovery
Modified: pypy/trunk/pypy/jit/metainterp/compile.py
==============================================================================
--- pypy/trunk/pypy/jit/metainterp/compile.py (original)
+++ pypy/trunk/pypy/jit/metainterp/compile.py Wed Nov 24 13:15:18 2010
@@ -1,4 +1,4 @@
-
+import weakref
from pypy.rpython.lltypesystem import lltype
from pypy.rpython.ootypesystem import ootype
from pypy.objspace.flow.model import Constant, Variable
@@ -48,6 +48,32 @@
loop_token.outermost_jitdriver_sd = jitdriver_sd
return loop_token
+def record_loop_or_bridge(loop):
+ """Do post-backend recordings and cleanups on 'loop'.
+ """
+ # get the original loop token (corresponding to 'loop', or if that is
+ # a bridge, to the loop that this bridge belongs to)
+ looptoken = loop.token
+ assert looptoken is not None
+ wref = weakref.ref(looptoken)
+ for op in loop.operations:
+ descr = op.getdescr()
+ if isinstance(descr, ResumeDescr):
+ descr.wref_original_loop_token = wref # stick it there
+ n = descr.index
+ if n >= 0: # we also record the resumedescr number
+ looptoken.compiled_loop_token.record_faildescr_index(n)
+ elif isinstance(descr, LoopToken):
+ # for a JUMP or a CALL_ASSEMBLER: record it as a potential jump.
+ # (the following test is not enough to prevent more complicated
+ # cases of cycles, but at least it helps in simple tests of
+ # test_memgr.py)
+ if descr is not looptoken:
+ looptoken.record_jump_to(descr)
+ op.setdescr(None) # clear reference, mostly for tests
+ # mostly for tests: make sure we don't keep a reference to the LoopToken
+ loop.token = None
+
# ____________________________________________________________
def compile_new_loop(metainterp, old_loop_tokens, start):
@@ -77,6 +103,7 @@
return old_loop_token
send_loop_to_backend(metainterp_sd, loop, "loop")
insert_loop_token(old_loop_tokens, loop_token)
+ record_loop_or_bridge(loop)
return loop_token
def insert_loop_token(old_loop_tokens, loop_token):
@@ -117,8 +144,11 @@
else:
loop._ignore_during_counting = True
metainterp_sd.log("compiled new " + type)
+ if metainterp_sd.warmrunnerdesc is not None: # for tests
+ metainterp_sd.warmrunnerdesc.memory_manager.keep_loop_alive(loop.token)
-def send_bridge_to_backend(metainterp_sd, faildescr, inputargs, operations):
+def send_bridge_to_backend(metainterp_sd, faildescr, inputargs, operations,
+ original_loop_token):
n = metainterp_sd.cpu.get_fail_descr_number(faildescr)
metainterp_sd.logger_ops.log_bridge(inputargs, operations, n)
if not we_are_translated():
@@ -127,7 +157,8 @@
metainterp_sd.profiler.start_backend()
debug_start("jit-backend")
try:
- metainterp_sd.cpu.compile_bridge(faildescr, inputargs, operations)
+ metainterp_sd.cpu.compile_bridge(faildescr, inputargs, operations,
+ original_loop_token)
finally:
debug_stop("jit-backend")
metainterp_sd.profiler.end_backend()
@@ -227,9 +258,6 @@
CNT_FLOAT = -0x60000000
CNT_MASK = 0x1FFFFFFF
- def __init__(self, metainterp_sd):
- self.metainterp_sd = metainterp_sd
-
def store_final_boxes(self, guard_op, boxes):
guard_op.setfailargs(boxes)
self.guard_opnum = guard_op.getopnum()
@@ -314,12 +342,14 @@
def compile_and_attach(self, metainterp, new_loop):
# We managed to create a bridge. Attach the new operations
- # to the corrsponding guard_op and compile from there
+ # to the corresponding guard_op and compile from there
+ assert metainterp.resumekey_original_loop_token is not None
+ new_loop.token = metainterp.resumekey_original_loop_token
inputargs = metainterp.history.inputargs
if not we_are_translated():
self._debug_suboperations = new_loop.operations
send_bridge_to_backend(metainterp.staticdata, self, inputargs,
- new_loop.operations)
+ new_loop.operations, new_loop.token)
def copy_all_attrbutes_into(self, res):
# XXX a bit ugly to have to list them all here
@@ -331,14 +361,14 @@
res.rd_pendingfields = self.rd_pendingfields
def _clone_if_mutable(self):
- res = ResumeGuardDescr(self.metainterp_sd)
+ res = ResumeGuardDescr()
self.copy_all_attrbutes_into(res)
return res
class ResumeGuardForcedDescr(ResumeGuardDescr):
def __init__(self, metainterp_sd, jitdriver_sd):
- ResumeGuardDescr.__init__(self, metainterp_sd)
+ self.metainterp_sd = metainterp_sd
self.jitdriver_sd = jitdriver_sd
def handle_fail(self, metainterp_sd, jitdriver_sd):
@@ -482,6 +512,8 @@
metainterp_sd = metainterp.staticdata
jitdriver_sd = metainterp.jitdriver_sd
redargs = new_loop.inputargs
+ # We make a new LoopToken for this entry bridge, and stick it
+ # to every guard in the loop.
new_loop_token = make_loop_token(len(redargs), jitdriver_sd)
new_loop.token = new_loop_token
send_loop_to_backend(metainterp_sd, new_loop, "entry bridge")
@@ -489,12 +521,14 @@
jitdriver_sd.warmstate.attach_unoptimized_bridge_from_interp(
self.original_greenkey,
new_loop_token)
- # store the new loop in compiled_merge_points too
+ # store the new loop in compiled_merge_points_wref too
old_loop_tokens = metainterp.get_compiled_merge_points(
self.original_greenkey)
# it always goes at the end of the list, as it is the most
# general loop token
old_loop_tokens.append(new_loop_token)
+ metainterp.set_compiled_merge_points(self.original_greenkey,
+ old_loop_tokens)
def reset_counter_from_failure(self):
pass
@@ -529,6 +563,7 @@
# know exactly what we must do (ResumeGuardDescr/ResumeFromInterpDescr)
prepare_last_operation(new_loop, target_loop_token)
resumekey.compile_and_attach(metainterp, new_loop)
+ record_loop_or_bridge(new_loop)
return target_loop_token
def prepare_last_operation(new_loop, target_loop_token):
@@ -555,7 +590,8 @@
propagate_exception_descr = PropagateExceptionDescr()
-def compile_tmp_callback(cpu, jitdriver_sd, greenboxes, redboxes):
+def compile_tmp_callback(cpu, jitdriver_sd, greenboxes, redboxes,
+ memory_manager=None):
"""Make a LoopToken that corresponds to assembler code that just
calls back the interpreter. Used temporarily: a fully compiled
version of the code may end up replacing it.
@@ -595,4 +631,6 @@
]
operations[1].setfailargs([])
cpu.compile_loop(inputargs, operations, loop_token, log=False)
+ if memory_manager is not None: # for tests
+ memory_manager.keep_loop_alive(loop_token)
return loop_token
Modified: pypy/trunk/pypy/jit/metainterp/history.py
==============================================================================
--- pypy/trunk/pypy/jit/metainterp/history.py (original)
+++ pypy/trunk/pypy/jit/metainterp/history.py Wed Nov 24 13:15:18 2010
@@ -4,7 +4,7 @@
from pypy.rpython.ootypesystem import ootype
from pypy.rlib.objectmodel import we_are_translated, r_dict, Symbolic
from pypy.rlib.objectmodel import compute_hash, compute_unique_id
-from pypy.rlib.rarithmetic import intmask
+from pypy.rlib.rarithmetic import intmask, r_longlong
from pypy.tool.uid import uid
from pypy.conftest import option
@@ -730,14 +730,28 @@
outermost_jitdriver_sd = None
# specnodes = ...
# and more data specified by the backend when the loop is compiled
- number = 0
+ number = -1
+ generation = r_longlong(0)
+ # one purpose of LoopToken is to keep alive the CompiledLoopToken
+ # returned by the backend. When the LoopToken goes away, the
+ # CompiledLoopToken has its __del__ called, which frees the assembler
+ # memory and the ResumeGuards.
+ compiled_loop_token = None
- def __init__(self, number=0):
- self.number = number
+ def __init__(self):
+ # For memory management of assembled loops
+ self._keepalive_target_looktokens = {} # set of other LoopTokens
+
+ def record_jump_to(self, target_loop_token):
+ self._keepalive_target_looktokens[target_loop_token] = None
+
+ def __repr__(self):
+ return '<Loop %d, gen=%d>' % (self.number, self.generation)
def repr_of_descr(self):
return '<Loop%d>' % self.number
+
class TreeLoop(object):
inputargs = None
operations = None
Modified: pypy/trunk/pypy/jit/metainterp/jitprof.py
==============================================================================
--- pypy/trunk/pypy/jit/metainterp/jitprof.py (original)
+++ pypy/trunk/pypy/jit/metainterp/jitprof.py Wed Nov 24 13:15:18 2010
@@ -24,6 +24,10 @@
NVIRTUALS
NVHOLES
NVREUSED
+TOTAL_COMPILED_LOOPS
+TOTAL_COMPILED_BRIDGES
+TOTAL_FREED_LOOPS
+TOTAL_FREED_BRIDGES
"""
def _setup():
@@ -35,6 +39,7 @@
_setup()
JITPROF_LINES = ncounters + 1 + 1 # one for TOTAL, 1 for calls, update if needed
+_CPU_LINES = 4 # the last 4 lines are stored on the cpu
class BaseProfiler(object):
pass
@@ -87,12 +92,13 @@
counters = None
calls = 0
current = None
+ cpu = None
def start(self):
self.starttime = self.timer()
self.t1 = self.starttime
self.times = [0, 0]
- self.counters = [0] * ncounters
+ self.counters = [0] * (ncounters - _CPU_LINES)
self.calls = 0
self.current = []
@@ -174,6 +180,16 @@
self._print_intline("nvirtuals", cnt[NVIRTUALS])
self._print_intline("nvholes", cnt[NVHOLES])
self._print_intline("nvreused", cnt[NVREUSED])
+ cpu = self.cpu
+ if cpu is not None: # for some tests
+ self._print_intline("Total # of loops",
+ cpu.total_compiled_loops)
+ self._print_intline("Total # of bridges",
+ cpu.total_compiled_bridges)
+ self._print_intline("Freed # of loops",
+ cpu.total_freed_loops)
+ self._print_intline("Freed # of bridges",
+ cpu.total_freed_bridges)
def _print_line_time(self, string, i, tim):
final = "%s:%s\t%d\t%f" % (string, " " * max(0, 13-len(string)), i, tim)
Modified: pypy/trunk/pypy/jit/metainterp/pyjitpl.py
==============================================================================
--- pypy/trunk/pypy/jit/metainterp/pyjitpl.py (original)
+++ pypy/trunk/pypy/jit/metainterp/pyjitpl.py Wed Nov 24 13:15:18 2010
@@ -14,7 +14,7 @@
from pypy.jit.metainterp.logger import Logger
from pypy.jit.metainterp.jitprof import EmptyProfiler
from pypy.jit.metainterp.jitprof import GUARDS, RECORDED_OPS, ABORT_ESCAPE
-from pypy.jit.metainterp.jitprof import ABORT_TOO_LONG
+from pypy.jit.metainterp.jitprof import ABORT_TOO_LONG, ABORT_BRIDGE
from pypy.jit.metainterp.jitexc import JitException, get_llexception
from pypy.rlib.rarithmetic import intmask
from pypy.rlib.objectmodel import specialize
@@ -1068,7 +1068,7 @@
resumedescr = compile.ResumeGuardForcedDescr(metainterp_sd,
metainterp.jitdriver_sd)
else:
- resumedescr = compile.ResumeGuardDescr(metainterp_sd)
+ resumedescr = compile.ResumeGuardDescr()
guard_op = metainterp.history.record(opnum, moreargs, None,
descr=resumedescr)
virtualizable_boxes = None
@@ -1222,6 +1222,7 @@
self.logger_ops = Logger(self, guard_number=True)
self.profiler = ProfilerClass()
+ self.profiler.cpu = cpu
self.warmrunnerdesc = warmrunnerdesc
backendmodule = self.cpu.__module__
@@ -1340,6 +1341,11 @@
return jitcode
return None
+ def try_to_free_some_loops(self):
+ # Increase here the generation recorded by the memory manager.
+ if self.warmrunnerdesc is not None: # for tests
+ self.warmrunnerdesc.memory_manager.next_generation()
+
# ---------------- logging ------------------------
def log(self, msg):
@@ -1636,6 +1642,7 @@
self.staticdata._setup_once()
self.staticdata.profiler.start_tracing()
assert jitdriver_sd is self.jitdriver_sd
+ self.staticdata.try_to_free_some_loops()
self.create_empty_history()
try:
original_boxes = self.initialize_original_boxes(jitdriver_sd, *args)
@@ -1664,10 +1671,15 @@
debug_start('jit-tracing')
self.staticdata.profiler.start_tracing()
assert isinstance(key, compile.ResumeGuardDescr)
+ # store the resumekey.wref_original_loop_token() on 'self' to make
+ # sure that it stays alive as long as this MetaInterp
+ self.resumekey_original_loop_token = key.wref_original_loop_token()
+ self.staticdata.try_to_free_some_loops()
self.initialize_state_from_guard_failure(key)
try:
return self._handle_guard_failure(key)
finally:
+ self.resumekey_original_loop_token = None
self.staticdata.profiler.end_tracing()
debug_stop('jit-tracing')
@@ -1677,6 +1689,8 @@
self.seen_loop_header_for_jdindex = -1
try:
self.prepare_resume_from_failure(key.guard_opnum)
+ if self.resumekey_original_loop_token is None: # very rare case
+ raise SwitchToBlackhole(ABORT_BRIDGE)
self.interpret()
except GenerateMergePoint, gmp:
return self.designate_target_loop(gmp)
@@ -1798,10 +1812,15 @@
raise NotImplementedError(opname[opnum])
def get_compiled_merge_points(self, greenkey):
+ """Get the list of looptokens corresponding to the greenkey.
+ Turns the (internal) list of weakrefs into regular refs.
+ """
+ cell = self.jitdriver_sd.warmstate.jit_cell_at_key(greenkey)
+ return cell.get_compiled_merge_points()
+
+ def set_compiled_merge_points(self, greenkey, looptokens):
cell = self.jitdriver_sd.warmstate.jit_cell_at_key(greenkey)
- if cell.compiled_merge_points is None:
- cell.compiled_merge_points = []
- return cell.compiled_merge_points
+ cell.set_compiled_merge_points(looptokens)
def compile(self, original_boxes, live_arg_boxes, start):
num_green_args = self.jitdriver_sd.num_green_args
@@ -1811,6 +1830,7 @@
self.history.record(rop.JUMP, live_arg_boxes[num_green_args:], None)
loop_token = compile.compile_new_loop(self, old_loop_tokens, start)
if loop_token is not None: # raise if it *worked* correctly
+ self.set_compiled_merge_points(greenkey, old_loop_tokens)
raise GenerateMergePoint(live_arg_boxes, loop_token)
self.history.operations.pop() # remove the JUMP
Modified: pypy/trunk/pypy/jit/metainterp/test/test_basic.py
==============================================================================
--- pypy/trunk/pypy/jit/metainterp/test/test_basic.py (original)
+++ pypy/trunk/pypy/jit/metainterp/test/test_basic.py Wed Nov 24 13:15:18 2010
@@ -19,7 +19,11 @@
from pypy.jit.metainterp import simple_optimize
class FakeJitCell:
- compiled_merge_points = None
+ __compiled_merge_points = []
+ def get_compiled_merge_points(self):
+ return self.__compiled_merge_points[:]
+ def set_compiled_merge_points(self, lst):
+ self.__compiled_merge_points = lst
class FakeWarmRunnerState:
def attach_unoptimized_bridge_from_interp(self, greenkey, newloop):
Modified: pypy/trunk/pypy/jit/metainterp/test/test_compile.py
==============================================================================
--- pypy/trunk/pypy/jit/metainterp/test/test_compile.py (original)
+++ pypy/trunk/pypy/jit/metainterp/test/test_compile.py Wed Nov 24 13:15:18 2010
@@ -53,6 +53,7 @@
stats = Stats()
profiler = jitprof.EmptyProfiler()
+ warmrunnerdesc = None
def log(self, msg, event_kind=None):
pass
@@ -206,14 +207,12 @@
class ExitFrameWithExceptionRef(Exception):
pass
FakeMetaInterpSD.cpu = cpu
- class FakeJitDriverSD:
- pass
cpu.set_future_value_int(0, -156)
cpu.set_future_value_int(1, -178)
cpu.set_future_value_int(2, -190)
fail_descr = cpu.execute_token(loop_token)
try:
- fail_descr.handle_fail(FakeMetaInterpSD(), FakeJitDriverSD())
+ fail_descr.handle_fail(FakeMetaInterpSD(), None)
except FakeMetaInterpSD.ExitFrameWithExceptionRef, e:
assert lltype.cast_opaque_ptr(lltype.Ptr(EXC), e.args[1]) == llexc
else:
Modified: pypy/trunk/pypy/jit/metainterp/test/test_logger.py
==============================================================================
--- pypy/trunk/pypy/jit/metainterp/test/test_logger.py (original)
+++ pypy/trunk/pypy/jit/metainterp/test/test_logger.py Wed Nov 24 13:15:18 2010
@@ -14,11 +14,20 @@
def capturing(func, *args, **kwds):
log_stream = StringIO()
- debug._stderr = log_stream
+ class MyDebugLog:
+ def debug_print(self, *args):
+ for arg in args:
+ print >> log_stream, arg,
+ print >> log_stream
+ def debug_start(self, *args):
+ pass
+ def debug_stop(self, *args):
+ pass
try:
+ debug._log = MyDebugLog()
func(*args, **kwds)
finally:
- debug._stderr = sys.stderr
+ debug._log = None
return log_stream.getvalue()
class Logger(logger.Logger):
@@ -112,7 +121,8 @@
equaloplists(loop.operations, oloop.operations)
def test_jump(self):
- namespace = {'target': LoopToken(3)}
+ namespace = {'target': LoopToken()}
+ namespace['target'].number = 3
inp = '''
[i0]
jump(i0, descr=target)
Modified: pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py
==============================================================================
--- pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py (original)
+++ pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py Wed Nov 24 13:15:18 2010
@@ -41,7 +41,7 @@
b1 = BoxInt()
opt = optimizeopt.Optimizer(FakeMetaInterpStaticData(LLtypeMixin.cpu),
None)
- fdescr = ResumeGuardDescr(None)
+ fdescr = ResumeGuardDescr()
op = ResOperation(rop.GUARD_TRUE, ['dummy'], None, descr=fdescr)
# setup rd data
fi0 = resume.FrameInfo(None, "code0", 11)
Modified: pypy/trunk/pypy/jit/metainterp/test/test_pyjitpl.py
==============================================================================
--- pypy/trunk/pypy/jit/metainterp/test/test_pyjitpl.py (original)
+++ pypy/trunk/pypy/jit/metainterp/test/test_pyjitpl.py Wed Nov 24 13:15:18 2010
@@ -17,6 +17,7 @@
portal.setup(None)
class FakeStaticData:
cpu = None
+ warmrunnerdesc = None
metainterp = pyjitpl.MetaInterp(FakeStaticData(), None)
metainterp.framestack = []
@@ -53,6 +54,7 @@
def test_remove_consts_and_duplicates():
class FakeStaticData:
cpu = None
+ warmrunnerdesc = None
def is_another_box_like(box, referencebox):
assert box is not referencebox
assert isinstance(box, referencebox.clonebox().__class__)
Modified: pypy/trunk/pypy/jit/metainterp/test/test_warmspot.py
==============================================================================
--- pypy/trunk/pypy/jit/metainterp/test/test_warmspot.py (original)
+++ pypy/trunk/pypy/jit/metainterp/test/test_warmspot.py Wed Nov 24 13:15:18 2010
@@ -293,23 +293,30 @@
exc_vtable = lltype.malloc(OBJECT_VTABLE, immortal=True)
cls.exc_vtable = exc_vtable
- class FakeFailDescr(object):
+ class FakeLoopToken:
def __init__(self, no):
self.no = no
+ self.generation = 0
+
+ class FakeFailDescr(object):
+ def __init__(self, looptoken):
+ assert isinstance(looptoken, FakeLoopToken)
+ self.looptoken = looptoken
def handle_fail(self, metainterp_sd, jitdrivers_sd):
- if self.no == 0:
+ no = self.looptoken.no
+ if no == 0:
raise metainterp_sd.warmrunnerdesc.DoneWithThisFrameInt(3)
- if self.no == 1:
+ if no == 1:
raise metainterp_sd.warmrunnerdesc.ContinueRunningNormally(
[0], [], [], [1], [], [])
- if self.no == 3:
+ if no == 3:
exc = lltype.malloc(OBJECT)
exc.typeptr = exc_vtable
raise metainterp_sd.warmrunnerdesc.ExitFrameWithExceptionRef(
metainterp_sd.cpu,
lltype.cast_opaque_ptr(llmemory.GCREF, exc))
- return self.no
+ return self.looptoken
class FakeDescr:
def as_vtable_size_descr(self):
@@ -334,11 +341,11 @@
sizeof = nodescr
def get_fail_descr_from_number(self, no):
- return FakeFailDescr(no)
+ return FakeFailDescr(FakeLoopToken(no))
def execute_token(self, token):
- assert token == 2
- return FakeFailDescr(1)
+ assert token.no == 2
+ return FakeFailDescr(FakeLoopToken(1))
driver = JitDriver(reds = ['red'], greens = ['green'])
Modified: pypy/trunk/pypy/jit/metainterp/test/test_warmstate.py
==============================================================================
--- pypy/trunk/pypy/jit/metainterp/test/test_warmstate.py (original)
+++ pypy/trunk/pypy/jit/metainterp/test/test_warmstate.py Wed Nov 24 13:15:18 2010
@@ -99,6 +99,8 @@
lltype.Float], lltype.Void))
class FakeWarmRunnerDesc:
rtyper = FakeRTyper()
+ cpu = None
+ memory_manager = None
class FakeJitDriverSD:
_get_jitcell_at_ptr = llhelper(GETTER, getter)
_set_jitcell_at_ptr = llhelper(SETTER, setter)
@@ -126,6 +128,7 @@
future_values[j] = "float", value
class FakeWarmRunnerDesc:
cpu = FakeCPU()
+ memory_manager = None
class FakeJitDriverSD:
_red_args_types = ["int", "float"]
virtualizable_info = None
@@ -154,16 +157,20 @@
_get_jitcell_at_ptr = None
state = WarmEnterState(None, FakeJitDriverSD())
get_jitcell = state.make_jitcell_getter()
+ class FakeLoopToken(object):
+ pass
+ looptoken = FakeLoopToken()
state.attach_unoptimized_bridge_from_interp([ConstInt(5),
ConstFloat(2.25)],
- "entry loop token")
+ looptoken)
cell1 = get_jitcell(True, 5, 2.25)
assert cell1.counter < 0
- assert cell1.entry_loop_token == "entry loop token"
+ assert cell1.get_entry_loop_token() is looptoken
def test_make_jitdriver_callbacks_1():
class FakeWarmRunnerDesc:
cpu = None
+ memory_manager = None
class FakeJitDriverSD:
_green_args_spec = [lltype.Signed, lltype.Float]
_get_printable_location_ptr = None
@@ -189,6 +196,7 @@
class FakeWarmRunnerDesc:
rtyper = None
cpu = None
+ memory_manager = None
class FakeJitDriverSD:
_green_args_spec = [lltype.Signed, lltype.Float]
_get_printable_location_ptr = llhelper(GET_LOCATION, get_location)
@@ -211,6 +219,7 @@
class FakeWarmRunnerDesc:
rtyper = None
cpu = None
+ memory_manager = None
class FakeJitDriverSD:
_green_args_spec = [lltype.Signed, lltype.Float]
_get_printable_location_ptr = None
@@ -233,6 +242,7 @@
class FakeWarmRunnerDesc:
rtyper = None
cpu = None
+ memory_manager = None
class FakeJitDriverSD:
_green_args_spec = [lltype.Signed, lltype.Float]
_get_printable_location_ptr = None
Modified: pypy/trunk/pypy/jit/metainterp/warmspot.py
==============================================================================
--- pypy/trunk/pypy/jit/metainterp/warmspot.py (original)
+++ pypy/trunk/pypy/jit/metainterp/warmspot.py Wed Nov 24 13:15:18 2010
@@ -12,11 +12,12 @@
from pypy.rlib.unroll import unrolling_iterable
from pypy.rlib.rarithmetic import r_uint, intmask
from pypy.rlib.debug import debug_print, fatalerror
+from pypy.rlib.debug import debug_start, debug_stop
from pypy.rpython.lltypesystem.lloperation import llop
from pypy.translator.simplify import get_funcobj, get_functype
from pypy.translator.unsimplify import call_final_function
-from pypy.jit.metainterp import history, pyjitpl, gc
+from pypy.jit.metainterp import history, pyjitpl, gc, memmgr
from pypy.jit.metainterp.pyjitpl import MetaInterpStaticData, MetaInterp
from pypy.jit.metainterp.typesystem import LLTypeHelper, OOTypeHelper
from pypy.jit.metainterp.jitprof import Profiler, EmptyProfiler
@@ -61,7 +62,7 @@
def jittify_and_run(interp, graph, args, repeat=1,
backendopt=False, trace_limit=sys.maxint,
- inline=False, **kwds):
+ inline=False, loop_longevity=0, **kwds):
from pypy.config.config import ConfigError
translator = interp.typer.annotator.translator
try:
@@ -78,6 +79,7 @@
jd.warmstate.set_param_trace_eagerness(2) # for tests
jd.warmstate.set_param_trace_limit(trace_limit)
jd.warmstate.set_param_inlining(inline)
+ jd.warmstate.set_param_loop_longevity(loop_longevity)
warmrunnerdesc.finish()
res = interp.eval_graph(graph, args)
if not kwds.get('translate_support_code', False):
@@ -145,6 +147,7 @@
optimizer=None, ProfilerClass=EmptyProfiler, **kwds):
pyjitpl._warmrunnerdesc = self # this is a global for debugging only!
self.set_translator(translator)
+ self.memory_manager = memmgr.MemoryManager()
self.build_cpu(CPUClass, **kwds)
self.find_portals()
self.codewriter = codewriter.CodeWriter(self.cpu, self.jitdrivers_sd)
@@ -707,7 +710,7 @@
loop_token = fail_descr.handle_fail(self.metainterp_sd, jd)
except JitException, e:
return handle_jitexception(e)
- fail_descr = self.cpu.execute_token(loop_token)
+ fail_descr = self.execute_token(loop_token)
jd._assembler_call_helper = assembler_call_helper # for debugging
jd._assembler_helper_ptr = self.helper_func(
@@ -801,3 +804,10 @@
py.test.skip("rewrite_force_virtual: port it to ootype")
all_graphs = self.translator.graphs
vrefinfo.replace_force_virtual_with_call(all_graphs)
+
+ # ____________________________________________________________
+
+ def execute_token(self, loop_token):
+ fail_descr = self.cpu.execute_token(loop_token)
+ self.memory_manager.keep_loop_alive(loop_token)
+ return fail_descr
Modified: pypy/trunk/pypy/jit/metainterp/warmstate.py
==============================================================================
--- pypy/trunk/pypy/jit/metainterp/warmstate.py (original)
+++ pypy/trunk/pypy/jit/metainterp/warmstate.py Wed Nov 24 13:15:18 2010
@@ -1,4 +1,4 @@
-import sys
+import sys, weakref
from pypy.rpython.lltypesystem import lltype, llmemory, rstr, rffi
from pypy.rpython.ootypesystem import ootype
from pypy.rpython.annlowlevel import hlstr, llstr, cast_base_ptr_to_instance
@@ -149,9 +149,34 @@
# counter == -1: there is an entry bridge for this cell
# counter == -2: tracing is currently going on for this cell
counter = 0
- compiled_merge_points = None
+ compiled_merge_points_wref = None # list of weakrefs to LoopToken
dont_trace_here = False
- entry_loop_token = None
+ wref_entry_loop_token = None # (possibly) one weakref to LoopToken
+
+ def get_compiled_merge_points(self):
+ result = []
+ if self.compiled_merge_points_wref is not None:
+ for wref in self.compiled_merge_points_wref:
+ looptoken = wref()
+ if looptoken is not None:
+ result.append(looptoken)
+ return result
+
+ def set_compiled_merge_points(self, looptokens):
+ self.compiled_merge_points_wref = [self._makeref(token)
+ for token in looptokens]
+
+ def get_entry_loop_token(self):
+ if self.wref_entry_loop_token is not None:
+ return self.wref_entry_loop_token()
+ return None
+
+ def set_entry_loop_token(self, looptoken):
+ self.wref_entry_loop_token = self._makeref(looptoken)
+
+ def _makeref(self, looptoken):
+ assert looptoken is not None
+ return weakref.ref(looptoken)
# ____________________________________________________________
@@ -164,6 +189,8 @@
"NOT_RPYTHON"
self.warmrunnerdesc = warmrunnerdesc
self.jitdriver_sd = jitdriver_sd
+ if warmrunnerdesc is not None: # for tests
+ self.cpu = warmrunnerdesc.cpu
try:
self.profiler = warmrunnerdesc.metainterp_sd.profiler
except AttributeError: # for tests
@@ -208,6 +235,12 @@
else:
raise ValueError("unknown optimizer")
+ def set_param_loop_longevity(self, value):
+ # note: it's a global parameter, not a per-jitdriver one
+ if (self.warmrunnerdesc is not None and
+ self.warmrunnerdesc.memory_manager is not None): # all for tests
+ self.warmrunnerdesc.memory_manager.set_max_age(value)
+
def disable_noninlinable_function(self, greenkey):
cell = self.jit_cell_at_key(greenkey)
cell.dont_trace_here = True
@@ -219,12 +252,15 @@
def attach_unoptimized_bridge_from_interp(self, greenkey,
entry_loop_token):
cell = self.jit_cell_at_key(greenkey)
- cell.counter = -1
- old_token = cell.entry_loop_token
- cell.entry_loop_token = entry_loop_token
+ old_token = cell.get_entry_loop_token()
+ cell.set_entry_loop_token(entry_loop_token)
+ cell.counter = -1 # valid entry bridge attached
if old_token is not None:
- cpu = self.warmrunnerdesc.cpu
- cpu.redirect_call_assembler(old_token, entry_loop_token)
+ self.cpu.redirect_call_assembler(old_token, entry_loop_token)
+ # entry_loop_token is also kept alive by any loop that used
+ # to point to old_token. Actually freeing old_token early
+ # is a pointless optimization (it is tiny).
+ old_token.record_jump_to(entry_loop_token)
# ----------
@@ -233,7 +269,8 @@
if hasattr(self, 'maybe_compile_and_run'):
return self.maybe_compile_and_run
- metainterp_sd = self.warmrunnerdesc.metainterp_sd
+ warmrunnerdesc = self.warmrunnerdesc
+ metainterp_sd = warmrunnerdesc.metainterp_sd
jitdriver_sd = self.jitdriver_sd
vinfo = jitdriver_sd.virtualizable_info
index_of_virtualizable = jitdriver_sd.index_of_virtualizable
@@ -291,23 +328,27 @@
assert cell.counter == -1
if not confirm_enter_jit(*args):
return
+ loop_token = cell.get_entry_loop_token()
+ if loop_token is None: # it was a weakref that has been freed
+ cell.counter = 0
+ return
# machine code was already compiled for these greenargs
# get the assembler and fill in the boxes
set_future_values(*args[num_green_args:])
- loop_token = cell.entry_loop_token
# ---------- execute assembler ----------
while True: # until interrupted by an exception
metainterp_sd.profiler.start_running()
debug_start("jit-running")
- fail_descr = metainterp_sd.cpu.execute_token(loop_token)
+ fail_descr = warmrunnerdesc.execute_token(loop_token)
debug_stop("jit-running")
metainterp_sd.profiler.end_running()
+ loop_token = None # for test_memmgr
if vinfo is not None:
vinfo.reset_vable_token(virtualizable)
loop_token = fail_descr.handle_fail(metainterp_sd,
jitdriver_sd)
-
+
maybe_compile_and_run._dont_inline_ = True
self.maybe_compile_and_run = maybe_compile_and_run
return maybe_compile_and_run
@@ -453,7 +494,7 @@
warmrunnerdesc = self.warmrunnerdesc
jitdriver_sd = self.jitdriver_sd
- cpu = warmrunnerdesc.cpu
+ cpu = self.cpu
vinfo = jitdriver_sd.virtualizable_info
red_args_types = unrolling_iterable(jitdriver_sd._red_args_types)
#
@@ -502,10 +543,11 @@
if hasattr(self, 'get_location_str'):
return
#
+ warmrunnerdesc = self.warmrunnerdesc
unwrap_greenkey = self.make_unwrap_greenkey()
jit_getter = self.make_jitcell_getter()
jd = self.jitdriver_sd
- cpu = self.warmrunnerdesc.cpu
+ cpu = self.cpu
def can_inline_greenargs(*greenargs):
if can_never_inline(*greenargs):
@@ -523,11 +565,16 @@
def get_assembler_token(greenkey, redboxes):
# 'redboxes' is only used to know the types of red arguments
cell = self.jit_cell_at_key(greenkey)
- if cell.entry_loop_token is None:
+ entry_loop_token = cell.get_entry_loop_token()
+ if entry_loop_token is None:
from pypy.jit.metainterp.compile import compile_tmp_callback
- cell.entry_loop_token = compile_tmp_callback(cpu, jd, greenkey,
- redboxes)
- return cell.entry_loop_token
+ if cell.counter == -1: # used to be a valid entry bridge,
+ cell.counter = 0 # but was freed in the meantime.
+ memmgr = warmrunnerdesc.memory_manager
+ entry_loop_token = compile_tmp_callback(cpu, jd, greenkey,
+ redboxes, memmgr)
+ cell.set_entry_loop_token(entry_loop_token)
+ return entry_loop_token
self.get_assembler_token = get_assembler_token
#
Modified: pypy/trunk/pypy/jit/tool/jitoutput.py
==============================================================================
--- pypy/trunk/pypy/jit/tool/jitoutput.py (original)
+++ pypy/trunk/pypy/jit/tool/jitoutput.py Wed Nov 24 13:15:18 2010
@@ -27,6 +27,10 @@
(('nvirtuals',), '^nvirtuals:\s+(\d+)$'),
(('nvholes',), '^nvholes:\s+(\d+)$'),
(('nvreused',), '^nvreused:\s+(\d+)$'),
+ (('total_compiled_loops',), '^Total # of loops:\s+(\d+)$'),
+ (('total_compiled_bridges',), '^Total # of bridges:\s+(\d+)$'),
+ (('total_freed_loops',), '^Freed # of loops:\s+(\d+)$'),
+ (('total_freed_bridges',), '^Freed # of bridges:\s+(\d+)$'),
]
class Ops(object):
Modified: pypy/trunk/pypy/jit/tool/test/test_jitoutput.py
==============================================================================
--- pypy/trunk/pypy/jit/tool/test/test_jitoutput.py (original)
+++ pypy/trunk/pypy/jit/tool/test/test_jitoutput.py Wed Nov 24 13:15:18 2010
@@ -63,6 +63,10 @@
nvirtuals: 13
nvholes: 14
nvreused: 15
+Total # of loops: 100
+Total # of bridges: 300
+Freed # of loops: 99
+Freed # of bridges: 299
'''
def test_parse():
Modified: pypy/trunk/pypy/module/__pypy__/__init__.py
==============================================================================
--- pypy/trunk/pypy/module/__pypy__/__init__.py (original)
+++ pypy/trunk/pypy/module/__pypy__/__init__.py Wed Nov 24 13:15:18 2010
@@ -11,6 +11,10 @@
'internal_repr' : 'interp_magic.internal_repr',
'bytebuffer' : 'bytebuffer.bytebuffer',
'identity_dict' : 'interp_identitydict.W_IdentityDict',
+ 'debug_start' : 'interp_debug.debug_start',
+ 'debug_print' : 'interp_debug.debug_print',
+ 'debug_stop' : 'interp_debug.debug_stop',
+ 'debug_print_once' : 'interp_debug.debug_print_once',
}
def setup_after_space_initialization(self):
Modified: pypy/trunk/pypy/module/_pickle_support/maker.py
==============================================================================
--- pypy/trunk/pypy/module/_pickle_support/maker.py (original)
+++ pypy/trunk/pypy/module/_pickle_support/maker.py Wed Nov 24 13:15:18 2010
@@ -67,11 +67,12 @@
return space.wrap(tb)
traceback_new.unwrap_spec = [ObjSpace]
-def generator_new(space, frame, running):
+def generator_new(space, w_frame, running):
+ frame = space.interp_w(PyFrame, w_frame, can_be_None=True)
new_generator = GeneratorIterator(frame)
new_generator.running = running
return space.wrap(new_generator)
-generator_new.unwrap_spec = [ObjSpace, PyFrame, int]
+generator_new.unwrap_spec = [ObjSpace, W_Root, int]
def xrangeiter_new(space, current, remaining, step):
from pypy.module.__builtin__.functional import W_XRangeIterator
Modified: pypy/trunk/pypy/rlib/debug.py
==============================================================================
--- pypy/trunk/pypy/rlib/debug.py (original)
+++ pypy/trunk/pypy/rlib/debug.py Wed Nov 24 13:15:18 2010
@@ -53,13 +53,11 @@
_log = None # patched from tests to be an object of class DebugLog
# or compatible
-_stderr = sys.stderr # alternatively, this is patched from tests
- # (redirects debug_print(), but not debug_start/stop)
def debug_print(*args):
for arg in args:
- print >> _stderr, arg,
- print >> _stderr
+ print >> sys.stderr, arg,
+ print >> sys.stderr
if _log is not None:
_log.debug_print(*args)
Modified: pypy/trunk/pypy/rlib/jit.py
==============================================================================
--- pypy/trunk/pypy/rlib/jit.py (original)
+++ pypy/trunk/pypy/rlib/jit.py Wed Nov 24 13:15:18 2010
@@ -265,6 +265,7 @@
'trace_limit': 10000,
'inlining': False,
'optimizer': OPTIMIZER_FULL,
+ 'loop_longevity': 1000,
}
unroll_parameters = unrolling_iterable(PARAMETERS.keys())
Modified: pypy/trunk/pypy/rpython/memory/gc/base.py
==============================================================================
--- pypy/trunk/pypy/rpython/memory/gc/base.py (original)
+++ pypy/trunk/pypy/rpython/memory/gc/base.py Wed Nov 24 13:15:18 2010
@@ -18,6 +18,7 @@
malloc_zero_filled = False
prebuilt_gc_objects_are_static_roots = True
object_minimal_size = 0
+ gcflag_extra = 0 # or a real GC flag that is always 0 when not collecting
def __init__(self, config, chunk_size=DEFAULT_CHUNK_SIZE,
translated_to_c=True):
Modified: pypy/trunk/pypy/rpython/memory/gc/inspector.py
==============================================================================
--- pypy/trunk/pypy/rpython/memory/gc/inspector.py (original)
+++ pypy/trunk/pypy/rpython/memory/gc/inspector.py Wed Nov 24 13:15:18 2010
@@ -107,15 +107,18 @@
def __init__(self, gc, fd):
self.gc = gc
+ self.gcflag = gc.gcflag_extra
self.fd = rffi.cast(rffi.INT, fd)
self.writebuffer = lltype.malloc(rffi.LONGP.TO, self.BUFSIZE,
flavor='raw')
self.buf_count = 0
- self.seen = AddressDict()
+ if self.gcflag == 0:
+ self.seen = AddressDict()
self.pending = AddressStack()
def delete(self):
- self.seen.delete()
+ if self.gcflag == 0:
+ self.seen.delete()
self.pending.delete()
lltype.free(self.writebuffer, flavor='raw')
free_non_gc_object(self)
@@ -140,6 +143,8 @@
self.flush()
write._always_inline_ = True
+ # ----------
+
def write_marker(self):
self.write(0)
self.write(0)
@@ -161,9 +166,15 @@
self.add(obj)
def add(self, obj):
- if not self.seen.contains(obj):
- self.seen.setitem(obj, obj)
- self.pending.append(obj)
+ if self.gcflag == 0:
+ if not self.seen.contains(obj):
+ self.seen.setitem(obj, obj)
+ self.pending.append(obj)
+ else:
+ hdr = self.gc.header(obj)
+ if (hdr.tid & self.gcflag) == 0:
+ hdr.tid |= self.gcflag
+ self.pending.append(obj)
def add_roots(self):
self.gc.enumerate_all_roots(_hd_add_root, self)
@@ -177,14 +188,50 @@
while pending.non_empty():
self.writeobj(pending.pop())
+ # ----------
+ # A simplified copy of the above, to make sure we walk again all the
+ # objects to clear the 'gcflag'.
+
+ def unwriteobj(self, obj):
+ gc = self.gc
+ gc.trace(obj, self._unwriteref, None)
+
+ def _unwriteref(self, pointer, _):
+ obj = pointer.address[0]
+ self.unadd(obj)
+
+ def unadd(self, obj):
+ assert self.gcflag != 0
+ hdr = self.gc.header(obj)
+ if (hdr.tid & self.gcflag) != 0:
+ hdr.tid &= ~self.gcflag
+ self.pending.append(obj)
+
+ def clear_gcflag_again(self):
+ self.gc.enumerate_all_roots(_hd_unadd_root, self)
+ pendingroots = self.pending
+ self.pending = AddressStack()
+ self.unwalk(pendingroots)
+ pendingroots.delete()
+
+ def unwalk(self, pending):
+ while pending.non_empty():
+ self.unwriteobj(pending.pop())
+
def _hd_add_root(obj, heap_dumper):
heap_dumper.add(obj)
+def _hd_unadd_root(obj, heap_dumper):
+ heap_dumper.unadd(obj)
+
def dump_rpy_heap(gc, fd):
heapdumper = HeapDumper(gc, fd)
heapdumper.add_roots()
heapdumper.walk(heapdumper.pending)
heapdumper.flush()
+ if heapdumper.gcflag != 0:
+ heapdumper.clear_gcflag_again()
+ heapdumper.unwalk(heapdumper.pending)
heapdumper.delete()
return True
Modified: pypy/trunk/pypy/rpython/memory/gc/minimark.py
==============================================================================
--- pypy/trunk/pypy/rpython/memory/gc/minimark.py (original)
+++ pypy/trunk/pypy/rpython/memory/gc/minimark.py Wed Nov 24 13:15:18 2010
@@ -26,7 +26,7 @@
to more than PYPY_GC_MAX_DELTA the amount really
used after a collection. Defaults to 1/8th of the
total RAM size (which is constrained to be at most
- 4GB on 32-bit systems). Try values like '200MB'.
+ 2/3/4GB on 32-bit systems). Try values like '200MB'.
PYPY_GC_MIN Don't collect while the memory size is below this
limit. Useful to avoid spending all the time in
@@ -102,6 +102,7 @@
needs_write_barrier = True
prebuilt_gc_objects_are_static_roots = False
malloc_zero_filled = True # xxx experiment with False
+ gcflag_extra = GCFLAG_FINALIZATION_ORDERING
# All objects start with a HDR, i.e. with a field 'tid' which contains
# a word. This word is divided in two halves: the lower half contains
Modified: pypy/trunk/pypy/rpython/memory/gc/semispace.py
==============================================================================
--- pypy/trunk/pypy/rpython/memory/gc/semispace.py (original)
+++ pypy/trunk/pypy/rpython/memory/gc/semispace.py Wed Nov 24 13:15:18 2010
@@ -42,6 +42,7 @@
inline_simple_malloc_varsize = True
malloc_zero_filled = True
first_unused_gcflag = first_gcflag << 5
+ gcflag_extra = GCFLAG_FINALIZATION_ORDERING
HDR = lltype.Struct('header', ('tid', lltype.Signed)) # XXX or rffi.INT?
typeid_is_in_field = 'tid'
Modified: pypy/trunk/pypy/translator/c/funcgen.py
==============================================================================
--- pypy/trunk/pypy/translator/c/funcgen.py (original)
+++ pypy/trunk/pypy/translator/c/funcgen.py Wed Nov 24 13:15:18 2010
@@ -1,3 +1,4 @@
+import sys
from pypy.translator.c.support import USESLOTS # set to False if necessary while refactoring
from pypy.translator.c.support import cdecl
from pypy.translator.c.support import llvalue_from_constant, gen_assignments
@@ -757,6 +758,16 @@
format.append('%s')
argv.append('(%s) ? "True" : "False"' % self.expr(arg))
continue
+ elif T == SignedLongLong:
+ if sys.platform == 'win32':
+ format.append('%I64d')
+ else:
+ format.append('%lld')
+ elif T == UnsignedLongLong:
+ if sys.platform == 'win32':
+ format.append('%I64u')
+ else:
+ format.append('%llu')
else:
raise Exception("don't know how to debug_print %r" % (T,))
argv.append(self.expr(arg))
@@ -765,17 +776,20 @@
"if (PYPY_HAVE_DEBUG_PRINTS) { fprintf(PYPY_DEBUG_FILE, %s); %s}"
% (', '.join(argv), free_line))
+ def _op_debug(self, opname, arg):
+ if isinstance(arg, Constant):
+ string_literal = c_string_constant(''.join(arg.value.chars))
+ return "%s(%s);" % (opname, string_literal)
+ else:
+ x = "%s(RPyString_AsCharP(%s));\n" % (opname, self.expr(arg))
+ x += "RPyString_FreeCache();"
+ return x
+
def OP_DEBUG_START(self, op):
- arg = op.args[0]
- assert isinstance(arg, Constant)
- return "PYPY_DEBUG_START(%s);" % (
- c_string_constant(''.join(arg.value.chars)),)
+ return self._op_debug('PYPY_DEBUG_START', op.args[0])
def OP_DEBUG_STOP(self, op):
- arg = op.args[0]
- assert isinstance(arg, Constant)
- return "PYPY_DEBUG_STOP(%s);" % (
- c_string_constant(''.join(arg.value.chars)),)
+ return self._op_debug('PYPY_DEBUG_STOP', op.args[0])
def OP_DEBUG_ASSERT(self, op):
return 'RPyAssert(%s, %s);' % (self.expr(op.args[0]),
Modified: pypy/trunk/pypy/translator/c/test/test_newgc.py
==============================================================================
--- pypy/trunk/pypy/translator/c/test/test_newgc.py (original)
+++ pypy/trunk/pypy/translator/c/test/test_newgc.py Wed Nov 24 13:15:18 2010
@@ -1064,14 +1064,16 @@
def test_get_rpy_type_index(self):
self.run("get_rpy_type_index")
- filename_dump = str(udir.join('test_dump_rpy_heap'))
+ filename1_dump = str(udir.join('test_dump_rpy_heap.1'))
+ filename2_dump = str(udir.join('test_dump_rpy_heap.2'))
def define_dump_rpy_heap(self):
U = lltype.GcForwardReference()
U.become(lltype.GcStruct('U', ('next', lltype.Ptr(U)),
('x', lltype.Signed)))
S = lltype.GcStruct('S', ('u', lltype.Ptr(U)))
A = lltype.GcArray(lltype.Ptr(S))
- filename = self.filename_dump
+ filename1 = self.filename1_dump
+ filename2 = self.filename2_dump
def fn():
s = lltype.malloc(S)
@@ -1081,20 +1083,31 @@
a = lltype.malloc(A, 1000)
s2 = lltype.malloc(S)
#
- fd = os.open(filename, os.O_WRONLY | os.O_CREAT, 0666)
- rgc.dump_rpy_heap(fd)
+ fd1 = os.open(filename1, os.O_WRONLY | os.O_CREAT, 0666)
+ fd2 = os.open(filename2, os.O_WRONLY | os.O_CREAT, 0666)
+ rgc.dump_rpy_heap(fd1)
+ rgc.dump_rpy_heap(fd2) # try twice in a row
keepalive_until_here(s2)
keepalive_until_here(s)
keepalive_until_here(a)
- os.close(fd)
+ os.close(fd1)
+ os.close(fd2)
return 0
return fn
def test_dump_rpy_heap(self):
self.run("dump_rpy_heap")
- assert os.path.exists(self.filename_dump)
- assert os.path.getsize(self.filename_dump) > 64
+ for fn in [self.filename1_dump, self.filename2_dump]:
+ assert os.path.exists(fn)
+ assert os.path.getsize(fn) > 64
+ f = open(self.filename1_dump)
+ data1 = f.read()
+ f.close()
+ f = open(self.filename2_dump)
+ data2 = f.read()
+ f.close()
+ assert data1 == data2
filename_dump_typeids_z = str(udir.join('test_typeids_z'))
def define_write_typeids_z(self):
Modified: pypy/trunk/pypy/translator/c/test/test_standalone.py
==============================================================================
--- pypy/trunk/pypy/translator/c/test/test_standalone.py (original)
+++ pypy/trunk/pypy/translator/c/test/test_standalone.py Wed Nov 24 13:15:18 2010
@@ -272,7 +272,7 @@
x = "got:"
debug_start ("mycat")
if have_debug_prints(): x += "b"
- debug_print ("foo", 2, "bar", 3)
+ debug_print ("foo", r_longlong(2), "bar", 3)
debug_start ("cat2")
if have_debug_prints(): x += "c"
debug_print ("baz")
@@ -403,6 +403,20 @@
assert not err
assert path.check(file=0)
+ def test_debug_print_start_stop_nonconst(self):
+ def entry_point(argv):
+ debug_start(argv[1])
+ debug_print(argv[2])
+ debug_stop(argv[1])
+ return 0
+ t, cbuilder = self.compile(entry_point)
+ out, err = cbuilder.cmdexec("foo bar", err=True, env={'PYPYLOG': ':-'})
+ lines = err.splitlines()
+ assert '{foo' in lines[0]
+ assert 'bar' == lines[1]
+ assert 'foo}' in lines[2]
+
+
def test_fatal_error(self):
def g(x):
if x == 1:
Modified: pypy/trunk/pypy/translator/driver.py
==============================================================================
--- pypy/trunk/pypy/translator/driver.py (original)
+++ pypy/trunk/pypy/translator/driver.py Wed Nov 24 13:15:18 2010
@@ -11,6 +11,7 @@
from pypy.annotation import policy as annpolicy
import optparse
from pypy.tool.udir import udir
+from pypy.tool.debug_print import debug_start, debug_print, debug_stop
from pypy.rlib.entrypoint import secondary_entrypoints
import py
@@ -275,6 +276,8 @@
return
else:
self.log.info("%s..." % title)
+ debug_start('translation-task')
+ debug_print('starting', goal)
self.timer.start_event(goal)
try:
instrument = False
@@ -292,11 +295,13 @@
assert False, 'we should not get here'
finally:
try:
+ debug_stop('translation-task')
self.timer.end_event(goal)
except (KeyboardInterrupt, SystemExit):
raise
except:
pass
+ #import gc; gc.dump_rpy_heap('rpyheap-after-%s.dump' % goal)
return res
def task_annotate(self):
Modified: pypy/trunk/pypy/translator/tool/reftracker.py
==============================================================================
--- pypy/trunk/pypy/translator/tool/reftracker.py (original)
+++ pypy/trunk/pypy/translator/tool/reftracker.py Wed Nov 24 13:15:18 2010
@@ -3,7 +3,7 @@
Usage: call track(obj).
"""
-import autopath, sys, os
+import autopath, sys, os, types
import gc
from pypy.translator.tool.graphpage import GraphPage, DotGen
from pypy.tool.uid import uid
@@ -39,7 +39,7 @@
if o2 is None:
continue
addedge(objectlist[i], o2)
- id2typename[uid(o2)] = type(o2).__name__
+ id2typename[uid(o2)] = self.shortrepr(o2)
del o2
for o2 in self.get_referrers(objectlist[i]):
if o2 is None:
@@ -47,7 +47,7 @@
if type(o2) is list and o2 and o2[0] is MARKER:
continue
addedge(o2, objectlist[i])
- id2typename[uid(o2)] = type(o2).__name__
+ id2typename[uid(o2)] = self.shortrepr(o2)
del o2
for ids, label in edges.items():
@@ -82,13 +82,23 @@
return self.newpage(objectlist)
def formatobject(self, o):
+ header = self.shortrepr(o, compact=False)
+ secondline = repr(o.__class__)
+ return header, secondline, repr(o)
+
+ def shortrepr(self, o, compact=True):
+ t = type(o)
+ if t is types.FrameType:
+ if compact:
+ return 'frame %r' % (o.f_code.co_name,)
+ else:
+ return 'frame %r' % (o.f_code,)
s = repr(o)
if len(s) > 50:
- linktext = s
s = s[:20] + ' ... ' + s[-20:]
- else:
- linktext = ''
- return type(o).__name__, s, linktext
+ if s.startswith('<') and s.endswith('>'):
+ s = s[1:-1]
+ return s
def edgelabel(self, o1, o2):
return ''
More information about the Pypy-commit
mailing list