[pypy-commit] pypy custom-trace: Step 1 (not translated so far) to add custom tracers.
arigo
noreply at buildbot.pypy.org
Fri Jul 22 13:24:00 CEST 2011
Author: Armin Rigo <arigo at tunes.org>
Branch: custom-trace
Changeset: r45868:775391fe3185
Date: 2011-07-21 21:13 +0200
http://bitbucket.org/pypy/pypy/changeset/775391fe3185/
Log: Step 1 (not translated so far) to add custom tracers.
diff --git a/pypy/rpython/lltypesystem/lltype.py b/pypy/rpython/lltypesystem/lltype.py
--- a/pypy/rpython/lltypesystem/lltype.py
+++ b/pypy/rpython/lltypesystem/lltype.py
@@ -362,7 +362,8 @@
about=self)._obj
Struct._install_extras(self, **kwds)
- def _attach_runtime_type_info_funcptr(self, funcptr, destrptr):
+ def _attach_runtime_type_info_funcptr(self, funcptr, destrptr,
+ customtraceptr):
if self._runtime_type_info is None:
raise TypeError("attachRuntimeTypeInfo: %r must have been built "
"with the rtti=True argument" % (self,))
@@ -376,7 +377,7 @@
raise TypeError("expected a runtime type info function "
"implementation, got: %s" % funcptr)
self._runtime_type_info.query_funcptr = funcptr
- if destrptr is not None :
+ if destrptr is not None:
T = typeOf(destrptr)
if (not isinstance(T, Ptr) or
not isinstance(T.TO, FuncType) or
@@ -386,6 +387,18 @@
raise TypeError("expected a destructor function "
"implementation, got: %s" % destrptr)
self._runtime_type_info.destructor_funcptr = destrptr
+ if customtraceptr is not None:
+ from pypy.rpython.lltypesystem import llmemory
+ T = typeOf(customtraceptr)
+ if (not isinstance(T, Ptr) or
+ not isinstance(T.TO, FuncType) or
+ len(T.TO.ARGS) != 2 or
+ T.TO.RESULT != llmemory.Address or
+ T.TO.ARGS[0] != llmemory.Address or
+ T.TO.ARGS[1] != llmemory.Address):
+ raise TypeError("expected a custom trace function "
+ "implementation, got: %s" % customtraceptr)
+ self._runtime_type_info.custom_trace_funcptr = customtraceptr
class GcStruct(RttiStruct):
_gckind = 'gc'
@@ -2039,10 +2052,12 @@
raise ValueError("only odd integers can be cast back to ptr")
return _ptr(PTRTYPE, oddint, solid=True)
-def attachRuntimeTypeInfo(GCSTRUCT, funcptr=None, destrptr=None):
+def attachRuntimeTypeInfo(GCSTRUCT, funcptr=None, destrptr=None,
+ customtraceptr=None):
if not isinstance(GCSTRUCT, RttiStruct):
raise TypeError, "expected a RttiStruct: %s" % GCSTRUCT
- GCSTRUCT._attach_runtime_type_info_funcptr(funcptr, destrptr)
+ GCSTRUCT._attach_runtime_type_info_funcptr(funcptr, destrptr,
+ customtraceptr)
return _ptr(Ptr(RuntimeTypeInfo), GCSTRUCT._runtime_type_info)
def getRuntimeTypeInfo(GCSTRUCT):
diff --git a/pypy/rpython/memory/gc/base.py b/pypy/rpython/memory/gc/base.py
--- a/pypy/rpython/memory/gc/base.py
+++ b/pypy/rpython/memory/gc/base.py
@@ -69,7 +69,10 @@
varsize_offsets_to_gcpointers_in_var_part,
weakpointer_offset,
member_index,
- is_rpython_class):
+ is_rpython_class,
+ has_custom_trace,
+ get_custom_trace,
+ fast_path_tracing):
self.getfinalizer = getfinalizer
self.is_varsize = is_varsize
self.has_gcptr_in_varsize = has_gcptr_in_varsize
@@ -83,6 +86,9 @@
self.weakpointer_offset = weakpointer_offset
self.member_index = member_index
self.is_rpython_class = is_rpython_class
+ self.has_custom_trace = has_custom_trace
+ self.get_custom_trace = get_custom_trace
+ self.fast_path_tracing = fast_path_tracing
def get_member_index(self, type_id):
return self.member_index(type_id)
@@ -181,8 +187,14 @@
Typically, 'callback' is a bound method and 'arg' can be None.
"""
typeid = self.get_type_id(obj)
+ #
+ # First, look if we need more than the simple fixed-size tracing
+ if not self.fast_path_tracing(typeid):
+ #
+ # Yes. Two cases: either we are just a GcArray(gcptr), for
+ # which we have a special case for performance, or we call
+ # the slow path version.
if self.is_gcarrayofgcptr(typeid):
- # a performance shortcut for GcArray(gcptr)
length = (obj + llmemory.gcarrayofptr_lengthoffset).signed[0]
item = obj + llmemory.gcarrayofptr_itemsoffset
while length > 0:
@@ -191,6 +203,9 @@
item += llmemory.gcarrayofptr_singleitemoffset
length -= 1
return
+ self._trace_slow_path(obj, callback, arg)
+ #
+ # Do the tracing on the fixed-size part of the object.
offsets = self.offsets_to_gc_pointers(typeid)
i = 0
while i < len(offsets):
@@ -198,6 +213,10 @@
if self.points_to_valid_gc_object(item):
callback(item, arg)
i += 1
+ trace._annspecialcase_ = 'specialize:arg(2)'
+
+ def _trace_slow_path(self, obj, callback, arg):
+ typeid = self.get_type_id(obj)
if self.has_gcptr_in_varsize(typeid):
item = obj + self.varsize_offset_to_variable_part(typeid)
length = (obj + self.varsize_offset_to_length(typeid)).signed[0]
@@ -212,7 +231,16 @@
j += 1
item += itemlength
length -= 1
- trace._annspecialcase_ = 'specialize:arg(2)'
+ if self.has_custom_trace(typeid):
+ generator = self.get_custom_trace(typeid)
+ item = llmemory.NULL
+ while True:
+ item = generator(obj, item)
+ if not item:
+ break
+ if self.points_to_valid_gc_object(item):
+ callback(item, arg)
+ _trace_slow_path._annspecialcase_ = 'specialize:arg(2)'
def trace_partial(self, obj, start, stop, callback, arg):
"""Like trace(), but only walk the array part, for indices in
@@ -317,7 +345,7 @@
break
obj = self.run_finalizers.popleft()
finalizer = self.getfinalizer(self.get_type_id(obj))
- finalizer(obj)
+ finalizer(obj, llmemory.NULL)
finally:
self.finalizer_lock_count -= 1
diff --git a/pypy/rpython/memory/gc/markcompact.py b/pypy/rpython/memory/gc/markcompact.py
--- a/pypy/rpython/memory/gc/markcompact.py
+++ b/pypy/rpython/memory/gc/markcompact.py
@@ -88,6 +88,9 @@
def __init__(self, config, space_size=4096,
min_next_collect_after=128, **kwds):
+ import py
+ py.test.skip("the 'markcompact' gc needs fixing for custom tracers")
+ #
MovingGCBase.__init__(self, config, **kwds)
self.space_size = space_size
self.min_next_collect_after = min_next_collect_after
diff --git a/pypy/rpython/memory/gc/marksweep.py b/pypy/rpython/memory/gc/marksweep.py
--- a/pypy/rpython/memory/gc/marksweep.py
+++ b/pypy/rpython/memory/gc/marksweep.py
@@ -450,7 +450,7 @@
hdr.next = self.malloced_objects
self.malloced_objects = hdr
#llop.debug_view(lltype.Void, self.malloced_objects, self.malloced_objects_with_finalizer, size_gc_header)
- finalizer(obj)
+ finalizer(obj, llmemory.NULL)
if not self.collect_in_progress: # another collection was caused?
debug_print("outer collect interrupted "
"by recursive collect")
diff --git a/pypy/rpython/memory/gctransform/framework.py b/pypy/rpython/memory/gctransform/framework.py
--- a/pypy/rpython/memory/gctransform/framework.py
+++ b/pypy/rpython/memory/gctransform/framework.py
@@ -1247,15 +1247,15 @@
assert not type_contains_pyobjs(TYPE), "not implemented"
if destrptr:
typename = TYPE.__name__
- def ll_finalizer(addr):
+ def ll_finalizer(addr, ignored):
v = llmemory.cast_adr_to_ptr(addr, DESTR_ARG)
ll_call_destructor(destrptr, v, typename)
+ return llmemory.NULL
fptr = self.transformer.annotate_finalizer(ll_finalizer,
- [llmemory.Address],
- lltype.Void)
+ [llmemory.Address, llmemory.Address], llmemory.Address)
+ return fptr
else:
- fptr = lltype.nullptr(gctypelayout.GCData.FINALIZERTYPE.TO)
- return fptr
+ return None
def gen_zero_gc_pointers(TYPE, v, llops, previous_steps=None):
diff --git a/pypy/rpython/memory/gctypelayout.py b/pypy/rpython/memory/gctypelayout.py
--- a/pypy/rpython/memory/gctypelayout.py
+++ b/pypy/rpython/memory/gctypelayout.py
@@ -17,13 +17,21 @@
_alloc_flavor_ = 'raw'
OFFSETS_TO_GC_PTR = lltype.Array(lltype.Signed)
- ADDRESS_VOID_FUNC = lltype.FuncType([llmemory.Address], lltype.Void)
- FINALIZERTYPE = lltype.Ptr(ADDRESS_VOID_FUNC)
+
+ # When used as a finalizer, the following functions only take one
+ # address and ignore the second, and return NULL. When used as a
+ # custom tracer (CT), it enumerates the addresses that contain GCREFs.
+ # It is called with the object as first argument, and the previous
+ # returned address (or NULL the first time) as the second argument.
+ FINALIZER_OR_CT_FUNC = lltype.FuncType([llmemory.Address,
+ llmemory.Address],
+ llmemory.Address)
+ FINALIZER_OR_CT = lltype.Ptr(FINALIZER_OR_CT_FUNC)
# structure describing the layout of a typeid
TYPE_INFO = lltype.Struct("type_info",
("infobits", lltype.Signed), # combination of the T_xxx consts
- ("finalizer", FINALIZERTYPE),
+ ("finalizer_or_customtrace", FINALIZER_OR_CT),
("fixedsize", lltype.Signed),
("ofstoptrs", lltype.Ptr(OFFSETS_TO_GC_PTR)),
hints={'immutable': True},
@@ -71,7 +79,11 @@
return (infobits & T_IS_GCARRAY_OF_GCPTR) != 0
def q_finalizer(self, typeid):
- return self.get(typeid).finalizer
+ typeinfo = self.get(typeid)
+ if typeinfo.infobits & T_HAS_FINALIZER:
+ return typeinfo.finalizer_or_customtrace
+ else:
+ return lltype.nullptr(GCData.FINALIZER_OR_CT_FUNC)
def q_offsets_to_gc_pointers(self, typeid):
return self.get(typeid).ofstoptrs
@@ -105,6 +117,25 @@
infobits = self.get(typeid).infobits
return infobits & T_IS_RPYTHON_INSTANCE != 0
+ def q_has_custom_trace(self, typeid):
+ infobits = self.get(typeid).infobits
+ return infobits & T_HAS_CUSTOM_TRACE != 0
+
+ def q_get_custom_trace(self, typeid):
+ ll_assert(self.q_has_custom_trace(typeid),
+ "T_HAS_CUSTOM_TRACE missing")
+ typeinfo = self.get(typeid)
+ return typeinfo.finalizer_or_customtrace
+
+ def q_fast_path_tracing(self, typeid):
+ # return True if none of the flags T_HAS_GCPTR_IN_VARSIZE,
+ # T_IS_GCARRAY_OF_GCPTR or T_HAS_CUSTOM_TRACE is set
+ T_ANY_SLOW_FLAG = (T_HAS_GCPTR_IN_VARSIZE |
+ T_IS_GCARRAY_OF_GCPTR |
+ T_HAS_CUSTOM_TRACE)
+ infobits = self.get(typeid).infobits
+ return infobits & T_ANY_SLOW_FLAG == 0
+
def set_query_functions(self, gc):
gc.set_query_functions(
self.q_is_varsize,
@@ -119,18 +150,23 @@
self.q_varsize_offsets_to_gcpointers_in_var_part,
self.q_weakpointer_offset,
self.q_member_index,
- self.q_is_rpython_class)
+ self.q_is_rpython_class,
+ self.q_has_custom_trace,
+ self.q_get_custom_trace,
+ self.q_fast_path_tracing)
# the lowest 16bits are used to store group member index
T_MEMBER_INDEX = 0xffff
-T_IS_VARSIZE = 0x10000
-T_HAS_GCPTR_IN_VARSIZE = 0x20000
-T_IS_GCARRAY_OF_GCPTR = 0x40000
-T_IS_WEAKREF = 0x80000
+T_IS_VARSIZE = 0x010000
+T_HAS_GCPTR_IN_VARSIZE = 0x020000
+T_IS_GCARRAY_OF_GCPTR = 0x040000
+T_IS_WEAKREF = 0x080000
T_IS_RPYTHON_INSTANCE = 0x100000 # the type is a subclass of OBJECT
+T_HAS_FINALIZER = 0x200000
+T_HAS_CUSTOM_TRACE = 0x400000
T_KEY_MASK = intmask(0xFF000000)
-T_KEY_VALUE = intmask(0x7A000000) # bug detection only
+T_KEY_VALUE = intmask(0x5A000000) # bug detection only
def _check_valid_type_info(p):
ll_assert(p.infobits & T_KEY_MASK == T_KEY_VALUE, "invalid type_id")
@@ -151,7 +187,18 @@
offsets = offsets_to_gc_pointers(TYPE)
infobits = index
info.ofstoptrs = builder.offsets2table(offsets, TYPE)
- info.finalizer = builder.make_finalizer_funcptr_for_type(TYPE)
+ #
+ kind_and_fptr = builder.finalizer_funcptr_for_type(TYPE)
+ if kind_and_fptr is not None:
+ kind, fptr = kind_and_fptr
+ info.finalizer_or_customtrace = fptr
+ if kind == "finalizer":
+ infobits |= T_HAS_FINALIZER
+ elif kind == "custom_trace":
+ infobits |= T_HAS_CUSTOM_TRACE
+ else:
+ assert 0, kind
+ #
if not TYPE._is_varsize():
info.fixedsize = llarena.round_up_for_allocation(
llmemory.sizeof(TYPE), builder.GCClass.object_minimal_size)
@@ -216,7 +263,7 @@
# for debugging, the following list collects all the prebuilt
# GcStructs and GcArrays
self.all_prebuilt_gc = []
- self.finalizer_funcptrs = {}
+ self._finalizer_funcptrs = {}
self.offsettable_cache = {}
def make_type_info_group(self):
@@ -318,15 +365,28 @@
return self.type_info_group
def finalizer_funcptr_for_type(self, TYPE):
- if TYPE in self.finalizer_funcptrs:
- return self.finalizer_funcptrs[TYPE]
- fptr = self.make_finalizer_funcptr_for_type(TYPE)
- self.finalizer_funcptrs[TYPE] = fptr
- return fptr
+ if TYPE in self._finalizer_funcptrs:
+ return self._finalizer_funcptrs[TYPE]
+ fptr1 = self.make_finalizer_funcptr_for_type(TYPE)
+ fptr2 = self.make_custom_trace_funcptr_for_type(TYPE)
+ assert not (fptr1 and fptr2), (
+ "type %r needs both a finalizer and a custom tracer" % (TYPE,))
+ if fptr1:
+ kind_and_fptr = "finalizer", fptr1
+ elif fptr2:
+ kind_and_fptr = "custom_trace", fptr2
+ else:
+ kind_and_fptr = None
+ self._finalizer_funcptrs[TYPE] = kind_and_fptr
+ return kind_and_fptr
def make_finalizer_funcptr_for_type(self, TYPE):
# must be overridden for proper finalizer support
- return lltype.nullptr(GCData.ADDRESS_VOID_FUNC)
+ return None
+
+ def make_custom_trace_funcptr_for_type(self, TYPE):
+ # must be overridden for proper custom tracer support
+ return None
def initialize_gc_query_function(self, gc):
return GCData(self.type_info_group).set_query_functions(gc)
diff --git a/pypy/rpython/memory/gcwrapper.py b/pypy/rpython/memory/gcwrapper.py
--- a/pypy/rpython/memory/gcwrapper.py
+++ b/pypy/rpython/memory/gcwrapper.py
@@ -196,17 +196,28 @@
DESTR_ARG = lltype.typeOf(destrptr).TO.ARGS[0]
destrgraph = destrptr._obj.graph
else:
- return lltype.nullptr(gctypelayout.GCData.FINALIZERTYPE.TO)
+ return None
assert not type_contains_pyobjs(TYPE), "not implemented"
- def ll_finalizer(addr):
+ def ll_finalizer(addr, dummy):
+ assert dummy == llmemory.NULL
try:
v = llmemory.cast_adr_to_ptr(addr, DESTR_ARG)
self.llinterp.eval_graph(destrgraph, [v], recursive=True)
except llinterp.LLException:
raise RuntimeError(
"a finalizer raised an exception, shouldn't happen")
- return llhelper(gctypelayout.GCData.FINALIZERTYPE, ll_finalizer)
+ return llmemory.NULL
+ return llhelper(gctypelayout.GCData.FINALIZER_OR_CT, ll_finalizer)
+
+ def make_custom_trace_funcptr_for_type(self, TYPE):
+ from pypy.rpython.memory.gctransform.support import get_rtti, \
+ type_contains_pyobjs
+ rtti = get_rtti(TYPE)
+ if rtti is not None and hasattr(rtti._obj, 'custom_trace_funcptr'):
+ return rtti._obj.custom_trace_funcptr
+ else:
+ return None
def collect_constants(graphs):
diff --git a/pypy/rpython/memory/test/test_gc.py b/pypy/rpython/memory/test/test_gc.py
--- a/pypy/rpython/memory/test/test_gc.py
+++ b/pypy/rpython/memory/test/test_gc.py
@@ -237,6 +237,46 @@
res = self.interpret(f, [5])
assert 160 <= res <= 165
+ def test_custom_trace(self):
+ from pypy.rpython.annlowlevel import llhelper
+ from pypy.rpython.lltypesystem import llmemory
+ from pypy.rpython.lltypesystem.llarena import ArenaError
+ #
+ S = lltype.GcStruct('S', ('x', llmemory.Address),
+ ('y', llmemory.Address), rtti=True)
+ T = lltype.GcStruct('T', ('z', lltype.Signed))
+ offset_of_x = llmemory.offsetof(S, 'x')
+ def customtrace(obj, prev):
+ if not prev:
+ return obj + offset_of_x
+ else:
+ return llmemory.NULL
+ CUSTOMTRACEFUNC = lltype.FuncType([llmemory.Address, llmemory.Address],
+ llmemory.Address)
+ customtraceptr = llhelper(lltype.Ptr(CUSTOMTRACEFUNC), customtrace)
+ lltype.attachRuntimeTypeInfo(S, customtraceptr=customtraceptr)
+ #
+ for attrname in ['x', 'y']:
+ def setup():
+ s1 = lltype.malloc(S)
+ tx = lltype.malloc(T)
+ tx.z = 42
+ ty = lltype.malloc(T)
+ s1.x = llmemory.cast_ptr_to_adr(tx)
+ s1.y = llmemory.cast_ptr_to_adr(ty)
+ return s1
+ def f():
+ s1 = setup()
+ llop.gc__collect(lltype.Void)
+ return llmemory.cast_adr_to_ptr(getattr(s1, attrname),
+ lltype.Ptr(T))
+ if attrname == 'x':
+ res = self.interpret(f, [])
+ assert res.z == 42
+ else:
+ py.test.raises((RuntimeError, ArenaError),
+ self.interpret, f, [])
+
def test_weakref(self):
import weakref, gc
class A(object):
More information about the pypy-commit
mailing list