[pypy-svn] r27525 - in pypy/dist/pypy/rpython/memory: . test
cfbolz at codespeak.net
cfbolz at codespeak.net
Sun May 21 11:01:42 CEST 2006
Author: cfbolz
Date: Sun May 21 11:01:39 2006
New Revision: 27525
Modified:
pypy/dist/pypy/rpython/memory/gc.py
pypy/dist/pypy/rpython/memory/gctransform.py
pypy/dist/pypy/rpython/memory/gcwrapper.py
pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py
Log:
first cut at implementing __del__ for the mark and sweep collector
Modified: pypy/dist/pypy/rpython/memory/gc.py
==============================================================================
--- pypy/dist/pypy/rpython/memory/gc.py (original)
+++ pypy/dist/pypy/rpython/memory/gc.py Sun May 21 11:01:39 2006
@@ -60,11 +60,13 @@
class GCBase(object):
_alloc_flavor_ = "raw"
- def set_query_functions(self, is_varsize, offsets_to_gc_pointers,
+ def set_query_functions(self, is_varsize, getfinalizer,
+ offsets_to_gc_pointers,
fixed_size, varsize_item_sizes,
varsize_offset_to_variable_part,
varsize_offset_to_length,
varsize_offsets_to_gcpointers_in_var_part):
+ self.getfinalizer = getfinalizer
self.is_varsize = is_varsize
self.offsets_to_gc_pointers = offsets_to_gc_pointers
self.fixed_size = fixed_size
@@ -133,8 +135,8 @@
self.bytes_malloced_threshold = start_heap_size
self.total_collection_time = 0.0
self.AddressLinkedList = AddressLinkedList
- #self.set_query_functions(None, None, None, None, None, None, None)
self.malloced_objects = lltype.nullptr(self.HDR)
+ self.malloced_objects_with_finalizer = lltype.nullptr(self.HDR)
self.get_roots = get_roots
self.gcheaderbuilder = GCHeaderBuilder(self.HDR)
# pools, for x_swap_pool():
@@ -172,8 +174,12 @@
result = raw_malloc(size_gc_header + size)
hdr = llmemory.cast_adr_to_ptr(result, self.HDRPTR)
hdr.typeid = typeid << 1
- hdr.next = self.malloced_objects
- self.malloced_objects = hdr
+ if not self.getfinalizer(typeid):
+ hdr.next = self.malloced_objects
+ self.malloced_objects = hdr
+ else:
+ hdr.next = self.malloced_objects_with_finalizer
+ self.malloced_objects_with_finalizer = hdr
self.bytes_malloced += raw_malloc_usage(size + size_gc_header)
result += size_gc_header
return llmemory.cast_adr_to_ptr(result, llmemory.GCREF)
@@ -193,14 +199,24 @@
(result + size_gc_header + offset_to_length).signed[0] = length
hdr = llmemory.cast_adr_to_ptr(result, self.HDRPTR)
hdr.typeid = typeid << 1
- hdr.next = self.malloced_objects
- self.malloced_objects = hdr
+ if not self.getfinalizer(typeid):
+ hdr.next = self.malloced_objects
+ self.malloced_objects = hdr
+ else:
+ hdr.next = self.malloced_objects_with_finalizer
+ self.malloced_objects_with_finalizer = hdr
self.bytes_malloced += raw_malloc_usage(size + size_gc_header)
result += size_gc_header
return llmemory.cast_adr_to_ptr(result, llmemory.GCREF)
def collect(self):
+ # 1. mark from the roots, and also the objects that objects-with-del
+ # point to (using the list of malloced_objects_with_finalizer)
+ # 2. walk the list of objects-without-del and free the ones not marked
+ # 3. walk the list of objects-with-del and for the ones not marked:
+ # call __del__, move the object to the list of object-without-del
import time
+ from pypy.rpython.lltypesystem.lloperation import llop
if DEBUG_PRINT:
llop.debug_print(lltype.Void, 'collecting...')
start_time = time.time()
@@ -208,10 +224,11 @@
size_gc_header = self.gcheaderbuilder.size_gc_header
## llop.debug_view(lltype.Void, self.malloced_objects, self.poolnodes,
## size_gc_header)
- objects = self.AddressLinkedList()
+
+ # push the roots on the mark stack
+ objects = self.AddressLinkedList() # mark stack
while 1:
curr = roots.pop()
-## print "root: ", curr
if curr == NULL:
break
# roots is a list of addresses to addresses:
@@ -225,35 +242,37 @@
# from this point onwards, no more mallocs should be possible
old_malloced = self.bytes_malloced
self.bytes_malloced = 0
+ curr_heap_size = 0
+ freed_size = 0
+
+ # mark objects reachable by objects with a finalizer, but not those
+ # themselves. add their size to curr_heap_size, since they always
+ # survive the collection
+ hdr = self.malloced_objects_with_finalizer
+ while hdr:
+ next = hdr.next
+ typeid = hdr.typeid >> 1
+ gc_info = llmemory.cast_ptr_to_adr(hdr)
+ obj = gc_info + size_gc_header
+ self.add_reachable_to_stack(obj, objects)
+ addr = llmemory.cast_ptr_to_adr(hdr)
+ size = self.fixed_size(typeid)
+ if self.is_varsize(typeid):
+ length = (obj + self.varsize_offset_to_length(typeid)).signed[0]
+ size += self.varsize_item_sizes(typeid) * length
+ estimate = raw_malloc_usage(size_gc_header + size)
+ curr_heap_size += estimate
+ hdr = next
+
+ # mark thinks on the mark stack and put their descendants onto the
+ # stack until the stack is empty
while objects.non_empty(): #mark
curr = objects.pop()
-## print "object: ", curr
+ self.add_reachable_to_stack(curr, objects)
gc_info = curr - size_gc_header
hdr = llmemory.cast_adr_to_ptr(gc_info, self.HDRPTR)
if hdr.typeid & 1:
continue
- typeid = hdr.typeid >> 1
- offsets = self.offsets_to_gc_pointers(typeid)
- i = 0
- while i < len(offsets):
- pointer = curr + offsets[i]
- objects.append(pointer.address[0])
- i += 1
- if self.is_varsize(typeid):
- offset = self.varsize_offset_to_variable_part(
- typeid)
- length = (curr + self.varsize_offset_to_length(typeid)).signed[0]
- curr += offset
- offsets = self.varsize_offsets_to_gcpointers_in_var_part(typeid)
- itemlength = self.varsize_item_sizes(typeid)
- i = 0
- while i < length:
- item = curr + itemlength * i
- j = 0
- while j < len(offsets):
- objects.append((item + offsets[j]).address[0])
- j += 1
- i += 1
hdr.typeid = hdr.typeid | 1
objects.delete()
# also mark self.curpool
@@ -262,8 +281,8 @@
hdr = llmemory.cast_adr_to_ptr(gc_info, self.HDRPTR)
hdr.typeid = hdr.typeid | 1
- curr_heap_size = 0
- freed_size = 0
+ # sweep: delete objects without del if they are not marked
+ # unmark objects without del that are marked
firstpoolnode = lltype.malloc(self.POOLNODE, flavor='raw')
firstpoolnode.linkedlist = self.malloced_objects
firstpoolnode.nextnode = self.poolnodes
@@ -328,10 +347,56 @@
## llop.debug_view(lltype.Void, self.malloced_objects, self.poolnodes,
## size_gc_header)
assert self.heap_usage + old_malloced == curr_heap_size + freed_size
+
+ # call finalizers if needed
self.heap_usage = curr_heap_size
+ hdr = self.malloced_objects_with_finalizer
+ self.malloced_objects_with_finalizer = lltype.nullptr(self.HDR)
+ while hdr:
+ next = hdr.next
+ if hdr.typeid & 1:
+ hdr.next = self.malloced_objects_with_finalizer
+ self.malloced_objects_with_finalizer = hdr
+ hdr.typeid = hdr.typeid & (~1)
+ else:
+ obj = llmemory.cast_ptr_to_adr(hdr) + size_gc_header
+ finalizer = self.getfinalizer(hdr.typeid >> 1)
+ finalizer(obj)
+ hdr.next = self.malloced_objects
+ self.malloced_objects = hdr
+ hdr = next
STATISTICS_NUMBERS = 2
+ def add_reachable_to_stack(self, obj, objects):
+ size_gc_header = self.gcheaderbuilder.size_gc_header
+ gc_info = obj - size_gc_header
+ hdr = llmemory.cast_adr_to_ptr(gc_info, self.HDRPTR)
+ if hdr.typeid & 1:
+ return
+ typeid = hdr.typeid >> 1
+ offsets = self.offsets_to_gc_pointers(typeid)
+ i = 0
+ while i < len(offsets):
+ pointer = obj + offsets[i]
+ objects.append(pointer.address[0])
+ i += 1
+ if self.is_varsize(typeid):
+ offset = self.varsize_offset_to_variable_part(
+ typeid)
+ length = (obj + self.varsize_offset_to_length(typeid)).signed[0]
+ obj += offset
+ offsets = self.varsize_offsets_to_gcpointers_in_var_part(typeid)
+ itemlength = self.varsize_item_sizes(typeid)
+ i = 0
+ while i < length:
+ item = obj + itemlength * i
+ j = 0
+ while j < len(offsets):
+ objects.append((item + offsets[j]).address[0])
+ j += 1
+ i += 1
+
def statistics(self):
return self.heap_usage, self.bytes_malloced
Modified: pypy/dist/pypy/rpython/memory/gctransform.py
==============================================================================
--- pypy/dist/pypy/rpython/memory/gctransform.py (original)
+++ pypy/dist/pypy/rpython/memory/gctransform.py Sun May 21 11:01:39 2006
@@ -388,6 +388,16 @@
ADDRESS_VOID_FUNC = lltype.FuncType([llmemory.Address], lltype.Void)
+def get_rtti(TYPE):
+ if isinstance(TYPE, lltype.GcStruct):
+ try:
+ return lltype.getRuntimeTypeInfo(TYPE)
+ except ValueError:
+ pass
+ return None
+
+
+
class RefcountingGCTransformer(GCTransformer):
HDR = lltype.Struct("header", ("refcount", lltype.Signed))
@@ -483,14 +493,6 @@
result.extend(self.pop_alive(oldval))
return result
- def get_rtti(self, TYPE):
- if isinstance(TYPE, lltype.GcStruct):
- try:
- return lltype.getRuntimeTypeInfo(TYPE)
- except ValueError:
- pass
- return None
-
def finish(self):
super(RefcountingGCTransformer, self).finish()
@@ -506,7 +508,7 @@
return self.static_deallocator_funcptrs[TYPE]
#print_call_chain(self)
- rtti = self.get_rtti(TYPE)
+ rtti = get_rtti(TYPE)
if rtti is not None and hasattr(rtti._obj, 'destructor_funcptr'):
destrptr = rtti._obj.destructor_funcptr
DESTR_ARG = lltype.typeOf(destrptr).TO.ARGS[0]
@@ -576,7 +578,7 @@
return self.dynamic_deallocator_funcptrs[TYPE]
#print_call_chain(self)
- rtti = self.get_rtti(TYPE)
+ rtti = get_rtti(TYPE)
if rtti is None:
p = self.static_deallocation_funcptr_for_type(TYPE)
self.dynamic_deallocator_funcptrs[TYPE] = p
@@ -661,14 +663,6 @@
""" for boehm it is enough to do nothing"""
return [SpaceOperation("same_as", [Constant(None, lltype.Void)], op.result)]
- def get_rtti(self, TYPE):
- if isinstance(TYPE, lltype.GcStruct):
- try:
- return lltype.getRuntimeTypeInfo(TYPE)
- except ValueError:
- pass
- return None
-
def finish(self):
super(BoehmGCTransformer, self).finish()
@@ -676,7 +670,7 @@
if TYPE in self.finalizer_funcptrs:
return self.finalizer_funcptrs[TYPE]
- rtti = self.get_rtti(TYPE)
+ rtti = get_rtti(TYPE)
if rtti is not None and hasattr(rtti._obj, 'destructor_funcptr'):
destrptr = rtti._obj.destructor_funcptr
DESTR_ARG = lltype.typeOf(destrptr).TO.ARGS[0]
@@ -751,11 +745,14 @@
super(FrameworkGCTransformer, self).__init__(translator, inline=True)
AddressLinkedList = get_address_linked_list()
GCClass = self.GCClass
+ self.finalizer_funcptrs = {}
+ self.FINALIZERTYPE = lltype.Ptr(ADDRESS_VOID_FUNC)
class GCData(object):
# types of the GC information tables
OFFSETS_TO_GC_PTR = lltype.Array(lltype.Signed)
TYPE_INFO = lltype.Struct("type_info",
("isvarsize", lltype.Bool),
+ ("finalyzer", self.FINALIZERTYPE),
("fixedsize", lltype.Signed),
("ofstoptrs", lltype.Ptr(OFFSETS_TO_GC_PTR)),
("varitemsize", lltype.Signed),
@@ -768,6 +765,9 @@
def q_is_varsize(typeid):
return gcdata.type_info_table[typeid].isvarsize
+ def q_finalyzer(typeid):
+ return gcdata.type_info_table[typeid].finalyzer
+
def q_offsets_to_gc_pointers(typeid):
return gcdata.type_info_table[typeid].ofstoptrs
@@ -818,6 +818,7 @@
gcdata.gc.setup()
gcdata.gc.set_query_functions(
q_is_varsize,
+ q_finalyzer,
q_offsets_to_gc_pointers,
q_fixed_size,
q_varsize_item_sizes,
@@ -984,6 +985,7 @@
self.id_of_type[TYPE] = type_id
offsets = offsets_to_gc_pointers(TYPE)
info["ofstoptrs"] = self.offsets2table(offsets, TYPE)
+ info["finalyzer"] = self.finalizer_funcptr_for_type(TYPE)
if not TYPE._is_varsize():
info["isvarsize"] = False
info["fixedsize"] = llmemory.sizeof(TYPE)
@@ -1018,6 +1020,31 @@
info["varitemsize"] = 0
return type_id
+ def finalizer_funcptr_for_type(self, TYPE):
+ if TYPE in self.finalizer_funcptrs:
+ return self.finalizer_funcptrs[TYPE]
+
+ rtti = get_rtti(TYPE)
+ if rtti is not None and hasattr(rtti._obj, 'destructor_funcptr'):
+ destrptr = rtti._obj.destructor_funcptr
+ DESTR_ARG = lltype.typeOf(destrptr).TO.ARGS[0]
+ else:
+ destrptr = None
+ DESTR_ARG = None
+
+ assert not type_contains_pyobjs(TYPE), "not implemented"
+ if destrptr:
+ def ll_finalizer(addr):
+ v = llmemory.cast_adr_to_ptr(addr, DESTR_ARG)
+ ll_call_destructor(destrptr, v)
+ g, fptr = self.annotate_helper(ll_finalizer, [llmemory.Address], lltype.Void)
+ else:
+ g = fptr = lltype.nullptr(ADDRESS_VOID_FUNC)
+ if g:
+ self.need_minimal_transform(g)
+ self.finalizer_funcptrs[TYPE] = fptr
+ return fptr
+
def consider_constant(self, TYPE, value):
if id(value) in self.seen_roots:
return
Modified: pypy/dist/pypy/rpython/memory/gcwrapper.py
==============================================================================
--- pypy/dist/pypy/rpython/memory/gcwrapper.py (original)
+++ pypy/dist/pypy/rpython/memory/gcwrapper.py Sun May 21 11:01:39 2006
@@ -27,6 +27,7 @@
def create_query_functions(self):
from pypy.rpython.lltypesystem import rstr
_is_varsize = []
+ _finalizers = []
_offsets_to_gc_pointers = []
_fixed_size = []
_varsize_item_sizes = []
@@ -39,6 +40,7 @@
for TYPE, typeid in tttid:
varsize = self.is_varsize(typeid)
_is_varsize.append(varsize)
+ _finalizers.append(None)
_offsets_to_gc_pointers.append(self.offsets_to_gc_pointers(typeid))
_fixed_size.append(self.fixed_size(typeid))
if varsize:
@@ -54,8 +56,12 @@
_varsize_offset_to_variable_part.append(0)
_varsize_offset_to_length.append(0)
_varsize_offsets_to_gcpointers_in_var_part.append([])
+ # trick to make the annotator see that the list can contain functions:
+ _finalizers.append(lambda addr: None)
def is_varsize(typeid):
return _is_varsize[typeid]
+ def getfinalizer(typeid):
+ return _finalizers[typeid]
def offsets_to_gc_pointers(typeid):
return _offsets_to_gc_pointers[typeid]
def fixed_size(typeid):
@@ -68,7 +74,7 @@
return _varsize_offset_to_length[typeid]
def varsize_offsets_to_gcpointers_in_var_part(typeid):
return _varsize_offsets_to_gcpointers_in_var_part[typeid]
- return (is_varsize, offsets_to_gc_pointers, fixed_size,
+ return (is_varsize, getfinalizer, offsets_to_gc_pointers, fixed_size,
varsize_item_sizes, varsize_offset_to_variable_part,
varsize_offset_to_length,
varsize_offsets_to_gcpointers_in_var_part)
@@ -80,6 +86,9 @@
(isinstance(TYPE, lltype.Struct) and
TYPE._arrayfld is not None))
+ def getfinalizer(self, typeid):
+ return None
+
def offsets_to_gc_pointers(self, typeid):
assert typeid >= 0
return lltypelayout.offsets_to_gc_pointers(self.types[typeid])
@@ -118,51 +127,12 @@
return 0
def get_setup_query_functions(self):
- return (self.is_varsize, self.offsets_to_gc_pointers, self.fixed_size,
+ return (self.is_varsize, self.getfinalizer,
+ self.offsets_to_gc_pointers, self.fixed_size,
self.varsize_item_sizes, self.varsize_offset_to_variable_part,
self.varsize_offset_to_length,
self.varsize_offsets_to_gcpointers_in_var_part)
-class SymbolicQueryTypes(QueryTypes):
- def fixed_size(self, typeid):
- assert typeid >= 0
- if self.types[typeid]._is_varsize():
- return llmemory.sizeof(self.types[typeid], 0)
- else:
- return llmemory.sizeof(self.types[typeid])
-
- def varsize_item_sizes(self, typeid):
- assert typeid >= 0
- if self.is_varsize(typeid):
- return llmemory.ItemOffset(self.types[typeid])
- else:
- return 0
-
- def varsize_offset_to_variable_part(self, typeid):
- assert typeid >= 0
- if self.is_varsize(typeid):
- return llmemory.ArrayItemsOffset(self.types[typeid])
- else:
- return 0
-
- def varsize_offset_to_length(self, typeid):
- assert typeid >= 0
- if self.is_varsize(typeid):
- TYPE = self.types[typeid]
- if isinstance(TYPE, lltype.Array):
- return 0
- else:
- return llmemory.FieldOffset(TYPE, TYPE._arrayfld)
- else:
- return 0
-
- def varsize_offsets_to_gcpointers_in_var_part(self, typeid):
- assert typeid >= 0
- if self.is_varsize(typeid):
- return lltypelayout.varsize_offsets_to_gcpointers_in_var_part(
- self.types[typeid])
- else:
- return 0
def getfunctionptr(annotator, graphfunc):
"""Make a functionptr from the given Python function."""
@@ -276,10 +246,10 @@
AddressLinkedList = self.AddressLinkedList
def instantiate_linked_list():
return AddressLinkedList()
- f1, f2, f3, f4, f5, f6, f7 = self.query_types.create_query_functions()
+ f1, f2, f3, f4, f5, f6, f7, f8 = self.query_types.create_query_functions()
the_gc = gc_class(AddressLinkedList)
def instantiate_gc():
- the_gc.set_query_functions(f1, f2, f3, f4, f5, f6, f7)
+ the_gc.set_query_functions(f1, f2, f3, f4, f5, f6, f7, f8)
the_gc.setup()
return the_gc
func, dummy_get_roots1, dummy_get_roots2 = gc.get_dummy_annotate(
Modified: pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py
==============================================================================
--- pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py (original)
+++ pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py Sun May 21 11:01:39 2006
@@ -293,6 +293,61 @@
heap_size = statistics().item0
assert heap_size < 16000 * INT_SIZE / 4 # xxx
+ def test_finalizer(self):
+ class B(object):
+ pass
+ b = B()
+ b.nextid = 0
+ b.num_deleted = 0
+ class A(object):
+ def __init__(self):
+ self.id = b.nextid
+ b.nextid += 1
+ def __del__(self):
+ b.num_deleted += 1
+ def f(x, y):
+ a = A()
+ i = 0
+ while i < x:
+ i += 1
+ a = A()
+ llop.gc__collect(lltype.Void)
+ llop.gc__collect(lltype.Void)
+ return b.num_deleted
+ run = self.runner(f, nbargs=2)
+ res = run([5, 42]) #XXX pure lazyness here too
+ assert res == 6
+
+ def test_finalizer_calls_malloc(self):
+ class B(object):
+ pass
+ b = B()
+ b.nextid = 0
+ b.num_deleted = 0
+ class A(object):
+ def __init__(self):
+ self.id = b.nextid
+ b.nextid += 1
+ def __del__(self):
+ b.num_deleted += 1
+ C()
+ class C(A):
+ def __del__(self):
+ b.num_deleted += 1
+ def f(x, y):
+ a = A()
+ c = C()
+ i = 0
+ while i < x:
+ i += 1
+ a = A()
+ llop.gc__collect(lltype.Void)
+ llop.gc__collect(lltype.Void)
+ return b.num_deleted
+ run = self.runner(f, nbargs=2)
+ res = run([5, 42]) #XXX pure lazyness here too
+ assert res == 13
+
def test_cloning(self):
B = lltype.GcStruct('B', ('x', lltype.Signed))
A = lltype.GcStruct('A', ('b', lltype.Ptr(B)),
More information about the Pypy-commit
mailing list