[pypy-svn] r27525 - in pypy/dist/pypy/rpython/memory: . test

cfbolz at codespeak.net cfbolz at codespeak.net
Sun May 21 11:01:42 CEST 2006


Author: cfbolz
Date: Sun May 21 11:01:39 2006
New Revision: 27525

Modified:
   pypy/dist/pypy/rpython/memory/gc.py
   pypy/dist/pypy/rpython/memory/gctransform.py
   pypy/dist/pypy/rpython/memory/gcwrapper.py
   pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py
Log:
first cut at implementing __del__ for the mark and sweep collector


Modified: pypy/dist/pypy/rpython/memory/gc.py
==============================================================================
--- pypy/dist/pypy/rpython/memory/gc.py	(original)
+++ pypy/dist/pypy/rpython/memory/gc.py	Sun May 21 11:01:39 2006
@@ -60,11 +60,13 @@
 class GCBase(object):
     _alloc_flavor_ = "raw"
 
-    def set_query_functions(self, is_varsize, offsets_to_gc_pointers,
+    def set_query_functions(self, is_varsize, getfinalizer,
+                            offsets_to_gc_pointers,
                             fixed_size, varsize_item_sizes,
                             varsize_offset_to_variable_part,
                             varsize_offset_to_length,
                             varsize_offsets_to_gcpointers_in_var_part):
+        self.getfinalizer = getfinalizer
         self.is_varsize = is_varsize
         self.offsets_to_gc_pointers = offsets_to_gc_pointers
         self.fixed_size = fixed_size
@@ -133,8 +135,8 @@
         self.bytes_malloced_threshold = start_heap_size
         self.total_collection_time = 0.0
         self.AddressLinkedList = AddressLinkedList
-        #self.set_query_functions(None, None, None, None, None, None, None)
         self.malloced_objects = lltype.nullptr(self.HDR)
+        self.malloced_objects_with_finalizer = lltype.nullptr(self.HDR)
         self.get_roots = get_roots
         self.gcheaderbuilder = GCHeaderBuilder(self.HDR)
         # pools, for x_swap_pool():
@@ -172,8 +174,12 @@
         result = raw_malloc(size_gc_header + size)
         hdr = llmemory.cast_adr_to_ptr(result, self.HDRPTR)
         hdr.typeid = typeid << 1
-        hdr.next = self.malloced_objects
-        self.malloced_objects = hdr
+        if not self.getfinalizer(typeid):
+            hdr.next = self.malloced_objects
+            self.malloced_objects = hdr
+        else:
+            hdr.next = self.malloced_objects_with_finalizer
+            self.malloced_objects_with_finalizer = hdr
         self.bytes_malloced += raw_malloc_usage(size + size_gc_header)
         result += size_gc_header
         return llmemory.cast_adr_to_ptr(result, llmemory.GCREF)
@@ -193,14 +199,24 @@
         (result + size_gc_header + offset_to_length).signed[0] = length
         hdr = llmemory.cast_adr_to_ptr(result, self.HDRPTR)
         hdr.typeid = typeid << 1
-        hdr.next = self.malloced_objects
-        self.malloced_objects = hdr
+        if not self.getfinalizer(typeid):
+            hdr.next = self.malloced_objects
+            self.malloced_objects = hdr
+        else:
+            hdr.next = self.malloced_objects_with_finalizer
+            self.malloced_objects_with_finalizer = hdr
         self.bytes_malloced += raw_malloc_usage(size + size_gc_header)
         result += size_gc_header
         return llmemory.cast_adr_to_ptr(result, llmemory.GCREF)
 
     def collect(self):
+        # 1. mark from the roots, and also the objects that objects-with-del
+        #    point to (using the list of malloced_objects_with_finalizer)
+        # 2. walk the list of objects-without-del and free the ones not marked
+        # 3. walk the list of objects-with-del and for the ones not marked:
+        #    call __del__, move the object to the list of object-without-del
         import time
+        from pypy.rpython.lltypesystem.lloperation import llop
         if DEBUG_PRINT:
             llop.debug_print(lltype.Void, 'collecting...')
         start_time = time.time()
@@ -208,10 +224,11 @@
         size_gc_header = self.gcheaderbuilder.size_gc_header
 ##        llop.debug_view(lltype.Void, self.malloced_objects, self.poolnodes,
 ##                        size_gc_header)
-        objects = self.AddressLinkedList()
+
+        # push the roots on the mark stack
+        objects = self.AddressLinkedList() # mark stack
         while 1:
             curr = roots.pop()
-##             print "root: ", curr
             if curr == NULL:
                 break
             # roots is a list of addresses to addresses:
@@ -225,35 +242,37 @@
         # from this point onwards, no more mallocs should be possible
         old_malloced = self.bytes_malloced
         self.bytes_malloced = 0
+        curr_heap_size = 0
+        freed_size = 0
+
+        # mark objects reachable by objects with a finalizer, but not those
+        # themselves. add their size to curr_heap_size, since they always
+        # survive the collection
+        hdr = self.malloced_objects_with_finalizer
+        while hdr:
+            next = hdr.next
+            typeid = hdr.typeid >> 1
+            gc_info = llmemory.cast_ptr_to_adr(hdr)
+            obj = gc_info + size_gc_header
+            self.add_reachable_to_stack(obj, objects)
+            addr = llmemory.cast_ptr_to_adr(hdr)
+            size = self.fixed_size(typeid)
+            if self.is_varsize(typeid):
+                length = (obj + self.varsize_offset_to_length(typeid)).signed[0]
+                size += self.varsize_item_sizes(typeid) * length
+            estimate = raw_malloc_usage(size_gc_header + size)
+            curr_heap_size += estimate
+            hdr = next
+
+        # mark thinks on the mark stack and put their descendants onto the
+        # stack until the stack is empty
         while objects.non_empty():  #mark
             curr = objects.pop()
-##             print "object: ", curr
+            self.add_reachable_to_stack(curr, objects)
             gc_info = curr - size_gc_header
             hdr = llmemory.cast_adr_to_ptr(gc_info, self.HDRPTR)
             if hdr.typeid & 1:
                 continue
-            typeid = hdr.typeid >> 1
-            offsets = self.offsets_to_gc_pointers(typeid)
-            i = 0
-            while i < len(offsets):
-                pointer = curr + offsets[i]
-                objects.append(pointer.address[0])
-                i += 1
-            if self.is_varsize(typeid):
-                offset = self.varsize_offset_to_variable_part(
-                    typeid)
-                length = (curr + self.varsize_offset_to_length(typeid)).signed[0]
-                curr += offset
-                offsets = self.varsize_offsets_to_gcpointers_in_var_part(typeid)
-                itemlength = self.varsize_item_sizes(typeid)
-                i = 0
-                while i < length:
-                    item = curr + itemlength * i
-                    j = 0
-                    while j < len(offsets):
-                        objects.append((item + offsets[j]).address[0])
-                        j += 1
-                    i += 1
             hdr.typeid = hdr.typeid | 1
         objects.delete()
         # also mark self.curpool
@@ -262,8 +281,8 @@
             hdr = llmemory.cast_adr_to_ptr(gc_info, self.HDRPTR)
             hdr.typeid = hdr.typeid | 1
 
-        curr_heap_size = 0
-        freed_size = 0
+        # sweep: delete objects without del if they are not marked
+        # unmark objects without del that are marked
         firstpoolnode = lltype.malloc(self.POOLNODE, flavor='raw')
         firstpoolnode.linkedlist = self.malloced_objects
         firstpoolnode.nextnode = self.poolnodes
@@ -328,10 +347,56 @@
 ##        llop.debug_view(lltype.Void, self.malloced_objects, self.poolnodes,
 ##                        size_gc_header)
         assert self.heap_usage + old_malloced == curr_heap_size + freed_size
+
+        # call finalizers if needed
         self.heap_usage = curr_heap_size
+        hdr = self.malloced_objects_with_finalizer
+        self.malloced_objects_with_finalizer = lltype.nullptr(self.HDR)
+        while hdr:
+            next = hdr.next
+            if hdr.typeid & 1:
+                hdr.next = self.malloced_objects_with_finalizer
+                self.malloced_objects_with_finalizer = hdr
+                hdr.typeid = hdr.typeid & (~1)
+            else:
+                obj = llmemory.cast_ptr_to_adr(hdr) + size_gc_header
+                finalizer = self.getfinalizer(hdr.typeid >> 1)
+                finalizer(obj)
+                hdr.next = self.malloced_objects
+                self.malloced_objects = hdr
+            hdr = next
 
     STATISTICS_NUMBERS = 2
 
+    def add_reachable_to_stack(self, obj, objects):
+        size_gc_header = self.gcheaderbuilder.size_gc_header
+        gc_info = obj - size_gc_header
+        hdr = llmemory.cast_adr_to_ptr(gc_info, self.HDRPTR)
+        if hdr.typeid & 1:
+            return
+        typeid = hdr.typeid >> 1
+        offsets = self.offsets_to_gc_pointers(typeid)
+        i = 0
+        while i < len(offsets):
+            pointer = obj + offsets[i]
+            objects.append(pointer.address[0])
+            i += 1
+        if self.is_varsize(typeid):
+            offset = self.varsize_offset_to_variable_part(
+                typeid)
+            length = (obj + self.varsize_offset_to_length(typeid)).signed[0]
+            obj += offset
+            offsets = self.varsize_offsets_to_gcpointers_in_var_part(typeid)
+            itemlength = self.varsize_item_sizes(typeid)
+            i = 0
+            while i < length:
+                item = obj + itemlength * i
+                j = 0
+                while j < len(offsets):
+                    objects.append((item + offsets[j]).address[0])
+                    j += 1
+                i += 1
+
     def statistics(self):
         return self.heap_usage, self.bytes_malloced
 

Modified: pypy/dist/pypy/rpython/memory/gctransform.py
==============================================================================
--- pypy/dist/pypy/rpython/memory/gctransform.py	(original)
+++ pypy/dist/pypy/rpython/memory/gctransform.py	Sun May 21 11:01:39 2006
@@ -388,6 +388,16 @@
 
 ADDRESS_VOID_FUNC = lltype.FuncType([llmemory.Address], lltype.Void)
 
+def get_rtti(TYPE):
+    if isinstance(TYPE, lltype.GcStruct):
+        try:
+            return lltype.getRuntimeTypeInfo(TYPE)
+        except ValueError:
+            pass
+    return None
+
+
+
 class RefcountingGCTransformer(GCTransformer):
 
     HDR = lltype.Struct("header", ("refcount", lltype.Signed))
@@ -483,14 +493,6 @@
         result.extend(self.pop_alive(oldval))
         return result
 
-    def get_rtti(self, TYPE):
-        if isinstance(TYPE, lltype.GcStruct):
-            try:
-                return lltype.getRuntimeTypeInfo(TYPE)
-            except ValueError:
-                pass
-        return None
-
     def finish(self):
         super(RefcountingGCTransformer, self).finish()
 
@@ -506,7 +508,7 @@
             return self.static_deallocator_funcptrs[TYPE]
         #print_call_chain(self)
 
-        rtti = self.get_rtti(TYPE) 
+        rtti = get_rtti(TYPE) 
         if rtti is not None and hasattr(rtti._obj, 'destructor_funcptr'):
             destrptr = rtti._obj.destructor_funcptr
             DESTR_ARG = lltype.typeOf(destrptr).TO.ARGS[0]
@@ -576,7 +578,7 @@
             return self.dynamic_deallocator_funcptrs[TYPE]
         #print_call_chain(self)
 
-        rtti = self.get_rtti(TYPE)
+        rtti = get_rtti(TYPE)
         if rtti is None:
             p = self.static_deallocation_funcptr_for_type(TYPE)
             self.dynamic_deallocator_funcptrs[TYPE] = p
@@ -661,14 +663,6 @@
         """ for boehm it is enough to do nothing"""
         return [SpaceOperation("same_as", [Constant(None, lltype.Void)], op.result)]
 
-    def get_rtti(self, TYPE):
-        if isinstance(TYPE, lltype.GcStruct):
-            try:
-                return lltype.getRuntimeTypeInfo(TYPE)
-            except ValueError:
-                pass
-        return None
-
     def finish(self):
         super(BoehmGCTransformer, self).finish()
 
@@ -676,7 +670,7 @@
         if TYPE in self.finalizer_funcptrs:
             return self.finalizer_funcptrs[TYPE]
 
-        rtti = self.get_rtti(TYPE)
+        rtti = get_rtti(TYPE)
         if rtti is not None and hasattr(rtti._obj, 'destructor_funcptr'):
             destrptr = rtti._obj.destructor_funcptr
             DESTR_ARG = lltype.typeOf(destrptr).TO.ARGS[0]
@@ -751,11 +745,14 @@
         super(FrameworkGCTransformer, self).__init__(translator, inline=True)
         AddressLinkedList = get_address_linked_list()
         GCClass = self.GCClass
+        self.finalizer_funcptrs = {}
+        self.FINALIZERTYPE = lltype.Ptr(ADDRESS_VOID_FUNC)
         class GCData(object):
             # types of the GC information tables
             OFFSETS_TO_GC_PTR = lltype.Array(lltype.Signed)
             TYPE_INFO = lltype.Struct("type_info",
                 ("isvarsize",   lltype.Bool),
+                ("finalyzer",   self.FINALIZERTYPE),
                 ("fixedsize",   lltype.Signed),
                 ("ofstoptrs",   lltype.Ptr(OFFSETS_TO_GC_PTR)),
                 ("varitemsize", lltype.Signed),
@@ -768,6 +765,9 @@
         def q_is_varsize(typeid):
             return gcdata.type_info_table[typeid].isvarsize
 
+        def q_finalyzer(typeid):
+            return gcdata.type_info_table[typeid].finalyzer
+
         def q_offsets_to_gc_pointers(typeid):
             return gcdata.type_info_table[typeid].ofstoptrs
 
@@ -818,6 +818,7 @@
             gcdata.gc.setup()
             gcdata.gc.set_query_functions(
                 q_is_varsize,
+                q_finalyzer,
                 q_offsets_to_gc_pointers,
                 q_fixed_size,
                 q_varsize_item_sizes,
@@ -984,6 +985,7 @@
             self.id_of_type[TYPE] = type_id
             offsets = offsets_to_gc_pointers(TYPE)
             info["ofstoptrs"] = self.offsets2table(offsets, TYPE)
+            info["finalyzer"] = self.finalizer_funcptr_for_type(TYPE)
             if not TYPE._is_varsize():
                 info["isvarsize"] = False
                 info["fixedsize"] = llmemory.sizeof(TYPE)
@@ -1018,6 +1020,31 @@
                     info["varitemsize"] = 0
             return type_id
 
+    def finalizer_funcptr_for_type(self, TYPE):
+        if TYPE in self.finalizer_funcptrs:
+            return self.finalizer_funcptrs[TYPE]
+
+        rtti = get_rtti(TYPE)
+        if rtti is not None and hasattr(rtti._obj, 'destructor_funcptr'):
+            destrptr = rtti._obj.destructor_funcptr
+            DESTR_ARG = lltype.typeOf(destrptr).TO.ARGS[0]
+        else:
+            destrptr = None
+            DESTR_ARG = None
+
+        assert not type_contains_pyobjs(TYPE), "not implemented"
+        if destrptr:
+            def ll_finalizer(addr):
+                v = llmemory.cast_adr_to_ptr(addr, DESTR_ARG)
+                ll_call_destructor(destrptr, v)
+            g, fptr = self.annotate_helper(ll_finalizer, [llmemory.Address], lltype.Void)
+        else:
+            g = fptr = lltype.nullptr(ADDRESS_VOID_FUNC)
+        if g:
+            self.need_minimal_transform(g)
+        self.finalizer_funcptrs[TYPE] = fptr
+        return fptr
+
     def consider_constant(self, TYPE, value):
         if id(value) in self.seen_roots:
             return

Modified: pypy/dist/pypy/rpython/memory/gcwrapper.py
==============================================================================
--- pypy/dist/pypy/rpython/memory/gcwrapper.py	(original)
+++ pypy/dist/pypy/rpython/memory/gcwrapper.py	Sun May 21 11:01:39 2006
@@ -27,6 +27,7 @@
     def create_query_functions(self):
         from pypy.rpython.lltypesystem import rstr
         _is_varsize = []
+        _finalizers = []
         _offsets_to_gc_pointers = []
         _fixed_size = []
         _varsize_item_sizes = []
@@ -39,6 +40,7 @@
         for TYPE, typeid in tttid:
             varsize = self.is_varsize(typeid)
             _is_varsize.append(varsize)
+            _finalizers.append(None)
             _offsets_to_gc_pointers.append(self.offsets_to_gc_pointers(typeid))
             _fixed_size.append(self.fixed_size(typeid))
             if varsize:
@@ -54,8 +56,12 @@
                 _varsize_offset_to_variable_part.append(0)
                 _varsize_offset_to_length.append(0)
                 _varsize_offsets_to_gcpointers_in_var_part.append([])
+        # trick to make the annotator see that the list can contain functions:
+        _finalizers.append(lambda addr: None)
         def is_varsize(typeid):
             return _is_varsize[typeid]
+        def getfinalizer(typeid):
+            return _finalizers[typeid]
         def offsets_to_gc_pointers(typeid):
             return _offsets_to_gc_pointers[typeid]
         def fixed_size(typeid):
@@ -68,7 +74,7 @@
             return _varsize_offset_to_length[typeid]
         def varsize_offsets_to_gcpointers_in_var_part(typeid):
             return _varsize_offsets_to_gcpointers_in_var_part[typeid]
-        return (is_varsize, offsets_to_gc_pointers, fixed_size,
+        return (is_varsize, getfinalizer, offsets_to_gc_pointers, fixed_size,
                 varsize_item_sizes, varsize_offset_to_variable_part,
                 varsize_offset_to_length,
                 varsize_offsets_to_gcpointers_in_var_part)
@@ -80,6 +86,9 @@
                 (isinstance(TYPE, lltype.Struct) and
                  TYPE._arrayfld is not None))
 
+    def getfinalizer(self, typeid):
+        return None
+
     def offsets_to_gc_pointers(self, typeid):
         assert typeid >= 0
         return lltypelayout.offsets_to_gc_pointers(self.types[typeid])
@@ -118,51 +127,12 @@
             return 0
 
     def get_setup_query_functions(self):
-        return (self.is_varsize, self.offsets_to_gc_pointers, self.fixed_size,
+        return (self.is_varsize, self.getfinalizer,
+                self.offsets_to_gc_pointers, self.fixed_size,
                 self.varsize_item_sizes, self.varsize_offset_to_variable_part,
                 self.varsize_offset_to_length,
                 self.varsize_offsets_to_gcpointers_in_var_part)
 
-class SymbolicQueryTypes(QueryTypes):
-    def fixed_size(self, typeid):
-        assert typeid >= 0
-        if self.types[typeid]._is_varsize():
-            return llmemory.sizeof(self.types[typeid], 0)
-        else:
-            return llmemory.sizeof(self.types[typeid])
-
-    def varsize_item_sizes(self, typeid):
-        assert typeid >= 0
-        if self.is_varsize(typeid):
-            return llmemory.ItemOffset(self.types[typeid])
-        else:
-            return 0
-
-    def varsize_offset_to_variable_part(self, typeid):
-        assert typeid >= 0
-        if self.is_varsize(typeid):
-            return llmemory.ArrayItemsOffset(self.types[typeid])
-        else:
-            return 0
-
-    def varsize_offset_to_length(self, typeid):
-        assert typeid >= 0
-        if self.is_varsize(typeid):
-            TYPE = self.types[typeid]
-            if isinstance(TYPE, lltype.Array):
-                return 0
-            else:
-                return llmemory.FieldOffset(TYPE, TYPE._arrayfld)
-        else:
-            return 0
-
-    def varsize_offsets_to_gcpointers_in_var_part(self, typeid):
-        assert typeid >= 0
-        if self.is_varsize(typeid):
-            return lltypelayout.varsize_offsets_to_gcpointers_in_var_part(
-                self.types[typeid])
-        else:
-            return 0
     
 def getfunctionptr(annotator, graphfunc):
     """Make a functionptr from the given Python function."""
@@ -276,10 +246,10 @@
         AddressLinkedList = self.AddressLinkedList
         def instantiate_linked_list():
             return AddressLinkedList()
-        f1, f2, f3, f4, f5, f6, f7 = self.query_types.create_query_functions()
+        f1, f2, f3, f4, f5, f6, f7, f8 = self.query_types.create_query_functions()
         the_gc = gc_class(AddressLinkedList)
         def instantiate_gc():
-            the_gc.set_query_functions(f1, f2, f3, f4, f5, f6, f7)
+            the_gc.set_query_functions(f1, f2, f3, f4, f5, f6, f7, f8)
             the_gc.setup()
             return the_gc
         func, dummy_get_roots1, dummy_get_roots2 = gc.get_dummy_annotate(

Modified: pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py
==============================================================================
--- pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py	(original)
+++ pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py	Sun May 21 11:01:39 2006
@@ -293,6 +293,61 @@
         heap_size = statistics().item0
         assert heap_size < 16000 * INT_SIZE / 4 # xxx
 
+    def test_finalizer(self):
+        class B(object):
+            pass
+        b = B()
+        b.nextid = 0
+        b.num_deleted = 0
+        class A(object):
+            def __init__(self):
+                self.id = b.nextid
+                b.nextid += 1
+            def __del__(self):
+                b.num_deleted += 1
+        def f(x, y):
+            a = A()
+            i = 0
+            while i < x:
+                i += 1
+                a = A()
+            llop.gc__collect(lltype.Void)
+            llop.gc__collect(lltype.Void)
+            return b.num_deleted
+        run = self.runner(f, nbargs=2)
+        res = run([5, 42]) #XXX pure lazyness here too
+        assert res == 6
+
+    def test_finalizer_calls_malloc(self):
+        class B(object):
+            pass
+        b = B()
+        b.nextid = 0
+        b.num_deleted = 0
+        class A(object):
+            def __init__(self):
+                self.id = b.nextid
+                b.nextid += 1
+            def __del__(self):
+                b.num_deleted += 1
+                C()
+        class C(A):
+            def __del__(self):
+                b.num_deleted += 1
+        def f(x, y):
+            a = A()
+            c = C()
+            i = 0
+            while i < x:
+                i += 1
+                a = A()
+            llop.gc__collect(lltype.Void)
+            llop.gc__collect(lltype.Void)
+            return b.num_deleted
+        run = self.runner(f, nbargs=2)
+        res = run([5, 42]) #XXX pure lazyness here too
+        assert res == 13
+
     def test_cloning(self):
         B = lltype.GcStruct('B', ('x', lltype.Signed))
         A = lltype.GcStruct('A', ('b', lltype.Ptr(B)),



More information about the Pypy-commit mailing list