[pypy-svn] r51484 - in pypy/branch/finalizer-order/pypy: rpython/memory/gc rpython/memory/test translator/c/test

arigo at codespeak.net arigo at codespeak.net
Thu Feb 14 13:54:48 CET 2008


Author: arigo
Date: Thu Feb 14 13:54:46 2008
New Revision: 51484

Added:
   pypy/branch/finalizer-order/pypy/rpython/memory/test/snippet.py
Modified:
   pypy/branch/finalizer-order/pypy/rpython/memory/gc/generation.py
   pypy/branch/finalizer-order/pypy/rpython/memory/gc/semispace.py
   pypy/branch/finalizer-order/pypy/rpython/memory/test/test_gc.py
   pypy/branch/finalizer-order/pypy/translator/c/test/test_newgc.py
Log:
(cfbolz, arigo)
The straightforward implementation of pypy/doc/discussion/finalizer-order.txt.
Tests pass.


Modified: pypy/branch/finalizer-order/pypy/rpython/memory/gc/generation.py
==============================================================================
--- pypy/branch/finalizer-order/pypy/rpython/memory/gc/generation.py	(original)
+++ pypy/branch/finalizer-order/pypy/rpython/memory/gc/generation.py	Thu Feb 14 13:54:46 2008
@@ -1,6 +1,5 @@
 import sys
-from pypy.rpython.memory.gc.semispace import SemiSpaceGC, GCFLAGSHIFT, \
-    GCFLAG_IMMORTAL
+from pypy.rpython.memory.gc.semispace import SemiSpaceGC, GCFLAG_IMMORTAL
 from pypy.rpython.lltypesystem.llmemory import NULL, raw_malloc_usage
 from pypy.rpython.lltypesystem import lltype, llmemory, llarena
 from pypy.rpython.memory.support import DEFAULT_CHUNK_SIZE
@@ -12,11 +11,11 @@
 # in the nursery.  It is initially set on all prebuilt and old objects,
 # and gets cleared by the write_barrier() when we write in them a
 # pointer to a young object.
-GCFLAG_NO_YOUNG_PTRS = 1 << (GCFLAGSHIFT+1)
+GCFLAG_NO_YOUNG_PTRS = SemiSpaceGC.first_unused_gcflag << 1
 
 # The following flag is set for static roots which are not on the list
 # of static roots yet, but will appear with write barrier
-GCFLAG_NO_HEAP_PTRS = 1 << (GCFLAGSHIFT+2)
+GCFLAG_NO_HEAP_PTRS = SemiSpaceGC.first_unused_gcflag << 2
 
 DEBUG_PRINT = False
 
@@ -30,6 +29,7 @@
     inline_simple_malloc_varsize = True
     needs_write_barrier = True
     prebuilt_gc_objects_are_static_roots = False
+    first_unused_gcflag = SemiSpaceGC.first_unused_gcflag << 3
 
     def __init__(self, chunk_size=DEFAULT_CHUNK_SIZE,
                  nursery_size=128,

Modified: pypy/branch/finalizer-order/pypy/rpython/memory/gc/semispace.py
==============================================================================
--- pypy/branch/finalizer-order/pypy/rpython/memory/gc/semispace.py	(original)
+++ pypy/branch/finalizer-order/pypy/rpython/memory/gc/semispace.py	Thu Feb 14 13:54:46 2008
@@ -2,7 +2,7 @@
 from pypy.rpython.lltypesystem.llmemory import raw_memcopy, raw_memclear
 from pypy.rpython.lltypesystem.llmemory import NULL, raw_malloc_usage
 from pypy.rpython.memory.support import DEFAULT_CHUNK_SIZE
-from pypy.rpython.memory.support import get_address_stack
+from pypy.rpython.memory.support import get_address_stack, get_address_deque
 from pypy.rpython.memory.gcheader import GCHeaderBuilder
 from pypy.rpython.lltypesystem import lltype, llmemory, llarena
 from pypy.rlib.objectmodel import free_non_gc_object
@@ -14,8 +14,9 @@
 import sys, os
 
 TYPEID_MASK = 0xffff
-GCFLAGSHIFT = 16
-GCFLAG_IMMORTAL = 1 << GCFLAGSHIFT
+first_gcflag = 1 << 16
+GCFLAG_IMMORTAL = first_gcflag
+GCFLAG_FINALIZATION_ORDERING = first_gcflag << 1
 
 memoryError = MemoryError()
 
@@ -24,6 +25,7 @@
     inline_simple_malloc = True
     inline_simple_malloc_varsize = True
     needs_zero_gc_pointers = False
+    first_unused_gcflag = first_gcflag << 2
 
     HDR = lltype.Struct('header', ('forw', llmemory.Address),
                                   ('tid', lltype.Signed))
@@ -35,6 +37,7 @@
         self.max_space_size = max_space_size
         self.gcheaderbuilder = GCHeaderBuilder(self.HDR)
         self.AddressStack = get_address_stack(chunk_size)
+        self.AddressDeque = get_address_deque(chunk_size)
 
     def setup(self):
         self.tospace = llarena.arena_malloc(self.space_size, True)
@@ -43,8 +46,8 @@
         self.fromspace = llarena.arena_malloc(self.space_size, True)
         ll_assert(bool(self.fromspace), "couldn't allocate fromspace")
         self.free = self.tospace
-        self.objects_with_finalizers = self.AddressStack()
-        self.run_finalizers = self.AddressStack()
+        self.objects_with_finalizers = self.AddressDeque()
+        self.run_finalizers = self.AddressDeque()
         self.objects_with_weakrefs = self.AddressStack()
         self.finalizer_lock_count = 0
         self.red_zone = 0
@@ -189,12 +192,11 @@
         self.top_of_space = tospace + self.space_size
         scan = self.free = tospace
         self.collect_roots()
-        scan = self.scan_copied(scan)
         if self.run_finalizers.non_empty():
             self.update_run_finalizers()
-        if self.objects_with_finalizers.non_empty():
-            self.deal_with_objects_with_finalizers()
         scan = self.scan_copied(scan)
+        if self.objects_with_finalizers.non_empty():
+            scan = self.deal_with_objects_with_finalizers(scan)
         if self.objects_with_weakrefs.non_empty():
             self.invalidate_weakrefs()
         self.notify_objects_just_moved()
@@ -308,20 +310,105 @@
         else:
             hdr.forw = NULL
 
-    def deal_with_objects_with_finalizers(self):
+    def deal_with_objects_with_finalizers(self, scan):
         # walk over list of objects with finalizers
         # if it is not copied, add it to the list of to-be-called finalizers
         # and copy it, to me make the finalizer runnable
-        # NOTE: the caller is calling scan_copied, so no need to do it here
-        new_with_finalizer = self.AddressStack()
+        # We try to run the finalizers in a "reasonable" order, like
+        # CPython does.  The details of this algorithm are in
+        # pypy/doc/discussion/finalizer-order.txt.
+        new_with_finalizer = self.AddressDeque()
+        marked = self.AddressDeque()
+        pending = self.AddressStack()
         while self.objects_with_finalizers.non_empty():
-            obj = self.objects_with_finalizers.pop()
-            if self.is_forwarded(obj):
-                new_with_finalizer.append(self.get_forwarding_address(obj))
+            x = self.objects_with_finalizers.popleft()
+            ll_assert(self._finalization_state(x) != 1, 
+                      "bad finalization state 1")
+            if self.is_forwarded(x):
+                new_with_finalizer.append(self.get_forwarding_address(x))
+                continue
+            marked.append(x)
+            pending.append(x)
+            while pending.non_empty():
+                y = pending.pop()
+                state = self._finalization_state(y)
+                if state == 0:
+                    self._bump_finalization_state_from_0_to_1(y)
+                elif state == 2:
+                    self._bump_finalization_state_from_2_to_3(y)
+                else:
+                    continue   # don't need to recurse inside y
+                self.trace(y, self._append_if_nonnull, pending)
+            scan = self._recursively_bump_finalization_state_from_1_to_2(
+                       x, scan)
+
+        while marked.non_empty():
+            x = marked.popleft()
+            state = self._finalization_state(x)
+            ll_assert(state >= 2, "unexpected finalization state < 2")
+            newx = self.get_forwarding_address(x)
+            if state == 2:
+                self.run_finalizers.append(newx)
+                # we must also fix the state from 2 to 3 here, otherwise
+                # we leave the GCFLAG_FINALIZATION_ORDERING bit behind
+                # which will confuse the next collection
+                pending.append(x)
+                while pending.non_empty():
+                    y = pending.pop()
+                    state = self._finalization_state(y)
+                    if state == 2:
+                        self._bump_finalization_state_from_2_to_3(y)
+                        self.trace(y, self._append_if_nonnull, pending)
             else:
-                self.run_finalizers.append(self.copy(obj))
+                new_with_finalizer.append(newx)
+
+        pending.delete()
+        marked.delete()
         self.objects_with_finalizers.delete()
         self.objects_with_finalizers = new_with_finalizer
+        return scan
+
+    def _append_if_nonnull(pointer, stack):
+        if pointer.address[0] != NULL:
+            stack.append(pointer.address[0])
+    _append_if_nonnull = staticmethod(_append_if_nonnull)
+
+    def _finalization_state(self, obj):
+        if self.is_forwarded(obj):
+            newobj = self.get_forwarding_address(obj)
+            hdr = self.header(newobj)
+            if hdr.tid & GCFLAG_FINALIZATION_ORDERING:
+                return 2
+            else:
+                return 3
+        else:
+            hdr = self.header(obj)
+            if hdr.tid & GCFLAG_FINALIZATION_ORDERING:
+                return 1
+            else:
+                return 0
+
+    def _bump_finalization_state_from_0_to_1(self, obj):
+        ll_assert(self._finalization_state(obj) == 0,
+                  "unexpected finalization state != 0")
+        hdr = self.header(obj)
+        hdr.tid |= GCFLAG_FINALIZATION_ORDERING
+
+    def _bump_finalization_state_from_2_to_3(self, obj):
+        ll_assert(self._finalization_state(obj) == 2,
+                  "unexpected finalization state != 2")
+        newobj = self.get_forwarding_address(obj)
+        hdr = self.header(newobj)
+        hdr.tid &= ~GCFLAG_FINALIZATION_ORDERING
+
+    def _recursively_bump_finalization_state_from_1_to_2(self, obj, scan):
+        # recursively convert objects from state 1 to state 2.
+        # Note that copy() copies all bits, including the
+        # GCFLAG_FINALIZATION_ORDERING.  The mapping between
+        # state numbers and the presence of this bit was designed
+        # for the following to work :-)
+        self.copy(obj)
+        return self.scan_copied(scan)
 
     def invalidate_weakrefs(self):
         # walk over list of objects that contain weakrefs
@@ -349,9 +436,9 @@
     def update_run_finalizers(self):
         # we are in an inner collection, caused by a finalizer
         # the run_finalizers objects need to be copied
-        new_run_finalizer = self.AddressStack()
+        new_run_finalizer = self.AddressDeque()
         while self.run_finalizers.non_empty():
-            obj = self.run_finalizers.pop()
+            obj = self.run_finalizers.popleft()
             new_run_finalizer.append(self.copy(obj))
         self.run_finalizers.delete()
         self.run_finalizers = new_run_finalizer
@@ -363,7 +450,7 @@
         try:
             while self.run_finalizers.non_empty():
                 #print "finalizer"
-                obj = self.run_finalizers.pop()
+                obj = self.run_finalizers.popleft()
                 finalizer = self.getfinalizer(self.get_type_id(obj))
                 finalizer(obj)
         finally:

Added: pypy/branch/finalizer-order/pypy/rpython/memory/test/snippet.py
==============================================================================
--- (empty file)
+++ pypy/branch/finalizer-order/pypy/rpython/memory/test/snippet.py	Thu Feb 14 13:54:46 2008
@@ -0,0 +1,121 @@
+from pypy.rpython.lltypesystem import lltype
+from pypy.rpython.lltypesystem.lloperation import llop
+
+class SemiSpaceGCTests:
+    large_tests_ok = False
+
+    def test_finalizer_order(self):
+        import random
+        from pypy.tool.algo import graphlib
+
+        examples = []
+        if self.large_tests_ok:
+            letters = 'abcdefghijklmnopqrstuvwxyz'
+            COUNT = 20
+        else:
+            letters = 'abcdefghijklm'
+            COUNT = 2
+        for i in range(COUNT):
+            input = []
+            edges = {}
+            for c in letters:
+                edges[c] = []
+            # make up a random graph
+            for c in letters:
+                for j in range(random.randrange(0, 4)):
+                    d = random.choice(letters)
+                    edges[c].append(graphlib.Edge(c, d))
+                    input.append((c, d))
+            # find the expected order in which destructors should be called
+            components = list(graphlib.strong_components(edges, edges))
+            head = {}
+            for component in components:
+                c = component.keys()[0]
+                for d in component:
+                    assert d not in head
+                    head[d] = c
+            assert len(head) == len(letters)
+            strict = []
+            for c, d in input:
+                if head[c] != head[d]:
+                    strict.append((c, d))
+            examples.append((input, components, strict))
+
+        class State:
+            pass
+        state = State()
+        class A:
+            def __init__(self, key):
+                self.key = key
+                self.refs = []
+            def __del__(self):
+                assert state.age[self.key] == -1
+                state.age[self.key] = state.time
+                state.progress = True
+
+        def build_example(input):
+            state.time = 0
+            state.age = {}
+            vertices = {}
+            for c in letters:
+                vertices[c] = A(c)
+                state.age[c] = -1
+            for c, d in input:
+                vertices[c].refs.append(vertices[d])
+
+        def f():
+            i = 0
+            while i < len(examples):
+                input, components, strict = examples[i]
+                build_example(input)
+                while state.time < len(letters):
+                    state.progress = False
+                    llop.gc__collect(lltype.Void)
+                    if not state.progress:
+                        break
+                    state.time += 1
+                # summarize the finalization order
+                lst = []
+                for c in letters:
+                    lst.append('%s:%d' % (c, state.age[c]))
+                summary = ', '.join(lst)
+
+                # check that all instances have been finalized
+                if -1 in state.age.values():
+                    return error(i, summary, "not all instances finalized")
+                # check that if a -> b and a and b are not in the same
+                # strong component, then a is finalized strictly before b
+                for c, d in strict:
+                    if state.age[c] >= state.age[d]:
+                        return error(i, summary,
+                                     "%s should be finalized before %s"
+                                     % (c, d))
+                # check that two instances in the same strong component
+                # are never finalized during the same collection
+                for component in components:
+                    seen = {}
+                    for c in component:
+                        age = state.age[c]
+                        if age in seen:
+                            d = seen[age]
+                            return error(i, summary,
+                                         "%s and %s should not be finalized"
+                                         " at the same time" % (c, d))
+                        seen[age] = c
+                i += 1
+            return "ok"
+
+        def error(i, summary, msg):
+            return '%d\n%s\n%s' % (i, summary, msg)
+
+        res = self.run(f)
+        if res != "ok":
+            i, summary, msg = res.split('\n')
+            i = int(i)
+            import pprint
+            print 'Example:'
+            pprint.pprint(examples[i])
+            print 'Finalization ages:'
+            print summary
+            print msg
+            py.test.fail(msg)

Modified: pypy/branch/finalizer-order/pypy/rpython/memory/test/test_gc.py
==============================================================================
--- pypy/branch/finalizer-order/pypy/rpython/memory/test/test_gc.py	(original)
+++ pypy/branch/finalizer-order/pypy/rpython/memory/test/test_gc.py	Thu Feb 14 13:54:46 2008
@@ -3,8 +3,10 @@
 
 #from pypy.rpython.memory.support import INT_SIZE
 from pypy.rpython.memory import gcwrapper
+from pypy.rpython.memory.test import snippet
 from pypy.rpython.test.test_llinterp import get_interpreter
 from pypy.rpython.lltypesystem import lltype
+from pypy.rpython.lltypesystem.rstr import STR
 from pypy.rpython.lltypesystem.lloperation import llop
 from pypy.rlib.objectmodel import we_are_translated
 from pypy.rlib.objectmodel import compute_unique_id
@@ -35,6 +37,12 @@
                                                self.GC_PARAMS)
         return interp.eval_graph(graph, values)
 
+    def run(self, func):      # for snippet.py
+        res = self.interpret(func, [])
+        if lltype.typeOf(res) == lltype.Ptr(STR):
+            res = ''.join(res.chars)
+        return res
+
     def test_llinterp_lists(self):
         #curr = simulator.current_size
         def malloc_a_lot():
@@ -384,108 +392,9 @@
 class TestMarkSweepGC(GCTest):
     from pypy.rpython.memory.gc.marksweep import MarkSweepGC as GCClass
 
-class TestSemiSpaceGC(GCTest):
+class TestSemiSpaceGC(GCTest, snippet.SemiSpaceGCTests):
     from pypy.rpython.memory.gc.semispace import SemiSpaceGC as GCClass
 
-    def test_finalizer_order(self):
-        py.test.skip("in-progress")
-        import random
-        from pypy.tool.algo import graphlib
-
-        examples = []
-        letters = 'abcdefghijklmnopqrstuvwxyz'
-        for i in range(20):
-            input = []
-            edges = {}
-            for c in letters:
-                edges[c] = []
-            # make up a random graph
-            for c in letters:
-                for j in range(random.randrange(0, 4)):
-                    d = random.choice(letters)
-                    edges[c].append(graphlib.Edge(c, d))
-                    input.append((c, d))
-            # find the expected order in which destructors should be called
-            components = list(graphlib.strong_components(edges, edges))
-            head = {}
-            for component in components:
-                c = component.keys()[0]
-                for d in component:
-                    assert d not in head
-                    head[d] = c
-            assert len(head) == len(letters)
-            strict = []
-            for c, d in input:
-                if head[c] != head[d]:
-                    strict.append((c, d))
-            examples.append((input, components, strict))
-
-        class State:
-            pass
-        state = State()
-        class A:
-            def __init__(self, key):
-                self.key = key
-                self.refs = []
-            def __del__(self):
-                assert state.age[self.key] == -1
-                state.age[self.key] = state.time
-                state.progress = True
-
-        def build_example(input):
-            state.time = 0
-            state.age = {}
-            vertices = {}
-            for c in letters:
-                vertices[c] = A(c)
-                state.age[c] = -1
-            for c, d in input:
-                vertices[c].refs.append(d)
-
-        def f():
-            i = 0
-            while i < len(examples):
-                input, components, strict = examples[i]
-                build_example(input)
-                while state.time < len(letters):
-                    state.progress = False
-                    llop.gc__collect(lltype.Void)
-                    if not state.progress:
-                        break
-                    state.time += 1
-                # check that all instances have been finalized
-                if -1 in state.age.values():
-                    return i * 10 + 1
-                # check that if a -> b and a and b are not in the same
-                # strong component, then a is finalized strictly before b
-                for c, d in strict:
-                    if state.age[c] >= state.age[d]:
-                        return i * 10 + 2
-                # check that two instances in the same strong component
-                # are never finalized during the same collection
-                for component in components:
-                    seen = {}
-                    for c in component:
-                        age = state.age[c]
-                        if age in seen:
-                            return i * 10 + 3
-                        seen[age] = True
-                i += 1
-            return 0
-
-        res = self.interpret(f, [])
-        if res != 0:
-            import pprint
-            pprint.pprint(examples[res / 10])
-            if res % 10 == 1:
-                py.test.fail("some instances have not been finalized at all")
-            if res % 10 == 2:
-                py.test.fail("the strict order is not respected")
-            if res % 10 == 3:
-                py.test.fail("two instances from the same component "
-                             "have been finalized together")
-            assert 0
-
 class TestGrowingSemiSpaceGC(TestSemiSpaceGC):
     GC_PARAMS = {'space_size': 64}
 

Modified: pypy/branch/finalizer-order/pypy/translator/c/test/test_newgc.py
==============================================================================
--- pypy/branch/finalizer-order/pypy/translator/c/test/test_newgc.py	(original)
+++ pypy/branch/finalizer-order/pypy/translator/c/test/test_newgc.py	Thu Feb 14 13:54:46 2008
@@ -9,6 +9,7 @@
 from pypy.translator.c import genc, gc
 from pypy.rpython.lltypesystem import lltype, llmemory
 from pypy.rpython.lltypesystem.lloperation import llop
+from pypy.rpython.memory.test import snippet
 from pypy import conftest
 
 def compile_func(fn, inputtypes, t=None, gcpolicy="ref"):
@@ -281,6 +282,12 @@
     gcpolicy = "marksweep"
     should_be_moving = False
 
+    # interface for snippet.py
+    large_tests_ok = True
+    def run(self, func):
+        fn = self.getcompiled(func)
+        return fn()
+
     def test_empty_collect(self):
         def f():
             llop.gc__collect(lltype.Void)
@@ -836,7 +843,7 @@
     def test_weakref(self):
         py.test.skip("fails for some reason I couldn't figure out yet :-(")
 
-class TestSemiSpaceGC(TestUsingFramework):
+class TestSemiSpaceGC(TestUsingFramework, snippet.SemiSpaceGCTests):
     gcpolicy = "semispace"
     should_be_moving = True
 



More information about the Pypy-commit mailing list