[pypy-svn] r47360 - in pypy/dist/pypy/rpython: lltypesystem memory memory/test

arigo at codespeak.net arigo at codespeak.net
Tue Oct 9 19:19:29 CEST 2007


Author: arigo
Date: Tue Oct  9 19:19:29 2007
New Revision: 47360

Modified:
   pypy/dist/pypy/rpython/lltypesystem/llarena.py
   pypy/dist/pypy/rpython/memory/gc.py
   pypy/dist/pypy/rpython/memory/gcwrapper.py
   pypy/dist/pypy/rpython/memory/test/test_gc.py
Log:
Expand space_size when memory runs out.  This allows us to start
with a smaller space_size, currently 8MB, and grow as needed.


Modified: pypy/dist/pypy/rpython/lltypesystem/llarena.py
==============================================================================
--- pypy/dist/pypy/rpython/lltypesystem/llarena.py	(original)
+++ pypy/dist/pypy/rpython/lltypesystem/llarena.py	Tue Oct  9 19:19:29 2007
@@ -279,7 +279,7 @@
 
 def llimpl_arena_malloc(nbytes, zero):
     addr = llmemory.raw_malloc(nbytes)
-    if zero:
+    if zero and bool(addr):
         clear_large_memory_chunk(addr, nbytes)
     return addr
 register_external(arena_malloc, [int, bool], llmemory.Address,

Modified: pypy/dist/pypy/rpython/memory/gc.py
==============================================================================
--- pypy/dist/pypy/rpython/memory/gc.py	(original)
+++ pypy/dist/pypy/rpython/memory/gc.py	Tue Oct  9 19:19:29 2007
@@ -957,14 +957,15 @@
                                   ('typeid', lltype.Signed))
 
     def __init__(self, AddressLinkedList, space_size=4096,
+                 max_space_size=sys.maxint//2+1,
                  get_roots=None):
         self.space_size = space_size
+        self.max_space_size = max_space_size
         self.get_roots = get_roots
         self.gcheaderbuilder = GCHeaderBuilder(self.HDR)
         self.AddressLinkedList = AddressLinkedList
 
     def setup(self):
-        self.bytes_malloced = 0
         self.tospace = llarena.arena_malloc(self.space_size, True)
         debug_assert(bool(self.tospace), "couldn't allocate tospace")
         self.top_of_space = self.tospace + self.space_size
@@ -973,6 +974,7 @@
         self.free = self.tospace
         self.objects_with_finalizers = self.AddressLinkedList()
         self.run_finalizers = self.AddressLinkedList()
+        self.executing_finalizers = False
         self.objects_with_weakrefs = self.AddressLinkedList()
 
     def malloc_fixedsize(self, typeid, size, can_collect, has_finalizer=False,
@@ -980,12 +982,7 @@
         size_gc_header = self.gcheaderbuilder.size_gc_header
         totalsize = size_gc_header + size
         if raw_malloc_usage(totalsize) > self.top_of_space - self.free:
-            if not can_collect:
-                raise memoryError
-            self.collect()
-            #XXX need to increase the space size if the object is too big
-            #for bonus points do big objects differently
-            if raw_malloc_usage(totalsize) > self.top_of_space - self.free:
+            if not can_collect or not self.obtain_free_space(totalsize):
                 raise memoryError
         result = self.free
         llarena.arena_reserve(result, totalsize)
@@ -1007,12 +1004,7 @@
         except OverflowError:
             raise memoryError
         if raw_malloc_usage(totalsize) > self.top_of_space - self.free:
-            if not can_collect:
-                raise memoryError
-            self.collect()
-            #XXX need to increase the space size if the object is too big
-            #for bonus points do big objects differently
-            if raw_malloc_usage(totalsize) > self.top_of_space - self.free:
+            if not can_collect or not self.obtain_free_space(totalsize):
                 raise memoryError
         result = self.free
         llarena.arena_reserve(result, totalsize)
@@ -1027,7 +1019,68 @@
     malloc_fixedsize_clear = malloc_fixedsize
     malloc_varsize_clear   = malloc_varsize
 
-    def collect(self):
+    def obtain_free_space(self, needed):
+        # XXX for bonus points do big objects differently
+        needed = raw_malloc_usage(needed)
+        self.collect()
+        missing = needed - (self.top_of_space - self.free)
+        if missing <= 0:
+            return True      # success
+        else:
+            # first check if the object could possibly fit
+            proposed_size = self.space_size
+            while missing > 0:
+                if proposed_size >= self.max_space_size:
+                    return False    # no way
+                missing -= proposed_size
+                proposed_size *= 2
+            # For address space fragmentation reasons, we double the space
+            # size possibly several times, moving the objects at each step,
+            # instead of going directly for the final size.  We assume that
+            # it's a rare case anyway.
+            while self.space_size < proposed_size:
+                if not self.double_space_size():
+                    return False
+            debug_assert(needed <= self.top_of_space - self.free,
+                         "double_space_size() failed to do its job")
+            return True
+
+    def double_space_size(self):
+        old_fromspace = self.fromspace
+        newsize = self.space_size * 2
+        newspace = llarena.arena_malloc(newsize, True)
+        if not newspace:
+            return False    # out of memory
+        llarena.arena_free(old_fromspace)
+        self.fromspace = newspace
+        # now self.tospace contains the existing objects and
+        # self.fromspace is the freshly allocated bigger space
+
+        self.collect(size_changing=True)
+        self.top_of_space = self.tospace + newsize
+        # now self.tospace is the freshly allocated bigger space,
+        # and self.fromspace is the old smaller space, now empty
+        llarena.arena_free(self.fromspace)
+
+        newspace = llarena.arena_malloc(newsize, True)
+        if not newspace:
+            # Complex failure case: we have in self.tospace a big chunk
+            # of memory, and the two smaller original spaces are already gone.
+            # Unsure if it's worth these efforts, but we can artificially
+            # split self.tospace in two again...
+            self.max_space_size = self.space_size    # don't try to grow again,
+            #              because doing arena_free(self.fromspace) would crash
+            self.fromspace = self.tospace + self.space_size
+            self.top_of_space = self.fromspace
+            debug_assert(self.free <= self.top_of_space,
+                         "unexpected growth of GC space usage during collect")
+            return False     # out of memory
+
+        self.fromspace = newspace
+        self.space_size = newsize
+        return True    # success
+
+    def collect(self, size_changing=False):
 ##         print "collecting"
         tospace = self.fromspace
         fromspace = self.tospace
@@ -1087,8 +1140,9 @@
         scan = self.scan_copied(scan)
         self.objects_with_finalizers.delete()
         self.objects_with_finalizers = new_with_finalizer
-        llarena.arena_reset(fromspace, self.space_size, True)
-        self.execute_finalizers()
+        if not size_changing:
+            llarena.arena_reset(fromspace, self.space_size, True)
+            self.execute_finalizers()
 
     def scan_copied(self, scan):
         while scan < self.free:
@@ -1184,11 +1238,17 @@
         hdr.typeid = typeid
 
     def execute_finalizers(self):
-        while self.run_finalizers.non_empty():
-            obj = self.run_finalizers.pop()
-            hdr = self.header(obj)
-            finalizer = self.getfinalizer(hdr.typeid)
-            finalizer(obj)
+        if self.executing_finalizers:
+            return    # the outer invocation of execute_finalizers() will do it
+        self.executing_finalizers = True
+        try:
+            while self.run_finalizers.non_empty():
+                obj = self.run_finalizers.pop()
+                hdr = self.header(obj)
+                finalizer = self.getfinalizer(hdr.typeid)
+                finalizer(obj)
+        finally:
+            self.executing_finalizers = False
 
     STATISTICS_NUMBERS = 0
 
@@ -1331,7 +1391,7 @@
         GC_PARAMS = {'start_heap_size': 8*1024*1024} # XXX adjust
         return MarkSweepGC, GC_PARAMS
     elif config.translation.frameworkgc == "semispace":
-        GC_PARAMS = {'space_size': 32*1024*1024} # XXX fixed at 32MB
+        GC_PARAMS = {'space_size': 8*1024*1024} # XXX adjust
         return SemiSpaceGC, GC_PARAMS
     else:
         raise ValueError("unknown value for frameworkgc: %r" % (

Modified: pypy/dist/pypy/rpython/memory/gcwrapper.py
==============================================================================
--- pypy/dist/pypy/rpython/memory/gcwrapper.py	(original)
+++ pypy/dist/pypy/rpython/memory/gcwrapper.py	Tue Oct  9 19:19:29 2007
@@ -7,9 +7,9 @@
 
 class GCManagedHeap(object):
 
-    def __init__(self, llinterp, flowgraphs, gc_class):
+    def __init__(self, llinterp, flowgraphs, gc_class, GC_PARAMS={}):
         self.AddressLinkedList = get_address_linked_list(10, hackishpop=True)
-        self.gc = gc_class(self.AddressLinkedList)
+        self.gc = gc_class(self.AddressLinkedList, **GC_PARAMS)
         self.gc.get_roots = self.get_roots_from_llinterp
         self.llinterp = llinterp
         self.constantroots = []
@@ -102,11 +102,12 @@
         def ll_finalizer(addr):
             old_active_frame = self.llinterp.active_frame
             try:
-                v = llmemory.cast_adr_to_ptr(addr, DESTR_ARG)
-                self.llinterp.eval_graph(destrgraph, [v])
-            except llinterp.LLException:
-                raise RuntimeError(
-                    "a finalizer raised an exception, shouldn't happen")
+                try:
+                    v = llmemory.cast_adr_to_ptr(addr, DESTR_ARG)
+                    self.llinterp.eval_graph(destrgraph, [v])
+                except llinterp.LLException:
+                    raise RuntimeError(
+                        "a finalizer raised an exception, shouldn't happen")
             finally:
                 self.llinterp.active_frame = old_active_frame
         return ll_finalizer
@@ -145,6 +146,6 @@
         if parent is not None:
             reccollect(constants, parent._as_ptr())
 
-def prepare_graphs_and_create_gc(llinterp, GCClass):
+def prepare_graphs_and_create_gc(llinterp, GCClass, GC_PARAMS={}):
     flowgraphs = llinterp.typer.annotator.translator.graphs[:]
-    llinterp.heap = GCManagedHeap(llinterp, flowgraphs, GCClass)
+    llinterp.heap = GCManagedHeap(llinterp, flowgraphs, GCClass, GC_PARAMS)

Modified: pypy/dist/pypy/rpython/memory/test/test_gc.py
==============================================================================
--- pypy/dist/pypy/rpython/memory/test/test_gc.py	(original)
+++ pypy/dist/pypy/rpython/memory/test/test_gc.py	Tue Oct  9 19:19:29 2007
@@ -17,6 +17,7 @@
 
 
 class GCTest(object):
+    GC_PARAMS = {}
 
     def setup_class(cls):
         cls._saved_logstate = py.log._getstate()
@@ -29,7 +30,8 @@
 
     def interpret(self, func, values, **kwds):
         interp, graph = get_interpreter(func, values, **kwds)
-        gcwrapper.prepare_graphs_and_create_gc(interp, self.GCClass)
+        gcwrapper.prepare_graphs_and_create_gc(interp, self.GCClass,
+                                               self.GC_PARAMS)
         return interp.eval_graph(graph, values)
 
     def test_llinterp_lists(self):
@@ -256,6 +258,9 @@
 class TestSemiSpaceGC(GCTest):
     from pypy.rpython.memory.gc import SemiSpaceGC as GCClass
 
+class TestGrowingSemiSpaceGC(TestSemiSpaceGC):
+    GC_PARAMS = {'space_size': 64}
+
 class TestDeferredRefcountingGC(GCTest):
     from pypy.rpython.memory.gc import DeferredRefcountingGC as GCClass
     def setup_class(cls):



More information about the Pypy-commit mailing list