[pypy-svn] r54994 - in pypy/branch/hybrid-io/pypy/rpython: lltypesystem memory/gc memory/gctransform

arigo at codespeak.net arigo at codespeak.net
Tue May 20 14:28:21 CEST 2008


Author: arigo
Date: Tue May 20 14:28:20 2008
New Revision: 54994

Modified:
   pypy/branch/hybrid-io/pypy/rpython/lltypesystem/lloperation.py
   pypy/branch/hybrid-io/pypy/rpython/memory/gc/hybrid.py
   pypy/branch/hybrid-io/pypy/rpython/memory/gctransform/framework.py
Log:
(fijal, arigo)
- Some renaming, some overflow checking, some other minor stuff.
- Collect unreachable resizable objects during a generation 2 collect.


Modified: pypy/branch/hybrid-io/pypy/rpython/lltypesystem/lloperation.py
==============================================================================
--- pypy/branch/hybrid-io/pypy/rpython/lltypesystem/lloperation.py	(original)
+++ pypy/branch/hybrid-io/pypy/rpython/lltypesystem/lloperation.py	Tue May 20 14:28:20 2008
@@ -326,8 +326,8 @@
     'malloc_nonmovable':    LLOp(canraise=(MemoryError,), canunwindgc=True),
     'malloc_nonmovable_varsize':LLOp(canraise=(MemoryError,),canunwindgc=True),
     'malloc_resizable_buffer': LLOp(canraise=(MemoryError,),canunwindgc=True),
-    'resize_buffer':        LLOp(canraise=(MemoryError,),canunwindgc=True),
-    'finish_building_buffer' : LLOp(),
+    'resize_buffer':        LLOp(canraise=(MemoryError,)),
+    'finish_building_buffer' : LLOp(canraise=(MemoryError,), canunwindgc=True),
     'zero_gc_pointers_inside': LLOp(),
     'free':                 LLOp(),
     'getfield':             LLOp(sideeffects=False, canrun=True),

Modified: pypy/branch/hybrid-io/pypy/rpython/memory/gc/hybrid.py
==============================================================================
--- pypy/branch/hybrid-io/pypy/rpython/memory/gc/hybrid.py	(original)
+++ pypy/branch/hybrid-io/pypy/rpython/memory/gc/hybrid.py	Tue May 20 14:28:20 2008
@@ -30,7 +30,7 @@
 # Object lists:
 #   * gen2_rawmalloced_objects
 #   * gen3_rawmalloced_objects
-#   * gen_resizable_objects
+#   * gen2_resizable_objects
 #   * old_objects_pointing_to_young: gen2or3 objs that point to gen1 objs
 #   * last_generation_root_objects: gen3 objs that point to gen1or2 objs
 #
@@ -43,12 +43,14 @@
 # Some invariants:
 #   * gen3 are either GCFLAG_NO_HEAP_PTRS or in 'last_generation_root_objects'
 #   * between collections, GCFLAG_UNVISITED set exactly for gen2_rawmalloced
+#   * objects in gen2_resizable_objects are part of the generation 2 but never
+#     explicitly listed in gen2_rawmalloced_objects.
 #
 # A malloc_varsize() of large objects returns objects that are external
 # but initially of generation 2.  Old objects from the semispaces are
 # moved to external objects directly as generation 3.
 
-# gen_resizable objects is for objects that are resizable
+# gen2_resizable_objects is for objects that are resizable
 
 # The "age" of an object is the number of times it survived a full
 # collections, without counting the step that moved it out of the nursery.
@@ -121,7 +123,7 @@
     def setup(self):
         self.gen2_rawmalloced_objects = self.AddressStack()
         self.gen3_rawmalloced_objects = self.AddressStack()
-        self.gen_resizable_objects = self.AddressStack()
+        self.gen2_resizable_objects = self.AddressStack()
         GenerationGC.setup(self)
 
     def set_max_heap_size(self, size):
@@ -174,7 +176,7 @@
                                                 llmemory.GCREF)
         return self.malloc_varsize_slowpath(typeid, length)
 
-    def malloc_varsize_slowpath(self, typeid, length, force_old=False,
+    def malloc_varsize_slowpath(self, typeid, length, force_nonmovable=False,
                                 resizable=False):
         # For objects that are too large, or when the nursery is exhausted.
         # In order to keep malloc_varsize_clear() as compact as possible,
@@ -193,7 +195,7 @@
             nonlarge_max = self.nonlarge_gcptrs_max
         else:
             nonlarge_max = self.nonlarge_max
-        if force_old or raw_malloc_usage(totalsize) > nonlarge_max:
+        if force_nonmovable or raw_malloc_usage(totalsize) > nonlarge_max:
             result = self.malloc_varsize_marknsweep(totalsize, resizable)
             flags = self.GCFLAGS_FOR_NEW_EXTERNAL_OBJECTS | GCFLAG_UNVISITED
         else:
@@ -204,7 +206,6 @@
         return llmemory.cast_adr_to_ptr(result+size_gc_header, llmemory.GCREF)
 
     malloc_varsize_slowpath._dont_inline_ = True
-    malloc_varsize_slowpath._annspecialcase_ = 'specialize:arg(3, 4)'
 
     def malloc_varsize_nonmovable(self, typeid, length):
         return self.malloc_varsize_slowpath(typeid, length, True)
@@ -220,13 +221,18 @@
             raise NotImplementedError("Not supported")
         return llmemory.cast_ptr_to_adr(gcref)
 
-    def realloc(self, ptr, newsize, const_size, itemsize, lengthofs, grow):
+    def realloc(self, ptr, newlength, fixedsize, itemsize, lengthofs, grow):
         size_gc_header = self.size_gc_header()
         addr = llmemory.cast_ptr_to_adr(ptr)
         tid = self.get_type_id(addr)
-        tot_size = size_gc_header + const_size + newsize * itemsize
-        oldsize = (addr + lengthofs).signed[0]
-        old_tot_size = size_gc_header + const_size + oldsize * itemsize
+        nonvarsize = size_gc_header + fixedsize
+        try:
+            varsize = ovfcheck(itemsize * newlength)
+            tot_size = ovfcheck(nonvarsize + varsize)
+        except OverflowError:
+            raise MemoryError()
+        oldlength = (addr + lengthofs).signed[0]
+        old_tot_size = size_gc_header + fixedsize + oldlength * itemsize
         source_addr = addr - size_gc_header
         self._remove_addr_from_resizable_objects(addr)
         if grow:
@@ -236,24 +242,27 @@
             result = llop.raw_realloc_shrink(llmemory.Address, source_addr,
                                              old_tot_size, tot_size)
         if not result:
+            self.gen2_resizable_objects.append(addr)
             raise MemoryError()
         if grow:
-            self.gen_resizable_objects.append(result + size_gc_header)
+            self.gen2_resizable_objects.append(result + size_gc_header)
         else:
             self.gen2_rawmalloced_objects.append(result + size_gc_header)
-            self._check_rawsize_alloced(tot_size)
-        (result + size_gc_header + lengthofs).signed[0] = newsize
+        self._check_rawsize_alloced(raw_malloc_usage(tot_size) -
+                                    raw_malloc_usage(old_tot_size),
+                                    can_collect = not grow)
+        (result + size_gc_header + lengthofs).signed[0] = newlength
         return llmemory.cast_adr_to_ptr(result + size_gc_header, llmemory.GCREF)
 
     def _remove_addr_from_resizable_objects(self, addr):
-        objects = self.gen_resizable_objects
+        objects = self.gen2_resizable_objects
         newstack = self.AddressStack()
         while objects.non_empty():
             obj = objects.pop()
             if obj != addr:
                 newstack.append(obj)
         objects.delete()
-        self.gen_resizable_objects = newstack
+        self.gen2_resizable_objects = newstack
 
     def can_move(self, addr):
         tid = self.header(addr).tid
@@ -268,9 +277,9 @@
             totalsize)
         return result
 
-    def _check_rawsize_alloced(self, totalsize):
-        self.large_objects_collect_trigger -= raw_malloc_usage(totalsize)
-        if self.large_objects_collect_trigger < 0:
+    def _check_rawsize_alloced(self, size_estimate, can_collect=True):
+        self.large_objects_collect_trigger -= size_estimate
+        if can_collect and self.large_objects_collect_trigger < 0:
             if DEBUG_PRINT:
                 llop.debug_print(lltype.Void, "allocated",
                                  self._initial_trigger -
@@ -286,7 +295,7 @@
         # XXX and adjust sizes based on it; otherwise we risk doing
         # XXX many many collections if the program allocates a lot
         # XXX more than the current self.space_size.
-        self._check_rawsize_alloced(totalsize)
+        self._check_rawsize_alloced(raw_malloc_usage(totalsize))
         result = self.allocate_external_object(totalsize)
         if not result:
             raise MemoryError()
@@ -295,11 +304,10 @@
         llmemory.raw_memclear(result, totalsize)
         size_gc_header = self.gcheaderbuilder.size_gc_header
         if resizable:
-            self.gen_resizable_objects.append(result + size_gc_header)
+            self.gen2_resizable_objects.append(result + size_gc_header)
         else:
             self.gen2_rawmalloced_objects.append(result + size_gc_header)
         return result
-    malloc_varsize_marknsweep._annspecialcase_ = 'specialize:arg(2)'
 
     def allocate_external_object(self, totalsize):
         # XXX maybe we should use arena_malloc() above a certain size?
@@ -455,6 +463,7 @@
         if self.is_collecting_gen3():
             self.sweep_rawmalloced_objects(generation=3)
         self.sweep_rawmalloced_objects(generation=2)
+        self.sweep_rawmalloced_objects(generation=-2)
         # As we just collected, it's fine to raw_malloc'ate up to space_size
         # bytes again before we should force another collect.
         self.large_objects_collect_trigger = self.space_size
@@ -473,7 +482,7 @@
             # to any gen2 object.  In this case, A remains a bit too long in
             # last_generation_root_objects, but this will be fixed by the
             # next collect_last_generation_roots().
-        else:
+        elif generation == 3:
             objects = self.gen3_rawmalloced_objects
             # generation 3 sweep: remove from last_generation_root_objects
             # all the objects that we are about to free
@@ -485,6 +494,11 @@
                     newgen3roots.append(obj)
             gen3roots.delete()
             self.last_generation_root_objects = newgen3roots
+        else:
+            # mostly a hack: the generation number -2 is the part of the
+            # generation 2 that lives in gen2_resizable_objects
+            ll_assert(generation == -2, "bogus 'generation'")
+            objects = self.gen2_resizable_objects
 
         surviving_objects = self.AddressStack()
         if DEBUG_PRINT: alive_count = alive_size = 0
@@ -502,7 +516,7 @@
                 if DEBUG_PRINT:alive_size+=raw_malloc_usage(self.get_size(obj))
                 if generation == 3:
                     surviving_objects.append(obj)
-                else:
+                elif generation == 2:
                     ll_assert((tid & GCFLAG_AGE_MASK) < GCFLAG_AGE_MAX,
                               "wrong age for generation 2 object")
                     tid += GCFLAG_AGE_ONE
@@ -516,11 +530,18 @@
                         tid |= GCFLAG_UNVISITED
                         surviving_objects.append(obj)
                     self.header(obj).tid = tid
+                elif generation == -2:
+                    # the object stays in generation -2
+                    tid |= GCFLAG_UNVISITED
+                    surviving_objects.append(obj)
+                    self.header(obj).tid = tid
         objects.delete()
         if generation == 2:
             self.gen2_rawmalloced_objects = surviving_objects
-        else:
+        elif generation == 3:
             self.gen3_rawmalloced_objects = surviving_objects
+        elif generation == -2:
+            self.gen2_resizable_objects = surviving_objects
         if DEBUG_PRINT:
             llop.debug_print(lltype.Void,
                              "| [hyb] gen", generation,

Modified: pypy/branch/hybrid-io/pypy/rpython/memory/gctransform/framework.py
==============================================================================
--- pypy/branch/hybrid-io/pypy/rpython/memory/gctransform/framework.py	(original)
+++ pypy/branch/hybrid-io/pypy/rpython/memory/gctransform/framework.py	Tue May 20 14:28:20 2008
@@ -536,8 +536,10 @@
                         c_itemsize, c_lengthofs, c_grow):
         vlist = [self.realloc_ptr, self.c_const_gc, v_ptr, v_newsize,
                  c_const_size, c_itemsize, c_lengthofs, c_grow]
-        # XXX don't really need push_roots in all cases
-        livevars = self.push_roots(hop)
+        if c_grow.value:
+            livevars = []    # collection not possible if grow=True
+        else:
+            livevars = self.push_roots(hop)
         v_result = hop.genop('direct_call', vlist,
                              resulttype=llmemory.GCREF)
         self.pop_roots(hop, livevars)



More information about the Pypy-commit mailing list