[pypy-svn] r77138 - in pypy/branch/gen2-gc/pypy: config rlib rpython/memory rpython/memory/gc rpython/memory/gc/test rpython/memory/test translator/c translator/c/test

arigo at codespeak.net arigo at codespeak.net
Fri Sep 17 14:58:03 CEST 2010


Author: arigo
Date: Fri Sep 17 14:58:01 2010
New Revision: 77138

Modified:
   pypy/branch/gen2-gc/pypy/config/translationoption.py
   pypy/branch/gen2-gc/pypy/rlib/rstring.py
   pypy/branch/gen2-gc/pypy/rpython/memory/gc/base.py
   pypy/branch/gen2-gc/pypy/rpython/memory/gc/minimark.py
   pypy/branch/gen2-gc/pypy/rpython/memory/gc/minimarkpage.py
   pypy/branch/gen2-gc/pypy/rpython/memory/gc/test/test_minimark.py
   pypy/branch/gen2-gc/pypy/rpython/memory/lltypelayout.py
   pypy/branch/gen2-gc/pypy/rpython/memory/test/test_gc.py
   pypy/branch/gen2-gc/pypy/rpython/memory/test/test_transformed_gc.py
   pypy/branch/gen2-gc/pypy/translator/c/funcgen.py
   pypy/branch/gen2-gc/pypy/translator/c/test/test_newgc.py
Log:
Translation.  Minor fixes, e.g. in the maximum size of UnicodeBuilder
in rstring.  Add support for set_max_heap_size().


Modified: pypy/branch/gen2-gc/pypy/config/translationoption.py
==============================================================================
--- pypy/branch/gen2-gc/pypy/config/translationoption.py	(original)
+++ pypy/branch/gen2-gc/pypy/config/translationoption.py	Fri Sep 17 14:58:01 2010
@@ -52,7 +52,7 @@
     # gc
     ChoiceOption("gc", "Garbage Collection Strategy",
                  ["boehm", "ref", "marksweep", "semispace", "statistics",
-                  "generation", "hybrid", "markcompact", "none"],
+                  "generation", "hybrid", "markcompact", "minimark", "none"],
                   "ref", requires={
                      "ref": [("translation.rweakref", False), # XXX
                              ("translation.gctransformer", "ref")],
@@ -65,6 +65,7 @@
                      "hybrid": [("translation.gctransformer", "framework")],
                      "boehm": [("translation.gctransformer", "boehm")],
                      "markcompact": [("translation.gctransformer", "framework")],
+                     "minimark": [("translation.gctransformer", "framework")],
                      },
                   cmdline="--gc"),
     ChoiceOption("gctransformer", "GC transformer that is used - internal",

Modified: pypy/branch/gen2-gc/pypy/rlib/rstring.py
==============================================================================
--- pypy/branch/gen2-gc/pypy/rlib/rstring.py	(original)
+++ pypy/branch/gen2-gc/pypy/rlib/rstring.py	Fri Sep 17 14:58:01 2010
@@ -46,7 +46,9 @@
 
 # -------------- public API ---------------------------------
 
-INIT_SIZE = 100 # XXX tweak
+# the following number is the maximum size of an RPython unicode
+# string that goes into the nursery of the minimark GC.
+INIT_SIZE = 56
 
 class AbstractStringBuilder(object):
     def __init__(self, init_size=INIT_SIZE):

Modified: pypy/branch/gen2-gc/pypy/rpython/memory/gc/base.py
==============================================================================
--- pypy/branch/gen2-gc/pypy/rpython/memory/gc/base.py	(original)
+++ pypy/branch/gen2-gc/pypy/rpython/memory/gc/base.py	Fri Sep 17 14:58:01 2010
@@ -5,6 +5,7 @@
 from pypy.rpython.memory.support import get_address_stack, get_address_deque
 from pypy.rpython.memory.support import AddressDict
 from pypy.rpython.lltypesystem.llmemory import NULL, raw_malloc_usage
+from pypy.rlib.rarithmetic import r_uint
 
 TYPEID_MAP = lltype.GcStruct('TYPEID_MAP', ('count', lltype.Signed),
                              ('size', lltype.Signed),
@@ -146,7 +147,7 @@
         return False
 
     def set_max_heap_size(self, size):
-        pass
+        raise NotImplementedError
 
     def x_swap_pool(self, newpool):
         return newpool
@@ -340,6 +341,7 @@
                "generation": "generation.GenerationGC",
                "hybrid": "hybrid.HybridGC",
                "markcompact" : "markcompact.MarkCompactGC",
+               "minimark" : "minimark.MiniMarkGC",
                }
     try:
         modulename, classname = classes[config.translation.gc].split('.')
@@ -351,10 +353,12 @@
     GCClass = getattr(module, classname)
     return GCClass, GCClass.TRANSLATION_PARAMS
 
-def read_from_env(varname):
+def _read_float_and_factor_from_env(varname):
     import os
     value = os.environ.get(varname)
     if value:
+        if len(value) > 1 and value[-1] in 'bB':
+            value = value[:-1]
         realvalue = value[:-1]
         if value[-1] in 'kK':
             factor = 1024
@@ -366,7 +370,21 @@
             factor = 1
             realvalue = value
         try:
-            return int(float(realvalue) * factor)
+            return (float(realvalue), factor)
         except ValueError:
             pass
-    return -1
+    return (0.0, 0)
+
+def read_from_env(varname):
+    value, factor = _read_float_and_factor_from_env(varname)
+    return int(value * factor)
+
+def read_uint_from_env(varname):
+    value, factor = _read_float_and_factor_from_env(varname)
+    return r_uint(value * factor)
+
+def read_float_from_env(varname):
+    value, factor = _read_float_and_factor_from_env(varname)
+    if factor != 1:
+        return 0.0
+    return value

Modified: pypy/branch/gen2-gc/pypy/rpython/memory/gc/minimark.py
==============================================================================
--- pypy/branch/gen2-gc/pypy/rpython/memory/gc/minimark.py	(original)
+++ pypy/branch/gen2-gc/pypy/rpython/memory/gc/minimark.py	Fri Sep 17 14:58:01 2010
@@ -2,13 +2,14 @@
 from pypy.rpython.lltypesystem import lltype, llmemory, llarena, llgroup
 from pypy.rpython.lltypesystem.lloperation import llop
 from pypy.rpython.memory.gc.base import GCBase, MovingGCBase
-from pypy.rpython.memory.gc import minimarkpage
+from pypy.rpython.memory.gc import minimarkpage, base, generation
 from pypy.rpython.memory.support import DEFAULT_CHUNK_SIZE
 from pypy.rlib.rarithmetic import ovfcheck, LONG_BIT, intmask, r_uint
 from pypy.rlib.debug import ll_assert, debug_print, debug_start, debug_stop
 from pypy.rlib.objectmodel import we_are_translated
 
 WORD = LONG_BIT // 8
+NULL = llmemory.NULL
 
 first_gcflag = 1 << (LONG_BIT//2)
 
@@ -39,9 +40,11 @@
 # collection.  See pypy/doc/discussion/finalizer-order.txt
 GCFLAG_FINALIZATION_ORDERING = first_gcflag << 4
 
-# Marker set to 'tid' during a minor collection when an object from
-# the nursery was forwarded.
-FORWARDED_MARKER = -1
+
+FORWARDSTUB = lltype.GcStruct('forwarding_stub',
+                              ('forw', llmemory.Address))
+FORWARDSTUBPTR = lltype.Ptr(FORWARDSTUB)
+
 
 # ____________________________________________________________
 
@@ -59,14 +62,14 @@
     # by GCFLAG_xxx above.
     HDR = lltype.Struct('header', ('tid', lltype.Signed))
     typeid_is_in_field = 'tid'
-    #withhash_flag_is_in_field = 'tid', GCFLAG_HAS_SHADOW
+    withhash_flag_is_in_field = 'tid', GCFLAG_HAS_SHADOW
     # ^^^ prebuilt objects may have the flag GCFLAG_HAS_SHADOW;
     #     then they are one word longer, the extra word storing the hash.
 
 
     # During a minor collection, the objects in the nursery that are
     # moved outside are changed in-place: their header is replaced with
-    # FORWARDED_MARKER, and the following word is set to the address of
+    # the value -1, and the following word is set to the address of
     # where the object was moved.  This means that all objects in the
     # nursery need to be at least 2 words long, but objects outside the
     # nursery don't need to.
@@ -75,10 +78,18 @@
 
 
     TRANSLATION_PARAMS = {
-        # The size of the nursery.  -1 means "auto", which means that it
-        # will look it up in the env var PYPY_GENERATIONGC_NURSERY and
-        # fall back to half the size of the L2 cache.
-        "nursery_size": -1,
+        # Automatically adjust the size of the nursery and the
+        # 'major_collection_threshold' from the environment.  For
+        # 'nursery_size' it will look it up in the env var
+        # PYPY_GC_NURSERY and fall back to half the size of
+        # the L2 cache.  For 'major_collection_threshold' it will look
+        # it up in the env var PYPY_GC_MAJOR_COLLECT.  It also sets
+        # 'max_heap_size' to PYPY_GC_MAX.
+        "read_from_env": True,
+
+        # The size of the nursery.  Note that this is only used as a
+        # fall-back number.
+        "nursery_size": 896*1024,
 
         # The system page size.  Like obmalloc.c, we assume that it is 4K,
         # which is OK for most systems.
@@ -90,8 +101,9 @@
 
         # The maximum size of an object allocated compactly.  All objects
         # that are larger are just allocated with raw_malloc().  The value
-        # chosen here is enough for a unicode string of length 100.
-        "small_request_threshold": 52*WORD,
+        # chosen here is enough for a unicode string of length 56 (on 64-bits)
+        # or 60 (on 32-bits).  See rlib.rstring.INIT_SIZE.
+        "small_request_threshold": 256-WORD,
 
         # Full collection threshold: after a major collection, we record
         # the total size consumed; and after every minor collection, if the
@@ -101,6 +113,7 @@
         }
 
     def __init__(self, config, chunk_size=DEFAULT_CHUNK_SIZE,
+                 read_from_env=False,
                  nursery_size=32*WORD,
                  page_size=16*WORD,
                  arena_size=64*WORD,
@@ -109,10 +122,17 @@
                  ArenaCollectionClass=None):
         MovingGCBase.__init__(self, config, chunk_size)
         assert small_request_threshold % WORD == 0
+        self.read_from_env = read_from_env
         self.nursery_size = nursery_size
         self.small_request_threshold = small_request_threshold
         self.major_collection_threshold = major_collection_threshold
-        self.nursery_hash_base = -1
+        self.num_major_collects = 0
+        self.max_heap_size = 0.0
+        self.max_heap_size_already_raised = False
+        #
+        self.nursery      = NULL
+        self.nursery_free = NULL
+        self.nursery_top  = NULL
         #
         # The ArenaCollection() handles the nonmovable objects allocation.
         if ArenaCollectionClass is None:
@@ -128,11 +148,6 @@
         # A list of all prebuilt GC objects that contain pointers to the heap
         self.prebuilt_root_objects = self.AddressStack()
         #
-        # Support for id and identityhash: map nursery objects with
-        # GCFLAG_HAS_SHADOW to their future location at the next
-        # minor collection.
-        self.young_objects_shadows = self.AddressDict()
-        #
         self._init_writebarrier_logic()
 
 
@@ -143,8 +158,6 @@
         # we implement differently anyway.  So directly call GCBase.setup().
         GCBase.setup(self)
         #
-        assert self.nursery_size > 0, "XXX"
-        #
         # A list of all raw_malloced objects (the objects too large)
         self.rawmalloced_objects = self.AddressStack()
         self.rawmalloced_total_size = r_uint(0)
@@ -159,6 +172,49 @@
         self.young_objects_with_weakrefs = self.AddressStack()
         self.old_objects_with_weakrefs = self.AddressStack()
         #
+        # Support for id and identityhash: map nursery objects with
+        # GCFLAG_HAS_SHADOW to their future location at the next
+        # minor collection.
+        self.young_objects_shadows = self.AddressDict()
+        #
+        # Allocate a nursery.  In case of auto_nursery_size, start by
+        # allocating a very small nursery, enough to do things like look
+        # up the env var, which requires the GC; and then really
+        # allocate the nursery of the final size.
+        if not self.read_from_env:
+            self.allocate_nursery()
+        else:
+            #
+            defaultsize = self.nursery_size
+            minsize = 18 * self.small_request_threshold
+            self.nursery_size = minsize
+            self.allocate_nursery()
+            #
+            # From there on, the GC is fully initialized and the code
+            # below can use it
+            newsize = base.read_from_env('PYPY_GC_NURSERY')
+            if newsize <= 0:
+                newsize = generation.estimate_best_nursery_size()
+                if newsize <= 0:
+                    newsize = defaultsize
+            #
+            major_coll = base.read_float_from_env('PYPY_GC_MAJOR_COLLECT')
+            if major_coll >= 1.0:
+                self.major_collection_threshold = major_coll
+            #
+            max_heap_size = base.read_uint_from_env('PYPY_GC_MAX')
+            if max_heap_size > 0:
+                self.max_heap_size = float(max_heap_size)
+            #
+            self.minor_collection()    # to empty the nursery
+            llarena.arena_free(self.nursery)
+            self.nursery_size = max(newsize, minsize)
+            self.allocate_nursery()
+
+
+    def allocate_nursery(self):
+        debug_start("gc-set-nursery-size")
+        debug_print("nursery size:", self.nursery_size)
         # the start of the nursery: we actually allocate a tiny bit more for
         # the nursery than really needed, to simplify pointer arithmetic
         # in malloc_fixedsize_clear().
@@ -167,12 +223,13 @@
         if not self.nursery:
             raise MemoryError("cannot allocate nursery")
         # the current position in the nursery:
-        self.nursery_next = self.nursery
+        self.nursery_free = self.nursery
         # the end of the nursery:
         self.nursery_top = self.nursery + self.nursery_size
         # initialize the threshold, a bit arbitrarily
         self.next_major_collection_threshold = (
             self.nursery_size * self.major_collection_threshold)
+        debug_stop("gc-set-nursery-size")
 
 
     def malloc_fixedsize_clear(self, typeid, size, can_collect=True,
@@ -205,9 +262,9 @@
             #
             # Get the memory from the nursery.  If there is not enough space
             # there, do a collect first.
-            result = self.nursery_next
-            self.nursery_next = result + totalsize
-            if self.nursery_next > self.nursery_top:
+            result = self.nursery_free
+            self.nursery_free = result + totalsize
+            if self.nursery_free > self.nursery_top:
                 result = self.collect_and_reserve(totalsize)
             #
             # Build the object.
@@ -256,9 +313,9 @@
             #
             # Get the memory from the nursery.  If there is not enough space
             # there, do a collect first.
-            result = self.nursery_next
-            self.nursery_next = result + totalsize
-            if self.nursery_next > self.nursery_top:
+            result = self.nursery_free
+            self.nursery_free = result + totalsize
+            if self.nursery_free > self.nursery_top:
                 result = self.collect_and_reserve(totalsize)
             #
             # Build the object.
@@ -277,7 +334,7 @@
             self.major_collection()
 
     def collect_and_reserve(self, totalsize):
-        """To call when nursery_next overflows nursery_top.
+        """To call when nursery_free overflows nursery_top.
         Do a minor collection, and possibly also a major collection,
         and finally reserve 'totalsize' bytes at the start of the
         now-empty nursery.
@@ -290,19 +347,21 @@
             # The nursery might not be empty now, because of
             # execute_finalizers().  If it is almost full again,
             # we need to fix it with another call to minor_collection().
-            if self.nursery_next + totalsize > self.nursery_top:
+            if self.nursery_free + totalsize > self.nursery_top:
                 self.minor_collection()
         #
-        result = self.nursery_next
-        self.nursery_next = result + totalsize
-        ll_assert(self.nursery_next <= self.nursery_top, "nursery overflow")
+        result = self.nursery_free
+        self.nursery_free = result + totalsize
+        ll_assert(self.nursery_free <= self.nursery_top, "nursery overflow")
         return result
     collect_and_reserve._dont_inline_ = True
 
 
-    def _full_collect_if_needed(self):
-        if self.get_total_memory_used() > self.next_major_collection_threshold:
-            self.collect()
+    def _full_collect_if_needed(self, reserving_size):
+        if (float(self.get_total_memory_used()) + reserving_size >
+                self.next_major_collection_threshold):
+            self.minor_collection()
+            self.major_collection(reserving_size)
 
     def _reserve_external_memory(self, totalsize):
         """Do a raw_malloc() to get some external memory.
@@ -322,7 +381,7 @@
         #
         # If somebody calls _external_malloc() a lot, we must eventually
         # force a full collection.
-        self._full_collect_if_needed()
+        self._full_collect_if_needed(totalsize)
         #
         result = self._reserve_external_memory(totalsize)
         llmemory.raw_memclear(result, totalsize)
@@ -336,7 +395,7 @@
         #
         # If somebody calls _malloc_nonmovable() a lot, we must eventually
         # force a full collection.
-        self._full_collect_if_needed()
+        self._full_collect_if_needed(totalsize)
         #
         rawtotalsize = llmemory.raw_malloc_usage(totalsize)
         if rawtotalsize <= self.small_request_threshold:
@@ -368,7 +427,10 @@
     # Other functions in the GC API
 
     def set_max_heap_size(self, size):
-        XXX
+        self.max_heap_size = float(size)
+        if self.max_heap_size > 0.0:
+            if self.max_heap_size < self.next_major_collection_threshold:
+                self.next_major_collection_threshold = self.max_heap_size
 
     def can_malloc_nonmovable(self):
         return True
@@ -399,11 +461,13 @@
 
 
     def malloc_fixedsize_nonmovable(self, typeid):
+        """NOT_RPYTHON: not tested translated"""
         size_gc_header = self.gcheaderbuilder.size_gc_header
         totalsize = size_gc_header + self.fixed_size(typeid)
         #
         result = self._malloc_nonmovable(typeid, totalsize)
-        return result + size_gc_header
+        obj = result + size_gc_header
+        return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF)
 
     def malloc_varsize_nonmovable(self, typeid, length):
         size_gc_header = self.gcheaderbuilder.size_gc_header
@@ -419,15 +483,15 @@
         result = self._malloc_nonmovable(typeid, totalsize)
         obj = result + size_gc_header
         (obj + offset_to_length).signed[0] = length
-        return obj
+        return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF)
 
     def malloc_nonmovable(self, typeid, length, zero):
         # helper for testing, same as GCBase.malloc
         if self.is_varsize(typeid):
-            obj = self.malloc_varsize_nonmovable(typeid, length)
+            gcref = self.malloc_varsize_nonmovable(typeid, length)
         else:
-            obj = self.malloc_fixedsize_nonmovable(typeid)
-        return obj
+            gcref = self.malloc_fixedsize_nonmovable(typeid)
+        return gcref
 
 
     # ----------
@@ -441,7 +505,6 @@
         return llop.combine_ushort(lltype.Signed, typeid16, flags)
 
     def init_gc_object(self, addr, typeid16, flags=0):
-        #print "init_gc_object(%r, 0x%x)" % (addr, flags)
         # The default 'flags' is zero.  The flags GCFLAG_NO_xxx_PTRS
         # have been chosen to allow 'flags' to be zero in the common
         # case (hence the 'NO' in their name).
@@ -460,13 +523,15 @@
         return self.nursery <= addr < self.nursery_top
 
     def is_forwarded(self, obj):
+        """Returns True if the nursery obj is marked as forwarded.
+        Implemented a bit obscurely by checking an unrelated flag
+        that can never be set on a young object -- except if tid == -1.
+        """
         assert self.is_in_nursery(obj)
-        tid = self.header(obj).tid
-        return isinstance(tid, int) and tid == FORWARDED_MARKER
+        return self.header(obj).tid & GCFLAG_FINALIZATION_ORDERING
 
     def get_forwarding_address(self, obj):
-        obj = llarena.getfakearenaaddress(obj)
-        return obj.address[0]
+        return llmemory.cast_adr_to_ptr(obj, FORWARDSTUBPTR).forw
 
     def get_total_memory_used(self):
         """Return the total memory used, not counting any object in the
@@ -481,6 +546,10 @@
         # similarily, all objects should have this flag:
         ll_assert(self.header(obj).tid & GCFLAG_NO_YOUNG_PTRS,
                   "missing GCFLAG_NO_YOUNG_PTRS")
+        # if we have GCFLAG_NO_HEAP_PTRS, then we have GCFLAG_NO_YOUNG_PTRS
+        if self.header(obj).tid & GCFLAG_NO_HEAP_PTRS:
+            ll_assert(self.header(obj).tid & GCFLAG_NO_YOUNG_PTRS,
+                      "GCFLAG_NO_HEAP_PTRS && !GCFLAG_NO_YOUNG_PTRS")
         # the GCFLAG_VISITED should not be set between collections
         ll_assert(self.header(obj).tid & GCFLAG_VISITED == 0,
                   "unexpected GCFLAG_VISITED")
@@ -547,17 +616,17 @@
 
 
     def assume_young_pointers(self, addr_struct):
-        """Called occasionally by the JIT to mean 'assume that 'addr_struct'
-        may now contain young pointers.
+        """Called occasionally by the JIT to mean ``assume that 'addr_struct'
+        may now contain young pointers.''
         """
-        XXX
         objhdr = self.header(addr_struct)
         if objhdr.tid & GCFLAG_NO_YOUNG_PTRS:
             self.old_objects_pointing_to_young.append(addr_struct)
             objhdr.tid &= ~GCFLAG_NO_YOUNG_PTRS
-        if objhdr.tid & GCFLAG_NO_HEAP_PTRS:
-            objhdr.tid &= ~GCFLAG_NO_HEAP_PTRS
-            self.last_generation_root_objects.append(addr_struct)
+            #
+            if objhdr.tid & GCFLAG_NO_HEAP_PTRS:
+                objhdr.tid &= ~GCFLAG_NO_HEAP_PTRS
+                self.prebuilt_root_objects.append(addr_struct)
 
     def writebarrier_before_copy(self, source_addr, dest_addr):
         """ This has the same effect as calling writebarrier over
@@ -618,7 +687,7 @@
         # All live nursery objects are out, and the rest dies.  Fill
         # the whole nursery with zero and reset the current nursery pointer.
         llarena.arena_reset(self.nursery, self.nursery_size, 2)
-        self.nursery_next = self.nursery
+        self.nursery_free = self.nursery
         #
         debug_print("minor collect, total memory used:",
                     self.get_total_memory_used())
@@ -633,9 +702,9 @@
         # then the write_barrier must have ensured that the prebuilt
         # GcStruct is in the list self.old_objects_pointing_to_young.
         self.root_walker.walk_roots(
-            MiniMarkGC._trace_drag_out,  # stack roots
-            MiniMarkGC._trace_drag_out,  # static in prebuilt non-gc
-            None)                        # static in prebuilt gc
+            MiniMarkGC._trace_drag_out1,  # stack roots
+            MiniMarkGC._trace_drag_out1,  # static in prebuilt non-gc
+            None)                         # static in prebuilt gc
 
     def collect_oldrefs_to_nursery(self):
         # Follow the old_objects_pointing_to_young list and move the
@@ -660,7 +729,10 @@
         self.trace(obj, self._trace_drag_out, None)
 
 
-    def _trace_drag_out(self, root, ignored=None):
+    def _trace_drag_out1(self, root):
+        self._trace_drag_out(root, None)
+
+    def _trace_drag_out(self, root, ignored):
         obj = root.address[0]
         #
         # If 'obj' is not in the nursery, nothing to change.
@@ -670,7 +742,6 @@
         # If 'obj' was already forwarded, change it to its forwarding address.
         if self.is_forwarded(obj):
             root.address[0] = self.get_forwarding_address(obj)
-            #print '(already forwarded)'
             return
         #
         # First visit to 'obj': we must move it out of the nursery.
@@ -686,10 +757,7 @@
         else:
             # The object has already a shadow.
             newobj = self.young_objects_shadows.get(obj)
-            ll_assert(newobj, "GCFLAG_HAS_SHADOW but not shadow found")
-            #print 'moving object %r into shadow %r' % (
-            #    llarena.getfakearenaaddress(obj),
-            #    llarena.getfakearenaaddress(newobj),)
+            ll_assert(newobj != NULL, "GCFLAG_HAS_SHADOW but no shadow found")
             newhdr = newobj - size_gc_header
             #
             # Remove the flag GCFLAG_HAS_SHADOW, so that it doesn't get
@@ -700,21 +768,19 @@
         # nursery are kept unchanged in this step.
         llmemory.raw_memcopy(obj - size_gc_header, newhdr, totalsize)
         #
-        # Set the old object's tid to FORWARDED_MARKER and replace
-        # the old object's content with the target address.
+        # Set the old object's tid to -1 (containing all flags) and
+        # replace the old object's content with the target address.
         # A bit of no-ops to convince llarena that we are changing
         # the layout, in non-translated versions.
-        llarena.arena_reset(obj - size_gc_header, totalsize, 0)
-        llarena.arena_reserve(obj - size_gc_header, llmemory.sizeof(self.HDR))
-        llarena.arena_reserve(obj, llmemory.sizeof(llmemory.Address))
-        self.header(obj).tid = FORWARDED_MARKER
         obj = llarena.getfakearenaaddress(obj)
+        llarena.arena_reset(obj - size_gc_header, totalsize, 0)
+        llarena.arena_reserve(obj - size_gc_header,
+                              size_gc_header + llmemory.sizeof(FORWARDSTUB))
+        self.header(obj).tid = -1
         newobj = newhdr + size_gc_header
-        obj.address[0] = newobj
+        llmemory.cast_adr_to_ptr(obj, FORWARDSTUBPTR).forw = newobj
         #
         # Change the original pointer to this object.
-        #print
-        #print '\t\t\t->', llarena.getfakearenaaddress(newobj - size_gc_header)
         root.address[0] = newobj
         #
         # Add the newobj to the list 'old_objects_pointing_to_young',
@@ -727,7 +793,7 @@
     # ----------
     # Full collection
 
-    def major_collection(self):
+    def major_collection(self, reserving_size=0):
         """Do a major collection.  Only for when the nursery is empty."""
         #
         debug_start("gc-collect")
@@ -740,7 +806,7 @@
                     self.rawmalloced_total_size, "bytes")
         #
         # Debugging checks
-        ll_assert(self.nursery_next == self.nursery,
+        ll_assert(self.nursery_free == self.nursery,
                   "nursery not empty in major_collection()")
         self.debug_check_consistency()
         #
@@ -779,17 +845,45 @@
         #
         self.debug_check_consistency()
         #
-        self.next_major_collection_threshold = (
-            self.get_total_memory_used() * self.major_collection_threshold)
-        #
+        self.num_major_collects += 1
         debug_print("| used after collection:")
         debug_print("|          in ArenaCollection:     ",
                     self.ac.total_memory_used, "bytes")
         debug_print("|          raw_malloced:           ",
                     self.rawmalloced_total_size, "bytes")
+        debug_print("| number of major collects:        ",
+                    self.num_major_collects)
         debug_print("`----------------------------------------------")
         debug_stop("gc-collect")
         #
+        # Set the threshold for the next major collection to be when we
+        # have allocated 'major_collection_threshold' times more than
+        # we currently have.
+        self.next_major_collection_threshold = (
+            (self.get_total_memory_used() * self.major_collection_threshold)
+            + reserving_size)
+        #
+        # Max heap size: gives an upper bound on the threshold.  If we
+        # already have at least this much allocated, raise MemoryError.
+        if (self.max_heap_size > 0.0 and
+                self.next_major_collection_threshold > self.max_heap_size):
+            #
+            self.next_major_collection_threshold = self.max_heap_size
+            if (float(self.get_total_memory_used()) + reserving_size >=
+                    self.next_major_collection_threshold):
+                #
+                # First raise MemoryError, giving the program a chance to
+                # quit cleanly.  It might still allocate in the nursery,
+                # which might eventually be emptied, triggering another
+                # major collect and (possibly) reaching here again with an
+                # even higher memory consumption.  To prevent it, if it's
+                # the second time we are here, then abort the program.
+                if self.max_heap_size_already_raised:
+                    llop.debug_fatalerror(lltype.Void,
+                                          "Using too much memory, aborting")
+                self.max_heap_size_already_raised = True
+                raise MemoryError
+        #
         # At the end, we can execute the finalizers of the objects
         # listed in 'run_finalizers'.  Note that this will typically do
         # more allocations.
@@ -805,7 +899,7 @@
         else:
             return True      # dies
 
-    def _reset_gcflag_visited(self, obj, ignored=None):
+    def _reset_gcflag_visited(self, obj, ignored):
         self.header(obj).tid &= ~GCFLAG_VISITED
 
     def free_unvisited_rawmalloc_objects(self):
@@ -848,7 +942,10 @@
     def _collect_obj(obj, objects_to_trace):
         objects_to_trace.append(obj)
 
-    def _collect_ref(self, root, ignored=None):
+    def _collect_ref(self, root):
+        self.objects_to_trace.append(root.address[0])
+
+    def _collect_ref_rec(self, root, ignored):
         self.objects_to_trace.append(root.address[0])
 
     def visit_all_objects(self):
@@ -877,7 +974,7 @@
         #
         # Trace the content of the object and put all objects it references
         # into the 'objects_to_trace' list.
-        self.trace(obj, self._collect_ref, None)
+        self.trace(obj, self._collect_ref_rec, None)
 
 
     # ----------
@@ -898,7 +995,8 @@
                 # collection
                 if self.header(obj).tid & GCFLAG_HAS_SHADOW:
                     shadow = self.young_objects_shadows.get(obj)
-                    ll_assert(shadow, "GCFLAG_HAS_SHADOW but not shadow found")
+                    ll_assert(shadow != NULL,
+                              "GCFLAG_HAS_SHADOW but no shadow found")
                 else:
                     size_gc_header = self.gcheaderbuilder.size_gc_header
                     size = self.get_size(obj)

Modified: pypy/branch/gen2-gc/pypy/rpython/memory/gc/minimarkpage.py
==============================================================================
--- pypy/branch/gen2-gc/pypy/rpython/memory/gc/minimarkpage.py	(original)
+++ pypy/branch/gen2-gc/pypy/rpython/memory/gc/minimarkpage.py	Fri Sep 17 14:58:01 2010
@@ -68,10 +68,11 @@
         self.nblocks_for_size = lltype.malloc(rffi.CArray(lltype.Signed),
                                               length, flavor='raw')
         self.hdrsize = llmemory.raw_malloc_usage(llmemory.sizeof(PAGE_HEADER))
+        self.nblocks_for_size[0] = 0    # unused
         for i in range(1, length):
             self.nblocks_for_size[i] = (page_size - self.hdrsize) // (WORD * i)
         #
-        self.uninitialized_pages = PAGE_NULL
+        self.uninitialized_pages = NULL
         self.num_uninitialized_pages = 0
         self.free_pages = NULL
         self.total_memory_used = r_uint(0)
@@ -328,7 +329,8 @@
 def start_of_page(addr, page_size):
     """Return the address of the start of the page that contains 'addr'."""
     if we_are_translated():
-        xxx
+        offset = llmemory.cast_adr_to_int(addr) % page_size
+        return addr - offset
     else:
         return _start_of_page_untranslated(addr, page_size)
 

Modified: pypy/branch/gen2-gc/pypy/rpython/memory/gc/test/test_minimark.py
==============================================================================
--- pypy/branch/gen2-gc/pypy/rpython/memory/gc/test/test_minimark.py	(original)
+++ pypy/branch/gen2-gc/pypy/rpython/memory/gc/test/test_minimark.py	Fri Sep 17 14:58:01 2010
@@ -13,8 +13,13 @@
     from pypy.rlib.rstring import INIT_SIZE
     from pypy.rpython.lltypesystem.rstr import STR, UNICODE
     #
+    size_gc_header = llmemory.raw_malloc_usage(
+        llmemory.sizeof(llmemory.Address))
+    #
     size1 = llmemory.raw_malloc_usage(llmemory.sizeof(STR, INIT_SIZE))
+    size1 = size_gc_header + size1
     assert size1 <= MiniMarkGC.TRANSLATION_PARAMS["small_request_threshold"]
     #
     size2 = llmemory.raw_malloc_usage(llmemory.sizeof(UNICODE, INIT_SIZE))
+    size2 = size_gc_header + size2
     assert size2 <= MiniMarkGC.TRANSLATION_PARAMS["small_request_threshold"]

Modified: pypy/branch/gen2-gc/pypy/rpython/memory/lltypelayout.py
==============================================================================
--- pypy/branch/gen2-gc/pypy/rpython/memory/lltypelayout.py	(original)
+++ pypy/branch/gen2-gc/pypy/rpython/memory/lltypelayout.py	Fri Sep 17 14:58:01 2010
@@ -7,7 +7,7 @@
 primitive_to_fmt = {lltype.Signed:          "l",
                     lltype.Unsigned:        "L",
                     lltype.Char:            "c",
-                    lltype.UniChar:         "H",     # maybe
+                    lltype.UniChar:         "i",     # 4 bytes
                     lltype.Bool:            "B",
                     lltype.Float:           "d",
                     llmemory.Address:       "P",

Modified: pypy/branch/gen2-gc/pypy/rpython/memory/test/test_gc.py
==============================================================================
--- pypy/branch/gen2-gc/pypy/rpython/memory/test/test_gc.py	(original)
+++ pypy/branch/gen2-gc/pypy/rpython/memory/test/test_gc.py	Fri Sep 17 14:58:01 2010
@@ -26,7 +26,7 @@
 class GCTest(object):
     GC_PARAMS = {}
     GC_CAN_MOVE = False
-    GC_CANNOT_MALLOC_NONMOVABLE = False
+    GC_CAN_MALLOC_NONMOVABLE = True
     GC_CAN_SHRINK_ARRAY = False
     GC_CAN_SHRINK_BIG_ARRAY = False
 
@@ -452,10 +452,10 @@
             a = rgc.malloc_nonmovable(TP, 3)
             if a:
                 assert not rgc.can_move(a)
-                return 0
-            return 1
+                return 1
+            return 0
 
-        assert self.interpret(func, []) == int(self.GC_CANNOT_MALLOC_NONMOVABLE)
+        assert self.interpret(func, []) == int(self.GC_CAN_MALLOC_NONMOVABLE)
 
     def test_malloc_nonmovable_fixsize(self):
         S = lltype.GcStruct('S', ('x', lltype.Float))
@@ -466,12 +466,12 @@
                 rgc.collect()
                 if a:
                     assert not rgc.can_move(a)
-                    return 0
-                return 1
+                    return 1
+                return 0
             except Exception, e:
                 return 2
 
-        assert self.interpret(func, []) == int(self.GC_CANNOT_MALLOC_NONMOVABLE)
+        assert self.interpret(func, []) == int(self.GC_CAN_MALLOC_NONMOVABLE)
 
     def test_shrink_array(self):
         from pypy.rpython.lltypesystem.rstr import STR
@@ -628,7 +628,7 @@
 class TestSemiSpaceGC(GCTest, snippet.SemiSpaceGCTests):
     from pypy.rpython.memory.gc.semispace import SemiSpaceGC as GCClass
     GC_CAN_MOVE = True
-    GC_CANNOT_MALLOC_NONMOVABLE = True
+    GC_CAN_MALLOC_NONMOVABLE = False
     GC_CAN_SHRINK_ARRAY = True
     GC_CAN_SHRINK_BIG_ARRAY = True
 
@@ -649,7 +649,7 @@
 
 class TestHybridGC(TestGenerationalGC):
     from pypy.rpython.memory.gc.hybrid import HybridGC as GCClass
-    GC_CANNOT_MALLOC_NONMOVABLE = False
+    GC_CAN_MALLOC_NONMOVABLE = True
     GC_CAN_SHRINK_BIG_ARRAY = False
 
     def test_ref_from_rawmalloced_to_regular(self):
@@ -720,7 +720,7 @@
     from pypy.rpython.memory.gc.hybrid import HybridGC as GCClass
     GC_CAN_MOVE = False # with this size of heap, stuff gets allocated
                         # in 3rd gen.
-    GC_CANNOT_MALLOC_NONMOVABLE = False
+    GC_CAN_MALLOC_NONMOVABLE = True
     GC_PARAMS = {'space_size': 48*WORD,
                  'min_nursery_size': 12*WORD,
                  'nursery_size': 12*WORD,
@@ -769,4 +769,4 @@
 class TestMiniMarkGC(TestSemiSpaceGC):
     from pypy.rpython.memory.gc.minimark import MiniMarkGC as GCClass
     GC_CAN_SHRINK_BIG_ARRAY = False
-    GC_CANNOT_MALLOC_NONMOVABLE = False
+    GC_CAN_MALLOC_NONMOVABLE = True

Modified: pypy/branch/gen2-gc/pypy/rpython/memory/test/test_transformed_gc.py
==============================================================================
--- pypy/branch/gen2-gc/pypy/rpython/memory/test/test_transformed_gc.py	(original)
+++ pypy/branch/gen2-gc/pypy/rpython/memory/test/test_transformed_gc.py	Fri Sep 17 14:58:01 2010
@@ -47,7 +47,7 @@
     gcpolicy = None
     stacklessgc = False
     GC_CAN_MOVE = False
-    GC_CANNOT_MALLOC_NONMOVABLE = False
+    GC_CAN_MALLOC_NONMOVABLE = True
     taggedpointers = False
     
     def setup_class(cls):
@@ -602,8 +602,8 @@
             rgc.collect()
             if a:
                 assert not rgc.can_move(a)
-                return 0
-            return 1
+                return 1
+            return 0
             #except Exception, e:
             #    return 2
 
@@ -611,7 +611,7 @@
     
     def test_malloc_nonmovable(self):
         run = self.runner("malloc_nonmovable")
-        assert int(self.GC_CANNOT_MALLOC_NONMOVABLE) == run([])
+        assert int(self.GC_CAN_MALLOC_NONMOVABLE) == run([])
 
     def define_malloc_nonmovable_fixsize(cls):
         S = lltype.GcStruct('S', ('x', lltype.Float))
@@ -622,8 +622,8 @@
                 rgc.collect()
                 if a:
                     assert not rgc.can_move(a)
-                    return 0
-                return 1
+                    return 1
+                return 0
             except Exception, e:
                 return 2
 
@@ -631,7 +631,7 @@
     
     def test_malloc_nonmovable_fixsize(self):
         run = self.runner("malloc_nonmovable_fixsize")
-        assert run([]) == int(self.GC_CANNOT_MALLOC_NONMOVABLE)
+        assert run([]) == int(self.GC_CAN_MALLOC_NONMOVABLE)
 
     def define_shrink_array(cls):
         from pypy.rpython.lltypesystem.rstr import STR
@@ -680,7 +680,8 @@
 
 class GenericMovingGCTests(GenericGCTests):
     GC_CAN_MOVE = True
-    GC_CANNOT_MALLOC_NONMOVABLE = True
+    GC_CAN_MALLOC_NONMOVABLE = False
+    GC_CAN_TEST_ID = False
 
     def define_many_ids(cls):
         class A(object):
@@ -710,7 +711,8 @@
         return f
     
     def test_many_ids(self):
-        py.test.skip("fails for bad reasons in lltype.py :-(")
+        if not self.GC_CAN_TEST_ID:
+            py.test.skip("fails for bad reasons in lltype.py :-(")
         run = self.runner("many_ids")
         run([])
 
@@ -856,7 +858,7 @@
         #     (and give fixedsize)
 
     def define_writebarrier_before_copy(cls):
-        S = lltype.GcStruct('S')
+        S = lltype.GcStruct('S', ('x', lltype.Char))
         TP = lltype.GcArray(lltype.Ptr(S))
         def fn():
             l = lltype.malloc(TP, 100)
@@ -1144,10 +1146,6 @@
             GC_PARAMS = {'space_size': 4096*WORD}
             root_stack_depth = 200
 
-    def test_writebarrier_before_copy(self):
-        py.test.skip("Not relevant, and crashes because llarena does not "
-                     "support empty GcStructs")
-
 class TestGenerationGC(GenericMovingGCTests):
     gcname = "generation"
     GC_CAN_SHRINK_ARRAY = True
@@ -1379,7 +1377,7 @@
 
 class TestHybridGC(TestGenerationGC):
     gcname = "hybrid"
-    GC_CANNOT_MALLOC_NONMOVABLE = False
+    GC_CAN_MALLOC_NONMOVABLE = True
 
     class gcpolicy(gc.FrameworkGcPolicy):
         class transformerclass(framework.FrameworkGCTransformer):
@@ -1444,6 +1442,21 @@
     def test_malloc_nonmovable_fixsize(self):
         py.test.skip("not supported")
 
+
+class TestMiniMarkGC(TestHybridGC):
+    gcname = "minimark"
+    GC_CAN_TEST_ID = True
+
+    class gcpolicy(gc.FrameworkGcPolicy):
+        class transformerclass(framework.FrameworkGCTransformer):
+            from pypy.rpython.memory.gc.minimark import MiniMarkGC as GCClass
+            GC_PARAMS = {'nursery_size': 32*WORD,
+                         'page_size': 16*WORD,
+                         'arena_size': 64*WORD,
+                         'small_request_threshold': 5*WORD,
+                         }
+            root_stack_depth = 200
+
 # ________________________________________________________________
 # tagged pointers
 

Modified: pypy/branch/gen2-gc/pypy/translator/c/funcgen.py
==============================================================================
--- pypy/branch/gen2-gc/pypy/translator/c/funcgen.py	(original)
+++ pypy/branch/gen2-gc/pypy/translator/c/funcgen.py	Fri Sep 17 14:58:01 2010
@@ -733,6 +733,8 @@
                 continue
             elif T == Signed:
                 format.append('%ld')
+            elif T == Unsigned:
+                format.append('%lu')
             elif T == Float:
                 format.append('%f')
             elif isinstance(T, Ptr) or T == Address:

Modified: pypy/branch/gen2-gc/pypy/translator/c/test/test_newgc.py
==============================================================================
--- pypy/branch/gen2-gc/pypy/translator/c/test/test_newgc.py	(original)
+++ pypy/branch/gen2-gc/pypy/translator/c/test/test_newgc.py	Fri Sep 17 14:58:01 2010
@@ -19,10 +19,11 @@
     removetypeptr = False
     taggedpointers = False
     GC_CAN_MOVE = False
-    GC_CANNOT_MALLOC_NONMOVABLE = False
+    GC_CAN_MALLOC_NONMOVABLE = True
     GC_CAN_SHRINK_ARRAY = False
 
     _isolated_func = None
+    c_allfuncs = None
 
     @classmethod
     def _makefunc_str_int(cls, f):
@@ -111,6 +112,7 @@
     def teardown_class(cls):
         if hasattr(cls.c_allfuncs, 'close_isolate'):
             cls.c_allfuncs.close_isolate()
+            cls.c_allfuncs = None
 
     def run(self, name, *args):
         if not args:
@@ -690,8 +692,8 @@
                 rgc.collect()
                 if a:
                     assert not rgc.can_move(a)
-                    return 0
-                return 1
+                    return 1
+                return 0
             except Exception, e:
                 return 2
 
@@ -699,7 +701,7 @@
 
     def test_malloc_nonmovable(self):
         res = self.run('malloc_nonmovable')
-        assert res == self.GC_CANNOT_MALLOC_NONMOVABLE
+        assert res == self.GC_CAN_MALLOC_NONMOVABLE
 
     def define_resizable_buffer(cls):
         from pypy.rpython.lltypesystem.rstr import STR
@@ -896,7 +898,7 @@
     gcpolicy = "semispace"
     should_be_moving = True
     GC_CAN_MOVE = True
-    GC_CANNOT_MALLOC_NONMOVABLE = True
+    GC_CAN_MALLOC_NONMOVABLE = False
     GC_CAN_SHRINK_ARRAY = True
 
     # for snippets
@@ -1055,7 +1057,7 @@
 class TestHybridGC(TestGenerationalGC):
     gcpolicy = "hybrid"
     should_be_moving = True
-    GC_CANNOT_MALLOC_NONMOVABLE = False
+    GC_CAN_MALLOC_NONMOVABLE = True
 
     def test_gc_set_max_heap_size(self):
         py.test.skip("not implemented")
@@ -1126,6 +1128,15 @@
         res = self.run("adding_a_hash")
         assert res == 0
 
+class TestMiniMarkGC(TestSemiSpaceGC):
+    gcpolicy = "minimark"
+    should_be_moving = True
+    GC_CAN_MALLOC_NONMOVABLE = True
+    GC_CAN_SHRINK_ARRAY = True
+
+    def test_gc_heap_stats(self):
+        py.test.skip("not implemented")
+
 # ____________________________________________________________________
 
 class TaggedPointersTest(object):



More information about the Pypy-commit mailing list