[pypy-commit] pypy default: Fix the test. This gives a first approximation of the .NET

arigo noreply at buildbot.pypy.org
Sat Jul 30 23:48:41 CEST 2011


Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r46116:e7121092d73f
Date: 2011-07-30 22:41 +0200
http://bitbucket.org/pypy/pypy/changeset/e7121092d73f/

Log:	Fix the test. This gives a first approximation of the .NET
	AddMemoryPressure(). This version is simpler than a counter that
	needs to be carefully incremented and decremented by the exact same
	amount. The idea is to use track_allocation=False to know when a
	raw malloc is going to be attached to a GC object (good enough for
	now). All such raw mallocs make the next major collection occur
	earlier. So the major collection is triggered when

	 sum( GC object surviving minor collections + rawmallocs
	with track_allocation=False ) > trigger

	The raw mallocs are attached to GC objects with a __del__, which are
	never allocated young, so they will survive at least until the major
	collection. But if they survive for longer, they are ignored for
	future major collections. This is again an approximation, but in
	the "safe" way. Indeed, *not* ignoring them would simply mean
	having a higher trigger, computed as (1.82*previous size).

diff --git a/pypy/rpython/memory/gc/minimark.py b/pypy/rpython/memory/gc/minimark.py
--- a/pypy/rpython/memory/gc/minimark.py
+++ b/pypy/rpython/memory/gc/minimark.py
@@ -390,6 +390,11 @@
         # initialize the threshold
         self.min_heap_size = max(self.min_heap_size, self.nursery_size *
                                               self.major_collection_threshold)
+        # the following two values are usually equal, but during raw mallocs
+        # of arrays, next_major_collection_threshold is decremented to make
+        # the next major collection arrive earlier.
+        # See translator/c/test/test_newgc, test_nongc_attached_to_gc
+        self.next_major_collection_initial = self.min_heap_size
         self.next_major_collection_threshold = self.min_heap_size
         self.set_major_threshold_from(0.0)
         debug_stop("gc-set-nursery-size")
@@ -397,7 +402,7 @@
 
     def set_major_threshold_from(self, threshold, reserving_size=0):
         # Set the next_major_collection_threshold.
-        threshold_max = (self.next_major_collection_threshold *
+        threshold_max = (self.next_major_collection_initial *
                          self.growth_rate_max)
         if threshold > threshold_max:
             threshold = threshold_max
@@ -412,6 +417,7 @@
         else:
             bounded = False
         #
+        self.next_major_collection_initial = threshold
         self.next_major_collection_threshold = threshold
         return bounded
 
@@ -718,9 +724,18 @@
     def set_max_heap_size(self, size):
         self.max_heap_size = float(size)
         if self.max_heap_size > 0.0:
+            if self.max_heap_size < self.next_major_collection_initial:
+                self.next_major_collection_initial = self.max_heap_size
             if self.max_heap_size < self.next_major_collection_threshold:
                 self.next_major_collection_threshold = self.max_heap_size
 
+    def raw_malloc_varsize_hint(self, sizehint):
+        self.next_major_collection_threshold -= sizehint
+        if self.next_major_collection_threshold < 0:
+            # cannot trigger a full collection now, but we can ensure
+            # that one will occur very soon
+            self.nursery_free = self.nursery_top
+
     def can_malloc_nonmovable(self):
         return True
 
@@ -1600,7 +1615,7 @@
         # Max heap size: gives an upper bound on the threshold.  If we
         # already have at least this much allocated, raise MemoryError.
         if bounded and (float(self.get_total_memory_used()) + reserving_size >=
-                        self.next_major_collection_threshold):
+                        self.next_major_collection_initial):
             #
             # First raise MemoryError, giving the program a chance to
             # quit cleanly.  It might still allocate in the nursery,
diff --git a/pypy/rpython/memory/gctransform/framework.py b/pypy/rpython/memory/gctransform/framework.py
--- a/pypy/rpython/memory/gctransform/framework.py
+++ b/pypy/rpython/memory/gctransform/framework.py
@@ -386,6 +386,18 @@
         else:
             self.malloc_varsize_nonmovable_ptr = None
 
+        if getattr(GCClass, 'raw_malloc_varsize_hint', False):
+            def raw_malloc_varsize_hint(length, itemsize):
+                totalmem = length * itemsize
+                if totalmem > 0:
+                    gcdata.gc.raw_malloc_varsize_hint(totalmem)
+                #else: probably an overflow -- the following rawmalloc
+                #      will fail then
+            self.raw_malloc_varsize_hint_ptr = getfn(
+                raw_malloc_varsize_hint,
+                [annmodel.SomeInteger(), annmodel.SomeInteger()],
+                annmodel.s_None, minimal_transform = False)
+
         self.identityhash_ptr = getfn(GCClass.identityhash.im_func,
                                       [s_gc, s_gcref],
                                       annmodel.SomeInteger(),
diff --git a/pypy/rpython/memory/gctransform/transform.py b/pypy/rpython/memory/gctransform/transform.py
--- a/pypy/rpython/memory/gctransform/transform.py
+++ b/pypy/rpython/memory/gctransform/transform.py
@@ -590,6 +590,16 @@
 
     def gct_fv_raw_malloc_varsize(self, hop, flags, TYPE, v_length, c_const_size, c_item_size,
                                                                     c_offset_to_length):
+        track_allocation = flags.get('track_allocation', True)
+        if not track_allocation:
+            # idea: raw mallocs with track_allocation=False correspond
+            # generally to raw mallocs of stuff that we store in GC objects.
+            # So we tell the GC about such raw mallocs, so that it can
+            # adjust its total size estimate.
+            if hasattr(self, 'raw_malloc_varsize_hint_ptr'):
+                hop.genop("direct_call",
+                          [self.raw_malloc_varsize_hint_ptr,
+                           v_length, c_item_size])
         if c_offset_to_length is None:
             if flags.get('zero'):
                 fnptr = self.raw_malloc_varsize_no_length_zero_ptr
@@ -605,7 +615,7 @@
                                [self.raw_malloc_varsize_ptr, v_length,
                                 c_const_size, c_item_size, c_offset_to_length],
                                resulttype=llmemory.Address)
-        if flags.get('track_allocation', True):
+        if track_allocation:
             hop.genop("track_alloc_start", [v_raw])
         return v_raw
 
diff --git a/pypy/translator/c/test/test_newgc.py b/pypy/translator/c/test/test_newgc.py
--- a/pypy/translator/c/test/test_newgc.py
+++ b/pypy/translator/c/test/test_newgc.py
@@ -1403,15 +1403,20 @@
             # allocate a total of ~77GB, but if the automatic gc'ing works,
             # it should never need more than a few MBs at once
             am1 = am2 = am3 = None
-            for i in range(100000):
+            res = 0
+            for i in range(1, 100001):
+                if am3 is not None:
+                    res += rffi.cast(lltype.Signed, am3.buf[0])
                 am3 = am2
                 am2 = am1
                 am1 = A(i * 4)
-            return 42
+                am1.buf[0] = rffi.cast(rffi.INT, i-50000)
+            return res
         return f
 
     def test_nongc_attached_to_gc(self):
-        self.run("nongc_attached_to_gc")
+        res = self.run("nongc_attached_to_gc")
+        assert res == -99997
 
 # ____________________________________________________________________
 


More information about the pypy-commit mailing list