[pypy-commit] pypy incremental-gc: alternate arena (probably will be removed when altenative created), more tests
andrewchambers
noreply at buildbot.pypy.org
Mon Aug 12 02:30:43 CEST 2013
Author: Andrew Chambers <andrewchamberss at gmail.com>
Branch: incremental-gc
Changeset: r66077:5e851020f0ec
Date: 2013-08-12 12:29 +1200
http://bitbucket.org/pypy/pypy/changeset/5e851020f0ec/
Log: alternate arena (probably will be removed when altenative created),
more tests
diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -139,8 +139,9 @@
# marking of objects can be done over multiple
STATE_MARKING = 1
STATE_SWEEPING_RAWMALLOC = 2
-STATE_SWEEPING_ARENA = 3
-STATE_FINALIZING = 4
+STATE_SWEEPING_ARENA_1 = 3
+STATE_SWEEPING_ARENA_2 = 4
+STATE_FINALIZING = 5
@@ -293,6 +294,8 @@
ArenaCollectionClass = minimarkpage.ArenaCollection
self.ac = ArenaCollectionClass(arena_size, page_size,
small_request_threshold)
+ self.ac_alternate = ArenaCollectionClass(arena_size, page_size,
+ small_request_threshold)
#
# Used by minor collection: a list of (mostly non-young) objects that
# (may) contain a pointer to a young object. Populated by
@@ -982,7 +985,9 @@
"""Return the total memory used, not counting any object in the
nursery: only objects in the ArenaCollection or raw-malloced.
"""
- return self.ac.total_memory_used + self.rawmalloced_total_size
+ return self.ac.total_memory_used + self.ac_alternate.total_memory_used \
+ + self.rawmalloced_total_size
+
def card_marking_words_for_length(self, length):
# --- Unoptimized version:
@@ -1025,7 +1030,9 @@
already_checked = True
elif self.gc_state == STATE_SWEEPING_RAWMALLOC:
pass
- elif self.gc_state == STATE_SWEEPING_ARENA:
+ elif self.gc_state == STATE_SWEEPING_ARENA_1:
+ pass
+ elif self.gc_state == STATE_SWEEPING_ARENA_2:
pass
elif self.gc_state == STATE_FINALIZING:
pass
@@ -1048,7 +1055,9 @@
self._debug_check_object_marking(obj)
elif self.gc_state == STATE_SWEEPING_RAWMALLOC:
self._debug_check_object_sweeping_rawmalloc(obj)
- elif self.gc_state == STATE_SWEEPING_ARENA:
+ elif self.gc_state == STATE_SWEEPING_ARENA_1:
+ self._debug_check_object_sweeping_arena(obj)
+ elif self.gc_state == STATE_SWEEPING_ARENA_2:
self._debug_check_object_sweeping_arena(obj)
elif self.gc_state == STATE_FINALIZING:
self._debug_check_object_finalizing(obj)
@@ -1772,14 +1781,24 @@
# XXX heuristic here to decide nobjects.
if self.free_unvisited_rawmalloc_objects_step(1):
#malloc objects freed
- self.gc_state = STATE_SWEEPING_ARENA
-
- elif self.gc_state == STATE_SWEEPING_ARENA:
+ self.gc_state = STATE_SWEEPING_ARENA_1
+
+ elif self.gc_state == STATE_SWEEPING_ARENA_1:
#
# Ask the ArenaCollection to visit all objects. Free the ones
# that have not been visited above, and reset GCFLAG_VISITED on
# the others.
- self.ac.mass_free(self._free_if_unvisited)
+ self.ac_alternate.mass_free(self._free_if_unvisited)
+ self.gc_state = STATE_SWEEPING_ARENA_2
+ #swap arenas and start clearing the other one
+ self.ac,self.ac_alternate = self.ac_alternate,self.ac
+
+ elif self.gc_state == STATE_SWEEPING_ARENA_2:
+
+ self.ac_alternate.mass_free(self._free_if_unvisited)
+
+ self.num_major_collects += 1
+
#
# We also need to reset the GCFLAG_VISITED on prebuilt GC objects.
self.prebuilt_root_objects.foreach(self._reset_gcflag_visited, None)
@@ -1810,12 +1829,13 @@
"Using too much memory, aborting")
self.max_heap_size_already_raised = True
raise MemoryError
+
self.gc_state = STATE_FINALIZING
- # END SWEEPING
# FINALIZING not yet incrementalised
# but it seems safe to allow mutator to run after sweeping and
# before finalizers are called. This is because run_finalizers
# is a different list to objects_with_finalizers.
+ # END SWEEPING
elif self.gc_state == STATE_FINALIZING:
# XXX This is considered rare,
# so should we make the calling incremental? or leave as is
@@ -1825,7 +1845,6 @@
self.gc_state = STATE_SCANNING
self.execute_finalizers()
- self.num_major_collects += 1
#END FINALIZING
else:
pass #XXX which exception to raise here. Should be unreachable.
diff --git a/rpython/translator/c/test/test_newgc.py b/rpython/translator/c/test/test_newgc.py
--- a/rpython/translator/c/test/test_newgc.py
+++ b/rpython/translator/c/test/test_newgc.py
@@ -1468,6 +1468,74 @@
res = self.run("nongc_opaque_attached_to_gc")
assert res == 0
+class TestIncrementalMiniMarkGC(TestSemiSpaceGC):
+ gcpolicy = "incminimark"
+ should_be_moving = True
+ GC_CAN_MALLOC_NONMOVABLE = True
+ GC_CAN_SHRINK_ARRAY = True
+
+ def test_gc_heap_stats(self):
+ py.test.skip("not implemented")
+
+ def define_nongc_attached_to_gc(cls):
+ from rpython.rtyper.lltypesystem import rffi
+ ARRAY = rffi.CArray(rffi.INT)
+ class A:
+ def __init__(self, n):
+ self.buf = lltype.malloc(ARRAY, n, flavor='raw',
+ add_memory_pressure=True)
+ def __del__(self):
+ lltype.free(self.buf, flavor='raw')
+ A(6)
+ def f():
+ # allocate a total of ~77GB, but if the automatic gc'ing works,
+ # it should never need more than a few MBs at once
+ am1 = am2 = am3 = None
+ res = 0
+ for i in range(1, 100001):
+ if am3 is not None:
+ res += rffi.cast(lltype.Signed, am3.buf[0])
+ am3 = am2
+ am2 = am1
+ am1 = A(i * 4)
+ am1.buf[0] = rffi.cast(rffi.INT, i - 50000)
+ return res
+ return f
+
+ def test_nongc_attached_to_gc(self):
+ res = self.run("nongc_attached_to_gc")
+ assert res == -99997
+
+ def define_nongc_opaque_attached_to_gc(cls):
+ from rpython.rlib import rgc, ropenssl
+
+ class A:
+ def __init__(self):
+ self.ctx = lltype.malloc(ropenssl.EVP_MD_CTX.TO,
+ flavor='raw')
+ digest = ropenssl.EVP_get_digestbyname('sha1')
+ ropenssl.EVP_DigestInit(self.ctx, digest)
+ rgc.add_memory_pressure(ropenssl.HASH_MALLOC_SIZE + 64)
+
+ def __del__(self):
+ ropenssl.EVP_MD_CTX_cleanup(self.ctx)
+ lltype.free(self.ctx, flavor='raw')
+ #A() --- can't call it here?? get glibc crashes on tannit64
+ def f():
+ am1 = am2 = am3 = None
+ for i in range(100000):
+ am3 = am2
+ am2 = am1
+ am1 = A()
+ # what can we use for the res?
+ return 0
+ return f
+
+ def test_nongc_opaque_attached_to_gc(self):
+ res = self.run("nongc_opaque_attached_to_gc")
+ assert res == 0
+
+
# ____________________________________________________________________
class TaggedPointersTest(object):
@@ -1560,3 +1628,6 @@
class TestMiniMarkGCMostCompact(TaggedPointersTest, TestMiniMarkGC):
removetypeptr = True
+
+class TestIncrementalMiniMarkGCMostCompact(TaggedPointersTest, TestIncrementalMiniMarkGC):
+ removetypeptr = True
More information about the pypy-commit
mailing list