[pypy-svn] r68960 - in pypy/branch/gc-jit-hack/pypy/rpython/memory/gc: . test
arigo at codespeak.net
arigo at codespeak.net
Wed Nov 4 10:55:33 CET 2009
Author: arigo
Date: Wed Nov 4 10:55:33 2009
New Revision: 68960
Modified:
pypy/branch/gc-jit-hack/pypy/rpython/memory/gc/generation.py
pypy/branch/gc-jit-hack/pypy/rpython/memory/gc/test/test_direct.py
Log:
Implement resizing the nursery at runtime.
Modified: pypy/branch/gc-jit-hack/pypy/rpython/memory/gc/generation.py
==============================================================================
--- pypy/branch/gc-jit-hack/pypy/rpython/memory/gc/generation.py (original)
+++ pypy/branch/gc-jit-hack/pypy/rpython/memory/gc/generation.py Wed Nov 4 10:55:33 2009
@@ -102,6 +102,16 @@
self.nursery_free = NULL
def set_nursery_size(self, newsize):
+ # Invariants: nursery <= nursery_free <= nursery_top
+ # nursery_size == nursery_top - nursery
+ #
+ # In the usual case: nursery_size == allocated_nursery_size
+ # == next_nursery_size
+ #
+ # In the presence of resize_nursery(), however, the nursery currently
+ # in use may be only a fraction of the larger allocated nursery.
+ # next_nursery_size <= nursery_size <= allocated_nursery_size
+ #
debug_start("gc-set-nursery-size")
if newsize < self.min_nursery_size:
newsize = self.min_nursery_size
@@ -111,6 +121,9 @@
# Compute the new bounds for how large young objects can be
# (larger objects are allocated directly old). XXX adjust
self.nursery_size = newsize
+ self.allocated_nursery_size = newsize
+ self.next_nursery_size = newsize
+ self.minimal_setting_for_nursery_size = newsize
self.largest_young_fixedsize = self.get_young_fixedsize(newsize)
self.largest_young_var_basesize = self.get_young_var_basesize(newsize)
scale = 0
@@ -321,42 +334,57 @@
# Implementation of nursery-only collections
def collect_nursery(self):
- if self.nursery_size > self.top_of_space - self.free:
+ while self.nursery_size > self.top_of_space - self.free:
# the semispace is running out, do a full collect
self.obtain_free_space(self.nursery_size)
- ll_assert(self.nursery_size <= self.top_of_space - self.free,
- "obtain_free_space failed to do its job")
+ # in rare cases it is possible that self.nursery_size changed
+ # when we ran the finalizers... that's why we loop.
if self.nursery:
debug_start("gc-minor")
- debug_print("--- minor collect ---")
- debug_print("nursery:", self.nursery, "to", self.nursery_top)
- # a nursery-only collection
- scan = beginning = self.free
- self.collect_oldrefs_to_nursery()
- self.collect_roots_in_nursery()
- scan = self.scan_objects_just_copied_out_of_nursery(scan)
- # at this point, all static and old objects have got their
- # GCFLAG_NO_YOUNG_PTRS set again by trace_and_drag_out_of_nursery
- if self.young_objects_with_weakrefs.non_empty():
- self.invalidate_young_weakrefs()
- if self.young_objects_with_id.length() > 0:
- self.update_young_objects_with_id()
+ self.minor_collection()
# mark the nursery as free and fill it with zeroes again
llarena.arena_reset(self.nursery, self.nursery_size, 2)
- debug_print("survived (fraction of the size):",
- float(scan - beginning) / self.nursery_size)
debug_stop("gc-minor")
#self.debug_check_consistency() # -- quite expensive
else:
# no nursery - this occurs after a full collect, triggered either
# just above or by some previous non-nursery-based allocation.
# Grab a piece of the current space for the nursery.
+ self.allocated_nursery_size = self.next_nursery_size
self.nursery = self.free
- self.nursery_top = self.nursery + self.nursery_size
- self.free = self.nursery_top
+ self.free += self.allocated_nursery_size
+ self.nursery_size = self.next_nursery_size
+ self.nursery_top = self.nursery + self.nursery_size
self.nursery_free = self.nursery
return self.nursery_free
+ def drop_nursery(self):
+ while self.nursery:
+ if self.nursery_size <= self.top_of_space - self.free:
+ self.minor_collection()
+ # mark as free, but don't bother filling it with zeroes
+ llarena.arena_reset(self.nursery, self.nursery_size, 0)
+ self.reset_nursery()
+ return
+ self.obtain_free_space(self.nursery_size)
+
+ def minor_collection(self):
+ debug_print("--- minor collect ---")
+ debug_print("nursery:", self.nursery, "to", self.nursery_top)
+ # a nursery-only collection
+ scan = beginning = self.free
+ self.collect_oldrefs_to_nursery()
+ self.collect_roots_in_nursery()
+ scan = self.scan_objects_just_copied_out_of_nursery(scan)
+ # at this point, all static and old objects have got their
+ # GCFLAG_NO_YOUNG_PTRS set again by trace_and_drag_out_of_nursery
+ if self.young_objects_with_weakrefs.non_empty():
+ self.invalidate_young_weakrefs()
+ if self.young_objects_with_id.length() > 0:
+ self.update_young_objects_with_id()
+ debug_print("survived (fraction of the size):",
+ float(scan - beginning) / self.nursery_size)
+
# NB. we can use self.copy() to move objects out of the nursery,
# but only if the object was really in the nursery.
@@ -564,6 +592,48 @@
else:
SemiSpaceGC.debug_check_can_copy(self, obj)
+ # ---------- resizing the nursery at runtime ----------
+
+ def resize_nursery(self, newsize):
+ debug_start("gc-resize-nursery")
+ if newsize < self.minimal_setting_for_nursery_size:
+ newsize = self.minimal_setting_for_nursery_size
+ if not self.nursery or newsize > self.allocated_nursery_size:
+ self.drop_nursery()
+ self.nursery_size = newsize
+ self.allocated_nursery_size = newsize
+ self.next_nursery_size = newsize
+ else:
+ # situation of the nursery: [###########.......|...........]
+ # \-- nursery_size --/
+ # \--- allocated_nursery_size ---/
+ #
+ # we have to change in-place the nursery_size and nursery_top.
+ #
+ self.next_nursery_size = newsize
+ if self.nursery_size < self.next_nursery_size:
+ # restore this invariant, i.e. grow nursery_top
+ self.nursery_size = self.next_nursery_size
+ self.nursery_top = self.nursery + self.nursery_size
+ else:
+ # this is the case where nursery_size is larger than
+ # next_nursery_size. If it is *much* larger, then it might
+ # still have room for more than newsize bytes (in addition
+ # to the already-allocated objects). In that case limit
+ # that extra space to newsize bytes.
+ currently_free = self.nursery_top - self.nursery_free
+ if currently_free > newsize:
+ self.nursery_top = self.nursery_free + newsize
+ self.nursery_size = self.nursery_top - self.nursery
+ ll_assert(self.next_nursery_size <= self.nursery_size,
+ "bogus invariant in resize_nursery (1)")
+ ll_assert(self.nursery + self.nursery_size == self.nursery_top,
+ "bogus invariant in resize_nursery (2)")
+ ll_assert(self.nursery_free <= self.nursery_top,
+ "bogus invariant in resize_nursery (3)")
+ #
+ debug_stop("gc-resize-nursery")
+
# ____________________________________________________________
import os
Modified: pypy/branch/gc-jit-hack/pypy/rpython/memory/gc/test/test_direct.py
==============================================================================
--- pypy/branch/gc-jit-hack/pypy/rpython/memory/gc/test/test_direct.py (original)
+++ pypy/branch/gc-jit-hack/pypy/rpython/memory/gc/test/test_direct.py Wed Nov 4 10:55:33 2009
@@ -271,6 +271,8 @@
class TestGenerationGC(TestSemiSpaceGC):
from pypy.rpython.memory.gc.generation import GenerationGC as GCClass
+ GC_PARAMS = {'nursery_size': 128}
+
def test_collect_gen(self):
gc = self.gc
old_semispace_collect = gc.semispace_collect
@@ -313,6 +315,107 @@
assert s0.next.x == 1
+ def test_resize_nursery(self):
+ if self.GC_PARAMS['nursery_size'] != 128:
+ py.test.skip("test only for nursery_size == 128")
+
+ gc = self.gc
+ old_minor_collection = gc.minor_collection
+ counters = [0, 0]
+
+ def minor_collection():
+ counters[1] += 1
+ return old_minor_collection()
+ gc.minor_collection = minor_collection
+
+ def alloc():
+ counters[0] += 1
+ self.stackroots.append(self.malloc(S))
+
+ # for a default nursery size of 128, and a space size of 4096
+ while counters[1] == 0:
+ alloc()
+ assert counters[1] == 1
+ assert counters[0] in (5, 9)
+ # nursery contains 1 object (out of a maximum of 4/8)
+
+ # almost fill the nursery again
+ for i in range(counters[0]-3):
+ alloc()
+ assert counters[1] == 1
+ assert counters[0] in (7, 15)
+ # nursery contains 3/7 objects
+
+ # now make the nursery grow to 1024
+ gc.resize_nursery(1024)
+ assert counters[1] == 2
+ assert counters[0] in (7, 15)
+ counters[0] = 0 # empty
+
+ # check its size
+ while counters[1] == 2:
+ alloc()
+ assert counters[1] == 3
+ assert counters[0] in (33, 65)
+
+ # almost fill it again
+ for i in range(counters[0]-3):
+ alloc()
+ assert counters[1] == 3
+ assert counters[0] in (63, 127)
+
+ # now mark it as "must shrink back"
+ gc.resize_nursery(0)
+ assert counters[1] == 3
+ assert counters[0] in (63, 127)
+
+ # check that it will shrink back after two allocations
+ alloc()
+ assert counters[1] == 3
+ alloc()
+ assert counters[1] == 4
+ counters[0] = 1 # only the most recently allocated object
+
+ # check its size
+ while counters[1] == 4:
+ alloc()
+ assert counters[1] == 5
+ assert counters[0] in (5, 9)
+
+ # flush the nursery
+ gc.collect()
+ assert counters[1] == 5
+ assert counters[0] in (5, 9)
+ counters[0] = 0 # nursery not allocated at all
+
+ # resize from a fresh nursery: should not do another collect
+ gc.resize_nursery(512)
+ assert counters[1] == 5
+
+ # fill it partially only
+ for i in range(20):
+ alloc()
+ assert counters[1] == 5
+ assert counters[0] == 20
+
+ # now mark it as "must shrink back",
+ # which should still leave room for 4/8 objects
+ gc.resize_nursery(0)
+ assert counters[1] == 5
+ assert counters[0] == 20
+
+ # check its size
+ while counters[1] == 5:
+ alloc()
+ assert counters[1] == 6
+ assert counters[0] in (25, 29)
+
+ # it should now have shrunk to the standard size of 4/8 objects
+ while counters[1] == 6:
+ alloc()
+ assert counters[1] == 7
+ assert counters[0] in (29, 37)
+
class TestHybridGC(TestGenerationGC):
from pypy.rpython.memory.gc.hybrid import HybridGC as GCClass
More information about the Pypy-commit
mailing list