[pypy-svn] r58228 - in pypy/branch/gc-experiments/pypy/rpython/memory/gc: . test
fijal at codespeak.net
fijal at codespeak.net
Thu Sep 18 18:23:24 CEST 2008
Author: fijal
Date: Thu Sep 18 18:23:23 2008
New Revision: 58228
Added:
pypy/branch/gc-experiments/pypy/rpython/memory/gc/markcompact.py (contents, props changed)
Modified:
pypy/branch/gc-experiments/pypy/rpython/memory/gc/base.py
pypy/branch/gc-experiments/pypy/rpython/memory/gc/generation.py
pypy/branch/gc-experiments/pypy/rpython/memory/gc/hybrid.py
pypy/branch/gc-experiments/pypy/rpython/memory/gc/marksweep.py
pypy/branch/gc-experiments/pypy/rpython/memory/gc/semispace.py
pypy/branch/gc-experiments/pypy/rpython/memory/gc/test/test_direct.py
Log:
A first try at mark&compact collector, a bit of shuffling around to avoid
repetition.
*IN-PROGRESS* doesn't work so far.
Modified: pypy/branch/gc-experiments/pypy/rpython/memory/gc/base.py
==============================================================================
--- pypy/branch/gc-experiments/pypy/rpython/memory/gc/base.py (original)
+++ pypy/branch/gc-experiments/pypy/rpython/memory/gc/base.py Thu Sep 18 18:23:23 2008
@@ -1,4 +1,4 @@
-from pypy.rpython.lltypesystem import lltype, llmemory
+from pypy.rpython.lltypesystem import lltype, llmemory, llarena
from pypy.rlib.debug import ll_assert
class GCBase(object):
@@ -182,12 +182,55 @@
pass
+TYPEID_MASK = 0xffff
+first_gcflag = 1 << 16
+GCFLAG_FORWARDED = first_gcflag
+# GCFLAG_EXTERNAL is set on objects not living in the semispace:
+# either immortal objects or (for HybridGC) externally raw_malloc'ed
+GCFLAG_EXTERNAL = first_gcflag << 1
+GCFLAG_FINALIZATION_ORDERING = first_gcflag << 2
+
class MovingGCBase(GCBase):
moving_gc = True
+ first_unused_gcflag = first_gcflag << 3
def can_move(self, addr):
return True
+ def header(self, addr):
+ addr -= self.gcheaderbuilder.size_gc_header
+ return llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR))
+
+ def get_type_id(self, addr):
+ tid = self.header(addr).tid
+ ll_assert(tid & (GCFLAG_FORWARDED|GCFLAG_EXTERNAL) != GCFLAG_FORWARDED,
+ "get_type_id on forwarded obj")
+ # Non-prebuilt forwarded objects are overwritten with a FORWARDSTUB.
+ # Although calling get_type_id() on a forwarded object works by itself,
+ # we catch it as an error because it's likely that what is then
+ # done with the typeid is bogus.
+ return tid & TYPEID_MASK
+
+ def init_gc_object(self, addr, typeid, flags=0):
+ hdr = llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR))
+ hdr.tid = typeid | flags
+
+ def init_gc_object_immortal(self, addr, typeid, flags=0):
+ hdr = llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR))
+ hdr.tid = typeid | flags | GCFLAG_EXTERNAL | GCFLAG_FORWARDED
+ # immortal objects always have GCFLAG_FORWARDED set;
+ # see get_forwarding_address().
+
+ def get_size(self, obj):
+ typeid = self.get_type_id(obj)
+ size = self.fixed_size(typeid)
+ if self.is_varsize(typeid):
+ lenaddr = obj + self.varsize_offset_to_length(typeid)
+ length = lenaddr.signed[0]
+ size += length * self.varsize_item_sizes(typeid)
+ size = llarena.round_up_for_allocation(size)
+ return size
+
def choose_gc_from_config(config):
"""Return a (GCClass, GC_PARAMS) from the given config object.
"""
Modified: pypy/branch/gc-experiments/pypy/rpython/memory/gc/generation.py
==============================================================================
--- pypy/branch/gc-experiments/pypy/rpython/memory/gc/generation.py (original)
+++ pypy/branch/gc-experiments/pypy/rpython/memory/gc/generation.py Thu Sep 18 18:23:23 2008
@@ -1,6 +1,6 @@
import sys
from pypy.rpython.memory.gc.semispace import SemiSpaceGC
-from pypy.rpython.memory.gc.semispace import GCFLAG_EXTERNAL, GCFLAG_FORWARDED
+from pypy.rpython.memory.gc.base import GCFLAG_EXTERNAL, GCFLAG_FORWARDED
from pypy.rpython.memory.gc.semispace import DEBUG_PRINT
from pypy.rpython.lltypesystem.llmemory import NULL, raw_malloc_usage
from pypy.rpython.lltypesystem import lltype, llmemory, llarena
Modified: pypy/branch/gc-experiments/pypy/rpython/memory/gc/hybrid.py
==============================================================================
--- pypy/branch/gc-experiments/pypy/rpython/memory/gc/hybrid.py (original)
+++ pypy/branch/gc-experiments/pypy/rpython/memory/gc/hybrid.py Thu Sep 18 18:23:23 2008
@@ -1,8 +1,8 @@
import sys
from pypy.rpython.memory.gc.semispace import SemiSpaceGC
from pypy.rpython.memory.gc.semispace import DEBUG_PRINT
-from pypy.rpython.memory.gc.generation import GenerationGC, GCFLAG_FORWARDED
-from pypy.rpython.memory.gc.semispace import GCFLAG_EXTERNAL
+from pypy.rpython.memory.gc.generation import GenerationGC
+from pypy.rpython.memory.gc.base import GCFLAG_EXTERNAL, GCFLAG_FORWARDED
from pypy.rpython.memory.gc.generation import GCFLAG_NO_YOUNG_PTRS
from pypy.rpython.memory.gc.generation import GCFLAG_NO_HEAP_PTRS
from pypy.rpython.lltypesystem import lltype, llmemory, llarena
Added: pypy/branch/gc-experiments/pypy/rpython/memory/gc/markcompact.py
==============================================================================
--- (empty file)
+++ pypy/branch/gc-experiments/pypy/rpython/memory/gc/markcompact.py Thu Sep 18 18:23:23 2008
@@ -0,0 +1,151 @@
+
+from pypy.rpython.lltypesystem import lltype, llmemory, llarena
+from pypy.rpython.memory.gc.base import MovingGCBase
+from pypy.rpython.memory.gcheader import GCHeaderBuilder
+from pypy.rlib.debug import ll_assert
+from pypy.rpython.memory.support import DEFAULT_CHUNK_SIZE
+from pypy.rpython.memory.support import get_address_stack, get_address_deque
+from pypy.rpython.memory.support import AddressDict
+from pypy.rpython.lltypesystem.llmemory import NULL, raw_malloc_usage
+from pypy.rlib.rarithmetic import ovfcheck
+from pypy.rpython.lltypesystem.lloperation import llop
+
+GCFLAG_MARKBIT = MovingGCBase.first_unused_gcflag
+
+memoryError = MemoryError()
+
+class MarkCompactGC(MovingGCBase):
+ HDR = lltype.Struct('header', ('tid', lltype.Signed))
+
+ def __init__(self, chunk_size=DEFAULT_CHUNK_SIZE, space_size=16*(1024**2)):
+ # space_size should be maximal available virtual memory.
+ # this way we'll never need to copy anything nor implement
+ # paging on our own
+ self.space_size = space_size
+ self.gcheaderbuilder = GCHeaderBuilder(self.HDR)
+ self.AddressStack = get_address_stack(chunk_size)
+ self.AddressDeque = get_address_deque(chunk_size)
+ self.AddressDict = AddressDict
+ self.counter = 0
+
+ def setup(self):
+ self.space = llarena.arena_malloc(self.space_size, True)
+ ll_assert(bool(self.space), "couldn't allocate arena")
+ self.spaceptr = self.space
+
+ def init_gc_object(self, addr, typeid, flags=0):
+ hdr = llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR))
+ hdr.tid = typeid | flags
+
+ def malloc_fixedsize_clear(self, typeid, size, can_collect,
+ has_finalizer=False, contains_weakptr=False):
+ assert can_collect
+ size_gc_header = self.gcheaderbuilder.size_gc_header
+ totalsize = size_gc_header + size
+ self.eventually_collect()
+ result = self.spaceptr
+ llarena.arena_reserve(result, totalsize)
+ self.init_gc_object(result, typeid)
+ self.spaceptr += totalsize
+ return llmemory.cast_adr_to_ptr(result+size_gc_header, llmemory.GCREF)
+
+ def malloc_varsize_clear(self, typeid, length, size, itemsize,
+ offset_to_length, can_collect,
+ has_finalizer=False):
+ assert can_collect
+ size_gc_header = self.gcheaderbuilder.size_gc_header
+ nonvarsize = size_gc_header + size
+ try:
+ varsize = ovfcheck(itemsize * length)
+ totalsize = ovfcheck(nonvarsize + varsize)
+ except OverflowError:
+ raise memoryError
+ self.eventually_collect()
+ result = self.spaceptr
+ llarena.arena_reserve(result, totalsize)
+ self.init_gc_object(result, typeid)
+ (result + size_gc_header + offset_to_length).signed[0] = length
+ self.spaceptr = result + llarena.round_up_for_allocation(totalsize)
+ # XXX has_finalizer etc.
+ return llmemory.cast_adr_to_ptr(result+size_gc_header, llmemory.GCREF)
+
+ def eventually_collect(self):
+ # XXX this is a very bad idea, come up with better heuristics
+ # XXX it also does not check can_collect flag
+ self.counter += 1
+ if self.counter == 1000:
+ self.collect()
+ self.counter = 0
+
+ def collect(self):
+ self.debug_check_consistency()
+ self.mark()
+ self.compact()
+ self.update_forward_refs()
+ self.debug_check_consistency()
+
+ def compact(self):
+ self.forwardrefs = self.AddressDict()
+ fromaddr = self.space
+ toaddr = self.space
+ size_gc_header = self.gcheaderbuilder.size_gc_header
+ while fromaddr < self.spaceptr:
+ hdr = llmemory.cast_adr_to_ptr(fromaddr, lltype.Ptr(self.HDR))
+ obj = fromaddr + size_gc_header
+ objsize = self.get_size(obj)
+ totalsize = size_gc_header + objsize
+ if not hdr.tid & GCFLAG_MARKBIT:
+ # this object dies, clear arena
+ llarena.arena_reset(fromaddr, totalsize, True)
+ else:
+ # this objects survives, clear MARKBIT
+ hdr.tid &= ~GCFLAG_MARKBIT
+ if fromaddr != toaddr:
+ # this object needs to be copied somewhere
+
+ # first approach - copy object if possible, otherwise
+ # copy it somewhere else and keep track of that
+ self.forwardrefs.setitem(fromaddr, toaddr)
+ if toaddr + totalsize > fromaddr:
+ # this is the worst possible scenario: object does
+ # not fit inside space to copy
+ xxx
+ else:
+ # object fits there: copy
+ llop.debug_print(lltype.Void, fromaddr, "copied to", toaddr,
+ "tid", self.header(obj).tid,
+ "size", totalsize)
+ llarena.arena_reserve(toaddr, totalsize)
+ llmemory.raw_memcopy(obj - size_gc_header, toaddr, totalsize)
+ llarena.arena_reset(fromaddr, totalsize, True)
+ toaddr += size_gc_header + objsize
+ fromaddr += size_gc_header + objsize
+ self.spaceptr = toaddr
+
+ def update_forward_refs(self):
+ ptr = self.space
+ size_gc_header = self.gcheaderbuilder.size_gc_header
+ while ptr < self.spaceptr:
+ obj = ptr + size_gc_header
+ objsize = self.get_size(obj)
+ totalsize = size_gc_header + objsize
+ self.trace(obj, self._trace_copy, None)
+ ptr += totalsize
+ self.forwardrefs = None
+
+ def _trace_copy(self, pointer, ignored):
+ addr = pointer.address[0]
+ if addr != NULL:
+ pointer.address[0] = self.forwardrefs.get(addr, addr)
+
+ def mark(self):
+ self.root_walker.walk_roots(
+ MarkCompactGC._mark_object, # stack roots
+ MarkCompactGC._mark_object, # static in prebuilt non-gc structures
+ MarkCompactGC._mark_object) # static in prebuilt gc objects
+
+ def _mark_object(self, pointer, ignored=None):
+ obj = pointer.address[0]
+ if obj != NULL:
+ self.header(obj).tid |= GCFLAG_MARKBIT
+ self.trace(obj, self._mark_object, None)
Modified: pypy/branch/gc-experiments/pypy/rpython/memory/gc/marksweep.py
==============================================================================
--- pypy/branch/gc-experiments/pypy/rpython/memory/gc/marksweep.py (original)
+++ pypy/branch/gc-experiments/pypy/rpython/memory/gc/marksweep.py Thu Sep 18 18:23:23 2008
@@ -22,8 +22,6 @@
DEBUG_PRINT = False
memoryError = MemoryError()
class MarkSweepGC(GCBase):
- _alloc_flavor_ = "raw"
-
HDR = lltype.ForwardReference()
HDRPTR = lltype.Ptr(HDR)
# need to maintain a linked list of malloced objects, since we used the
Modified: pypy/branch/gc-experiments/pypy/rpython/memory/gc/semispace.py
==============================================================================
--- pypy/branch/gc-experiments/pypy/rpython/memory/gc/semispace.py (original)
+++ pypy/branch/gc-experiments/pypy/rpython/memory/gc/semispace.py Thu Sep 18 18:23:23 2008
@@ -10,18 +10,12 @@
from pypy.rlib.debug import ll_assert
from pypy.rpython.lltypesystem.lloperation import llop
from pypy.rlib.rarithmetic import ovfcheck
-from pypy.rpython.memory.gc.base import MovingGCBase
+from pypy.rpython.memory.gc.base import MovingGCBase, GCFLAG_EXTERNAL
+from pypy.rpython.memory.gc.base import GCFLAG_FORWARDED
+from pypy.rpython.memory.gc.base import GCFLAG_FINALIZATION_ORDERING
import sys, os
-TYPEID_MASK = 0xffff
-first_gcflag = 1 << 16
-GCFLAG_FORWARDED = first_gcflag
-# GCFLAG_EXTERNAL is set on objects not living in the semispace:
-# either immortal objects or (for HybridGC) externally raw_malloc'ed
-GCFLAG_EXTERNAL = first_gcflag << 1
-GCFLAG_FINALIZATION_ORDERING = first_gcflag << 2
-
DEBUG_PRINT = False
memoryError = MemoryError()
@@ -30,7 +24,7 @@
inline_simple_malloc = True
inline_simple_malloc_varsize = True
malloc_zero_filled = True
- first_unused_gcflag = first_gcflag << 3
+
total_collection_time = 0.0
total_collection_count = 0
@@ -388,40 +382,6 @@
stub = llmemory.cast_adr_to_ptr(obj, self.FORWARDSTUBPTR)
stub.forw = newobj
- def get_size(self, obj):
- typeid = self.get_type_id(obj)
- size = self.fixed_size(typeid)
- if self.is_varsize(typeid):
- lenaddr = obj + self.varsize_offset_to_length(typeid)
- length = lenaddr.signed[0]
- size += length * self.varsize_item_sizes(typeid)
- size = llarena.round_up_for_allocation(size)
- return size
-
- def header(self, addr):
- addr -= self.gcheaderbuilder.size_gc_header
- return llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR))
-
- def get_type_id(self, addr):
- tid = self.header(addr).tid
- ll_assert(tid & (GCFLAG_FORWARDED|GCFLAG_EXTERNAL) != GCFLAG_FORWARDED,
- "get_type_id on forwarded obj")
- # Non-prebuilt forwarded objects are overwritten with a FORWARDSTUB.
- # Although calling get_type_id() on a forwarded object works by itself,
- # we catch it as an error because it's likely that what is then
- # done with the typeid is bogus.
- return tid & TYPEID_MASK
-
- def init_gc_object(self, addr, typeid, flags=0):
- hdr = llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR))
- hdr.tid = typeid | flags
-
- def init_gc_object_immortal(self, addr, typeid, flags=0):
- hdr = llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR))
- hdr.tid = typeid | flags | GCFLAG_EXTERNAL | GCFLAG_FORWARDED
- # immortal objects always have GCFLAG_FORWARDED set;
- # see get_forwarding_address().
-
def deal_with_objects_with_finalizers(self, scan):
# walk over list of objects with finalizers
# if it is not copied, add it to the list of to-be-called finalizers
Modified: pypy/branch/gc-experiments/pypy/rpython/memory/gc/test/test_direct.py
==============================================================================
--- pypy/branch/gc-experiments/pypy/rpython/memory/gc/test/test_direct.py (original)
+++ pypy/branch/gc-experiments/pypy/rpython/memory/gc/test/test_direct.py Thu Sep 18 18:23:23 2008
@@ -279,3 +279,7 @@
'large_object_gcptrs': 12,
'generation3_collect_threshold': 5,
}
+
+class TestMarkCompactGC(DirectGCTest):
+ from pypy.rpython.memory.gc.markcompact import MarkCompactGC as GCClass
+
More information about the Pypy-commit
mailing list