[pypy-svn] r27296 - in pypy/dist/pypy: rpython/lltypesystem rpython/memory rpython/memory/test translator/c/test translator/stackless
arigo at codespeak.net
arigo at codespeak.net
Tue May 16 20:50:15 CEST 2006
Author: arigo
Date: Tue May 16 20:50:13 2006
New Revision: 27296
Modified:
pypy/dist/pypy/rpython/lltypesystem/llmemory.py
pypy/dist/pypy/rpython/memory/gc.py
pypy/dist/pypy/rpython/memory/gctransform.py
pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py
pypy/dist/pypy/translator/c/test/test_newgc.py
pypy/dist/pypy/translator/stackless/frame.py
Log:
Stackless GC tests now passing! Both on llinterp and in translated C.
* introduced a generic GCREF opaque pointer type for things like
the return value of the malloc() functions
* display more information during a collect
Modified: pypy/dist/pypy/rpython/lltypesystem/llmemory.py
==============================================================================
--- pypy/dist/pypy/rpython/lltypesystem/llmemory.py (original)
+++ pypy/dist/pypy/rpython/lltypesystem/llmemory.py Tue May 16 20:50:13 2006
@@ -385,6 +385,9 @@
NULL.intaddress = 0 # this is to make memory.lladdress more happy
Address = lltype.Primitive("Address", NULL)
+# GCREF is similar to Address but it is GC-aware
+GCREF = lltype.Ptr(lltype.GcOpaqueType('GCREF'))
+
class _fakeaccessor(object):
def __init__(self, addr):
Modified: pypy/dist/pypy/rpython/memory/gc.py
==============================================================================
--- pypy/dist/pypy/rpython/memory/gc.py (original)
+++ pypy/dist/pypy/rpython/memory/gc.py Tue May 16 20:50:13 2006
@@ -112,8 +112,9 @@
_size_gc_header = GCHeaderOffset(HDR)
def __init__(self, AddressLinkedList, start_heap_size=4096, get_roots=None):
- self.bytes_malloced = 0
- self.heap_size = start_heap_size
+ self.heap_usage = 0 # at the end of the latest collection
+ self.bytes_malloced = 0 # since the latest collection
+ self.bytes_malloced_threshold = start_heap_size
#need to maintain a list of malloced objects, since we used the systems
#allocator and can't walk the heap
self.malloced_objects = None
@@ -125,18 +126,20 @@
self.malloced_objects = self.AddressLinkedList()
def malloc(self, typeid, length=0):
- if self.bytes_malloced > self.heap_size:
- self.collect()
size = self.fixed_size(typeid)
if self.is_varsize(typeid):
itemsize = self.varsize_item_sizes(typeid)
offset_to_length = self.varsize_offset_to_length(typeid)
- return self.malloc_varsize(typeid, length, size, itemsize,
- offset_to_length, True)
- return self.malloc_fixedsize(typeid, size, True)
+ ref = self.malloc_varsize(typeid, length, size, itemsize,
+ offset_to_length, True)
+ else:
+ ref = self.malloc_fixedsize(typeid, size, True)
+ # XXX lots of cast and reverse-cast around, but this malloc()
+ # should eventually be killed
+ return lltype.cast_ptr_to_adr(ref)
def malloc_fixedsize(self, typeid, size, can_collect):
- if can_collect and self.bytes_malloced > self.heap_size:
+ if can_collect and self.bytes_malloced > self.bytes_malloced_threshold:
self.collect()
size_gc_header = MarkSweepGC._size_gc_header
result = raw_malloc(size_gc_header + size)
@@ -144,11 +147,12 @@
hdr.typeid = typeid << 1
self.malloced_objects.append(result)
self.bytes_malloced += raw_malloc_usage(size + size_gc_header)
- return result + size_gc_header
+ result += size_gc_header
+ return llmemory.cast_adr_to_ptr(result, llmemory.GCREF)
def malloc_varsize(self, typeid, length, size, itemsize, offset_to_length,
can_collect):
- if can_collect and self.bytes_malloced > self.heap_size:
+ if can_collect and self.bytes_malloced > self.bytes_malloced_threshold:
self.collect()
try:
varsize = rarithmetic.ovfcheck(itemsize * length)
@@ -163,13 +167,12 @@
hdr.typeid = typeid << 1
self.malloced_objects.append(result)
self.bytes_malloced += raw_malloc_usage(size + size_gc_header)
- return result + size_gc_header
+ result += size_gc_header
+ return llmemory.cast_adr_to_ptr(result, llmemory.GCREF)
def collect(self):
import os
- os.write(2, 'collecting... ')
- old_malloced = self.bytes_malloced
- self.bytes_malloced = 0
+ os.write(2, 'collecting...\n')
roots = self.get_roots()
objects = self.AddressLinkedList()
while 1:
@@ -184,6 +187,9 @@
## gc_info = curr.address[0] - MarkSweepGC._size_gc_header
## gc_info.signed[0] = gc_info.signed[0] & (~1)
free_non_gc_object(roots)
+ # from this point onwards, no more mallocs should be possible
+ old_malloced = self.bytes_malloced
+ self.bytes_malloced = 0
while 1: #mark
curr = objects.pop()
## print "object: ", curr
@@ -240,17 +246,25 @@
raw_free(curr)
free_non_gc_object(self.malloced_objects)
self.malloced_objects = newmo
- if curr_heap_size > self.heap_size:
- self.heap_size = curr_heap_size
+ if curr_heap_size > self.bytes_malloced_threshold:
+ self.bytes_malloced_threshold = curr_heap_size
# warning, the following debug print allocates memory to manipulate
# the strings! so it must be at the end
- os.write(2, "malloced before this collection %s bytes. " % old_malloced)
- os.write(2, "freed %s bytes. the heap is now %s bytes.\n" % (freed_size, curr_heap_size))
+ os.write(2, " malloced since previous collection: %s bytes\n" %
+ old_malloced)
+ os.write(2, " heap usage at start of collection: %s bytes\n" %
+ (self.heap_usage + old_malloced))
+ os.write(2, " freed: %s bytes\n" %
+ freed_size)
+ os.write(2, " new heap usage: %s bytes\n" %
+ curr_heap_size)
+ assert self.heap_usage + old_malloced == curr_heap_size + freed_size
+ self.heap_usage = curr_heap_size
STATISTICS_NUMBERS = 2
def statistics(self):
- return self.heap_size, self.bytes_malloced
+ return self.heap_usage, self.bytes_malloced
def size_gc_header(self, typeid=0):
return MarkSweepGC._size_gc_header
Modified: pypy/dist/pypy/rpython/memory/gctransform.py
==============================================================================
--- pypy/dist/pypy/rpython/memory/gctransform.py (original)
+++ pypy/dist/pypy/rpython/memory/gctransform.py Tue May 16 20:50:13 2006
@@ -859,16 +859,17 @@
classdef = bk.getuniqueclassdef(GCClass)
s_gcdata = annmodel.SomeInstance(classdef)
+ s_gcref = annmodel.SomePtr(llmemory.GCREF)
self.malloc_fixedsize_ptr = getfn(
GCClass.malloc_fixedsize.im_func,
[s_gcdata, annmodel.SomeInteger(nonneg=True),
annmodel.SomeInteger(nonneg=True),
- annmodel.SomeBool()], annmodel.SomeAddress(),
+ annmodel.SomeBool()], s_gcref,
inline = True)
self.malloc_varsize_ptr = getfn(
GCClass.malloc_varsize.im_func,
[s_gcdata] + [annmodel.SomeInteger(nonneg=True) for i in range(5)]
- + [annmodel.SomeBool()], annmodel.SomeAddress())
+ + [annmodel.SomeBool()], s_gcref)
self.collect_ptr = getfn(GCClass.collect.im_func,
[s_gcdata], annmodel.s_None)
@@ -1110,7 +1111,6 @@
assert PTRTYPE.TO == TYPE
type_id = self.get_type_id(TYPE)
- v = varoftype(llmemory.Address)
c_type_id = rmodel.inputconst(lltype.Signed, type_id)
info = self.type_info_list[type_id]
c_size = rmodel.inputconst(lltype.Signed, info["fixedsize"])
@@ -1124,10 +1124,11 @@
args = [self.malloc_varsize_ptr, self.c_const_gc, c_type_id,
v_length, c_size, c_varitemsize, c_ofstolength,
c_can_collect]
+ v = varoftype(llmemory.GCREF)
newop = SpaceOperation("direct_call", args, v)
ops, index = self.protect_roots(newop, livevars, block,
block.operations.index(op))
- ops.append(SpaceOperation("cast_adr_to_ptr", [v], op.result))
+ ops.append(SpaceOperation("cast_opaque_ptr", [v], op.result))
return ops
replace_malloc_varsize = replace_malloc
Modified: pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py
==============================================================================
--- pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py (original)
+++ pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py Tue May 16 20:50:13 2006
@@ -155,8 +155,11 @@
## gclltype.use_gc = cls.old
from pypy.translator.c import gc
+from pypy.annotation import model as annmodel
+from pypy.rpython.lltypesystem import lltype
from pypy.rpython.memory import gctransform
from pypy.rpython.memory.support import INT_SIZE
+from pypy import conftest
def rtype(func, inputtypes, specialize=True):
@@ -165,33 +168,38 @@
t.buildannotator().build_types(func, inputtypes)
if specialize:
t.buildrtyper().specialize(t)
+ if conftest.option.view:
+ t.view()
return t
class GCTest(object):
gcpolicy = None
- def runner(self, f, withargs=False, statistics=False):
- if withargs:
- def entrypoint(argv):
- x = int(argv[0])
- y = int(argv[1])
+ def runner(self, f, nbargs=0, statistics=False):
+ if nbargs == 2:
+ def entrypoint(args):
+ x = args[0]
+ y = args[1]
r = f(x, y)
return r
- else:
- def entrypoint(argv):
+ elif nbargs == 0:
+ def entrypoint(args):
return f()
-
+ else:
+ raise NotImplementedError("pure laziness")
+
from pypy.rpython.llinterp import LLInterpreter
from pypy.translator.c.genc import CStandaloneBuilder
- from pypy.annotation.listdef import s_list_of_strings
- t = rtype(entrypoint, [s_list_of_strings])
+ ARGS = lltype.FixedSizeArray(lltype.Signed, nbargs)
+ s_args = annmodel.SomePtr(lltype.Ptr(ARGS))
+ t = rtype(entrypoint, [s_args])
cbuild = CStandaloneBuilder(t, entrypoint, self.gcpolicy)
db = cbuild.generate_graphs_for_llinterp()
entrypointptr = cbuild.getentrypointptr()
entrygraph = entrypointptr._obj.graph
-
- r_list_of_strings = t.rtyper.getrepr(s_list_of_strings)
+ if conftest.option.view:
+ t.view()
llinterp = LLInterpreter(t.rtyper)
@@ -199,8 +207,10 @@
setupgraph = db.gctransformer.frameworkgc_setup_ptr.value._obj.graph
llinterp.eval_graph(setupgraph, [])
def run(args):
- ll_argv = r_list_of_strings.convert_const([repr(x) for x in args])
- res = llinterp.eval_graph(entrygraph, [ll_argv])
+ ll_args = lltype.malloc(ARGS, immortal=True)
+ for i in range(nbargs):
+ ll_args[i] = args[i]
+ res = llinterp.eval_graph(entrygraph, [ll_args])
return res
if statistics:
@@ -259,7 +269,7 @@
def append_to_list(i, j):
box.lst.append([i] * 50)
return box.lst[j][0]
- run = self.runner(append_to_list, withargs=True)
+ run = self.runner(append_to_list, nbargs=2)
res = run([0, 0])
assert res == 0
for i in range(1, 15):
@@ -273,15 +283,14 @@
for i in range(j):
lst.append(str(i))
return len("".join(lst))
- res = self.runner(concat, withargs=True)([100, 0])
- run, statistics = self.runner(concat, withargs=True, statistics=True)
+ run, statistics = self.runner(concat, nbargs=2, statistics=True)
res = run([100, 0])
assert res == concat(100, 0)
heap_size = statistics().item0
assert heap_size < 16000 * INT_SIZE / 4 # xxx
-class INPROGRESS_TestStacklessMarkSweepGC(TestMarkSweepGC):
+class TestStacklessMarkSweepGC(TestMarkSweepGC):
class gcpolicy(gc.StacklessFrameworkGcPolicy):
class transformerclass(gctransform.StacklessFrameworkGCTransformer):
Modified: pypy/dist/pypy/translator/c/test/test_newgc.py
==============================================================================
--- pypy/dist/pypy/translator/c/test/test_newgc.py (original)
+++ pypy/dist/pypy/translator/c/test/test_newgc.py Tue May 16 20:50:13 2006
@@ -563,14 +563,22 @@
from pypy.translator.c.gc import StacklessFrameworkGcPolicy as gcpolicy
def getcompiled(self, f):
- py.test.skip('in-progress')
# XXX quick hack
from pypy.translator.c.test.test_stackless import StacklessTest
runner = StacklessTest()
runner.gcpolicy = self.gcpolicy
runner.stacklessmode = True
- res = runner.wrap_stackless_function(f)
+ try:
+ res = runner.wrap_stackless_function(f)
+ except py.process.cmdexec.Error, e:
+ if 'Fatal PyPy error: MemoryError' in e.err:
+ res = MemoryError
+ else:
+ raise
self.t = runner.t
def compiled():
- return res
+ if res is MemoryError:
+ raise MemoryError
+ else:
+ return res
return compiled
Modified: pypy/dist/pypy/translator/stackless/frame.py
==============================================================================
--- pypy/dist/pypy/translator/stackless/frame.py (original)
+++ pypy/dist/pypy/translator/stackless/frame.py Tue May 16 20:50:13 2006
@@ -10,7 +10,7 @@
# ____________________________________________________________
# generic data types
-SAVED_REFERENCE = lltype.Ptr(lltype.GcOpaqueType('stackless.saved_ref'))
+SAVED_REFERENCE = llmemory.GCREF
null_saved_ref = lltype.nullptr(SAVED_REFERENCE.TO)
STORAGE_TYPES = [lltype.Void, SAVED_REFERENCE, llmemory.Address,
More information about the Pypy-commit
mailing list