[pypy-svn] r27296 - in pypy/dist/pypy: rpython/lltypesystem rpython/memory rpython/memory/test translator/c/test translator/stackless

arigo at codespeak.net arigo at codespeak.net
Tue May 16 20:50:15 CEST 2006


Author: arigo
Date: Tue May 16 20:50:13 2006
New Revision: 27296

Modified:
   pypy/dist/pypy/rpython/lltypesystem/llmemory.py
   pypy/dist/pypy/rpython/memory/gc.py
   pypy/dist/pypy/rpython/memory/gctransform.py
   pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py
   pypy/dist/pypy/translator/c/test/test_newgc.py
   pypy/dist/pypy/translator/stackless/frame.py
Log:
Stackless GC tests now passing!  Both on llinterp and in translated C.

* introduced a generic GCREF opaque pointer type for things like
    the return value of the malloc() functions

* display more information during a collect



Modified: pypy/dist/pypy/rpython/lltypesystem/llmemory.py
==============================================================================
--- pypy/dist/pypy/rpython/lltypesystem/llmemory.py	(original)
+++ pypy/dist/pypy/rpython/lltypesystem/llmemory.py	Tue May 16 20:50:13 2006
@@ -385,6 +385,9 @@
 NULL.intaddress = 0      # this is to make memory.lladdress more happy
 Address = lltype.Primitive("Address", NULL)
 
+# GCREF is similar to Address but it is GC-aware
+GCREF = lltype.Ptr(lltype.GcOpaqueType('GCREF'))
+
 
 class _fakeaccessor(object):
     def __init__(self, addr):

Modified: pypy/dist/pypy/rpython/memory/gc.py
==============================================================================
--- pypy/dist/pypy/rpython/memory/gc.py	(original)
+++ pypy/dist/pypy/rpython/memory/gc.py	Tue May 16 20:50:13 2006
@@ -112,8 +112,9 @@
     _size_gc_header = GCHeaderOffset(HDR)
 
     def __init__(self, AddressLinkedList, start_heap_size=4096, get_roots=None):
-        self.bytes_malloced = 0
-        self.heap_size = start_heap_size
+        self.heap_usage = 0          # at the end of the latest collection
+        self.bytes_malloced = 0      # since the latest collection
+        self.bytes_malloced_threshold = start_heap_size
         #need to maintain a list of malloced objects, since we used the systems
         #allocator and can't walk the heap
         self.malloced_objects = None
@@ -125,18 +126,20 @@
         self.malloced_objects = self.AddressLinkedList()
 
     def malloc(self, typeid, length=0):
-        if self.bytes_malloced > self.heap_size:
-            self.collect()
         size = self.fixed_size(typeid)
         if self.is_varsize(typeid):
             itemsize = self.varsize_item_sizes(typeid)
             offset_to_length = self.varsize_offset_to_length(typeid)
-            return self.malloc_varsize(typeid, length, size, itemsize,
-                                       offset_to_length, True)
-        return self.malloc_fixedsize(typeid, size, True)
+            ref = self.malloc_varsize(typeid, length, size, itemsize,
+                                      offset_to_length, True)
+        else:
+            ref = self.malloc_fixedsize(typeid, size, True)
+        # XXX lots of cast and reverse-cast around, but this malloc()
+        # should eventually be killed
+        return lltype.cast_ptr_to_adr(ref)
 
     def malloc_fixedsize(self, typeid, size, can_collect):
-        if can_collect and self.bytes_malloced > self.heap_size:
+        if can_collect and self.bytes_malloced > self.bytes_malloced_threshold:
             self.collect()
         size_gc_header = MarkSweepGC._size_gc_header
         result = raw_malloc(size_gc_header + size)
@@ -144,11 +147,12 @@
         hdr.typeid = typeid << 1
         self.malloced_objects.append(result)
         self.bytes_malloced += raw_malloc_usage(size + size_gc_header)
-        return result + size_gc_header
+        result += size_gc_header
+        return llmemory.cast_adr_to_ptr(result, llmemory.GCREF)
 
     def malloc_varsize(self, typeid, length, size, itemsize, offset_to_length,
                        can_collect):
-        if can_collect and self.bytes_malloced > self.heap_size:
+        if can_collect and self.bytes_malloced > self.bytes_malloced_threshold:
             self.collect()
         try:
             varsize = rarithmetic.ovfcheck(itemsize * length)
@@ -163,13 +167,12 @@
         hdr.typeid = typeid << 1
         self.malloced_objects.append(result)
         self.bytes_malloced += raw_malloc_usage(size + size_gc_header)
-        return result + size_gc_header
+        result += size_gc_header
+        return llmemory.cast_adr_to_ptr(result, llmemory.GCREF)
 
     def collect(self):
         import os
-        os.write(2, 'collecting... ')
-        old_malloced = self.bytes_malloced
-        self.bytes_malloced = 0
+        os.write(2, 'collecting...\n')
         roots = self.get_roots()
         objects = self.AddressLinkedList()
         while 1:
@@ -184,6 +187,9 @@
 ##            gc_info = curr.address[0] - MarkSweepGC._size_gc_header
 ##            gc_info.signed[0] = gc_info.signed[0] & (~1)
         free_non_gc_object(roots)
+        # from this point onwards, no more mallocs should be possible
+        old_malloced = self.bytes_malloced
+        self.bytes_malloced = 0
         while 1:  #mark
             curr = objects.pop()
 ##             print "object: ", curr
@@ -240,17 +246,25 @@
                 raw_free(curr)
         free_non_gc_object(self.malloced_objects)
         self.malloced_objects = newmo
-        if curr_heap_size > self.heap_size:
-            self.heap_size = curr_heap_size
+        if curr_heap_size > self.bytes_malloced_threshold:
+            self.bytes_malloced_threshold = curr_heap_size
         # warning, the following debug print allocates memory to manipulate
         # the strings!  so it must be at the end
-        os.write(2, "malloced before this collection %s bytes. " % old_malloced)
-        os.write(2, "freed %s bytes. the heap is now %s bytes.\n" % (freed_size, curr_heap_size))
+        os.write(2, "  malloced since previous collection: %s bytes\n" %
+                 old_malloced)
+        os.write(2, "  heap usage at start of collection:  %s bytes\n" %
+                 (self.heap_usage + old_malloced))
+        os.write(2, "  freed:                              %s bytes\n" %
+                 freed_size)
+        os.write(2, "  new heap usage:                     %s bytes\n" %
+                 curr_heap_size)
+        assert self.heap_usage + old_malloced == curr_heap_size + freed_size
+        self.heap_usage = curr_heap_size
 
     STATISTICS_NUMBERS = 2
 
     def statistics(self):
-        return self.heap_size, self.bytes_malloced
+        return self.heap_usage, self.bytes_malloced
 
     def size_gc_header(self, typeid=0):
         return MarkSweepGC._size_gc_header

Modified: pypy/dist/pypy/rpython/memory/gctransform.py
==============================================================================
--- pypy/dist/pypy/rpython/memory/gctransform.py	(original)
+++ pypy/dist/pypy/rpython/memory/gctransform.py	Tue May 16 20:50:13 2006
@@ -859,16 +859,17 @@
 
         classdef = bk.getuniqueclassdef(GCClass)
         s_gcdata = annmodel.SomeInstance(classdef)
+        s_gcref = annmodel.SomePtr(llmemory.GCREF)
         self.malloc_fixedsize_ptr = getfn(
             GCClass.malloc_fixedsize.im_func,
             [s_gcdata, annmodel.SomeInteger(nonneg=True),
              annmodel.SomeInteger(nonneg=True),
-             annmodel.SomeBool()], annmodel.SomeAddress(),
+             annmodel.SomeBool()], s_gcref,
             inline = True)
         self.malloc_varsize_ptr = getfn(
             GCClass.malloc_varsize.im_func,
             [s_gcdata] + [annmodel.SomeInteger(nonneg=True) for i in range(5)]
-            + [annmodel.SomeBool()], annmodel.SomeAddress())
+            + [annmodel.SomeBool()], s_gcref)
         self.collect_ptr = getfn(GCClass.collect.im_func,
             [s_gcdata], annmodel.s_None)
 
@@ -1110,7 +1111,6 @@
         assert PTRTYPE.TO == TYPE
         type_id = self.get_type_id(TYPE)
 
-        v = varoftype(llmemory.Address)
         c_type_id = rmodel.inputconst(lltype.Signed, type_id)
         info = self.type_info_list[type_id]
         c_size = rmodel.inputconst(lltype.Signed, info["fixedsize"])
@@ -1124,10 +1124,11 @@
             args = [self.malloc_varsize_ptr, self.c_const_gc, c_type_id,
                     v_length, c_size, c_varitemsize, c_ofstolength,
                     c_can_collect]
+        v = varoftype(llmemory.GCREF)
         newop = SpaceOperation("direct_call", args, v)
         ops, index = self.protect_roots(newop, livevars, block,
                                         block.operations.index(op))
-        ops.append(SpaceOperation("cast_adr_to_ptr", [v], op.result))
+        ops.append(SpaceOperation("cast_opaque_ptr", [v], op.result))
         return ops
 
     replace_malloc_varsize = replace_malloc

Modified: pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py
==============================================================================
--- pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py	(original)
+++ pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py	Tue May 16 20:50:13 2006
@@ -155,8 +155,11 @@
 ##         gclltype.use_gc = cls.old
 
 from pypy.translator.c import gc
+from pypy.annotation import model as annmodel
+from pypy.rpython.lltypesystem import lltype
 from pypy.rpython.memory import gctransform
 from pypy.rpython.memory.support import INT_SIZE
+from pypy import conftest
 
 
 def rtype(func, inputtypes, specialize=True):
@@ -165,33 +168,38 @@
     t.buildannotator().build_types(func, inputtypes)
     if specialize:
         t.buildrtyper().specialize(t)
+    if conftest.option.view:
+        t.view()
     return t
 
 class GCTest(object):
     gcpolicy = None
 
-    def runner(self, f, withargs=False, statistics=False):
-        if withargs:
-            def entrypoint(argv):
-                x = int(argv[0])
-                y = int(argv[1])
+    def runner(self, f, nbargs=0, statistics=False):
+        if nbargs == 2:
+            def entrypoint(args):
+                x = args[0]
+                y = args[1]
                 r = f(x, y)
                 return r
-        else:
-            def entrypoint(argv):
+        elif nbargs == 0:
+            def entrypoint(args):
                 return f()
-            
+        else:
+            raise NotImplementedError("pure laziness")
+
         from pypy.rpython.llinterp import LLInterpreter
         from pypy.translator.c.genc import CStandaloneBuilder
-        from pypy.annotation.listdef import s_list_of_strings
 
-        t = rtype(entrypoint, [s_list_of_strings])
+        ARGS = lltype.FixedSizeArray(lltype.Signed, nbargs)
+        s_args = annmodel.SomePtr(lltype.Ptr(ARGS))
+        t = rtype(entrypoint, [s_args])
         cbuild = CStandaloneBuilder(t, entrypoint, self.gcpolicy)
         db = cbuild.generate_graphs_for_llinterp()
         entrypointptr = cbuild.getentrypointptr()
         entrygraph = entrypointptr._obj.graph
-
-        r_list_of_strings = t.rtyper.getrepr(s_list_of_strings)
+        if conftest.option.view:
+            t.view()
 
         llinterp = LLInterpreter(t.rtyper)
 
@@ -199,8 +207,10 @@
         setupgraph = db.gctransformer.frameworkgc_setup_ptr.value._obj.graph
         llinterp.eval_graph(setupgraph, [])
         def run(args):
-            ll_argv = r_list_of_strings.convert_const([repr(x) for x in args])
-            res = llinterp.eval_graph(entrygraph, [ll_argv])
+            ll_args = lltype.malloc(ARGS, immortal=True)
+            for i in range(nbargs):
+                ll_args[i] = args[i]
+            res = llinterp.eval_graph(entrygraph, [ll_args])
             return res
 
         if statistics:
@@ -259,7 +269,7 @@
         def append_to_list(i, j):
             box.lst.append([i] * 50)
             return box.lst[j][0]
-        run = self.runner(append_to_list, withargs=True)
+        run = self.runner(append_to_list, nbargs=2)
         res = run([0, 0])
         assert res == 0
         for i in range(1, 15):
@@ -273,15 +283,14 @@
             for i in range(j):
                 lst.append(str(i))
             return len("".join(lst))
-        res = self.runner(concat, withargs=True)([100, 0])
-        run, statistics = self.runner(concat, withargs=True, statistics=True)
+        run, statistics = self.runner(concat, nbargs=2, statistics=True)
         res = run([100, 0])
         assert res == concat(100, 0)
         heap_size = statistics().item0
         assert heap_size < 16000 * INT_SIZE / 4 # xxx
 
 
-class INPROGRESS_TestStacklessMarkSweepGC(TestMarkSweepGC):
+class TestStacklessMarkSweepGC(TestMarkSweepGC):
 
     class gcpolicy(gc.StacklessFrameworkGcPolicy):
         class transformerclass(gctransform.StacklessFrameworkGCTransformer):

Modified: pypy/dist/pypy/translator/c/test/test_newgc.py
==============================================================================
--- pypy/dist/pypy/translator/c/test/test_newgc.py	(original)
+++ pypy/dist/pypy/translator/c/test/test_newgc.py	Tue May 16 20:50:13 2006
@@ -563,14 +563,22 @@
     from pypy.translator.c.gc import StacklessFrameworkGcPolicy as gcpolicy
 
     def getcompiled(self, f):
-        py.test.skip('in-progress')
         # XXX quick hack
         from pypy.translator.c.test.test_stackless import StacklessTest
         runner = StacklessTest()
         runner.gcpolicy = self.gcpolicy
         runner.stacklessmode = True
-        res = runner.wrap_stackless_function(f)
+        try:
+            res = runner.wrap_stackless_function(f)
+        except py.process.cmdexec.Error, e:
+            if 'Fatal PyPy error: MemoryError' in e.err:
+                res = MemoryError
+            else:
+                raise
         self.t = runner.t
         def compiled():
-            return res
+            if res is MemoryError:
+                raise MemoryError
+            else:
+                return res
         return compiled

Modified: pypy/dist/pypy/translator/stackless/frame.py
==============================================================================
--- pypy/dist/pypy/translator/stackless/frame.py	(original)
+++ pypy/dist/pypy/translator/stackless/frame.py	Tue May 16 20:50:13 2006
@@ -10,7 +10,7 @@
 # ____________________________________________________________
 # generic data types
 
-SAVED_REFERENCE = lltype.Ptr(lltype.GcOpaqueType('stackless.saved_ref'))
+SAVED_REFERENCE = llmemory.GCREF
 null_saved_ref = lltype.nullptr(SAVED_REFERENCE.TO)
 
 STORAGE_TYPES = [lltype.Void, SAVED_REFERENCE, llmemory.Address,



More information about the Pypy-commit mailing list