[pypy-svn] r54432 - in pypy/branch/io-improvements/pypy/rpython: . memory/gc memory/gctransform memory/test

fijal at codespeak.net fijal at codespeak.net
Mon May 5 10:44:07 CEST 2008


Author: fijal
Date: Mon May  5 10:44:06 2008
New Revision: 54432

Modified:
   pypy/branch/io-improvements/pypy/rpython/llinterp.py
   pypy/branch/io-improvements/pypy/rpython/memory/gc/base.py
   pypy/branch/io-improvements/pypy/rpython/memory/gctransform/boehm.py
   pypy/branch/io-improvements/pypy/rpython/memory/gctransform/framework.py
   pypy/branch/io-improvements/pypy/rpython/memory/gctransform/transform.py
   pypy/branch/io-improvements/pypy/rpython/memory/test/test_transformed_gc.py
Log:
* Enough support to make boehm realloc.
* Implement boehm realloc via rffi, looks cleaner to me
* Default, copying implementation that works on every gc.

This whole speeds up certain use cases. Next step would be to have nicer
interface for string builder


Modified: pypy/branch/io-improvements/pypy/rpython/llinterp.py
==============================================================================
--- pypy/branch/io-improvements/pypy/rpython/llinterp.py	(original)
+++ pypy/branch/io-improvements/pypy/rpython/llinterp.py	Mon May  5 10:44:06 2008
@@ -697,7 +697,7 @@
         zero = flags.get('zero', False)
         return self.heap.malloc_nonmovable(obj, size, zero=zero)
 
-    def op_malloc_resizable_buffer(self, obj, size):
+    def op_malloc_resizable_buffer(self, obj, flags, size):
         return self.heap.malloc_resizable_buffer(obj, size)
 
     def op_resize_buffer(self, obj, new_size):

Modified: pypy/branch/io-improvements/pypy/rpython/memory/gc/base.py
==============================================================================
--- pypy/branch/io-improvements/pypy/rpython/memory/gc/base.py	(original)
+++ pypy/branch/io-improvements/pypy/rpython/memory/gc/base.py	Mon May  5 10:44:06 2008
@@ -7,6 +7,7 @@
     needs_write_barrier = False
     malloc_zero_filled = False
     prebuilt_gc_objects_are_static_roots = True
+    can_realloc = False
 
     def set_query_functions(self, is_varsize, has_gcptr_in_varsize,
                             is_gcarrayofgcptr,

Modified: pypy/branch/io-improvements/pypy/rpython/memory/gctransform/boehm.py
==============================================================================
--- pypy/branch/io-improvements/pypy/rpython/memory/gctransform/boehm.py	(original)
+++ pypy/branch/io-improvements/pypy/rpython/memory/gctransform/boehm.py	Mon May  5 10:44:06 2008
@@ -4,6 +4,8 @@
 from pypy.rpython.lltypesystem import lltype, llmemory
 from pypy.objspace.flow.model import Constant
 from pypy.rpython.lltypesystem.lloperation import llop
+from pypy.rpython.lltypesystem import rffi
+from pypy.rpython import rmodel
 
 class BoehmGCTransformer(GCTransformer):
     malloc_zero_filled = True
@@ -19,12 +21,19 @@
 
         mh = mallocHelpers()
         mh.allocate = lambda size: llop.boehm_malloc(llmemory.Address, size)
+        c_realloc = rffi.llexternal('GC_REALLOC', [rffi.VOIDP, rffi.INT],
+                                    rffi.VOIDP)
+        def _realloc(ptr, size):
+            return llmemory.cast_ptr_to_adr(c_realloc(rffi.cast(rffi.VOIDP, ptr), size))
+        mh.realloc = _realloc
         ll_malloc_fixedsize = mh._ll_malloc_fixedsize
 
         # XXX, do we need/want an atomic version of this function?
         ll_malloc_varsize_no_length = mh.ll_malloc_varsize_no_length
         ll_malloc_varsize = mh.ll_malloc_varsize
 
+        ll_realloc = mh.ll_realloc
+
         if self.translator:
             self.malloc_fixedsize_ptr = self.inittime_helper(
                 ll_malloc_fixedsize, [lltype.Signed], llmemory.Address)
@@ -39,6 +48,9 @@
                 inline=False)
             self.weakref_deref_ptr = self.inittime_helper(
                 ll_weakref_deref, [llmemory.WeakRefPtr], llmemory.Address)
+            self.realloc_ptr = self.inittime_helper(
+                ll_realloc, [llmemory.Address] + [lltype.Signed] * 4,
+                llmemory.Address)
             self.mixlevelannotator.finish()   # for now
             self.mixlevelannotator.backend_optimize()
 
@@ -48,6 +60,15 @@
     def pop_alive_nopyobj(self, var, llops):
         pass
 
+    def _can_realloc(self):
+        return True
+
+    def perform_realloc(self, hop, c_const_size, c_item_size, c_lengthofs):
+        op = hop.spaceop
+        args = [self.realloc_ptr, op.args[0], op.args[1], c_const_size,
+                c_item_size, c_lengthofs]
+        return hop.genop('direct_call', args, resulttype=llmemory.Address)
+
     def gct_fv_gc_malloc(self, hop, flags, TYPE, c_size):
         # XXX same behavior for zero=True: in theory that's wrong
         if TYPE._is_atomic():
@@ -63,7 +84,6 @@
             hop.genop("boehm_register_finalizer", [v_raw, c_finalizer_ptr])
         return v_raw
 
-
     def gct_fv_gc_malloc_varsize(self, hop, flags, TYPE, v_length, c_const_size, c_item_size,
                                                                    c_offset_to_length):
         # XXX same behavior for zero=True: in theory that's wrong        

Modified: pypy/branch/io-improvements/pypy/rpython/memory/gctransform/framework.py
==============================================================================
--- pypy/branch/io-improvements/pypy/rpython/memory/gctransform/framework.py	(original)
+++ pypy/branch/io-improvements/pypy/rpython/memory/gctransform/framework.py	Mon May  5 10:44:06 2008
@@ -442,7 +442,7 @@
         has_finalizer = bool(self.finalizer_funcptr_for_type(TYPE))
         c_has_finalizer = rmodel.inputconst(lltype.Bool, has_finalizer)
 
-        if not op.opname.endswith('_varsize'):
+        if not op.opname.endswith('_varsize') and not flags.get('varsize'):
             #malloc_ptr = self.malloc_fixedsize_ptr
             zero = flags.get('zero', False)
             if (self.malloc_fast_ptr is not None and
@@ -489,6 +489,9 @@
         hop.genop("direct_call", [self.can_move_ptr, self.c_const_gc, v_addr],
                   resultvar=op.result)
 
+    def _can_realloc(self):
+        return self.gcdata.gc.can_realloc
+
     def gct_gc__disable_finalizers(self, hop):
         # cannot collect()
         op = hop.spaceop

Modified: pypy/branch/io-improvements/pypy/rpython/memory/gctransform/transform.py
==============================================================================
--- pypy/branch/io-improvements/pypy/rpython/memory/gctransform/transform.py	(original)
+++ pypy/branch/io-improvements/pypy/rpython/memory/gctransform/transform.py	Mon May  5 10:44:06 2008
@@ -455,6 +455,15 @@
         return result
     mh.ll_malloc_varsize_no_length_zero = _ll_malloc_varsize_no_length_zero
 
+    def ll_realloc(ptr, length, constsize, itemsize, lengthoffset):
+        size = constsize + length * itemsize
+        result = mh.realloc(ptr, size)
+        if not result:
+            raise MemoryError()
+        (result + lengthoffset).signed[0] = length
+        return result
+    mh.ll_realloc = ll_realloc
+
     return mh
 
 class GCTransformer(BaseGCTransformer):
@@ -522,23 +531,52 @@
 
     gct_malloc_nonmovable = gct_malloc
     gct_malloc_nonmovable_varsize = gct_malloc_varsize
-    
-    gct_malloc_resizable_buffer = gct_malloc_varsize
+
+    def gct_malloc_resizable_buffer(self, hop):
+        flags = hop.spaceop.args[1].value
+        flags['varsize'] = True
+        flavor = flags['flavor']
+        assert flavor != 'cpy', "cannot malloc CPython objects directly"
+        meth = getattr(self, 'gct_fv_%s_malloc_varsize' % flavor, None)
+        assert meth, "%s has no support for malloc_varsize with flavor %r" % (self, flavor) 
+        return self.varsize_malloc_helper(hop, flags, meth, [])
 
     def gct_resize_buffer(self, hop):
+        if self._can_realloc():
+            self._gct_resize_buffer_realloc(hop)
+        else:
+            self._gct_resize_buffer_no_realloc(hop)
+
+    def _can_realloc(self):
+        return False
+
+    def _gct_resize_buffer_realloc(self, hop):
+        def intconst(c): return rmodel.inputconst(lltype.Signed, c)
+        op = hop.spaceop
+        flags = {'flavor':'gc', 'varsize': True}
+        TYPE = op.args[0].concretetype.TO
+        ARRAY = TYPE._flds[TYPE._arrayfld]
+        offset_to_length = llmemory.FieldOffset(TYPE, TYPE._arrayfld) + \
+                           llmemory.ArrayLengthOffset(ARRAY)
+        c_const_size = intconst(llmemory.sizeof(TYPE, 0))
+        c_item_size = intconst(llmemory.sizeof(ARRAY.OF))
+
+        c_lengthofs = intconst(offset_to_length)
+        v_raw = self.perform_realloc(hop, c_const_size, c_item_size,
+                                     c_lengthofs)
+        hop.cast_result(v_raw)
+
+    def _gct_resize_buffer_no_realloc(self, hop):
         op = hop.spaceop
         meth = self.gct_fv_gc_malloc_varsize
-        flags = {'flavor':'gc'}
+        flags = {'flavor':'gc', 'varsize': True}
         self.varsize_malloc_helper(hop, flags, meth, [])
         # fish resvar
         v_newbuf = hop.llops[-1].result
         v_src = op.args[0]
         TYPE = v_src.concretetype.TO
-        v_arr = hop.genop('getfield', [v_src, rmodel.inputconst(lltype.Void,
-                                                            TYPE._arrayfld)],
-                          resulttype=getattr(TYPE, TYPE._arrayfld))
-        v_lgt = hop.genop('getfield', [v_arr, rmodel.inputconst(lltype.Void,
-                                                             'length')],
+        c_fldname = rmodel.inputconst(lltype.Void, TYPE._arrayfld)
+        v_lgt = hop.genop('getinteriorarraysize', [v_src, c_fldname],
                           resulttype=lltype.Signed)
         v_adrsrc = hop.genop('cast_ptr_to_adr', [v_src],
                              resulttype=llmemory.Address)
@@ -546,14 +584,21 @@
                                 resulttype=llmemory.Address)
         ofs = (llmemory.offsetof(TYPE, TYPE._arrayfld) +
                llmemory.itemoffsetof(getattr(TYPE, TYPE._arrayfld), 0))
-        v_ofs = rmodel.inputconst(lltype.Void, ofs)
+        v_ofs = rmodel.inputconst(lltype.Signed, ofs)
         v_adrsrc = hop.genop('adr_add', [v_adrsrc, v_ofs],
                              resulttype=llmemory.Address)
         v_adrnewbuf = hop.genop('adr_add', [v_adrnewbuf, v_ofs],
                                 resulttype=llmemory.Address)
-        vlist = [v_adrsrc, v_adrnewbuf, v_lgt]
+        size = llmemory.sizeof(getattr(TYPE, TYPE._arrayfld).OF)
+        c_size = rmodel.inputconst(lltype.Signed, size)
+        v_lgtsym = hop.genop('int_mul', [c_size, v_lgt],
+                             resulttype=lltype.Signed) 
+        vlist = [v_adrsrc, v_adrnewbuf, v_lgtsym]
         hop.genop('raw_memcopy', vlist)
-        return v_newbuf
+
+    def gct_finish_building_buffer(self, hop):
+        op = hop.spaceop
+        return hop.genop('same_as', op.args, resultvar=op.result)
 
     def varsize_malloc_helper(self, hop, flags, meth, extraargs):
         def intconst(c): return rmodel.inputconst(lltype.Signed, c)

Modified: pypy/branch/io-improvements/pypy/rpython/memory/test/test_transformed_gc.py
==============================================================================
--- pypy/branch/io-improvements/pypy/rpython/memory/test/test_transformed_gc.py	(original)
+++ pypy/branch/io-improvements/pypy/rpython/memory/test/test_transformed_gc.py	Mon May  5 10:44:06 2008
@@ -493,7 +493,6 @@
         assert run([]) == int(self.GC_CAN_MOVE)
 
     def test_resizable_buffer(self):
-        py.test.skip("Does not work")
         from pypy.rpython.lltypesystem.rstr import STR
         from pypy.rpython.annlowlevel import hlstr
         from pypy.rlib import rgc



More information about the Pypy-commit mailing list