[pypy-svn] r32351 - in pypy/branch/kill-keepalives/pypy: annotation rpython rpython/lltypesystem rpython/memory rpython/rctypes
arigo at codespeak.net
arigo at codespeak.net
Fri Sep 15 12:44:07 CEST 2006
Author: arigo
Date: Fri Sep 15 12:44:04 2006
New Revision: 32351
Modified:
pypy/branch/kill-keepalives/pypy/annotation/builtin.py
pypy/branch/kill-keepalives/pypy/rpython/lltypesystem/llmemory.py
pypy/branch/kill-keepalives/pypy/rpython/memory/gctransform.py
pypy/branch/kill-keepalives/pypy/rpython/rbuiltin.py
pypy/branch/kill-keepalives/pypy/rpython/rctypes/astringbuf.py
pypy/branch/kill-keepalives/pypy/rpython/rctypes/rchar_p.py
pypy/branch/kill-keepalives/pypy/rpython/rctypes/rmodel.py
Log:
Intermediate check-in:
* a more complete llmemory.raw_memclear()
* finally allow ll helpers to call the llmemory.raw_*()
interface instead of the lladdress.raw_*() one
* progress on rctypes in the direction of raw_malloc/raw_free
of the C data, instead of keeping it inside the GcStruct boxes.
Modified: pypy/branch/kill-keepalives/pypy/annotation/builtin.py
==============================================================================
--- pypy/branch/kill-keepalives/pypy/annotation/builtin.py (original)
+++ pypy/branch/kill-keepalives/pypy/annotation/builtin.py Fri Sep 15 12:44:04 2006
@@ -556,6 +556,7 @@
# memory address
from pypy.rpython.memory import lladdress
+from pypy.rpython.lltypesystem import llmemory
def raw_malloc(s_size):
assert isinstance(s_size, SomeInteger) #XXX add noneg...?
@@ -587,11 +588,15 @@
BUILTIN_ANALYZERS[lladdress.raw_memclear] = raw_memclear
BUILTIN_ANALYZERS[lladdress.raw_memcopy] = raw_memcopy
+BUILTIN_ANALYZERS[llmemory.raw_malloc] = raw_malloc
+BUILTIN_ANALYZERS[llmemory.raw_malloc_usage] = raw_malloc_usage
+BUILTIN_ANALYZERS[llmemory.raw_free] = raw_free
+BUILTIN_ANALYZERS[llmemory.raw_memclear] = raw_memclear
+BUILTIN_ANALYZERS[llmemory.raw_memcopy] = raw_memcopy
+
#_________________________________
# offsetof/sizeof
-from pypy.rpython.lltypesystem import llmemory
-
def offsetof(TYPE, fldname):
return SomeInteger()
Modified: pypy/branch/kill-keepalives/pypy/rpython/lltypesystem/llmemory.py
==============================================================================
--- pypy/branch/kill-keepalives/pypy/rpython/lltypesystem/llmemory.py (original)
+++ pypy/branch/kill-keepalives/pypy/rpython/lltypesystem/llmemory.py Fri Sep 15 12:44:04 2006
@@ -21,7 +21,7 @@
return NotImplemented
return CompositeOffset(self, other)
- def raw_malloc(self, rest):
+ def raw_malloc(self, rest, zero=False):
raise NotImplementedError("raw_malloc(%r, %r)" % (self, rest))
@@ -51,16 +51,16 @@
index = firstitemref.index + self.repeat
return _arrayitemref(array, index)
- def raw_malloc(self, rest):
+ def raw_malloc(self, rest, zero=False):
assert not rest
if (isinstance(self.TYPE, lltype.ContainerType)
and self.TYPE._gckind == 'gc'):
assert self.repeat == 1
- p = lltype.malloc(self.TYPE, flavor='raw')
+ p = lltype.malloc(self.TYPE, flavor='raw', zero=zero)
return cast_ptr_to_adr(p)
else:
T = lltype.FixedSizeArray(self.TYPE, self.repeat)
- p = lltype.malloc(T, flavor='raw')
+ p = lltype.malloc(T, flavor='raw', zero=zero)
array_adr = cast_ptr_to_adr(p)
return array_adr + ArrayItemsOffset(T)
@@ -80,11 +80,13 @@
struct = lltype.cast_pointer(lltype.Ptr(self.TYPE), struct)
return _structfieldref(struct, self.fldname)
- def raw_malloc(self, rest, parenttype=None):
+ def raw_malloc(self, rest, parenttype=None, zero=False):
if self.fldname != self.TYPE._arrayfld:
- return AddressOffset.raw_malloc(self, rest) # for the error msg
+ return AddressOffset.raw_malloc(self, rest, # for the error msg
+ zero=zero)
assert rest
- return rest[0].raw_malloc(rest[1:], parenttype=parenttype or self.TYPE)
+ return rest[0].raw_malloc(rest[1:], parenttype=parenttype or self.TYPE,
+ zero=zero)
class CompositeOffset(AddressOffset):
@@ -122,8 +124,8 @@
ref = item.ref(ref)
return ref
- def raw_malloc(self, rest):
- return self.offsets[0].raw_malloc(self.offsets[1:] + rest)
+ def raw_malloc(self, rest, zero=False):
+ return self.offsets[0].raw_malloc(self.offsets[1:] + rest, zero=zero)
class ArrayItemsOffset(AddressOffset):
@@ -139,7 +141,7 @@
assert lltype.rawTypeOf(array).TO == self.TYPE
return _arrayitemref(array, index=0)
- def raw_malloc(self, rest, parenttype=None):
+ def raw_malloc(self, rest, parenttype=None, zero=False):
if rest:
assert len(rest) == 1
assert isinstance(rest[0], ItemOffset)
@@ -149,7 +151,7 @@
count = 0
if self.TYPE._hints.get('isrpystring'):
count -= 1 # because malloc() will give us the extra char for free
- p = lltype.malloc(parenttype or self.TYPE, count,
+ p = lltype.malloc(parenttype or self.TYPE, count, zero=zero,
immortal = self.TYPE._gckind == 'raw')
return cast_ptr_to_adr(p)
@@ -183,11 +185,11 @@
gcptr = self.gcheaderbuilder.object_from_header(header)
return _obref(gcptr)
- def raw_malloc(self, rest):
+ def raw_malloc(self, rest, zero=False):
assert rest
if isinstance(rest[0], GCHeaderAntiOffset):
- return rest[1].raw_malloc(rest[2:]) # just for fun
- gcobjadr = rest[0].raw_malloc(rest[1:])
+ return rest[1].raw_malloc(rest[2:], zero=zero) # just for fun
+ gcobjadr = rest[0].raw_malloc(rest[1:], zero=zero)
headerptr = self.gcheaderbuilder.new_header(gcobjadr.get())
return cast_ptr_to_adr(headerptr)
@@ -207,10 +209,10 @@
headerptr = self.gcheaderbuilder.header_of_object(gcptr)
return _obref(headerptr)
- def raw_malloc(self, rest):
+ def raw_malloc(self, rest, zero=False):
assert len(rest) >= 2
assert isinstance(rest[0], GCHeaderOffset)
- return rest[1].raw_malloc(rest[2:])
+ return rest[1].raw_malloc(rest[2:], zero=zero)
class _arrayitemref(object):
@@ -560,15 +562,11 @@
return size
def raw_memclear(adr, size):
- # hack hack hack
- # stab stab stab
- assert (adr.offset is None or
- (isinstance(adr.offset, ArrayItemsOffset)
- and isinstance(lltype.typeOf(adr.ob).TO, lltype.FixedSizeArray)))
- TYPE = lltype.typeOf(adr.ob)
- fresh = lltype.malloc(TYPE.TO, zero=True, flavor='raw')
+ if not isinstance(size, AddressOffset):
+ raise NotImplementedError(size)
+ fresh = size.raw_malloc([], zero=True)
from pypy.rpython.rctypes.rmodel import reccopy
- reccopy(fresh, adr.ob)
+ reccopy(fresh.get(), adr.get())
def raw_memcopy(source, dest, size):
source = source.get()
@@ -603,8 +601,9 @@
self.unitsize = unitsize
self.n = n
- def raw_malloc(self, rest):
+ def raw_malloc(self, rest, zero=False):
assert not rest
+ assert not zero # for now
return fakeaddress(_arena(self), ArenaItem(0))
def arena(TYPE, n):
Modified: pypy/branch/kill-keepalives/pypy/rpython/memory/gctransform.py
==============================================================================
--- pypy/branch/kill-keepalives/pypy/rpython/memory/gctransform.py (original)
+++ pypy/branch/kill-keepalives/pypy/rpython/memory/gctransform.py Fri Sep 15 12:44:04 2006
@@ -1018,6 +1018,7 @@
if ARRAY.OF != lltype.Void:
info["ofstovar"] = ofs1 + llmemory.itemoffsetof(ARRAY, 0)
else:
+ # XXX ?
info["fixedsize"] = ofs1 + llmemory.sizeof(lltype.Signed)
if ARRAY._hints.get('isrpystring'):
info["fixedsize"] = llmemory.sizeof(TYPE, 1)
@@ -1034,6 +1035,7 @@
info["varofstoptrs"] = self.offsets2table(offsets, ARRAY.OF)
info["varitemsize"] = llmemory.sizeof(ARRAY.OF)
else:
+ # XXX ?
info["varofstoptrs"] = self.offsets2table((), lltype.Void)
info["varitemsize"] = llmemory.sizeof(ARRAY.OF)
return type_id
Modified: pypy/branch/kill-keepalives/pypy/rpython/rbuiltin.py
==============================================================================
--- pypy/branch/kill-keepalives/pypy/rpython/rbuiltin.py (original)
+++ pypy/branch/kill-keepalives/pypy/rpython/rbuiltin.py Fri Sep 15 12:44:04 2006
@@ -577,6 +577,12 @@
BUILTIN_TYPER[lladdress.raw_memclear] = rtype_raw_memclear
BUILTIN_TYPER[lladdress.raw_memcopy] = rtype_raw_memcopy
+BUILTIN_TYPER[llmemory.raw_malloc] = rtype_raw_malloc
+BUILTIN_TYPER[llmemory.raw_malloc_usage] = rtype_raw_malloc_usage
+BUILTIN_TYPER[llmemory.raw_free] = rtype_raw_free
+BUILTIN_TYPER[llmemory.raw_memclear] = rtype_raw_memclear
+BUILTIN_TYPER[llmemory.raw_memcopy] = rtype_raw_memcopy
+
def rtype_offsetof(hop):
TYPE, field = hop.inputargs(lltype.Void, lltype.Void)
return hop.inputconst(lltype.Signed,
Modified: pypy/branch/kill-keepalives/pypy/rpython/rctypes/astringbuf.py
==============================================================================
--- pypy/branch/kill-keepalives/pypy/rpython/rctypes/astringbuf.py (original)
+++ pypy/branch/kill-keepalives/pypy/rpython/rctypes/astringbuf.py Fri Sep 15 12:44:04 2006
@@ -34,11 +34,7 @@
[v_length] = hop.inputargs(lltype.Signed)
r_stringbuf = hop.r_result
hop.exception_cannot_occur()
- return hop.genop("zero_malloc_varsize", [
- hop.inputconst(lltype.Void, r_stringbuf.lowleveltype.TO),
- v_length,
- ], resulttype=r_stringbuf.lowleveltype,
- )
+ return r_stringbuf.allocate_instance_varsize(hop.llops, v_length)
class ObjEntry(CTypesObjEntry):
Modified: pypy/branch/kill-keepalives/pypy/rpython/rctypes/rchar_p.py
==============================================================================
--- pypy/branch/kill-keepalives/pypy/rpython/rctypes/rchar_p.py (original)
+++ pypy/branch/kill-keepalives/pypy/rpython/rctypes/rchar_p.py Fri Sep 15 12:44:04 2006
@@ -1,5 +1,5 @@
from pypy.rpython.rmodel import inputconst
-from pypy.rpython.lltypesystem import lltype
+from pypy.rpython.lltypesystem import lltype, llmemory
from pypy.rpython.rstr import AbstractStringRepr
from pypy.rpython.lltypesystem.rstr import string_repr
from pypy.rpython.rctypes.rmodel import CTypesValueRepr, C_ZERO
@@ -12,6 +12,8 @@
class CCharPRepr(CTypesValueRepr):
+ autofree_fields = ("keepalive",)
+
def return_c_data(self, llops, v_c_data):
"""Read out the RPython string from a raw C pointer.
Used when the data is returned from an operation or C function call.
@@ -25,8 +27,8 @@
return llops.gendirectcall(ll_charp2str, v_value)
def get_content_keepalive_type(self):
- "An extra keepalive used for the RPython string."
- return string_repr.lowleveltype
+ "An extra keepalive used for the raw copy the RPython string."
+ return lltype.Ptr(CHARSCOPY) # raw_malloc'ed, automatically raw_free'd
def getstring(self, llops, v_box):
return llops.gendirectcall(ll_getstring, v_box)
@@ -111,9 +113,6 @@
i += 1
return i
-def ll_str2charp(s):
- return lltype.direct_arrayitems(s.chars)
-
def ll_charp2str(p):
if not p:
return lltype.nullptr(string_repr.lowleveltype.TO)
@@ -127,14 +126,7 @@
def ll_getstring(box):
p = box.c_data[0]
if p:
- if box.keepalive and ll_str2charp(box.keepalive) == p:
- maxlen = len(box.keepalive.chars)
- length = ll_strnlen(p, maxlen)
- if length == maxlen:
- # no embedded zero in the string
- return box.keepalive
- else:
- length = ll_strlen(p)
+ length = ll_strlen(p)
newstr = lltype.malloc(string_repr.lowleveltype.TO, length)
newstr.hash = 0
for i in range(length):
@@ -144,8 +136,24 @@
return lltype.nullptr(string_repr.lowleveltype.TO)
def ll_setstring(box, string):
- if string:
- box.c_data[0] = ll_str2charp(string)
- else:
- box.c_data[0] = lltype.nullptr(CCHARP.TO)
- box.keepalive = string
+ # XXX the copying of the string is often not needed, but it is
+ # very hard to avoid in the general case of a moving GC that can
+ # move 'string' around unpredictably.
+ n = len(string.chars)
+ buffer = llmemory.raw_malloc(NULL_HEADER + SIZEOF_CHAR * (n+1))
+ charscopy = llmemory.cast_adr_to_ptr(buffer, PCHARSCOPY)
+ for i in range(n):
+ charscopy[i] = string.chars[i]
+ charscopy[n] = '\x00'
+ # store a 'char *' pointer in the box.c_data
+ box.c_data[0] = lltype.direct_arrayitems(charscopy)
+ # keep the ownership of the buffer in the box.keepalive field
+ prev = box.keepalive
+ box.keepalive = charscopy
+ if prev:
+ llmemory.raw_free(llmemory.cast_ptr_to_adr(prev))
+
+CHARSCOPY = lltype.Array(lltype.Char, hints={'nolength': True})
+PCHARSCOPY = lltype.Ptr(CHARSCOPY)
+NULL_HEADER = llmemory.itemoffsetof(CHARSCOPY)
+SIZEOF_CHAR = llmemory.sizeof(lltype.Char)
Modified: pypy/branch/kill-keepalives/pypy/rpython/rctypes/rmodel.py
==============================================================================
--- pypy/branch/kill-keepalives/pypy/rpython/rctypes/rmodel.py (original)
+++ pypy/branch/kill-keepalives/pypy/rpython/rctypes/rmodel.py Fri Sep 15 12:44:04 2006
@@ -29,6 +29,8 @@
# * 'r_memoryowner.lowleveltype' is the lowleveltype of the repr for the
# same ctype but for ownsmemory=True.
+ autofree_fields = ()
+
def __init__(self, rtyper, s_ctypesobject, ll_type):
# s_ctypesobject: the annotation to represent
# ll_type: the low-level type representing the raw
@@ -42,33 +44,58 @@
self.c_data_type = self.get_c_data_type(ll_type)
- fields = []
+ fields = [("c_data", lltype.Ptr(self.c_data_type))]
content_keepalive_type = self.get_content_keepalive_type()
if content_keepalive_type:
fields.append(( "keepalive", content_keepalive_type ))
if self.ownsmemory:
+ self.autofree_fields = ("c_data",) + self.__class__.autofree_fields
self.r_memoryowner = self
- fields.append(( "c_data", self.c_data_type ))
else:
s_memoryowner = SomeCTypesObject(ctype, ownsmemory=True)
self.r_memoryowner = rtyper.getrepr(s_memoryowner)
- fields += [
- ( "c_data_owner_keepalive", self.r_memoryowner.lowleveltype ),
- ( "c_data", lltype.Ptr(self.c_data_type) ),
- ]
+ fields.append(
+ ( "c_data_owner_keepalive", self.r_memoryowner.lowleveltype ))
+
+ keywords = {'hints': {'autofree_fields': self.autofree_fields}}
self.lowleveltype = lltype.Ptr(
lltype.GcStruct( "CtypesBox_%s" % (ctype.__name__,),
- *fields
+ *fields,
+ **keywords
)
)
+ if self.autofree_fields:
+ lltype.attachRuntimeTypeInfo(self.lowleveltype.TO)
self.const_cache = {} # store generated const values+original value
+ def _setup_repr_final(self):
+ if self.autofree_fields:
+ # XXX should be done by the gctransform from the GcStruct hint
+ from pypy.annotation import model as annmodel
+ from pypy.rpython.unroll import unrolling_iterable
+ autofree_fields = unrolling_iterable(self.autofree_fields)
+
+ def ll_rctypes_free(box):
+ for fieldname in autofree_fields:
+ p = getattr(box, fieldname)
+ if p:
+ llmemory.raw_free(llmemory.cast_ptr_to_adr(p))
+
+ args_s = [annmodel.SomePtr(self.lowleveltype)]
+ graph = self.rtyper.annotate_helper(ll_rctypes_free, args_s)
+ destrptr = self.rtyper.getcallable(graph)
+ lltype.attachRuntimeTypeInfo(self.lowleveltype.TO,
+ destrptr = destrptr)
+
def get_content_keepalive_type(self):
"""Return the type of the extra keepalive field used for the content
of this object."""
return None
+ def get_extra_autofree_fields(self):
+ return []
+
def ctypecheck(self, value):
return isinstance(value, self.ctype)
@@ -86,28 +113,29 @@
return self.const_cache[key][0]
except KeyError:
self.setup()
- p = lltype.malloc(self.r_memoryowner.lowleveltype.TO, zero=True)
+ p = lltype.malloc(self.r_memoryowner.lowleveltype.TO,
+ immortal = True)
+ p.c_data = lltype.malloc(self.r_memoryowner.c_data_type,
+ immortal = True,
+ zero = True)
self.initialize_const(p, value)
if self.ownsmemory:
result = p
else:
# we must return a non-memory-owning box that keeps the
# memory-owning box alive
- result = lltype.malloc(self.lowleveltype.TO, zero=True)
+ result = lltype.malloc(self.lowleveltype.TO,
+ immortal = True,
+ zero = True)
result.c_data = p.c_data # initialize c_data pointer
result.c_data_owner_keepalive = p
self.const_cache[key] = result, keepalive
return result
def get_c_data(self, llops, v_box):
- if self.ownsmemory:
- inputargs = [v_box, inputconst(lltype.Void, "c_data")]
- return llops.genop('getsubstruct', inputargs,
- lltype.Ptr(self.c_data_type) )
- else:
- inputargs = [v_box, inputconst(lltype.Void, "c_data")]
- return llops.genop('getfield', inputargs,
- lltype.Ptr(self.c_data_type) )
+ inputargs = [v_box, inputconst(lltype.Void, "c_data")]
+ return llops.genop('getfield', inputargs,
+ lltype.Ptr(self.c_data_type))
def get_c_data_owner(self, llops, v_box):
if self.ownsmemory:
@@ -120,18 +148,55 @@
def allocate_instance(self, llops):
TYPE = self.lowleveltype.TO
+ c1 = inputconst(lltype.Void, TYPE)
+ v_box = llops.genop("zero_malloc", [c1], resulttype=self.lowleveltype)
+ # XXX don't use zero_malloc, but just make sure there is a NULL
+ # in the autofreed field(s)
+
+ TYPE = self.c_data_type
if TYPE._is_varsize():
raise TyperError("allocating array with unknown length")
- c1 = inputconst(lltype.Void, TYPE)
- return llops.genop("zero_malloc", [c1], resulttype=self.lowleveltype)
+ if self.ownsmemory:
+ # XXX use zero=True instead, and malloc instead of raw_malloc?
+ c_size = inputconst(lltype.Signed, llmemory.sizeof(TYPE))
+ v_rawaddr = llops.genop("raw_malloc", [c_size],
+ resulttype=llmemory.Address)
+ llops.genop("raw_memclear", [v_rawaddr, c_size])
+ v_rawdata = llops.genop("cast_adr_to_ptr", [v_rawaddr],
+ resulttype=lltype.Ptr(TYPE))
+ c_datafieldname = inputconst(lltype.Void, "c_data")
+ llops.genop("setfield", [v_box, c_datafieldname, v_rawdata])
+ return v_box
def allocate_instance_varsize(self, llops, v_length):
TYPE = self.lowleveltype.TO
- if not TYPE._is_varsize():
- raise TyperError("allocating non-array with a specified length")
c1 = inputconst(lltype.Void, TYPE)
- return llops.genop("zero_malloc_varsize", [c1, v_length],
- resulttype=self.lowleveltype)
+ v_box = llops.genop("zero_malloc", [c1], resulttype=self.lowleveltype)
+ # XXX don't use zero_malloc, but just make sure there is a NULL
+ # in the autofreed field(s)
+
+ TYPE = self.c_data_type
+ if not TYPE._is_varsize():
+ raise TyperError("allocating non-array with specified length")
+ if self.ownsmemory:
+ # XXX use zero=True instead, and malloc instead of raw_malloc?
+ assert isinstance(TYPE, lltype.Array)
+ c_fixedsize = inputconst(lltype.Signed, llmemory.sizeof(TYPE, 0))
+ c_itemsize = inputconst(lltype.Signed, llmemory.sizeof(TYPE.OF))
+ v_varsize = llops.genop("int_mul", [c_itemsize, v_length],
+ resulttype=lltype.Signed)
+ v_size = llops.genop("int_add", [c_fixedsize, v_varsize],
+ resulttype=lltype.Signed)
+ v_rawaddr = llops.genop("raw_malloc", [v_size],
+ resulttype=llmemory.Address)
+ llops.genop("raw_memclear", [v_rawaddr, v_size])
+ v_rawdata = llops.genop("cast_adr_to_ptr", [v_rawaddr],
+ resulttype=lltype.Ptr(TYPE))
+ c_datafieldname = inputconst(lltype.Void, "c_data")
+ llops.genop("setfield", [v_box, c_datafieldname, v_rawdata])
+ else:
+ raise TyperError("allocate_instance_varsize on an alias box")
+ return v_box
def allocate_instance_ref(self, llops, v_c_data, v_c_data_owner=None):
"""Only if self.ownsmemory is false. This allocates a new instance
More information about the Pypy-commit
mailing list