[pypy-svn] r51623 - in pypy/branch/no-forw-ptr/pypy/rpython: lltypesystem lltypesystem/test memory
arigo at codespeak.net
arigo at codespeak.net
Tue Feb 19 12:30:15 CET 2008
Author: arigo
Date: Tue Feb 19 12:30:14 2008
New Revision: 51623
Modified:
pypy/branch/no-forw-ptr/pypy/rpython/lltypesystem/llarena.py
pypy/branch/no-forw-ptr/pypy/rpython/lltypesystem/llmemory.py
pypy/branch/no-forw-ptr/pypy/rpython/lltypesystem/lltype.py
pypy/branch/no-forw-ptr/pypy/rpython/lltypesystem/test/test_llarena.py
pypy/branch/no-forw-ptr/pypy/rpython/memory/gcheader.py
Log:
This seems to be the amount of hacking needed to allow an arena to
replace an object with another, and still have the old address work in
order to go to the new object's header. This works only to some extend
-- many operations on the old address still raise RuntimeError as
before, in the hope that it will not allow nasty bugs to go unnoticed.
Modified: pypy/branch/no-forw-ptr/pypy/rpython/lltypesystem/llarena.py
==============================================================================
--- pypy/branch/no-forw-ptr/pypy/rpython/lltypesystem/llarena.py (original)
+++ pypy/branch/no-forw-ptr/pypy/rpython/lltypesystem/llarena.py Tue Feb 19 12:30:14 2008
@@ -1,4 +1,4 @@
-import array
+import array, weakref
from pypy.rpython.lltypesystem import lltype, llmemory
# An "arena" is a large area of memory which can hold a number of
@@ -15,6 +15,7 @@
class Arena(object):
object_arena_location = {} # {container: (arena, offset)}
+ old_object_arena_location = weakref.WeakKeyDictionary()
def __init__(self, nbytes, zero):
self.nbytes = nbytes
@@ -29,7 +30,7 @@
if size is None:
stop = self.nbytes
else:
- stop = start + size
+ stop = start + llmemory.raw_malloc_usage(size)
assert 0 <= start <= stop <= self.nbytes
for offset, ptr in self.objectptrs.items():
size = self.objectsizes[offset]
@@ -79,9 +80,7 @@
addr2 = size._raw_malloc([], zero=zero)
pattern = 'X' + 'x'*(bytes-1)
self.usagemap[offset:offset+bytes] = array.array('c', pattern)
- self.objectptrs[offset] = addr2.ptr
- self.objectsizes[offset] = bytes
- Arena.object_arena_location[addr2.ptr._obj] = self, offset
+ self.setobject(addr2, offset, bytes)
# common case: 'size' starts with a GCHeaderOffset. In this case
# we can also remember that the real object starts after the header.
while isinstance(size, RoundedUpForAllocation):
@@ -91,12 +90,17 @@
objaddr = addr2 + size.offsets[0]
hdrbytes = llmemory.raw_malloc_usage(size.offsets[0])
objoffset = offset + hdrbytes
- assert objoffset not in self.objectptrs
- self.objectptrs[objoffset] = objaddr.ptr
- self.objectsizes[objoffset] = bytes - hdrbytes
- Arena.object_arena_location[objaddr.ptr._obj] = self, objoffset
+ self.setobject(objaddr, objoffset, bytes - hdrbytes)
return addr2
+ def setobject(self, objaddr, offset, bytes):
+ assert offset not in self.objectptrs
+ self.objectptrs[offset] = objaddr.ptr
+ self.objectsizes[offset] = bytes
+ container = objaddr.ptr._obj
+ Arena.object_arena_location[container] = self, offset
+ Arena.old_object_arena_location[container] = self, offset
+
class fakearenaaddress(llmemory.fakeaddress):
def __init__(self, arena, offset):
@@ -205,6 +209,30 @@
return self.arena._getid() + self.offset
+def _getfakearenaaddress(addr):
+ """Logic to handle test_replace_object_with_stub()."""
+ if isinstance(addr, fakearenaaddress):
+ return addr
+ else:
+ assert isinstance(addr, llmemory.fakeaddress)
+ assert addr, "NULL address"
+ # it must be possible to use the address of an already-freed
+ # arena object
+ obj = addr.ptr._getobj(check=False)
+ return _oldobj_to_address(obj)
+
+def _oldobj_to_address(obj):
+ obj = obj._normalizedcontainer(check=False)
+ try:
+ arena, offset = Arena.old_object_arena_location[obj]
+ except KeyError:
+ if obj._was_freed():
+ msg = "taking address of %r, but it was freed"
+ else:
+ msg = "taking address of %r, but it is not in an arena"
+ raise RuntimeError(msg % (obj,))
+ return arena.getaddr(offset)
+
class RoundedUpForAllocation(llmemory.AddressOffset):
"""A size that is rounded up in order to preserve alignment of objects
following it. For arenas containing heterogenous objects.
@@ -247,7 +275,7 @@
"""Free all objects in the arena, which can then be reused.
The arena is filled with zeroes if 'zero' is True. This can also
be used on a subrange of the arena."""
- assert isinstance(arena_addr, fakearenaaddress)
+ arena_addr = _getfakearenaaddress(arena_addr)
arena_addr.arena.reset(zero, arena_addr.offset, size)
def arena_reserve(addr, size, check_alignment=True):
@@ -256,7 +284,7 @@
overlap. The size must be symbolic; in non-translated version
this is used to know what type of lltype object to allocate."""
from pypy.rpython.memory.lltypelayout import memory_alignment
- assert isinstance(addr, fakearenaaddress)
+ addr = _getfakearenaaddress(addr)
if check_alignment and (addr.offset & (memory_alignment-1)) != 0:
raise ArenaError("object at offset %d would not be correctly aligned"
% (addr.offset,))
Modified: pypy/branch/no-forw-ptr/pypy/rpython/lltypesystem/llmemory.py
==============================================================================
--- pypy/branch/no-forw-ptr/pypy/rpython/lltypesystem/llmemory.py (original)
+++ pypy/branch/no-forw-ptr/pypy/rpython/lltypesystem/llmemory.py Tue Feb 19 12:30:14 2008
@@ -339,7 +339,9 @@
# NOTE: the 'ptr' in the addresses must be normalized.
# Use cast_ptr_to_adr() instead of directly fakeaddress() if unsure.
def __init__(self, ptr):
- self.ptr = ptr or None # null ptr => None
+ if ptr is not None and ptr._obj0 is None:
+ ptr = None # null ptr => None
+ self.ptr = ptr
def __repr__(self):
if self.ptr is None:
@@ -374,8 +376,8 @@
def __eq__(self, other):
if isinstance(other, fakeaddress):
- obj1 = self.ptr
- obj2 = other.ptr
+ obj1 = self._fixup().ptr
+ obj2 = other._fixup().ptr
if obj1 is not None: obj1 = obj1._obj
if obj2 is not None: obj2 = obj2._obj
return obj1 == obj2
@@ -412,8 +414,9 @@
return self.ptr
def _cast_to_ptr(self, EXPECTED_TYPE):
- if self:
- return cast_any_ptr(EXPECTED_TYPE, self.ptr)
+ addr = self._fixup()
+ if addr:
+ return cast_any_ptr(EXPECTED_TYPE, addr.ptr)
else:
return lltype.nullptr(EXPECTED_TYPE.TO)
@@ -423,6 +426,14 @@
else:
return 0
+ def _fixup(self):
+ if self.ptr is not None and self.ptr._was_freed():
+ # hack to support llarena.test_replace_object_with_stub()
+ from pypy.rpython.lltypesystem import llarena
+ return llarena._getfakearenaaddress(self)
+ else:
+ return self
+
# ____________________________________________________________
class NullAddressError(Exception):
Modified: pypy/branch/no-forw-ptr/pypy/rpython/lltypesystem/lltype.py
==============================================================================
--- pypy/branch/no-forw-ptr/pypy/rpython/lltypesystem/lltype.py (original)
+++ pypy/branch/no-forw-ptr/pypy/rpython/lltypesystem/lltype.py Tue Feb 19 12:30:14 2008
@@ -968,7 +968,7 @@
self._set_solid(solid)
self._set_obj0(obj0)
- def _getobj(self):
+ def _getobj(self, check=True):
obj = self._obj0
if obj is not None:
if self._weak:
@@ -977,12 +977,17 @@
raise RuntimeError("accessing already garbage collected %r"
% (self._T,))
if isinstance(obj, _container):
- obj._check()
+ if check:
+ obj._check()
elif isinstance(obj, str) and obj.startswith("delayed!"):
raise DelayedPointer
return obj
_obj = property(_getobj)
+ def _was_freed(self):
+ return (self._obj0 is not None and
+ self._getobj(check=False)._was_freed())
+
def __getattr__(self, field_name): # ! can only return basic or ptr !
if isinstance(self._T, Struct):
if field_name in self._T._flds:
@@ -1178,6 +1183,10 @@
from pypy.rpython.lltypesystem import llmemory
if isinstance(self._T, FuncType):
return llmemory.fakeaddress(self)
+ elif self._was_freed():
+ # hack to support llarena.test_replace_object_with_stub()
+ from pypy.rpython.lltypesystem import llarena
+ return llarena._oldobj_to_address(self._getobj(check=False))
elif isinstance(self._obj, _subarray):
return llmemory.fakeaddress(self)
## # return an address built as an offset in the whole array
@@ -1192,8 +1201,8 @@
def _as_ptr(self):
return self
- def _as_obj(self):
- return self._obj
+ def _as_obj(self, check=True):
+ return self._getobj(check=check)
def _expose(self, offset, val):
"""XXX A nice docstring here"""
@@ -1258,13 +1267,13 @@
class _container(object):
__slots__ = ()
- def _parentstructure(self):
+ def _parentstructure(self, check=True):
return None
def _check(self):
pass
def _as_ptr(self):
return _ptr(Ptr(self._TYPE), self, True)
- def _as_obj(self):
+ def _as_obj(self, check=True):
return self
def _normalizedcontainer(self):
return self
@@ -1295,7 +1304,16 @@
self._storage = None
def _was_freed(self):
- return self._storage is None
+ if self._storage is None:
+ return True
+ if self._wrparent is None:
+ return False
+ parent = self._wrparent()
+ if parent is None:
+ raise RuntimeError("accessing sub%s %r,\n"
+ "but already garbage collected parent %r"
+ % (self._kind, self, self._parent_type))
+ return parent._was_freed()
def _setparentstructure(self, parent, parentindex):
self._wrparent = weakref.ref(parent)
@@ -1307,14 +1325,15 @@
# keep strong reference to parent, we share the same allocation
self._keepparent = parent
- def _parentstructure(self):
+ def _parentstructure(self, check=True):
if self._wrparent is not None:
parent = self._wrparent()
if parent is None:
raise RuntimeError("accessing sub%s %r,\n"
"but already garbage collected parent %r"
% (self._kind, self, self._parent_type))
- parent._check()
+ if check:
+ parent._check()
return parent
return None
@@ -1323,14 +1342,15 @@
raise RuntimeError("accessing freed %r" % self._TYPE)
self._parentstructure()
- def _normalizedcontainer(self):
+ def _normalizedcontainer(self, check=True):
# if we are the first inlined substructure of a structure,
# return the whole (larger) structure instead
container = self
while True:
- parent, index = parentlink(container)
+ parent = container._parentstructure(check=check)
if parent is None:
break
+ index = container._parent_index
T = typeOf(parent)
if (not isinstance(T, Struct) or T._first_struct()[0] != index
or isinstance(T, FixedSizeArray)):
@@ -1525,7 +1545,7 @@
def __repr__(self):
return '<_subarray at %r in %r>' % (self._parent_index,
- self._parentstructure())
+ self._parentstructure(check=False))
def getlength(self):
assert isinstance(self._TYPE, FixedSizeArray)
Modified: pypy/branch/no-forw-ptr/pypy/rpython/lltypesystem/test/test_llarena.py
==============================================================================
--- pypy/branch/no-forw-ptr/pypy/rpython/lltypesystem/test/test_llarena.py (original)
+++ pypy/branch/no-forw-ptr/pypy/rpython/lltypesystem/test/test_llarena.py Tue Feb 19 12:30:14 2008
@@ -165,6 +165,8 @@
plist.append(llmemory.cast_adr_to_ptr(b, SPTR))
# clear blist[1] and blist[2] but not blist[0] nor blist[3]
arena_reset(blist[1], llmemory.raw_malloc_usage(precomputed_size)*2, False)
+ py.test.raises(RuntimeError, "plist[1].x") # marked as freed
+ py.test.raises(RuntimeError, "plist[2].x") # marked as freed
# re-reserve object at index 1 and 2
blist[1] = reserve(1)
blist[2] = reserve(2)
@@ -173,6 +175,10 @@
assert plist[3].x == 103
py.test.raises(RuntimeError, "plist[1].x") # marked as freed
py.test.raises(RuntimeError, "plist[2].x") # marked as freed
+ # but we can still cast the old ptrs to addresses, which compare equal
+ # to the new ones we gotq
+ assert llmemory.cast_ptr_to_adr(plist[1]) == blist[1]
+ assert llmemory.cast_ptr_to_adr(plist[2]) == blist[2]
# check via addresses
assert (blist[0] + llmemory.offsetof(SX, 'x')).signed[0] == 100
assert (blist[3] + llmemory.offsetof(SX, 'x')).signed[0] == 103
@@ -204,6 +210,45 @@
assert llmemory.cast_adr_to_int(a) == llmemory.cast_adr_to_int(a1)
assert llmemory.cast_adr_to_int(a+1) == llmemory.cast_adr_to_int(a1) + 1
+def test_replace_object_with_stub():
+ from pypy.rpython.memory.gcheader import GCHeaderBuilder
+ HDR = lltype.Struct('HDR', ('x', lltype.Signed))
+ S = lltype.GcStruct('S', ('y', lltype.Signed), ('z', lltype.Signed))
+ STUB = lltype.GcStruct('STUB', ('t', lltype.Char))
+ gcheaderbuilder = GCHeaderBuilder(HDR)
+ size_gc_header = gcheaderbuilder.size_gc_header
+
+ a = arena_malloc(50, True)
+ hdraddr = a + 12
+ arena_reserve(hdraddr, size_gc_header + llmemory.sizeof(S))
+ hdr = llmemory.cast_adr_to_ptr(hdraddr, lltype.Ptr(HDR))
+ hdr.x = 42
+ obj = llmemory.cast_adr_to_ptr(hdraddr + size_gc_header, lltype.Ptr(S))
+ obj.y = -5
+ obj.z = -6
+
+ hdraddr = llmemory.cast_ptr_to_adr(obj) - size_gc_header
+ arena_reset(hdraddr, size_gc_header + llmemory.sizeof(S), False)
+ arena_reserve(hdraddr, size_gc_header + llmemory.sizeof(STUB))
+
+ # check that it possible to reach the newly reserved HDR+STUB
+ # via the header of the old 'obj' pointer, both via the existing
+ # 'hdraddr':
+ hdr = llmemory.cast_adr_to_ptr(hdraddr, lltype.Ptr(HDR))
+ hdr.x = 46
+ stub = llmemory.cast_adr_to_ptr(hdraddr + size_gc_header, lltype.Ptr(STUB))
+ stub.t = '!'
+
+ # and via a (now-invalid) pointer to the old 'obj': (this is needed
+ # because during a garbage collection there are still pointers to
+ # the old 'obj' around to be fixed)
+ hdraddr = llmemory.cast_ptr_to_adr(obj) - size_gc_header
+ hdr = llmemory.cast_adr_to_ptr(hdraddr, lltype.Ptr(HDR))
+ assert hdr.x == 46
+ stub = llmemory.cast_adr_to_ptr(hdraddr + size_gc_header,
+ lltype.Ptr(STUB))
+ assert stub.t == '!'
+
def test_llinterpreted():
from pypy.rpython.test.test_llinterp import interpret
Modified: pypy/branch/no-forw-ptr/pypy/rpython/memory/gcheader.py
==============================================================================
--- pypy/branch/no-forw-ptr/pypy/rpython/memory/gcheader.py (original)
+++ pypy/branch/no-forw-ptr/pypy/rpython/memory/gcheader.py Tue Feb 19 12:30:14 2008
@@ -16,17 +16,17 @@
def header_of_object(self, gcptr):
# XXX hackhackhack
- gcptr = gcptr._as_obj()
+ gcptr = gcptr._as_obj(check=False)
if isinstance(gcptr, llmemory._gctransformed_wref):
- return self.obj2header[gcptr._ptr._as_obj()]
+ return self.obj2header[gcptr._ptr._as_obj(check=False)]
return self.obj2header[gcptr]
def object_from_header(headerptr):
- return header2obj[headerptr._as_obj()]
+ return header2obj[headerptr._as_obj(check=False)]
object_from_header = staticmethod(object_from_header)
def get_header(self, gcptr):
- return self.obj2header.get(gcptr._as_obj(), None)
+ return self.obj2header.get(gcptr._as_obj(check=False), None)
def attach_header(self, gcptr, headerptr):
gcobj = gcptr._as_obj()
More information about the Pypy-commit
mailing list