[pypy-commit] pypy missing-tp_new: merge default into branch
mattip
pypy.commits at gmail.com
Fri Jan 6 07:07:22 EST 2017
Author: Matti Picus <matti.picus at gmail.com>
Branch: missing-tp_new
Changeset: r89388:ed458f15fed3
Date: 2017-01-05 07:19 +0200
http://bitbucket.org/pypy/pypy/changeset/ed458f15fed3/
Log: merge default into branch
diff too long, truncating to 2000 out of 2328 lines
diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -28,7 +28,7 @@
DEALINGS IN THE SOFTWARE.
-PyPy Copyright holders 2003-2016
+PyPy Copyright holders 2003-2017
-----------------------------------
Except when otherwise stated (look for LICENSE files or information at
diff --git a/lib-python/2.7/ctypes/test/test_frombuffer.py b/lib-python/2.7/ctypes/test/test_frombuffer.py
--- a/lib-python/2.7/ctypes/test/test_frombuffer.py
+++ b/lib-python/2.7/ctypes/test/test_frombuffer.py
@@ -32,7 +32,7 @@
del a; gc.collect(); gc.collect(); gc.collect()
self.assertEqual(x[:], expected)
- self.assertRaises((TypeError, ValueError),
+ self.assertRaises(TypeError,
(c_char * 16).from_buffer, "a" * 16)
def test_fom_buffer_with_offset(self):
diff --git a/lib_pypy/_ctypes/basics.py b/lib_pypy/_ctypes/basics.py
--- a/lib_pypy/_ctypes/basics.py
+++ b/lib_pypy/_ctypes/basics.py
@@ -85,7 +85,17 @@
def from_buffer(self, obj, offset=0):
size = self._sizeofinstances()
+ if isinstance(obj, (str, unicode)):
+ # hack, buffer(str) will always return a readonly buffer.
+ # CPython calls PyObject_AsWriteBuffer(...) here!
+ # str cannot be modified, thus raise a type error in this case
+ raise TypeError("Cannot use %s as modifiable buffer" % str(type(obj)))
+
+ # why not just call memoryview(obj)[offset:]?
+ # array in Python 2.7 does not support the buffer protocol and will
+ # fail, even though buffer is supported
buf = buffer(obj, offset, size)
+
if len(buf) < size:
raise ValueError(
"Buffer size too small (%d instead of at least %d bytes)"
diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -428,11 +428,6 @@
``datetime.date`` is the superclass of ``datetime.datetime``).
Anyway, the proper fix is arguably to use a regular method call in
the first place: ``datetime.date.today().strftime(...)``
-
-* the ``__dict__`` attribute of new-style classes returns a normal dict, as
- opposed to a dict proxy like in CPython. Mutating the dict will change the
- type and vice versa. For builtin types, a dictionary will be returned that
- cannot be changed (but still looks and behaves like a normal dictionary).
* some functions and attributes of the ``gc`` module behave in a
slightly different way: for example, ``gc.enable`` and
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -76,3 +76,14 @@
PyMemoryViewObject with a PyBuffer attached so that the call to
``PyMemoryView_GET_BUFFER`` does not leak a PyBuffer-sized piece of memory.
Properly call ``bf_releasebuffer`` when not ``NULL``.
+
+.. branch: boehm-rawrefcount
+
+Support translations of cpyext with the Boehm GC (for special cases like
+revdb).
+
+.. branch: strbuf-as-buffer
+
+Implement StringBuffer.get_raw_address (missing feature for the buffer protocol).
+More generally it is now possible to obtain the address of any object (if it
+is readonly) without pinning it.
diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -305,9 +305,9 @@
config.objspace.lonepycfiles = False
if config.objspace.usemodules.cpyext:
- if config.translation.gc != 'incminimark':
+ if config.translation.gc not in ('incminimark', 'boehm'):
raise Exception("The 'cpyext' module requires the 'incminimark'"
- " GC. You need either 'targetpypystandalone.py"
+ " 'boehm' GC. You need either 'targetpypystandalone.py"
" --withoutmod-cpyext' or '--gc=incminimark'")
config.translating = True
diff --git a/pypy/module/__pypy__/bytebuffer.py b/pypy/module/__pypy__/bytebuffer.py
--- a/pypy/module/__pypy__/bytebuffer.py
+++ b/pypy/module/__pypy__/bytebuffer.py
@@ -4,6 +4,7 @@
from rpython.rlib.buffer import Buffer
from pypy.interpreter.gateway import unwrap_spec
+from rpython.rlib.rgc import nonmoving_raw_ptr_for_resizable_list
class ByteBuffer(Buffer):
@@ -22,6 +23,8 @@
def setitem(self, index, char):
self.data[index] = char
+ def get_raw_address(self):
+ return nonmoving_raw_ptr_for_resizable_list(self.data)
@unwrap_spec(length=int)
def bytebuffer(space, length):
diff --git a/pypy/module/_cffi_backend/func.py b/pypy/module/_cffi_backend/func.py
--- a/pypy/module/_cffi_backend/func.py
+++ b/pypy/module/_cffi_backend/func.py
@@ -136,6 +136,9 @@
return _from_buffer(space, w_ctype, w_x)
def _from_buffer(space, w_ctype, w_x):
+ if space.isinstance_w(w_x, space.w_unicode):
+ raise oefmt(space.w_TypeError,
+ "from_buffer() cannot return the address a unicode")
buf = _fetch_as_read_buffer(space, w_x)
if space.isinstance_w(w_x, space.w_str):
_cdata = get_raw_address_of_string(space, w_x)
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -3419,24 +3419,28 @@
BCharA = new_array_type(BCharP, None)
p1 = from_buffer(BCharA, b"foo")
assert p1 == from_buffer(BCharA, b"foo")
- import gc; gc.collect()
- assert p1 == from_buffer(BCharA, b"foo")
py.test.raises(TypeError, from_buffer, BCharA, u+"foo")
try:
from __builtin__ import buffer
except ImportError:
pass
else:
- # from_buffer(buffer(b"foo")) does not work, because it's not
- # implemented on pypy; only from_buffer(b"foo") works.
- py.test.raises(TypeError, from_buffer, BCharA, buffer(b"foo"))
- py.test.raises(TypeError, from_buffer, BCharA, buffer(u+"foo"))
+ contents = from_buffer(BCharA, buffer(b"foo"))
+ for i in range(len(contents)):
+ assert contents[i] == p1[i]
+ p4 = from_buffer(BCharA, b"f\x00\x00\x00o\x00\x00\x00o\x00\x00\x00")
+ contents = from_buffer(BCharA, buffer(u+"foo"))
+ for i in range(len(contents)):
+ assert contents[i] == p4[i]
try:
from __builtin__ import memoryview
except ImportError:
pass
else:
- py.test.raises(TypeError, from_buffer, BCharA, memoryview(b"foo"))
+ contents = from_buffer(BCharA, memoryview(b"foo"))
+ for i in range(len(contents)):
+ assert contents[i] == p1[i]
+
def test_from_buffer_bytearray():
a = bytearray(b"xyz")
diff --git a/pypy/module/_io/interp_bufferedio.py b/pypy/module/_io/interp_bufferedio.py
--- a/pypy/module/_io/interp_bufferedio.py
+++ b/pypy/module/_io/interp_bufferedio.py
@@ -4,6 +4,7 @@
from pypy.interpreter.typedef import (
TypeDef, GetSetProperty, generic_new_descr, interp_attrproperty_w)
from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault
+from rpython.rlib.rgc import nonmoving_raw_ptr_for_resizable_list
from rpython.rlib.buffer import Buffer
from rpython.rlib.rstring import StringBuilder
from rpython.rlib.rarithmetic import r_longlong, intmask
@@ -120,6 +121,9 @@
def setitem(self, index, char):
self.buf[self.start + index] = char
+ def get_raw_address(self):
+ return nonmoving_raw_ptr_for_resizable_list(self.buf)
+
class BufferedMixin:
_mixin_ = True
diff --git a/pypy/module/_socket/interp_func.py b/pypy/module/_socket/interp_func.py
--- a/pypy/module/_socket/interp_func.py
+++ b/pypy/module/_socket/interp_func.py
@@ -1,6 +1,6 @@
from rpython.rlib import rsocket
from rpython.rlib.rsocket import SocketError, INVALID_SOCKET
-from rpython.rlib.rarithmetic import intmask
+from rpython.rlib.rarithmetic import intmask, r_longlong, r_uint32
from pypy.interpreter.error import OperationError, oefmt
from pypy.interpreter.gateway import unwrap_spec, WrappedDefault
@@ -98,7 +98,8 @@
proto = space.str_w(w_proto)
if port < 0 or port > 0xffff:
- raise oefmt(space.w_ValueError, "getservbyport: port must be 0-65535.")
+ raise oefmt(space.w_OverflowError,
+ "getservbyport: port must be 0-65535.")
try:
service = rsocket.getservbyport(port, proto)
@@ -163,40 +164,58 @@
space.wrap(W_Socket(space, sock2))
])
-# The following 4 functions refuse all negative numbers, like CPython 2.6.
-# They could also check that the argument is not too large, but CPython 2.6
-# is not doing that consistently.
- at unwrap_spec(x="c_uint")
+# The following 4 functions refuse all negative numbers.
+# They also check that the argument is not too large, but note that
+# CPython 2.7 is not doing that consistently (CPython 3.x does).
+LONGLONG_UINT32_MAX = r_longlong(2**32-1)
+
+ at unwrap_spec(x="c_int")
def ntohs(space, x):
"""ntohs(integer) -> integer
Convert a 16-bit integer from network to host byte order.
"""
+ if x < 0:
+ raise oefmt(space.w_OverflowError,
+ "can't convert negative number to unsigned long")
return space.wrap(rsocket.ntohs(intmask(x)))
- at unwrap_spec(x="c_uint")
+ at unwrap_spec(x=r_longlong)
def ntohl(space, x):
"""ntohl(integer) -> integer
Convert a 32-bit integer from network to host byte order.
"""
- return space.wrap(rsocket.ntohl(x))
+ if x < r_longlong(0):
+ raise oefmt(space.w_OverflowError,
+ "can't convert negative number to unsigned long")
+ if x > LONGLONG_UINT32_MAX:
+ raise oefmt(space.w_OverflowError, "long int larger than 32 bits")
+ return space.wrap(rsocket.ntohl(r_uint32(x)))
- at unwrap_spec(x="c_uint")
+ at unwrap_spec(x="c_int")
def htons(space, x):
"""htons(integer) -> integer
Convert a 16-bit integer from host to network byte order.
"""
- return space.wrap(rsocket.htons(intmask(x)))
+ if x < 0:
+ raise oefmt(space.w_OverflowError,
+ "can't convert negative number to unsigned long")
+ return space.wrap(rsocket.htons(x))
- at unwrap_spec(x="c_uint")
+ at unwrap_spec(x=r_longlong)
def htonl(space, x):
"""htonl(integer) -> integer
Convert a 32-bit integer from host to network byte order.
"""
- return space.wrap(rsocket.htonl(x))
+ if x < r_longlong(0):
+ raise oefmt(space.w_OverflowError,
+ "can't convert negative number to unsigned long")
+ if x > LONGLONG_UINT32_MAX:
+ raise oefmt(space.w_OverflowError, "long int larger than 32 bits")
+ return space.wrap(rsocket.htonl(r_uint32(x)))
@unwrap_spec(ip=str)
def inet_aton(space, ip):
diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py
--- a/pypy/module/_socket/interp_socket.py
+++ b/pypy/module/_socket/interp_socket.py
@@ -424,7 +424,7 @@
w_addr = w_param3
try:
addr = self.addr_from_object(space, w_addr)
- count = self.sock.sendto(data, flags, addr)
+ count = self.sock.sendto(data, len(data), flags, addr)
except SocketError as e:
raise converted_error(space, e)
return space.wrap(count)
diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py
--- a/pypy/module/_socket/test/test_sock_app.py
+++ b/pypy/module/_socket/test/test_sock_app.py
@@ -83,11 +83,6 @@
"(_socket, port): return _socket.getservbyport(port)")
assert space.unwrap(name) == "smtp"
- from pypy.interpreter.error import OperationError
- exc = raises(OperationError, space.appexec,
- [w_socket], "(_socket): return _socket.getservbyport(-1)")
- assert exc.value.match(space, space.w_ValueError)
-
def test_getprotobyname():
name = "tcp"
w_n = space.appexec([w_socket, space.wrap(name)],
@@ -325,6 +320,11 @@
assert _socket.socket.__name__ == 'socket'
assert _socket.socket.__module__ == '_socket'
+ def test_overflow_errors(self):
+ import _socket
+ raises(OverflowError, _socket.getservbyport, -1)
+ raises(OverflowError, _socket.getservbyport, 65536)
+
def test_ntoa_exception(self):
import _socket
raises(_socket.error, _socket.inet_ntoa, b"ab")
@@ -495,7 +495,8 @@
def test_socket_connect_typeerrors(self):
tests = [
"",
- ("80"),
+ "80",
+ ("80",),
("80", "80"),
(80, 80),
]
@@ -519,44 +520,25 @@
def test_NtoH(self):
import sys
import _socket as socket
- # This just checks that htons etc. are their own inverse,
- # when looking at the lower 16 or 32 bits.
+ # This checks that htons etc. are their own inverse,
+ # when looking at the lower 16 or 32 bits. It also
+ # checks that we get OverflowErrors when calling with -1,
+ # or (for XtoXl()) with too large values. For XtoXs()
+ # large values are silently truncated instead, like CPython.
sizes = {socket.htonl: 32, socket.ntohl: 32,
socket.htons: 16, socket.ntohs: 16}
for func, size in sizes.items():
mask = (1 << size) - 1
- for i in (0, 1, 0xffff, ~0xffff, 2, 0x01234567, 0x76543210):
+ for i in (0, 1, 0xffff, 0xffff0000, 2, 0x01234567, 0x76543210):
assert i & mask == func(func(i&mask)) & mask
swapped = func(mask)
assert swapped & mask == mask
- try:
- func(-1)
- except (OverflowError, ValueError):
- pass
- else:
- assert False
- try:
- func(sys.maxint*2+2)
- except OverflowError:
- pass
- else:
- assert False
-
- def test_NtoH_overflow(self):
- skip("we are not checking for overflowing values yet")
- import _socket as socket
- # Checks that we cannot give too large values to htons etc.
- # Skipped for now; CPython 2.6 is also not consistent.
- sizes = {socket.htonl: 32, socket.ntohl: 32,
- socket.htons: 16, socket.ntohs: 16}
- for func, size in sizes.items():
- try:
- func(1 << size)
- except OverflowError:
- pass
- else:
- assert False
+ raises(OverflowError, func, -1)
+ raises(OverflowError, func, -1L)
+ if size > 16: # else, values too large are ignored
+ raises(OverflowError, func, 2 ** size)
+ raises(OverflowError, func, 2L ** size)
def test_newsocket(self):
import socket
diff --git a/pypy/module/cppyy/test/conftest.py b/pypy/module/cppyy/test/conftest.py
--- a/pypy/module/cppyy/test/conftest.py
+++ b/pypy/module/cppyy/test/conftest.py
@@ -2,12 +2,6 @@
@py.test.mark.tryfirst
def pytest_runtest_setup(item):
- if 'linux' in sys.platform:
- # tests require minimally std=c++11
- cc_info = py.process.cmdexec('gcc -v --help')
- if not '-std=c++11' in cc_info:
- py.test.skip('skipping tests because gcc does not support C++11')
-
if py.path.local.sysfind('genreflex') is None:
import pypy.module.cppyy.capi.loadable_capi as lcapi
if 'dummy' in lcapi.reflection_library:
diff --git a/pypy/module/cppyy/test/support.py b/pypy/module/cppyy/test/support.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/cppyy/test/support.py
@@ -0,0 +1,16 @@
+import py, sys, subprocess
+
+currpath = py.path.local(__file__).dirpath()
+
+
+def setup_make(targetname):
+ if sys.platform == 'win32':
+ py.test.skip("win32 not supported so far")
+ import pypy.module.cppyy.capi.loadable_capi as lcapi
+ popen = subprocess.Popen(["make", targetname], cwd=str(currpath),
+ stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+ stdout, _ = popen.communicate()
+ if popen.returncode:
+ if '-std=c++11' in stdout:
+ py.test.skip("gcc does not seem to support -std=c++11")
+ raise OSError("'make' failed:\n%s" % (stdout,))
diff --git a/pypy/module/cppyy/test/test_cppyy.py b/pypy/module/cppyy/test/test_cppyy.py
--- a/pypy/module/cppyy/test/test_cppyy.py
+++ b/pypy/module/cppyy/test/test_cppyy.py
@@ -1,18 +1,15 @@
import py, os, sys
+import subprocess
from pypy.module.cppyy import interp_cppyy, executor
+from .support import setup_make
currpath = py.path.local(__file__).dirpath()
test_dct = str(currpath.join("example01Dict.so"))
def setup_module(mod):
- if sys.platform == 'win32':
- py.test.skip("win32 not supported so far")
- import pypy.module.cppyy.capi.loadable_capi as lcapi
- err = os.system("cd '%s' && make example01Dict.so" % currpath)
- if err:
- raise OSError("'make' failed (see stderr)")
+ setup_make("example01Dict.so")
class TestCPPYYImplementation:
def test01_class_query(self, space):
diff --git a/pypy/module/cppyy/test/test_datatypes.py b/pypy/module/cppyy/test/test_datatypes.py
--- a/pypy/module/cppyy/test/test_datatypes.py
+++ b/pypy/module/cppyy/test/test_datatypes.py
@@ -1,15 +1,12 @@
import py, os, sys
+from .support import setup_make
currpath = py.path.local(__file__).dirpath()
test_dct = str(currpath.join("datatypesDict.so"))
def setup_module(mod):
- if sys.platform == 'win32':
- py.test.skip("win32 not supported so far")
- err = os.system("cd '%s' && make datatypesDict.so" % currpath)
- if err:
- raise OSError("'make' failed (see stderr)")
+ setup_make("datatypesDict.so")
class AppTestDATATYPES:
spaceconfig = dict(usemodules=['cppyy', '_rawffi', 'itertools'])
diff --git a/pypy/module/cppyy/test/test_pythonify.py b/pypy/module/cppyy/test/test_pythonify.py
--- a/pypy/module/cppyy/test/test_pythonify.py
+++ b/pypy/module/cppyy/test/test_pythonify.py
@@ -1,17 +1,14 @@
import py, os, sys
from pypy.module.cppyy import interp_cppyy, executor
+from .support import setup_make
currpath = py.path.local(__file__).dirpath()
test_dct = str(currpath.join("example01Dict.so"))
def setup_module(mod):
- if sys.platform == 'win32':
- py.test.skip("win32 not supported so far")
- err = os.system("cd '%s' && make example01Dict.so" % currpath)
- if err:
- raise OSError("'make' failed (see stderr)")
+ setup_make("example01Dict.so")
class AppTestPYTHONIFY:
spaceconfig = dict(usemodules=['cppyy', '_rawffi', 'itertools'])
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -1,6 +1,5 @@
import ctypes
import sys, os
-import atexit
import py
@@ -18,7 +17,6 @@
from rpython.translator.tool.cbuild import ExternalCompilationInfo
from rpython.translator.gensupp import NameManager
from rpython.tool.udir import udir
-from rpython.translator import platform
from pypy.module.cpyext.state import State
from pypy.interpreter.error import OperationError, oefmt
from pypy.interpreter.baseobjspace import W_Root
@@ -485,6 +483,12 @@
TYPES[configname] = forward
return forward
+GLOBALS = {}
+def register_global(name, typ, expr, header=None):
+ if header is not None:
+ name = '%s#%s' % (name, header)
+ GLOBALS[name] = (typ, expr)
+
INTERPLEVEL_API = {}
FUNCTIONS = {}
FUNCTIONS_BY_HEADER = {}
@@ -545,18 +549,23 @@
'_Py_QnewFlag', 'Py_Py3kWarningFlag', 'Py_HashRandomizationFlag', '_Py_PackageContext',
]
TYPES = {}
-GLOBALS = { # this needs to include all prebuilt pto, otherwise segfaults occur
- '_Py_NoneStruct#%s' % pypy_decl: ('PyObject*', 'space.w_None'),
- '_Py_TrueStruct#%s' % pypy_decl: ('PyIntObject*', 'space.w_True'),
- '_Py_ZeroStruct#%s' % pypy_decl: ('PyIntObject*', 'space.w_False'),
- '_Py_NotImplementedStruct#%s' % pypy_decl: ('PyObject*', 'space.w_NotImplemented'),
- '_Py_EllipsisObject#%s' % pypy_decl: ('PyObject*', 'space.w_Ellipsis'),
- 'PyDateTimeAPI': ('PyDateTime_CAPI*', 'None'),
- }
FORWARD_DECLS = []
INIT_FUNCTIONS = []
BOOTSTRAP_FUNCTIONS = []
+# this needs to include all prebuilt pto, otherwise segfaults occur
+register_global('_Py_NoneStruct',
+ 'PyObject*', 'space.w_None', header=pypy_decl)
+register_global('_Py_TrueStruct',
+ 'PyIntObject*', 'space.w_True', header=pypy_decl)
+register_global('_Py_ZeroStruct',
+ 'PyIntObject*', 'space.w_False', header=pypy_decl)
+register_global('_Py_NotImplementedStruct',
+ 'PyObject*', 'space.w_NotImplemented', header=pypy_decl)
+register_global('_Py_EllipsisObject',
+ 'PyObject*', 'space.w_Ellipsis', header=pypy_decl)
+register_global('PyDateTimeAPI', 'PyDateTime_CAPI*', 'None')
+
def build_exported_objects():
# Standard exceptions
# PyExc_BaseException, PyExc_Exception, PyExc_ValueError, PyExc_KeyError,
@@ -565,7 +574,7 @@
# PyExc_NameError, PyExc_MemoryError, PyExc_RuntimeError,
# PyExc_UnicodeEncodeError, PyExc_UnicodeDecodeError, ...
for exc_name in exceptions.Module.interpleveldefs.keys():
- GLOBALS['PyExc_' + exc_name] = (
+ register_global('PyExc_' + exc_name,
'PyTypeObject*',
'space.gettypeobject(interp_exceptions.W_%s.typedef)'% (exc_name, ))
@@ -600,7 +609,7 @@
'PyCFunction_Type': 'space.gettypeobject(cpyext.methodobject.W_PyCFunctionObject.typedef)',
'PyWrapperDescr_Type': 'space.gettypeobject(cpyext.methodobject.W_PyCMethodObject.typedef)'
}.items():
- GLOBALS['%s#%s' % (cpyname, pypy_decl)] = ('PyTypeObject*', pypyexpr)
+ register_global(cpyname, 'PyTypeObject*', pypyexpr, header=pypy_decl)
for cpyname in '''PyMethodObject PyListObject PyLongObject
PyClassObject'''.split():
@@ -1021,14 +1030,12 @@
def build_bridge(space):
"NOT_RPYTHON"
from pypy.module.cpyext.pyobject import make_ref
+ from rpython.translator.c.database import LowLevelDatabase
+ use_micronumpy = setup_micronumpy(space)
+ db = LowLevelDatabase()
+ prefix ='cpyexttest'
- use_micronumpy = setup_micronumpy(space)
-
- export_symbols = list(FUNCTIONS) + SYMBOLS_C + list(GLOBALS)
- from rpython.translator.c.database import LowLevelDatabase
- db = LowLevelDatabase()
-
- generate_macros(export_symbols, prefix='cpyexttest')
+ functions = generate_decls_and_callbacks(db, prefix=prefix)
# Structure declaration code
members = []
@@ -1049,9 +1056,6 @@
RPY_EXTERN struct PyPyAPI* pypyAPI = &_pypyAPI;
""" % dict(members=structmembers)
- functions = generate_decls_and_callbacks(db, export_symbols,
- prefix='cpyexttest')
-
global_objects = []
for name, (typ, expr) in GLOBALS.iteritems():
if '#' in name:
@@ -1078,7 +1082,7 @@
'\n' +
'\n'.join(functions))
- eci = build_eci(True, export_symbols, code, use_micronumpy)
+ eci = build_eci(True, code, use_micronumpy)
eci = eci.compile_shared_lib(
outputfilename=str(udir / "module_cache" / "pypyapi"))
modulename = py.path.local(eci.libraries[-1])
@@ -1099,7 +1103,6 @@
run_bootstrap_functions(space)
# load the bridge, and init structure
- import ctypes
bridge = ctypes.CDLL(str(modulename), mode=ctypes.RTLD_GLOBAL)
space.fromcache(State).install_dll(eci)
@@ -1119,7 +1122,7 @@
INTERPLEVEL_API[name] = w_obj
- name = name.replace('Py', 'cpyexttest')
+ name = name.replace('Py', prefix)
if isptr:
ptr = ctypes.c_void_p.in_dll(bridge, name)
if typ == 'PyObject*':
@@ -1147,12 +1150,6 @@
pypyAPI = ctypes.POINTER(ctypes.c_void_p).in_dll(bridge, 'pypyAPI')
# implement structure initialization code
- #for name, func in FUNCTIONS.iteritems():
- # if name.startswith('cpyext_'): # XXX hack
- # continue
- # pypyAPI[structindex[name]] = ctypes.cast(
- # ll2ctypes.lltype2ctypes(func.get_llhelper(space)),
- # ctypes.c_void_p)
for header, header_functions in FUNCTIONS_BY_HEADER.iteritems():
for name, func in header_functions.iteritems():
if name.startswith('cpyext_') or func is None: # XXX hack
@@ -1242,13 +1239,13 @@
else:
return None
-def generate_macros(export_symbols, prefix):
+def generate_decls_and_callbacks(db, api_struct=True, prefix=''):
"NOT_RPYTHON"
pypy_macros = []
- renamed_symbols = []
+ export_symbols = sorted(FUNCTIONS) + sorted(SYMBOLS_C) + sorted(GLOBALS)
for name in export_symbols:
if '#' in name:
- name,header = name.split('#')
+ name, header = name.split('#')
else:
header = pypy_decl
newname = mangle_name(prefix, name)
@@ -1257,8 +1254,6 @@
pypy_macros.append('#define %s %s' % (name, newname))
if name.startswith("PyExc_"):
pypy_macros.append('#define _%s _%s' % (name, newname))
- renamed_symbols.append(newname)
- export_symbols[:] = renamed_symbols
# Generate defines
for macro_name, size in [
@@ -1278,8 +1273,6 @@
pypy_macros_h = udir.join('pypy_macros.h')
pypy_macros_h.write('\n'.join(pypy_macros))
-def generate_decls_and_callbacks(db, export_symbols, api_struct=True, prefix=''):
- "NOT_RPYTHON"
# implement function callbacks and generate function decls
functions = []
decls = {}
@@ -1365,7 +1358,7 @@
source_dir / "pymem.c",
]
-def build_eci(building_bridge, export_symbols, code, use_micronumpy=False):
+def build_eci(building_bridge, code, use_micronumpy=False):
"NOT_RPYTHON"
# Build code and get pointer to the structure
kwds = {}
@@ -1434,31 +1427,29 @@
return use_micronumpy
# import registers api functions by side-effect, we also need HEADER
from pypy.module.cpyext.ndarrayobject import HEADER
- global GLOBALS, FUNCTIONS_BY_HEADER, separate_module_files
+ global FUNCTIONS_BY_HEADER, separate_module_files
for func_name in ['PyArray_Type', '_PyArray_FILLWBYTE', '_PyArray_ZEROS']:
FUNCTIONS_BY_HEADER.setdefault(HEADER, {})[func_name] = None
- GLOBALS["PyArray_Type#%s" % HEADER] = ('PyTypeObject*', "space.gettypeobject(W_NDimArray.typedef)")
+ register_global("PyArray_Type",
+ 'PyTypeObject*', "space.gettypeobject(W_NDimArray.typedef)",
+ header=HEADER)
separate_module_files.append(source_dir / "ndarrayobject.c")
return use_micronumpy
def setup_library(space):
"NOT_RPYTHON"
+ from rpython.translator.c.database import LowLevelDatabase
use_micronumpy = setup_micronumpy(space)
- export_symbols = sorted(FUNCTIONS) + sorted(SYMBOLS_C) + sorted(GLOBALS)
- from rpython.translator.c.database import LowLevelDatabase
db = LowLevelDatabase()
prefix = 'PyPy'
- generate_macros(export_symbols, prefix=prefix)
-
- functions = generate_decls_and_callbacks(db, [], api_struct=False,
- prefix=prefix)
+ functions = generate_decls_and_callbacks(db, api_struct=False, prefix=prefix)
code = "#include <Python.h>\n"
if use_micronumpy:
code += "#include <pypy_numpy.h> /* api.py line 1290 */\n"
code += "\n".join(functions)
- eci = build_eci(False, export_symbols, code, use_micronumpy)
+ eci = build_eci(False, code, use_micronumpy)
space.fromcache(State).install_dll(eci)
@@ -1610,7 +1601,7 @@
@specialize.memo()
def make_generic_cpy_call(FT, expect_null):
- from pypy.module.cpyext.pyobject import make_ref, from_ref, Py_DecRef
+ from pypy.module.cpyext.pyobject import make_ref, from_ref
from pypy.module.cpyext.pyobject import is_pyobj, as_pyobj
from pypy.module.cpyext.pyobject import get_w_obj_and_decref
from pypy.module.cpyext.pyerrors import PyErr_Occurred
diff --git a/pypy/module/cpyext/state.py b/pypy/module/cpyext/state.py
--- a/pypy/module/cpyext/state.py
+++ b/pypy/module/cpyext/state.py
@@ -1,7 +1,7 @@
from rpython.rlib.objectmodel import we_are_translated
from rpython.rtyper.lltypesystem import rffi, lltype
from pypy.interpreter.error import OperationError, oefmt
-from pypy.interpreter.executioncontext import AsyncAction
+from pypy.interpreter import executioncontext
from rpython.rtyper.lltypesystem import lltype
from rpython.rtyper.annlowlevel import llhelper
from rpython.rlib.rdynload import DLLHANDLE
@@ -14,8 +14,9 @@
self.reset()
self.programname = lltype.nullptr(rffi.CCHARP.TO)
self.version = lltype.nullptr(rffi.CCHARP.TO)
- pyobj_dealloc_action = PyObjDeallocAction(space)
- self.dealloc_trigger = lambda: pyobj_dealloc_action.fire()
+ if space.config.translation.gc != "boehm":
+ pyobj_dealloc_action = PyObjDeallocAction(space)
+ self.dealloc_trigger = lambda: pyobj_dealloc_action.fire()
def reset(self):
from pypy.module.cpyext.modsupport import PyMethodDef
@@ -67,6 +68,11 @@
state.api_lib = str(api.build_bridge(self.space))
else:
api.setup_library(self.space)
+ #
+ if self.space.config.translation.gc == "boehm":
+ action = BoehmPyObjDeallocAction(self.space)
+ self.space.actionflag.register_periodic_action(action,
+ use_bytecode_counter=True)
def install_dll(self, eci):
"""NOT_RPYTHON
@@ -84,8 +90,10 @@
from pypy.module.cpyext.api import init_static_data_translated
if we_are_translated():
- rawrefcount.init(llhelper(rawrefcount.RAWREFCOUNT_DEALLOC_TRIGGER,
- self.dealloc_trigger))
+ if space.config.translation.gc != "boehm":
+ rawrefcount.init(
+ llhelper(rawrefcount.RAWREFCOUNT_DEALLOC_TRIGGER,
+ self.dealloc_trigger))
init_static_data_translated(space)
setup_new_method_def(space)
@@ -143,15 +151,23 @@
self.extensions[path] = w_copy
-class PyObjDeallocAction(AsyncAction):
+def _rawrefcount_perform(space):
+ from pypy.module.cpyext.pyobject import PyObject, decref
+ while True:
+ py_obj = rawrefcount.next_dead(PyObject)
+ if not py_obj:
+ break
+ decref(space, py_obj)
+
+class PyObjDeallocAction(executioncontext.AsyncAction):
"""An action that invokes _Py_Dealloc() on the dying PyObjects.
"""
+ def perform(self, executioncontext, frame):
+ _rawrefcount_perform(self.space)
+class BoehmPyObjDeallocAction(executioncontext.PeriodicAsyncAction):
+ # This variant is used with Boehm, which doesn't have the explicit
+ # callback. Instead we must periodically check ourselves.
def perform(self, executioncontext, frame):
- from pypy.module.cpyext.pyobject import PyObject, decref
-
- while True:
- py_obj = rawrefcount.next_dead(PyObject)
- if not py_obj:
- break
- decref(self.space, py_obj)
+ if we_are_translated():
+ _rawrefcount_perform(self.space)
diff --git a/pypy/module/cpyext/stubgen.py b/pypy/module/cpyext/stubgen.py
deleted file mode 100644
--- a/pypy/module/cpyext/stubgen.py
+++ /dev/null
@@ -1,100 +0,0 @@
-# -*- coding: utf-8 -*-
-from os import path
-
-from pypy.module.cpyext import api
-
-from sphinx import addnodes
-
-
-TEMPLATE = """
- at cpython_api([%(paramtypes)s], %(rettype)s)
-def %(functionname)s(%(params)s):
-%(docstring)s raise NotImplementedError
- %(borrows)s
-"""
-
-C_TYPE_TO_PYPY_TYPE = {
- "void": "lltype.Void",
- "int": "rffi.INT_real",
- "PyTypeObject*": "PyTypeObjectPtr",
- "PyVarObject*": "PyObject",
- "const char*": "rffi.CCHARP",
- "double": "rffi.DOUBLE",
- "PyObject*": "PyObject",
- "PyObject**": "PyObjectP",
- "char*": "rffi.CCHARP",
- "PyMethodDef*": "PyMethodDef",
- "Py_ssize_t": "Py_ssize_t",
- "Py_ssize_t*": "Py_ssize_t",
- "size_t": "rffi.SIZE_T",
- "...": "...",
- "char": "lltype.Char",
- "long": "lltype.Signed",
- "Py_buffer*": "Py_buffer",
- "": "",
- }
-
-C_TYPE_TO_PYPY_TYPE_ARGS = C_TYPE_TO_PYPY_TYPE.copy()
-C_TYPE_TO_PYPY_TYPE_ARGS.update({
- "void": "rffi.VOIDP",
- })
-
-
-def c_param_to_type_and_name(string, is_arg=True):
- string = string.replace(" **", "** ").replace(" *", "* ")
- try:
- typ, name = string.rsplit(" ", 1)
- except ValueError:
- typ = string
- name = ""
- return [C_TYPE_TO_PYPY_TYPE, C_TYPE_TO_PYPY_TYPE_ARGS][is_arg]\
- .get(typ, "{" + typ + "}"), name
-
-
-def process_doctree(app, doctree):
- for node in doctree.traverse(addnodes.desc_content):
- par = node.parent
- if par['desctype'] != 'cfunction':
- continue
- if not par[0].has_key('names') or not par[0]['names']:
- continue
- functionname = par[0]['names'][0]
- if (functionname in api.FUNCTIONS or
- functionname in api.SYMBOLS_C):
- print "Wow, you implemented already", functionname
- continue
- borrows = docstring = ""
- crettype, _, cparameters = par[0]
- crettype = crettype.astext()
- cparameters = cparameters.astext()
- rettype, _ = c_param_to_type_and_name(crettype, False)
- params = ["space"]
- paramtypes = []
- for param in cparameters.split(","):
- typ, name = c_param_to_type_and_name(param.strip())
- params.append(name)
- paramtypes.append(typ)
- params = ", ".join(params)
- paramtypes = ", ".join(paramtypes)
- docstring = node.astext()
- entry = app._refcounts.get(functionname)
- if entry and entry.result_type in ("PyObject*", "PyVarObject*"):
- if entry.result_refs is None:
- docstring += "\nReturn value: always NULL."
- else:
- borrows = ("borrow_from()", "")[entry.result_refs]
- docstring = "\n ".join(docstring.splitlines())
- if docstring:
- docstring = ' """%s"""\n' % (docstring,)
- code = TEMPLATE % locals()
- app._stubgen_f.write(code)
-
-
-def init_apidump(app):
- fname = path.join(path.dirname(api.__file__), "stubs.py")
- app._stubgen_f = file(fname, "w")
- app.connect('doctree-read', process_doctree)
-
-
-def setup(app):
- app.connect('builder-inited', init_apidump)
diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py
--- a/pypy/module/cpyext/test/test_memoryobject.py
+++ b/pypy/module/cpyext/test/test_memoryobject.py
@@ -36,6 +36,30 @@
assert space.eq_w(space.getattr(w_mv, w_f),
space.getattr(w_memoryview, w_f))
+
+class AppTestPyBuffer_FillInfo(AppTestCpythonExtensionBase):
+ def test_fillWithObject(self):
+ module = self.import_extension('foo', [
+ ("fillinfo", "METH_VARARGS",
+ """
+ Py_buffer buf;
+ PyObject *str = PyBytes_FromString("hello, world.");
+ if (PyBuffer_FillInfo(&buf, str, PyBytes_AsString(str), 13,
+ 0, 0)) {
+ return NULL;
+ }
+
+ /* Get rid of our own reference to the object, but
+ * the Py_buffer should still have a reference.
+ */
+ Py_DECREF(str);
+
+ return PyMemoryView_FromBuffer(&buf);
+ """)])
+ result = module.fillinfo()
+ assert b"hello, world." == result
+ del result
+
class AppTestBufferProtocol(AppTestCpythonExtensionBase):
def test_buffer_protocol_app(self):
import struct
@@ -62,7 +86,7 @@
return NULL;
vlen = view.len / view.itemsize;
PyBuffer_Release(&view);
- return PyInt_FromLong(vlen);
+ return PyLong_FromLong(vlen);
"""),
("test_buffer", "METH_VARARGS",
"""
@@ -70,10 +94,10 @@
PyObject* obj = PyTuple_GetItem(args, 0);
PyObject* memoryview = PyMemoryView_FromObject(obj);
if (memoryview == NULL)
- return PyInt_FromLong(-1);
+ return PyLong_FromLong(-1);
view = PyMemoryView_GET_BUFFER(memoryview);
Py_DECREF(memoryview);
- return PyInt_FromLong(view->len / view->itemsize);
+ return PyLong_FromLong(view->len / view->itemsize);
""")])
module = self.import_module(name='buffer_test')
arr = module.PyMyArray(10)
diff --git a/pypy/objspace/std/bufferobject.py b/pypy/objspace/std/bufferobject.py
--- a/pypy/objspace/std/bufferobject.py
+++ b/pypy/objspace/std/bufferobject.py
@@ -17,9 +17,6 @@
assert isinstance(buf, Buffer)
self.buf = buf
- def _finalize_(self):
- return self.buf.releasebuffer()
-
def buffer_w(self, space, flags):
space.check_buf_flags(flags, self.buf.readonly)
return self.buf
diff --git a/pypy/objspace/std/test/test_bufferobject.py b/pypy/objspace/std/test/test_bufferobject.py
--- a/pypy/objspace/std/test/test_bufferobject.py
+++ b/pypy/objspace/std/test/test_bufferobject.py
@@ -199,7 +199,9 @@
raises(TypeError, "buf[MyInt(0):MyInt(5)]")
def test_pypy_raw_address_base(self):
- raises(ValueError, buffer("foobar")._pypy_raw_address)
- raises(ValueError, buffer(u"foobar")._pypy_raw_address)
- a = buffer(bytearray("foobar"))._pypy_raw_address()
+ a = buffer("foobar")._pypy_raw_address()
assert a != 0
+ b = buffer(u"foobar")._pypy_raw_address()
+ assert b != 0
+ c = buffer(bytearray("foobar"))._pypy_raw_address()
+ assert c != 0
diff --git a/pypy/objspace/std/test/test_memoryobject.py b/pypy/objspace/std/test/test_memoryobject.py
--- a/pypy/objspace/std/test/test_memoryobject.py
+++ b/pypy/objspace/std/test/test_memoryobject.py
@@ -56,6 +56,7 @@
assert u"abc" != memoryview("abc")
def test_pypy_raw_address_base(self):
- raises(ValueError, memoryview("foobar")._pypy_raw_address)
- a = memoryview(bytearray("foobar"))._pypy_raw_address()
+ a = memoryview("foobar")._pypy_raw_address()
assert a != 0
+ b = memoryview(bytearray("foobar"))._pypy_raw_address()
+ assert b != 0
diff --git a/rpython/annotator/description.py b/rpython/annotator/description.py
--- a/rpython/annotator/description.py
+++ b/rpython/annotator/description.py
@@ -398,6 +398,8 @@
s_result = self.specialize(inputcells, op)
if isinstance(s_result, FunctionGraph):
s_result = s_result.getreturnvar().annotation
+ if s_result is None:
+ s_result = s_ImpossibleValue
s_result = unionof(s_result, s_previous_result)
return s_result
diff --git a/rpython/jit/codewriter/assembler.py b/rpython/jit/codewriter/assembler.py
--- a/rpython/jit/codewriter/assembler.py
+++ b/rpython/jit/codewriter/assembler.py
@@ -5,6 +5,7 @@
from rpython.jit.codewriter.jitcode import SwitchDictDescr, JitCode
from rpython.jit.codewriter import heaptracker, longlong
from rpython.rlib.objectmodel import ComputedIntSymbolic
+from rpython.rlib.rarithmetic import r_int
from rpython.flowspace.model import Constant
from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
from rpython.rtyper import rclass
@@ -82,6 +83,8 @@
if not isinstance(value, (llmemory.AddressAsInt,
ComputedIntSymbolic)):
value = lltype.cast_primitive(lltype.Signed, value)
+ if type(value) is r_int:
+ value = int(value)
if allow_short:
try:
short_num = -128 <= value <= 127
diff --git a/rpython/jit/codewriter/jitcode.py b/rpython/jit/codewriter/jitcode.py
--- a/rpython/jit/codewriter/jitcode.py
+++ b/rpython/jit/codewriter/jitcode.py
@@ -1,6 +1,7 @@
from rpython.jit.metainterp.history import AbstractDescr, ConstInt
from rpython.jit.codewriter import heaptracker
from rpython.rlib.objectmodel import we_are_translated
+from rpython.rlib.rarithmetic import base_int
class JitCode(AbstractDescr):
@@ -21,6 +22,10 @@
liveness=None, startpoints=None, alllabels=None,
resulttypes=None):
self.code = code
+ for x in constants_i:
+ assert not isinstance(x, base_int), (
+ "found constant %r of type %r, must not appear in "
+ "JitCode.constants_i" % (x, type(x)))
# if the following lists are empty, use a single shared empty list
self.constants_i = constants_i or self._empty_i
self.constants_r = constants_r or self._empty_r
diff --git a/rpython/jit/codewriter/test/test_assembler.py b/rpython/jit/codewriter/test/test_assembler.py
--- a/rpython/jit/codewriter/test/test_assembler.py
+++ b/rpython/jit/codewriter/test/test_assembler.py
@@ -7,6 +7,7 @@
from rpython.jit.metainterp.history import AbstractDescr
from rpython.flowspace.model import Constant
from rpython.rtyper.lltypesystem import lltype, llmemory
+from rpython.rlib.rarithmetic import r_int, r_uint
def test_assemble_simple():
@@ -239,3 +240,17 @@
]
assembler = Assembler()
py.test.raises(AssemblerError, assembler.assemble, ssarepr)
+
+def test_assemble_r_int():
+ # r_int is a strange type, which the jit should replace with int.
+ # r_uint is also replaced with int.
+ ssarepr = SSARepr("test")
+ i0, i1, i2 = Register('int', 0), Register('int', 1), Register('int', 2)
+ ssarepr.insns = [
+ ('uint_add', i0, Constant(r_uint(42424242), lltype.Unsigned), '->', i1),
+ ('int_add', i0, Constant(r_int(42424243), lltype.Signed), '->', i2),
+ ]
+ assembler = Assembler()
+ jitcode = assembler.assemble(ssarepr)
+ assert jitcode.constants_i == [42424242, 42424243]
+ assert map(type, jitcode.constants_i) == [int, int]
diff --git a/rpython/jit/metainterp/blackhole.py b/rpython/jit/metainterp/blackhole.py
--- a/rpython/jit/metainterp/blackhole.py
+++ b/rpython/jit/metainterp/blackhole.py
@@ -6,6 +6,7 @@
from rpython.jit.metainterp.history import MissingValue
from rpython.rlib import longlong2float
from rpython.rlib.debug import ll_assert, make_sure_not_resized
+from rpython.rlib.debug import check_annotation
from rpython.rlib.objectmodel import we_are_translated, specialize
from rpython.rlib.rarithmetic import intmask, LONG_BIT, r_uint, ovfcheck
from rpython.rlib.unroll import unrolling_iterable
@@ -183,7 +184,7 @@
if lltype.typeOf(result) is lltype.Bool:
result = int(result)
assert lltype.typeOf(result) is lltype.Signed
- self.registers_i[ord(code[position])] = result
+ self.registers_i[ord(code[position])] = plain_int(result)
position += 1
elif resulttype == 'r':
# argcode should be 'r' too
@@ -213,7 +214,7 @@
if lltype.typeOf(result) is lltype.Bool:
result = int(result)
assert lltype.typeOf(result) is lltype.Signed
- self.registers_i[ord(code[position])] = result
+ self.registers_i[ord(code[position])] = plain_int(result)
position += 1
elif resulttype == 'L':
assert result >= 0
@@ -251,6 +252,23 @@
if b < 0 or b >= LONG_BIT:
raise ValueError("Shift count, %d, not in valid range, 0 .. %d." % (b, LONG_BIT-1))
+def check_list_of_plain_integers(s_arg, bookkeeper):
+ """Check that 'BlackhopeInterpreter.registers_i' is annotated as a
+ non-resizable list of plain integers (and not r_int's for example)."""
+ from rpython.annotator import model as annmodel
+ assert isinstance(s_arg, annmodel.SomeList)
+ s_arg.listdef.never_resize()
+ assert s_arg.listdef.listitem.s_value.knowntype is int
+
+def _check_int(s_arg, bookkeeper):
+ assert s_arg.knowntype is int
+
+def plain_int(x):
+ """Check that 'x' is annotated as a plain integer (and not r_int)"""
+ check_annotation(x, _check_int)
+ return x
+
+
class BlackholeInterpreter(object):
def __init__(self, builder, count_interpreter):
@@ -277,6 +295,7 @@
self.tmpreg_r = default_r
self.tmpreg_f = default_f
self.jitcode = None
+ check_annotation(self.registers_i, check_list_of_plain_integers)
def __repr__(self):
return '<BHInterp #%d>' % self.count_interpreter
@@ -295,7 +314,7 @@
def setarg_i(self, index, value):
assert lltype.typeOf(value) is lltype.Signed
- self.registers_i[index] = value
+ self.registers_i[index] = plain_int(value)
def setarg_r(self, index, value):
assert lltype.typeOf(value) == llmemory.GCREF
@@ -1573,7 +1592,8 @@
# 'xxx_call_yyy' instructions from the caller frame
def _setup_return_value_i(self, result):
assert lltype.typeOf(result) is lltype.Signed
- self.registers_i[ord(self.jitcode.code[self.position-1])] = result
+ self.registers_i[ord(self.jitcode.code[self.position-1])] = plain_int(
+ result)
def _setup_return_value_r(self, result):
assert lltype.typeOf(result) == llmemory.GCREF
self.registers_r[ord(self.jitcode.code[self.position-1])] = result
diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -158,7 +158,11 @@
# record that ignore_finalizer() has been called
GCFLAG_IGNORE_FINALIZER = first_gcflag << 10
-_GCFLAG_FIRST_UNUSED = first_gcflag << 11 # the first unused bit
+# shadow objects can have its memory initialized when it is created.
+# It does not need an additional copy in trace out
+GCFLAG_SHADOW_INITIALIZED = first_gcflag << 11
+
+_GCFLAG_FIRST_UNUSED = first_gcflag << 12 # the first unused bit
# States for the incremental GC
@@ -729,6 +733,16 @@
obj = self.external_malloc(typeid, length, alloc_young=True)
return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF)
+ def move_out_of_nursery(self, obj):
+ # called twice, it should return the same shadow object,
+ # and not creating another shadow object
+ if self.header(obj).tid & GCFLAG_HAS_SHADOW:
+ shadow = self.nursery_objects_shadows.get(obj)
+ ll_assert(shadow != llmemory.NULL,
+ "GCFLAG_HAS_SHADOW but no shadow found")
+ return shadow
+
+ return self._allocate_shadow(obj, copy=True)
def collect(self, gen=2):
"""Do a minor (gen=0), start a major (gen=1), or do a full
@@ -1982,6 +1996,9 @@
and self.young_rawmalloced_objects.contains(obj)):
self._visit_young_rawmalloced_object(obj)
return
+ # copy the contents of the object? usually yes, but not for some
+ # shadow objects
+ copy = True
#
size_gc_header = self.gcheaderbuilder.size_gc_header
if self.header(obj).tid & (GCFLAG_HAS_SHADOW | GCFLAG_PINNED) == 0:
@@ -2037,13 +2054,18 @@
# Remove the flag GCFLAG_HAS_SHADOW, so that it doesn't get
# copied to the shadow itself.
self.header(obj).tid &= ~GCFLAG_HAS_SHADOW
+ tid = self.header(obj).tid
+ if (tid & GCFLAG_SHADOW_INITIALIZED) != 0:
+ copy = False
+ self.header(obj).tid &= ~GCFLAG_SHADOW_INITIALIZED
#
totalsize = size_gc_header + self.get_size(obj)
self.nursery_surviving_size += raw_malloc_usage(totalsize)
#
# Copy it. Note that references to other objects in the
# nursery are kept unchanged in this step.
- llmemory.raw_memcopy(obj - size_gc_header, newhdr, totalsize)
+ if copy:
+ llmemory.raw_memcopy(obj - size_gc_header, newhdr, totalsize)
#
# Set the old object's tid to -42 (containing all flags) and
# replace the old object's content with the target address.
@@ -2570,7 +2592,8 @@
# ----------
# id() and identityhash() support
- def _allocate_shadow(self, obj):
+ @specialize.arg(2)
+ def _allocate_shadow(self, obj, copy=False):
size_gc_header = self.gcheaderbuilder.size_gc_header
size = self.get_size(obj)
shadowhdr = self._malloc_out_of_nursery(size_gc_header +
@@ -2592,6 +2615,12 @@
#
self.header(obj).tid |= GCFLAG_HAS_SHADOW
self.nursery_objects_shadows.setitem(obj, shadow)
+
+ if copy:
+ self.header(obj).tid |= GCFLAG_SHADOW_INITIALIZED
+ totalsize = size_gc_header + self.get_size(obj)
+ llmemory.raw_memcopy(obj - size_gc_header, shadow, totalsize)
+
return shadow
def _find_shadow(self, obj):
diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py
--- a/rpython/memory/gctransform/framework.py
+++ b/rpython/memory/gctransform/framework.py
@@ -551,6 +551,13 @@
[s_gc, SomeAddress()],
annmodel.s_None)
+ self.move_out_of_nursery_ptr = None
+ if hasattr(GCClass, 'move_out_of_nursery'):
+ self.move_out_of_nursery_ptr = getfn(GCClass.move_out_of_nursery,
+ [s_gc, SomeAddress()],
+ SomeAddress())
+
+
def create_custom_trace_funcs(self, gc, rtyper):
custom_trace_funcs = tuple(rtyper.custom_trace_funcs)
rtyper.custom_trace_funcs = custom_trace_funcs
@@ -1585,6 +1592,17 @@
hop.genop("direct_call", [self.ignore_finalizer_ptr,
self.c_const_gc, v_adr])
+ def gct_gc_move_out_of_nursery(self, hop):
+ if self.move_out_of_nursery_ptr is not None:
+ v_adr = hop.genop("cast_ptr_to_adr", [hop.spaceop.args[0]],
+ resulttype=llmemory.Address)
+ v_ret = hop.genop("direct_call", [self.move_out_of_nursery_ptr,
+ self.c_const_gc, v_adr],
+ resulttype=llmemory.Address)
+ hop.genop("cast_adr_to_ptr", [v_ret],
+ resultvar = hop.spaceop.result)
+
+
class TransformerLayoutBuilder(gctypelayout.TypeLayoutBuilder):
diff --git a/rpython/rlib/buffer.py b/rpython/rlib/buffer.py
--- a/rpython/rlib/buffer.py
+++ b/rpython/rlib/buffer.py
@@ -2,6 +2,8 @@
Buffer protocol support.
"""
from rpython.rlib import jit
+from rpython.rlib.rgc import (resizable_list_supporting_raw_ptr,
+ nonmoving_raw_ptr_for_resizable_list)
class Buffer(object):
@@ -84,7 +86,7 @@
def __init__(self, value):
self.value = value
- self.readonly = True
+ self.readonly = 1
def getlength(self):
return len(self.value)
@@ -108,6 +110,9 @@
return self.value[start:stop]
return Buffer.getslice(self, start, stop, step, size)
+ def get_raw_address(self):
+ from rpython.rtyper.lltypesystem import rffi
+ return rffi.get_raw_address_of_string(self.value)
class SubBuffer(Buffer):
_attrs_ = ['buffer', 'offset', 'size', 'readonly']
diff --git a/rpython/rlib/rawrefcount.py b/rpython/rlib/rawrefcount.py
--- a/rpython/rlib/rawrefcount.py
+++ b/rpython/rlib/rawrefcount.py
@@ -4,10 +4,11 @@
# This is meant for pypy's cpyext module, but is a generally
# useful interface over our GC. XXX "pypy" should be removed here
#
-import sys, weakref
-from rpython.rtyper.lltypesystem import lltype, llmemory
+import sys, weakref, py
+from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
from rpython.rlib.objectmodel import we_are_translated, specialize, not_rpython
from rpython.rtyper.extregistry import ExtRegistryEntry
+from rpython.translator.tool.cbuild import ExternalCompilationInfo
from rpython.rlib import rgc
@@ -245,6 +246,11 @@
v_p, v_ob = hop.inputargs(*hop.args_r)
hop.exception_cannot_occur()
hop.genop(name, [_unspec_p(hop, v_p), _unspec_ob(hop, v_ob)])
+ #
+ if hop.rtyper.annotator.translator.config.translation.gc == "boehm":
+ c_func = hop.inputconst(lltype.typeOf(func_boehm_eci),
+ func_boehm_eci)
+ hop.genop('direct_call', [c_func])
class Entry(ExtRegistryEntry):
@@ -297,3 +303,10 @@
v_ob = hop.genop('gc_rawrefcount_next_dead', [],
resulttype = llmemory.Address)
return _spec_ob(hop, v_ob)
+
+src_dir = py.path.local(__file__).dirpath() / 'src'
+boehm_eci = ExternalCompilationInfo(
+ post_include_bits = [(src_dir / 'boehm-rawrefcount.h').read()],
+ separate_module_files = [(src_dir / 'boehm-rawrefcount.c')],
+)
+func_boehm_eci = rffi.llexternal_use_eci(boehm_eci)
diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py
--- a/rpython/rlib/rgc.py
+++ b/rpython/rlib/rgc.py
@@ -535,6 +535,25 @@
from rpython.rtyper.lltypesystem.lloperation import llop
llop.gc_ignore_finalizer(lltype.Void, obj)
+ at jit.dont_look_inside
+def move_out_of_nursery(obj):
+ """ Returns another object which is a copy of obj; but at any point
+ (either now or in the future) the returned object might suddenly
+ become identical to the one returned.
+
+ NOTE: Only use for immutable objects!
+ """
+ pass
+
+class MoveOutOfNurseryEntry(ExtRegistryEntry):
+ _about_ = move_out_of_nursery
+
+ def compute_result_annotation(self, s_obj):
+ return s_obj
+
+ def specialize_call(self, hop):
+ hop.exception_cannot_occur()
+ return hop.genop('gc_move_out_of_nursery', hop.args_v, resulttype=hop.r_result)
# ____________________________________________________________
diff --git a/rpython/rlib/rsocket.py b/rpython/rlib/rsocket.py
--- a/rpython/rlib/rsocket.py
+++ b/rpython/rlib/rsocket.py
@@ -997,12 +997,12 @@
if signal_checker is not None:
signal_checker()
- def sendto(self, data, flags, address):
+ def sendto(self, data, length, flags, address):
"""Like send(data, flags) but allows specifying the destination
address. (Note that 'flags' is mandatory here.)"""
self.wait_for_data(True)
addr = address.lock()
- res = _c.sendto(self.fd, data, len(data), flags,
+ res = _c.sendto(self.fd, data, length, flags,
addr, address.addrlen)
address.unlock()
if res < 0:
diff --git a/rpython/rlib/src/boehm-rawrefcount.c b/rpython/rlib/src/boehm-rawrefcount.c
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/src/boehm-rawrefcount.c
@@ -0,0 +1,282 @@
+#include <stdlib.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <assert.h>
+#include <limits.h>
+#include <gc/gc.h>
+#include <gc/gc_mark.h>
+
+#ifdef TEST_BOEHM_RAWREFCOUNT
+# define RPY_EXTERN /* nothing */
+#else
+# include "common_header.h"
+#endif
+
+
+#define REFCNT_FROM_PYPY (LONG_MAX / 4 + 1)
+
+typedef struct pypy_header0 gcobj_t; /* opaque here */
+
+#ifndef _WIN32
+typedef intptr_t Py_ssize_t;
+#else
+typedef long Py_ssize_t;
+#endif
+
+/* this is the first two words of the PyObject structure used in
+ pypy/module/cpyext */
+typedef struct {
+ Py_ssize_t ob_refcnt;
+ Py_ssize_t ob_pypy_link;
+} pyobj_t;
+
+struct link_s {
+ pyobj_t *pyobj; /* NULL if entry unused */
+ uintptr_t gcenc;
+ struct link_s *next_in_bucket;
+};
+
+#define MARKER_LIST_START ((pyobj_t *)-1)
+
+static struct link_s **hash_buckets, *hash_list, *hash_free_list;
+static uintptr_t hash_mask_bucket;
+static intptr_t hash_list_walk_next = -1;
+
+static uintptr_t hash_get_hash(gcobj_t *gcobj)
+{
+ assert(gcobj != NULL);
+ uintptr_t h = (uintptr_t)gcobj;
+ assert((h & 1) == 0);
+ h -= (h >> 6);
+ return h & hash_mask_bucket;
+}
+
+static gcobj_t *decode_gcenc(uintptr_t gcenc)
+{
+ if (gcenc & 1)
+ gcenc = ~gcenc;
+ return (gcobj_t *)gcenc;
+}
+
+static void hash_link(struct link_s *lnk)
+{
+ uintptr_t h = hash_get_hash(decode_gcenc(lnk->gcenc));
+ lnk->next_in_bucket = hash_buckets[h];
+ hash_buckets[h] = lnk;
+}
+
+static void boehm_is_about_to_collect(void);
+
+static void hash_grow_table(void)
+{
+ static int rec = 0;
+ assert(!rec); /* recursive hash_grow_table() */
+ rec = 1;
+
+ if (hash_buckets == NULL)
+ GC_set_start_callback(boehm_is_about_to_collect);
+
+ uintptr_t i, num_buckets = (hash_mask_bucket + 1) * 2;
+ if (num_buckets < 16) num_buckets = 16;
+ assert((num_buckets & (num_buckets - 1)) == 0); /* power of two */
+
+ /* The new hash_buckets: an array of pointers to struct link_s, of
+ length a power of two, used as a dictionary hash table. It is
+ not allocated with Boehm because there is no point in Boehm looking
+ in it.
+ */
+ struct link_s **new_buckets = calloc(num_buckets, sizeof(struct link_s *));
+ assert(new_buckets);
+
+ /* The new hash_list: the array of all struct link_s. Their order
+ is irrelevant. There is a GC_register_finalizer() on the 'gcenc'
+ field, so we don't move the array; instead we allocate a new array
+ to use in addition to the old one. There are a total of 2 to 4
+ times as many 'struct link_s' as the length of 'buckets'.
+ */
+ uintptr_t num_list = num_buckets * 2;
+ struct link_s *new_list = GC_MALLOC(num_list * sizeof(struct link_s));
+ for (i = num_list; i-- > 1; ) {
+ new_list[i].next_in_bucket = hash_free_list;
+ hash_free_list = &new_list[i];
+ }
+ /* list[0] is abused to store a pointer to the previous list and
+ the length of the current list */
+ struct link_s *old_list = hash_list;
+ new_list[0].next_in_bucket = old_list;
+ new_list[0].gcenc = num_list;
+ new_list[0].pyobj = MARKER_LIST_START;
+
+ hash_list = new_list;
+ free(hash_buckets);
+ hash_buckets = new_buckets;
+ hash_mask_bucket = num_buckets - 1;
+ hash_list_walk_next = hash_mask_bucket;
+
+ /* re-add all old 'struct link_s' to the hash_buckets */
+ struct link_s *plist = old_list;
+ while (plist != NULL) {
+ uintptr_t count = plist[0].gcenc;
+ for (i = 1; i < count; i++) {
+ if (plist[i].gcenc != 0)
+ hash_link(&plist[i]);
+ }
+ plist = plist[0].next_in_bucket;
+ }
+ GC_reachable_here(old_list);
+
+ rec = 0;
+}
+
+static void hash_add_entry(gcobj_t *gcobj, pyobj_t *pyobj)
+{
+ if (hash_free_list == NULL) {
+ hash_grow_table();
+ }
+ assert(pyobj->ob_pypy_link == 0);
+
+ struct link_s *lnk = hash_free_list;
+ hash_free_list = lnk->next_in_bucket;
+ lnk->pyobj = pyobj;
+ lnk->gcenc = (uintptr_t)gcobj;
+ pyobj->ob_pypy_link = (Py_ssize_t)lnk;
+
+ hash_link(lnk);
+
+ if (GC_base(gcobj) == NULL) {
+ /* 'gcobj' is probably a prebuilt object - it makes no */
+ /* sense to register it then, and it crashes Boehm in */
+ /* quite obscure ways */
+ }
+ else {
+ int j = GC_general_register_disappearing_link(
+ (void **)&lnk->gcenc, gcobj);
+ assert(j == GC_SUCCESS);
+ }
+}
+
+static pyobj_t *hash_get_entry(gcobj_t *gcobj)
+{
+ if (hash_buckets == NULL)
+ return NULL;
+ uintptr_t h = hash_get_hash(gcobj);
+ struct link_s *lnk = hash_buckets[h];
+ while (lnk != NULL) {
+ assert(lnk->pyobj != NULL);
+ if (decode_gcenc(lnk->gcenc) == gcobj)
+ return lnk->pyobj;
+ lnk = lnk->next_in_bucket;
+ }
+ return NULL;
+}
+
+
+RPY_EXTERN
+/*pyobj_t*/void *gc_rawrefcount_next_dead(void)
+{
+ while (hash_list_walk_next >= 0) {
+ struct link_s *p, **pp = &hash_buckets[hash_list_walk_next];
+ while (1) {
+ p = *pp;
+ if (p == NULL)
+ break;
+ assert(p->pyobj != NULL);
+ if (p->gcenc == 0) {
+ /* quadratic time on the number of links from the same
+ bucket chain, but it should be small with very high
+ probability */
+ pyobj_t *result = p->pyobj;
+#ifdef TEST_BOEHM_RAWREFCOUNT
+ printf("next_dead: %p\n", result);
+#endif
+ assert(result->ob_refcnt == REFCNT_FROM_PYPY);
+ result->ob_refcnt = 1;
+ p->pyobj = NULL;
+ *pp = p->next_in_bucket;
+ p->next_in_bucket = hash_free_list;
+ hash_free_list = p;
+ return result;
+ }
+ else {
+ assert(p->gcenc != ~(uintptr_t)0);
+ pp = &p->next_in_bucket;
+ }
+ }
+ hash_list_walk_next--;
+ }
+ return NULL;
+}
+
+RPY_EXTERN
+void gc_rawrefcount_create_link_pypy(/*gcobj_t*/void *gcobj,
+ /*pyobj_t*/void *pyobj)
+{
+ gcobj_t *gcobj1 = (gcobj_t *)gcobj;
+ pyobj_t *pyobj1 = (pyobj_t *)pyobj;
+
+ assert(pyobj1->ob_pypy_link == 0);
+ /*assert(pyobj1->ob_refcnt >= REFCNT_FROM_PYPY);*/
+ /*^^^ could also be fixed just after the call to create_link_pypy()*/
+
+ hash_add_entry(gcobj1, pyobj1);
+}
+
+RPY_EXTERN
+/*pyobj_t*/void *gc_rawrefcount_from_obj(/*gcobj_t*/void *gcobj)
+{
+ return hash_get_entry((gcobj_t *)gcobj);
+}
+
+RPY_EXTERN
+/*gcobj_t*/void *gc_rawrefcount_to_obj(/*pyobj_t*/void *pyobj)
+{
+ pyobj_t *pyobj1 = (pyobj_t *)pyobj;
+
+ if (pyobj1->ob_pypy_link == 0)
+ return NULL;
+
+ struct link_s *lnk = (struct link_s *)pyobj1->ob_pypy_link;
+ assert(lnk->pyobj == pyobj1);
+
+ gcobj_t *g = decode_gcenc(lnk->gcenc);
+ assert(g != NULL);
+ return g;
+}
+
+static void boehm_is_about_to_collect(void)
+{
+ struct link_s *plist = hash_list;
+ uintptr_t gcenc_union = 0;
+ while (plist != NULL) {
+ uintptr_t i, count = plist[0].gcenc;
+ for (i = 1; i < count; i++) {
+ if (plist[i].gcenc == 0)
+ continue;
+
+ pyobj_t *p = plist[i].pyobj;
+ assert(p != NULL);
+ assert(p->ob_refcnt >= REFCNT_FROM_PYPY);
+
+#ifdef TEST_BOEHM_RAWREFCOUNT
+ printf("plist[%d].gcenc: %p ", (int)i, (void *)plist[i].gcenc);
+#endif
+
+ if ((plist[i].gcenc & 1) ^ (p->ob_refcnt == REFCNT_FROM_PYPY)) {
+ /* ob_refcnt > FROM_PYPY: non-zero regular refcnt,
+ the gc obj must stay alive. decode gcenc.
+ ---OR---
+ ob_refcnt == FROM_PYPY: no refs from C code, the
+ gc obj must not (necessarily) stay alive. encode gcenc.
+ */
+ plist[i].gcenc = ~plist[i].gcenc;
+ }
+ gcenc_union |= plist[i].gcenc;
+#ifdef TEST_BOEHM_RAWREFCOUNT
+ printf("-> %p\n", (void *)plist[i].gcenc);
+#endif
+ }
+ plist = plist[0].next_in_bucket;
+ }
+ if (gcenc_union & 1) /* if there is at least one item potentially dead */
+ hash_list_walk_next = hash_mask_bucket;
+}
diff --git a/rpython/rlib/src/boehm-rawrefcount.h b/rpython/rlib/src/boehm-rawrefcount.h
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/src/boehm-rawrefcount.h
@@ -0,0 +1,26 @@
+
+/* Missing:
+ OP_GC_RAWREFCOUNT_INIT(callback, r): the callback is not supported here
+ OP_GC_RAWREFCOUNT_CREATE_LINK_PYOBJ(): not implemented, maybe not needed
+*/
+
+#define OP_GC_RAWREFCOUNT_CREATE_LINK_PYPY(gcobj, pyobj, r) \
+ gc_rawrefcount_create_link_pypy(gcobj, pyobj)
+
+#define OP_GC_RAWREFCOUNT_FROM_OBJ(gcobj, r) \
+ r = gc_rawrefcount_from_obj(gcobj)
+
+#define OP_GC_RAWREFCOUNT_TO_OBJ(pyobj, r) \
+ r = gc_rawrefcount_to_obj(pyobj)
+
+#define OP_GC_RAWREFCOUNT_NEXT_DEAD(r) \
+ r = gc_rawrefcount_next_dead()
+
+#define OP_GC_RAWREFCOUNT_MARK_DEALLOCATING(gcobj, pyobj, r) /* nothing */
+
+
+RPY_EXTERN void gc_rawrefcount_create_link_pypy(/*gcobj_t*/void *gcobj,
+ /*pyobj_t*/void *pyobj);
+RPY_EXTERN /*pyobj_t*/void *gc_rawrefcount_from_obj(/*gcobj_t*/void *gcobj);
+RPY_EXTERN /*gcobj_t*/void *gc_rawrefcount_to_obj(/*pyobj_t*/void *pyobj);
+RPY_EXTERN /*pyobj_t*/void *gc_rawrefcount_next_dead(void);
diff --git a/rpython/rlib/test/test_buffer.py b/rpython/rlib/test/test_buffer.py
--- a/rpython/rlib/test/test_buffer.py
+++ b/rpython/rlib/test/test_buffer.py
@@ -1,4 +1,4 @@
-from rpython.rlib.buffer import *
+from rpython.rlib.buffer import StringBuffer, SubBuffer, Buffer
from rpython.annotator.annrpython import RPythonAnnotator
from rpython.annotator.model import SomeInteger
@@ -64,3 +64,10 @@
for i in range(9999, 9, -1):
buf = SubBuffer(buf, 1, i)
assert buf.getlength() == 10
+
+def test_string_buffer_as_buffer():
+ buf = StringBuffer(b'hello world')
+ addr = buf.get_raw_address()
+ assert addr[0] == b'h'
+ assert addr[4] == b'o'
+ assert addr[6] == b'w'
diff --git a/rpython/rlib/test/test_rawrefcount.py b/rpython/rlib/test/test_rawrefcount.py
--- a/rpython/rlib/test/test_rawrefcount.py
+++ b/rpython/rlib/test/test_rawrefcount.py
@@ -1,7 +1,7 @@
import weakref
from rpython.rlib import rawrefcount, objectmodel, rgc
from rpython.rlib.rawrefcount import REFCNT_FROM_PYPY, REFCNT_FROM_PYPY_LIGHT
-from rpython.rtyper.lltypesystem import lltype, llmemory
+from rpython.rtyper.lltypesystem import lltype
from rpython.rtyper.annlowlevel import llhelper
from rpython.translator.c.test.test_standalone import StandaloneTests
from rpython.config.translationoption import get_combined_translation_config
@@ -264,6 +264,9 @@
if rawrefcount.next_dead(PyObject) != ob:
print "NEXT_DEAD != OB"
return 1
+ if ob.c_ob_refcnt != 1:
+ print "next_dead().ob_refcnt != 1"
+ return 1
if rawrefcount.next_dead(PyObject) != lltype.nullptr(PyObjectS):
print "NEXT_DEAD second time != NULL"
return 1
diff --git a/rpython/rlib/test/test_rawrefcount_boehm.py b/rpython/rlib/test/test_rawrefcount_boehm.py
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/test/test_rawrefcount_boehm.py
@@ -0,0 +1,308 @@
+import itertools, os, subprocess, py
+from hypothesis import given, strategies
+from rpython.tool.udir import udir
+from rpython.rlib import rawrefcount, rgc
+from rpython.rlib.rawrefcount import REFCNT_FROM_PYPY
+from rpython.rlib.test.test_rawrefcount import W_Root, PyObject, PyObjectS
+from rpython.rtyper.lltypesystem import lltype
+from rpython.translator.c.test.test_standalone import StandaloneTests
+from rpython.config.translationoption import get_combined_translation_config
+
+
+def compile_test(basename):
+ srcdir = os.path.dirname(os.path.dirname(
+ os.path.abspath(os.path.join(__file__))))
+ srcdir = os.path.join(srcdir, 'src')
+
+ err = os.system("cd '%s' && gcc -Werror -lgc -I%s -o %s %s.c"
+ % (udir, srcdir, basename, basename))
+ return err
+
+def setup_module():
+ filename = str(udir.join("test-rawrefcount-boehm-check.c"))
+ with open(filename, "w") as f:
+ print >> f, '#include "gc/gc_mark.h"'
+ print >> f, '#include <stdio.h>'
+ print >> f, 'int main(void) {'
+ print >> f, ' printf("%p", &GC_set_start_callback);'
+ print >> f, ' return 0;'
+ print >> f, '}'
+
+ if compile_test("test-rawrefcount-boehm-check") != 0:
+ py.test.skip("Boehm GC not installed or too old version")
+
+
+
+TEST_CODE = r"""
+#define TEST_BOEHM_RAWREFCOUNT
+#include "boehm-rawrefcount.c"
+
+static gcobj_t *alloc_gcobj(void) /* for tests */
+{
+ gcobj_t *g = GC_MALLOC(1000);
+ printf("gc obj: %p\n", g);
+ return g;
+}
+
+static pyobj_t *alloc_pyobj(void) /* for tests */
+{
+ pyobj_t *p = malloc(1000);
+ p->ob_refcnt = 1;
+ p->ob_pypy_link = 0;
+ printf("py obj: %p\n", p);
+ return p;
+}
+
+static void decref(pyobj_t *p) /* for tests */
+{
+ p->ob_refcnt--;
+ if (p->ob_refcnt == 0) {
+ printf("decref to zero: %p\n", p);
+ free(p);
+ }
+ assert(p->ob_refcnt >= REFCNT_FROM_PYPY ||
+ p->ob_refcnt < REFCNT_FROM_PYPY * 0.99);
+}
+
+void run_test(void); /* forward declaration, produced by the test */
+
+int main(void)
+{
+ run_test();
+ while (gc_rawrefcount_next_dead() != NULL)
+ ;
+ return 0;
+}
+"""
+
+
+operations = strategies.sampled_from([
+ 'new_pyobj',
+ 'new_gcobj',
+ 'create_link',
+ 'from_obj',
+ 'to_obj',
+ 'forget_pyobj',
+ 'forget_gcobj',
+ 'collect',
+ 'dead',
+ ])
+
+
+ at strategies.composite
+def make_code(draw):
+ code = []
+ pyobjs = []
+ gcobjs = []
+ num_gcobj = itertools.count()
+ num_pyobj = itertools.count()
+ links_g2p = {}
+ links_p2g = {}
+
+ def new_gcobj():
+ varname = 'g%d' % next(num_gcobj)
+ code.append('gcobj_t *volatile %s = alloc_gcobj();' % varname)
+ gcobjs.append(varname)
+ return varname
+
+ def new_pyobj():
+ varname = 'p%d' % next(num_pyobj)
+ code.append('pyobj_t *%s = alloc_pyobj();' % varname)
+ pyobjs.append(varname)
+ return varname
+
+ for op in draw(strategies.lists(operations, average_size=250)):
+ if op == 'new_gcobj':
+ new_gcobj()
+ elif op == 'new_pyobj':
+ new_pyobj()
+ elif op == 'create_link':
+ gvars = [varname for varname in gcobjs if varname not in links_g2p]
+ if gvars == []:
+ gvars.append(new_gcobj())
+ pvars = [varname for varname in pyobjs if varname not in links_p2g]
+ if pvars == []:
+ pvars.append(new_pyobj())
+ gvar = draw(strategies.sampled_from(gvars))
+ pvar = draw(strategies.sampled_from(pvars))
+ code.append(r'printf("create_link %%p-%%p\n", %s, %s); '
+ % (gvar, pvar) +
+ "%s->ob_refcnt += REFCNT_FROM_PYPY; " % pvar +
+ "gc_rawrefcount_create_link_pypy(%s, %s);"
+ % (gvar, pvar))
+ links_g2p[gvar] = pvar
+ links_p2g[pvar] = gvar
+ elif op == 'from_obj':
+ if gcobjs:
+ prnt = False
+ gvar = draw(strategies.sampled_from(gcobjs))
+ if gvar not in links_g2p:
+ check = "== NULL"
+ elif links_g2p[gvar] in pyobjs:
+ check = "== %s" % (links_g2p[gvar],)
+ else:
+ check = "!= NULL"
+ prnt = True
+ code.append("assert(gc_rawrefcount_from_obj(%s) %s);"
+ % (gvar, check))
+ if prnt:
+ code.append(r'printf("link %%p-%%p\n", %s, '
+ 'gc_rawrefcount_from_obj(%s));' % (gvar, gvar))
+ elif op == 'to_obj':
+ if pyobjs:
+ prnt = False
+ pvar = draw(strategies.sampled_from(pyobjs))
+ if pvar not in links_p2g:
+ check = "== NULL"
+ elif links_p2g[pvar] in gcobjs:
+ check = "== %s" % (links_p2g[pvar],)
+ else:
+ check = "!= NULL"
+ prnt = True
+ code.append("assert(gc_rawrefcount_to_obj(%s) %s);"
+ % (pvar, check))
+ if prnt:
+ code.append(r'printf("link %%p-%%p\n", '
+ 'gc_rawrefcount_to_obj(%s), %s);' % (pvar, pvar))
+ elif op == 'forget_pyobj':
+ if pyobjs:
+ index = draw(strategies.sampled_from(range(len(pyobjs))))
+ pvar = pyobjs.pop(index)
+ code.append(r'printf("-p%%p\n", %s); ' % pvar +
+ "decref(%s); %s = NULL;" % (pvar, pvar))
+ elif op == 'forget_gcobj':
+ if gcobjs:
+ index = draw(strategies.sampled_from(range(len(gcobjs))))
+ gvar = gcobjs.pop(index)
+ code.append(r'printf("-g%%p\n", %s); ' % gvar +
+ "%s = NULL;" % (gvar,))
+ elif op == 'collect':
+ code.append("GC_gcollect();")
+ elif op == 'dead':
+ code.append('gc_rawrefcount_next_dead();')
+ else:
+ assert False, op
+
+ return '\n'.join(code)
+
+
+ at given(make_code())
+def test_random(code):
+ filename = str(udir.join("test-rawrefcount-boehm.c"))
+ with open(filename, "w") as f:
+ print >> f, TEST_CODE
+ print >> f, 'void run_test(void) {'
+ print >> f, code
+ print >> f, '}'
+
+ err = compile_test("test-rawrefcount-boehm")
+ if err != 0:
+ raise OSError("gcc failed")
+ p = subprocess.Popen("./test-rawrefcount-boehm", stdout=subprocess.PIPE,
+ cwd=str(udir))
+ stdout, _ = p.communicate()
+ assert p.wait() == 0
+
+ gcobjs = {}
+ pyobjs = {}
+ links_p2g = {}
+ links_g2p = {}
+ for line in stdout.splitlines():
+ if line.startswith('py obj: '):
+ p = line[8:]
+ assert not pyobjs.get(p)
+ pyobjs[p] = True
+ assert p not in links_p2g
+ elif line.startswith('gc obj: '):
+ g = line[8:]
+ assert not gcobjs.get(g)
+ gcobjs[g] = True
+ if g in links_g2p: del links_g2p[g]
More information about the pypy-commit
mailing list