[pypy-commit] pypy more_strategies: Merge default.
ltratt
noreply at buildbot.pypy.org
Mon Nov 11 21:59:16 CET 2013
Author: Laurence Tratt <laurie at tratt.net>
Branch: more_strategies
Changeset: r67960:21ecaa0104be
Date: 2013-11-11 20:47 +0000
http://bitbucket.org/pypy/pypy/changeset/21ecaa0104be/
Log: Merge default.
diff too long, truncating to 2000 out of 4612 lines
diff --git a/lib-python/2.7/json/encoder.py b/lib-python/2.7/json/encoder.py
--- a/lib-python/2.7/json/encoder.py
+++ b/lib-python/2.7/json/encoder.py
@@ -4,6 +4,21 @@
from __pypy__.builders import StringBuilder, UnicodeBuilder
+class StringOrUnicodeBuilder(object):
+ def __init__(self):
+ self._builder = StringBuilder()
+ def append(self, string):
+ try:
+ self._builder.append(string)
+ except UnicodeEncodeError:
+ ub = UnicodeBuilder()
+ ub.append(self._builder.build())
+ self._builder = ub
+ ub.append(string)
+ def build(self):
+ return self._builder.build()
+
+
ESCAPE = re.compile(r'[\x00-\x1f\\"\b\f\n\r\t]')
ESCAPE_ASCII = re.compile(r'([\\"]|[^\ -~])')
HAS_UTF8 = re.compile(r'[\x80-\xff]')
@@ -192,7 +207,7 @@
if self.ensure_ascii:
builder = StringBuilder()
else:
- builder = UnicodeBuilder()
+ builder = StringOrUnicodeBuilder()
self.__encode(o, markers, builder, 0)
return builder.build()
diff --git a/lib-python/2.7/weakref.py b/lib-python/2.7/weakref.py
--- a/lib-python/2.7/weakref.py
+++ b/lib-python/2.7/weakref.py
@@ -48,7 +48,14 @@
def remove(wr, selfref=ref(self)):
self = selfref()
if self is not None:
- del self.data[wr.key]
+ # Changed this for PyPy: made more resistent. The
+ # issue is that in some corner cases, self.data
+ # might already be changed or removed by the time
+ # this weakref's callback is called. If that is
+ # the case, we don't want to randomly kill an
+ # unrelated entry.
+ if self.data.get(wr.key) is wr:
+ del self.data[wr.key]
self._remove = remove
UserDict.UserDict.__init__(self, *args, **kw)
@@ -160,22 +167,26 @@
try:
o = self.data.pop(key)()
except KeyError:
+ o = None
+ if o is None:
if args:
return args[0]
- raise
- if o is None:
raise KeyError, key
else:
return o
+ # The logic above was fixed in PyPy
def setdefault(self, key, default=None):
try:
- wr = self.data[key]
+ o = self.data[key]()
except KeyError:
+ o = None
+ if o is None:
self.data[key] = KeyedRef(default, self._remove, key)
return default
else:
- return wr()
+ return o
+ # The logic above was fixed in PyPy
def update(self, dict=None, **kwargs):
d = self.data
diff --git a/lib_pypy/cffi.egg-info b/lib_pypy/cffi.egg-info
--- a/lib_pypy/cffi.egg-info
+++ b/lib_pypy/cffi.egg-info
@@ -1,6 +1,6 @@
Metadata-Version: 1.0
Name: cffi
-Version: 0.7
+Version: 0.8
Summary: Foreign Function Interface for Python calling C code.
Home-page: http://cffi.readthedocs.org
Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,5 +4,5 @@
from .api import FFI, CDefError, FFIError
from .ffiplatform import VerificationError, VerificationMissing
-__version__ = "0.7.2"
-__version_info__ = (0, 7, 2)
+__version__ = "0.8"
+__version_info__ = (0, 8)
diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py
--- a/lib_pypy/cffi/api.py
+++ b/lib_pypy/cffi/api.py
@@ -1,4 +1,5 @@
import types
+from .lock import allocate_lock
try:
callable
@@ -61,6 +62,7 @@
# rely on it! It's probably not going to work well.)
self._backend = backend
+ self._lock = allocate_lock()
self._parser = cparser.Parser()
self._cached_btypes = {}
self._parsed_types = types.ModuleType('parsed_types').__dict__
@@ -74,7 +76,8 @@
if name.startswith('RTLD_'):
setattr(self, name, getattr(backend, name))
#
- self.BVoidP = self._get_cached_btype(model.voidp_type)
+ with self._lock:
+ self.BVoidP = self._get_cached_btype(model.voidp_type)
if isinstance(backend, types.ModuleType):
# _cffi_backend: attach these constants to the class
if not hasattr(FFI, 'NULL'):
@@ -95,11 +98,12 @@
if not isinstance(csource, basestring):
raise TypeError("cdef() argument must be a string")
csource = csource.encode('ascii')
- self._parser.parse(csource, override=override)
- self._cdefsources.append(csource)
- if override:
- for cache in self._function_caches:
- cache.clear()
+ with self._lock:
+ self._parser.parse(csource, override=override)
+ self._cdefsources.append(csource)
+ if override:
+ for cache in self._function_caches:
+ cache.clear()
def dlopen(self, name, flags=0):
"""Load and return a dynamic library identified by 'name'.
@@ -109,31 +113,47 @@
library we only look for the actual (untyped) symbols.
"""
assert isinstance(name, basestring) or name is None
- lib, function_cache = _make_ffi_library(self, name, flags)
- self._function_caches.append(function_cache)
- self._libraries.append(lib)
+ with self._lock:
+ lib, function_cache = _make_ffi_library(self, name, flags)
+ self._function_caches.append(function_cache)
+ self._libraries.append(lib)
return lib
+ def _typeof_locked(self, cdecl):
+ # call me with the lock!
+ key = cdecl
+ if key in self._parsed_types:
+ return self._parsed_types[key]
+ #
+ if not isinstance(cdecl, str): # unicode, on Python 2
+ cdecl = cdecl.encode('ascii')
+ #
+ type = self._parser.parse_type(cdecl)
+ really_a_function_type = type.is_raw_function
+ if really_a_function_type:
+ type = type.as_function_pointer()
+ btype = self._get_cached_btype(type)
+ result = btype, really_a_function_type
+ self._parsed_types[key] = result
+ return result
+
def _typeof(self, cdecl, consider_function_as_funcptr=False):
# string -> ctype object
try:
- btype, cfaf = self._parsed_types[cdecl]
- if consider_function_as_funcptr and not cfaf:
- raise KeyError
+ result = self._parsed_types[cdecl]
except KeyError:
- key = cdecl
- if not isinstance(cdecl, str): # unicode, on Python 2
- cdecl = cdecl.encode('ascii')
- cfaf = consider_function_as_funcptr
- type = self._parser.parse_type(cdecl,
- consider_function_as_funcptr=cfaf)
- btype = self._get_cached_btype(type)
- self._parsed_types[key] = btype, cfaf
+ with self._lock:
+ result = self._typeof_locked(cdecl)
+ #
+ btype, really_a_function_type = result
+ if really_a_function_type and not consider_function_as_funcptr:
+ raise CDefError("the type %r is a function type, not a "
+ "pointer-to-function type" % (cdecl,))
return btype
def typeof(self, cdecl):
"""Parse the C type given as a string and return the
- corresponding Python type: <class 'ffi.CData<...>'>.
+ corresponding <ctype> object.
It can also be used on 'cdata' instance to get its C type.
"""
if isinstance(cdecl, basestring):
@@ -144,6 +164,10 @@
res = _builtin_function_type(cdecl)
if res is not None:
return res
+ if (isinstance(cdecl, types.FunctionType)
+ and hasattr(cdecl, '_cffi_base_type')):
+ with self._lock:
+ return self._get_cached_btype(cdecl._cffi_base_type)
raise TypeError(type(cdecl))
def sizeof(self, cdecl):
@@ -280,14 +304,17 @@
data. Later, when this new cdata object is garbage-collected,
'destructor(old_cdata_object)' will be called.
"""
- try:
- gc_weakrefs = self.gc_weakrefs
- except AttributeError:
- from .gc_weakref import GcWeakrefs
- gc_weakrefs = self.gc_weakrefs = GcWeakrefs(self)
- return gc_weakrefs.build(cdata, destructor)
+ with self._lock:
+ try:
+ gc_weakrefs = self.gc_weakrefs
+ except AttributeError:
+ from .gc_weakref import GcWeakrefs
+ gc_weakrefs = self.gc_weakrefs = GcWeakrefs(self)
+ return gc_weakrefs.build(cdata, destructor)
def _get_cached_btype(self, type):
+ assert self._lock.acquire(False) is False
+ # call me with the lock!
try:
BType = self._cached_btypes[type]
except KeyError:
@@ -322,7 +349,8 @@
def _pointer_to(self, ctype):
from . import model
- return model.pointer_cache(self, ctype)
+ with self._lock:
+ return model.pointer_cache(self, ctype)
def addressof(self, cdata, field=None):
"""Return the address of a <cdata 'struct-or-union'>.
@@ -342,10 +370,12 @@
variables, which must anyway be accessed directly from the
lib object returned by the original FFI instance.
"""
- self._parser.include(ffi_to_include._parser)
- self._cdefsources.append('[')
- self._cdefsources.extend(ffi_to_include._cdefsources)
- self._cdefsources.append(']')
+ with ffi_to_include._lock:
+ with self._lock:
+ self._parser.include(ffi_to_include._parser)
+ self._cdefsources.append('[')
+ self._cdefsources.extend(ffi_to_include._cdefsources)
+ self._cdefsources.append(']')
def new_handle(self, x):
return self._backend.newp_handle(self.BVoidP, x)
@@ -372,7 +402,7 @@
backendlib = backend.load_library(path, flags)
copied_enums = []
#
- def make_accessor(name):
+ def make_accessor_locked(name):
key = 'function ' + name
if key in ffi._parser._declarations:
tp = ffi._parser._declarations[key]
@@ -404,11 +434,17 @@
if enumname not in library.__dict__:
library.__dict__[enumname] = enumval
copied_enums.append(True)
+ if name in library.__dict__:
+ return
#
- if name in library.__dict__: # copied from an enum value just above,
- return # or multithread's race condition
raise AttributeError(name)
#
+ def make_accessor(name):
+ with ffi._lock:
+ if name in library.__dict__ or name in FFILibrary.__dict__:
+ return # added by another thread while waiting for the lock
+ make_accessor_locked(name)
+ #
class FFILibrary(object):
def __getattr__(self, name):
make_accessor(name)
@@ -444,4 +480,5 @@
except (KeyError, AttributeError, TypeError):
return None
else:
- return ffi._get_cached_btype(tp)
+ with ffi._lock:
+ return ffi._get_cached_btype(tp)
diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py
--- a/lib_pypy/cffi/cparser.py
+++ b/lib_pypy/cffi/cparser.py
@@ -142,7 +142,7 @@
if 1 <= linenum <= len(csourcelines):
line = csourcelines[linenum-1]
if line:
- msg = 'cannot parse "%s"\n%s' % (line, msg)
+ msg = 'cannot parse "%s"\n%s' % (line.strip(), msg)
else:
msg = 'parse error\n%s' % (msg,)
raise api.CDefError(msg)
@@ -217,19 +217,18 @@
#
if decl.name:
tp = self._get_type(node, partial_length_ok=True)
- if self._is_constant_declaration(node):
+ if self._is_constant_globalvar(node):
self._declare('constant ' + decl.name, tp)
else:
self._declare('variable ' + decl.name, tp)
- def parse_type(self, cdecl, consider_function_as_funcptr=False):
+ def parse_type(self, cdecl):
ast, macros = self._parse('void __dummy(\n%s\n);' % cdecl)
assert not macros
exprnode = ast.ext[-1].type.args.params[0]
if isinstance(exprnode, pycparser.c_ast.ID):
raise api.CDefError("unknown identifier '%s'" % (exprnode.name,))
- return self._get_type(exprnode.type,
- consider_function_as_funcptr=consider_function_as_funcptr)
+ return self._get_type(exprnode.type)
def _declare(self, name, obj):
if name in self._declarations:
@@ -249,28 +248,17 @@
return model.ConstPointerType(type)
return model.PointerType(type)
- def _get_type(self, typenode, convert_array_to_pointer=False,
- name=None, partial_length_ok=False,
- consider_function_as_funcptr=False):
+ def _get_type(self, typenode, name=None, partial_length_ok=False):
# first, dereference typedefs, if we have it already parsed, we're good
if (isinstance(typenode, pycparser.c_ast.TypeDecl) and
isinstance(typenode.type, pycparser.c_ast.IdentifierType) and
len(typenode.type.names) == 1 and
('typedef ' + typenode.type.names[0]) in self._declarations):
type = self._declarations['typedef ' + typenode.type.names[0]]
- if isinstance(type, model.ArrayType):
- if convert_array_to_pointer:
- return type.item
- else:
- if (consider_function_as_funcptr and
- isinstance(type, model.RawFunctionType)):
- return type.as_function_pointer()
return type
#
if isinstance(typenode, pycparser.c_ast.ArrayDecl):
# array type
- if convert_array_to_pointer:
- return self._get_type_pointer(self._get_type(typenode.type))
if typenode.dim is None:
length = None
else:
@@ -331,10 +319,7 @@
#
if isinstance(typenode, pycparser.c_ast.FuncDecl):
# a function type
- result = self._parse_function_type(typenode, name)
- if consider_function_as_funcptr:
- result = result.as_function_pointer()
- return result
+ return self._parse_function_type(typenode, name)
#
# nested anonymous structs or unions end up here
if isinstance(typenode, pycparser.c_ast.Struct):
@@ -365,21 +350,24 @@
isinstance(params[0].type.type, pycparser.c_ast.IdentifierType)
and list(params[0].type.type.names) == ['void']):
del params[0]
- args = [self._get_type(argdeclnode.type,
- convert_array_to_pointer=True,
- consider_function_as_funcptr=True)
+ args = [self._as_func_arg(self._get_type(argdeclnode.type))
for argdeclnode in params]
result = self._get_type(typenode.type)
return model.RawFunctionType(tuple(args), result, ellipsis)
- def _is_constant_declaration(self, typenode, const=False):
- if isinstance(typenode, pycparser.c_ast.ArrayDecl):
- return self._is_constant_declaration(typenode.type)
+ def _as_func_arg(self, type):
+ if isinstance(type, model.ArrayType):
+ return model.PointerType(type.item)
+ elif isinstance(type, model.RawFunctionType):
+ return type.as_function_pointer()
+ else:
+ return type
+
+ def _is_constant_globalvar(self, typenode):
if isinstance(typenode, pycparser.c_ast.PtrDecl):
- const = 'const' in typenode.quals
- return self._is_constant_declaration(typenode.type, const)
+ return 'const' in typenode.quals
if isinstance(typenode, pycparser.c_ast.TypeDecl):
- return const or 'const' in typenode.quals
+ return 'const' in typenode.quals
return False
def _get_struct_union_enum_type(self, kind, type, name=None, nested=False):
@@ -491,7 +479,7 @@
return tp
def _make_partial(self, tp, nested):
- if not isinstance(tp, model.StructType):
+ if not isinstance(tp, model.StructOrUnion):
raise api.CDefError("%s cannot be partial" % (tp,))
if not tp.has_c_name() and not nested:
raise NotImplementedError("%s is partial but has no C name" %(tp,))
@@ -511,7 +499,7 @@
if (isinstance(exprnode, pycparser.c_ast.ID) and
exprnode.name == '__dotdotdotarray__'):
self._partial_length = True
- return None
+ return '...'
#
raise api.FFIError("unsupported expression: expected a "
"simple numeric constant")
diff --git a/lib_pypy/cffi/gc_weakref.py b/lib_pypy/cffi/gc_weakref.py
--- a/lib_pypy/cffi/gc_weakref.py
+++ b/lib_pypy/cffi/gc_weakref.py
@@ -14,6 +14,6 @@
def build(self, cdata, destructor):
# make a new cdata of the same type as the original one
- new_cdata = self.ffi.cast(self.ffi.typeof(cdata), cdata)
+ new_cdata = self.ffi.cast(self.ffi._backend.typeof(cdata), cdata)
self.data[ref(new_cdata, self.remove)] = destructor, cdata
return new_cdata
diff --git a/lib_pypy/cffi/lock.py b/lib_pypy/cffi/lock.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/cffi/lock.py
@@ -0,0 +1,30 @@
+import sys
+
+if sys.version_info < (3,):
+ try:
+ from thread import allocate_lock
+ except ImportError:
+ from dummy_thread import allocate_lock
+else:
+ try:
+ from _thread import allocate_lock
+ except ImportError:
+ from _dummy_thread import allocate_lock
+
+
+##import sys
+##l1 = allocate_lock
+
+##class allocate_lock(object):
+## def __init__(self):
+## self._real = l1()
+## def __enter__(self):
+## for i in range(4, 0, -1):
+## print sys._getframe(i).f_code
+## print
+## return self._real.__enter__()
+## def __exit__(self, *args):
+## return self._real.__exit__(*args)
+## def acquire(self, f):
+## assert f is False
+## return self._real.acquire(f)
diff --git a/lib_pypy/cffi/model.py b/lib_pypy/cffi/model.py
--- a/lib_pypy/cffi/model.py
+++ b/lib_pypy/cffi/model.py
@@ -1,7 +1,10 @@
import weakref
+from .lock import allocate_lock
+
class BaseTypeByIdentity(object):
is_array_type = False
+ is_raw_function = False
def get_c_name(self, replace_with='', context='a C file'):
result = self.c_name_with_marker
@@ -146,6 +149,7 @@
# a function, but not a pointer-to-function. The backend has no
# notion of such a type; it's used temporarily by parsing.
_base_pattern = '(&)(%s)'
+ is_raw_function = True
def build_backend_type(self, ffi, finishlist):
from . import api
@@ -192,10 +196,6 @@
_base_pattern = " const *&"
_base_pattern_array = "(const *&)"
- def build_backend_type(self, ffi, finishlist):
- BPtr = PointerType(self.totype).get_cached_btype(ffi, finishlist)
- return BPtr
-
const_voidp_type = ConstPointerType(void_type)
@@ -216,10 +216,12 @@
self.item = item
self.length = length
#
- if self.length is None:
+ if length is None:
brackets = '&[]'
+ elif length == '...':
+ brackets = '&[/*...*/]'
else:
- brackets = '&[%d]' % self.length
+ brackets = '&[%d]' % length
self.c_name_with_marker = (
self.item.c_name_with_marker.replace('&', brackets))
@@ -227,6 +229,10 @@
return ArrayType(self.item, newlength)
def build_backend_type(self, ffi, finishlist):
+ if self.length == '...':
+ from . import api
+ raise api.CDefError("cannot render the type %r: unknown length" %
+ (self,))
self.item.get_cached_btype(ffi, finishlist) # force the item BType
BPtrItem = PointerType(self.item).get_cached_btype(ffi, finishlist)
return global_cache(self, ffi, 'new_array_type', BPtrItem, self.length)
@@ -252,6 +258,7 @@
class StructOrUnion(StructOrUnionOrEnum):
fixedlayout = None
completed = False
+ partial = False
def __init__(self, name, fldnames, fldtypes, fldbitsize):
self.name = name
@@ -303,20 +310,21 @@
return # not completing it: it's an opaque struct
#
self.completed = 1
- fldtypes = tuple(tp.get_cached_btype(ffi, finishlist)
- for tp in self.fldtypes)
#
if self.fixedlayout is None:
+ fldtypes = [tp.get_cached_btype(ffi, finishlist)
+ for tp in self.fldtypes]
lst = list(zip(self.fldnames, fldtypes, self.fldbitsize))
ffi._backend.complete_struct_or_union(BType, lst, self)
#
else:
+ fldtypes = []
fieldofs, fieldsize, totalsize, totalalignment = self.fixedlayout
for i in range(len(self.fldnames)):
fsize = fieldsize[i]
ftype = self.fldtypes[i]
#
- if isinstance(ftype, ArrayType) and ftype.length is None:
+ if isinstance(ftype, ArrayType) and ftype.length == '...':
# fix the length to match the total size
BItemType = ftype.item.get_cached_btype(ffi, finishlist)
nlen, nrest = divmod(fsize, ffi.sizeof(BItemType))
@@ -327,18 +335,20 @@
ftype = ftype.resolve_length(nlen)
self.fldtypes = (self.fldtypes[:i] + (ftype,) +
self.fldtypes[i+1:])
- BArrayType = ftype.get_cached_btype(ffi, finishlist)
- fldtypes = (fldtypes[:i] + (BArrayType,) +
- fldtypes[i+1:])
- continue
#
- bitemsize = ffi.sizeof(fldtypes[i])
- if bitemsize != fsize:
- self._verification_error(
- "field '%s.%s' is declared as %d bytes, but is "
- "really %d bytes" % (self.name,
- self.fldnames[i] or '{}',
- bitemsize, fsize))
+ BFieldType = ftype.get_cached_btype(ffi, finishlist)
+ if isinstance(ftype, ArrayType) and ftype.length is None:
+ assert fsize == 0
+ else:
+ bitemsize = ffi.sizeof(BFieldType)
+ if bitemsize != fsize:
+ self._verification_error(
+ "field '%s.%s' is declared as %d bytes, but is "
+ "really %d bytes" % (self.name,
+ self.fldnames[i] or '{}',
+ bitemsize, fsize))
+ fldtypes.append(BFieldType)
+ #
lst = list(zip(self.fldnames, fldtypes, self.fldbitsize, fieldofs))
ffi._backend.complete_struct_or_union(BType, lst, self,
totalsize, totalalignment)
@@ -348,11 +358,6 @@
from .ffiplatform import VerificationError
raise VerificationError(msg)
-
-class StructType(StructOrUnion):
- kind = 'struct'
- partial = False
-
def check_not_partial(self):
if self.partial and self.fixedlayout is None:
from . import ffiplatform
@@ -361,19 +366,18 @@
def build_backend_type(self, ffi, finishlist):
self.check_not_partial()
finishlist.append(self)
-
- return global_cache(self, ffi, 'new_struct_type',
+ #
+ return global_cache(self, ffi, 'new_%s_type' % self.kind,
self.get_official_name(), key=self)
+class StructType(StructOrUnion):
+ kind = 'struct'
+
+
class UnionType(StructOrUnion):
kind = 'union'
- def build_backend_type(self, ffi, finishlist):
- finishlist.append(self)
- return global_cache(self, ffi, 'new_union_type',
- self.get_official_name(), key=self)
-
class EnumType(StructOrUnionOrEnum):
kind = 'enum'
@@ -387,6 +391,12 @@
self.baseinttype = baseinttype
self.build_c_name_with_marker()
+ def force_the_name(self, forcename):
+ StructOrUnionOrEnum.force_the_name(self, forcename)
+ if self.forcename is None:
+ name = self.get_official_name()
+ self.forcename = '$' + name.replace(' ', '_')
+
def check_not_partial(self):
if self.partial and not self.partial_resolved:
from . import ffiplatform
@@ -444,6 +454,9 @@
tp = StructType(structname, None, None, None)
return NamedPointerType(tp, name)
+
+global_lock = allocate_lock()
+
def global_cache(srctype, ffi, funcname, *args, **kwds):
key = kwds.pop('key', (funcname, args))
assert not kwds
@@ -464,8 +477,17 @@
res = getattr(ffi._backend, funcname)(*args)
except NotImplementedError as e:
raise NotImplementedError("%r: %s" % (srctype, e))
- ffi._backend.__typecache[key] = res
- return res
+ # note that setdefault() on WeakValueDictionary is not atomic
+ # and contains a rare bug (http://bugs.python.org/issue19542);
+ # we have to use a lock and do it ourselves
+ cache = ffi._backend.__typecache
+ with global_lock:
+ res1 = cache.get(key)
+ if res1 is None:
+ cache[key] = res
+ return res
+ else:
+ return res1
def pointer_cache(ffi, BType):
return global_cache('?', ffi, 'new_pointer_type', BType)
diff --git a/lib_pypy/cffi/vengine_cpy.py b/lib_pypy/cffi/vengine_cpy.py
--- a/lib_pypy/cffi/vengine_cpy.py
+++ b/lib_pypy/cffi/vengine_cpy.py
@@ -15,7 +15,7 @@
def patch_extension_kwds(self, kwds):
pass
- def find_module(self, module_name, path, so_suffix):
+ def find_module(self, module_name, path, so_suffixes):
try:
f, filename, descr = imp.find_module(module_name, path)
except ImportError:
@@ -25,7 +25,7 @@
# Note that after a setuptools installation, there are both .py
# and .so files with the same basename. The code here relies on
# imp.find_module() locating the .so in priority.
- if descr[0] != so_suffix:
+ if descr[0] not in so_suffixes:
return None
return filename
@@ -160,7 +160,10 @@
def __dir__(self):
return FFILibrary._cffi_dir + list(self.__dict__)
library = FFILibrary()
- module._cffi_setup(lst, ffiplatform.VerificationError, library)
+ if module._cffi_setup(lst, ffiplatform.VerificationError, library):
+ import warnings
+ warnings.warn("reimporting %r might overwrite older definitions"
+ % (self.verifier.get_module_name()))
#
# finally, call the loaded_cpy_xxx() functions. This will perform
# the final adjustments, like copying the Python->C wrapper
@@ -280,8 +283,8 @@
return '_cffi_from_c_pointer((char *)%s, _cffi_type(%d))' % (
var, self._gettypenum(tp))
elif isinstance(tp, model.ArrayType):
- return '_cffi_from_c_deref((char *)%s, _cffi_type(%d))' % (
- var, self._gettypenum(tp))
+ return '_cffi_from_c_pointer((char *)%s, _cffi_type(%d))' % (
+ var, self._gettypenum(model.PointerType(tp.item)))
elif isinstance(tp, model.StructType):
if tp.fldnames is None:
raise TypeError("'%s' is used as %s, but is opaque" % (
@@ -464,11 +467,14 @@
prnt(' static Py_ssize_t nums[] = {')
prnt(' sizeof(%s),' % cname)
prnt(' offsetof(struct _cffi_aligncheck, y),')
- for fname, _, fbitsize in tp.enumfields():
+ for fname, ftype, fbitsize in tp.enumfields():
if fbitsize >= 0:
continue # xxx ignore fbitsize for now
prnt(' offsetof(%s, %s),' % (cname, fname))
- prnt(' sizeof(((%s *)0)->%s),' % (cname, fname))
+ if isinstance(ftype, model.ArrayType) and ftype.length is None:
+ prnt(' 0, /* %s */' % ftype._get_c_name())
+ else:
+ prnt(' sizeof(((%s *)0)->%s),' % (cname, fname))
prnt(' -1')
prnt(' };')
prnt(' return _cffi_get_struct_layout(nums);')
@@ -491,7 +497,7 @@
#
function = getattr(module, layoutfuncname)
layout = function()
- if isinstance(tp, model.StructType) and tp.partial:
+ if isinstance(tp, model.StructOrUnion) and tp.partial:
# use the function()'s sizes and offsets to guide the
# layout of the struct
totalsize = layout[0]
@@ -528,9 +534,10 @@
continue # xxx ignore fbitsize for now
check(layout[i], ffi.offsetof(BStruct, fname),
"wrong offset for field %r" % (fname,))
- BField = ffi._get_cached_btype(ftype)
- check(layout[i+1], ffi.sizeof(BField),
- "wrong size for field %r" % (fname,))
+ if layout[i+1] != 0:
+ BField = ffi._get_cached_btype(ftype)
+ check(layout[i+1], ffi.sizeof(BField),
+ "wrong size for field %r" % (fname,))
i += 2
assert i == len(layout)
@@ -566,7 +573,7 @@
# constants, likely declared with '#define'
def _generate_cpy_const(self, is_int, name, tp=None, category='const',
- vartp=None, delayed=True):
+ vartp=None, delayed=True, size_too=False):
prnt = self._prnt
funcname = '_cffi_%s_%s' % (category, name)
prnt('static int %s(PyObject *lib)' % funcname)
@@ -597,6 +604,15 @@
'(unsigned long long)(%s));' % (name,))
prnt(' if (o == NULL)')
prnt(' return -1;')
+ if size_too:
+ prnt(' {')
+ prnt(' PyObject *o1 = o;')
+ prnt(' o = Py_BuildValue("On", o1, (Py_ssize_t)sizeof(%s));'
+ % (name,))
+ prnt(' Py_DECREF(o1);')
+ prnt(' if (o == NULL)')
+ prnt(' return -1;')
+ prnt(' }')
prnt(' res = PyObject_SetAttrString(lib, "%s", o);' % name)
prnt(' Py_DECREF(o);')
prnt(' if (res < 0)')
@@ -633,12 +649,23 @@
prnt('static int %s(PyObject *lib)' % funcname)
prnt('{')
for enumerator, enumvalue in zip(tp.enumerators, tp.enumvalues):
- prnt(' if (%s != %d) {' % (enumerator, enumvalue))
+ if enumvalue < 0:
+ prnt(' if ((%s) >= 0 || (long)(%s) != %dL) {' % (
+ enumerator, enumerator, enumvalue))
+ else:
+ prnt(' if ((%s) < 0 || (unsigned long)(%s) != %dUL) {' % (
+ enumerator, enumerator, enumvalue))
+ prnt(' char buf[64];')
+ prnt(' if ((%s) < 0)' % enumerator)
+ prnt(' snprintf(buf, 63, "%%ld", (long)(%s));' % enumerator)
+ prnt(' else')
+ prnt(' snprintf(buf, 63, "%%lu", (unsigned long)(%s));' %
+ enumerator)
prnt(' PyErr_Format(_cffi_VerificationError,')
- prnt(' "enum %s: %s has the real value %d, '
- 'not %d",')
- prnt(' "%s", "%s", (int)%s, %d);' % (
- name, enumerator, enumerator, enumvalue))
+ prnt(' "enum %s: %s has the real value %s, '
+ 'not %s",')
+ prnt(' "%s", "%s", buf, "%d");' % (
+ name, enumerator, enumvalue))
prnt(' return -1;')
prnt(' }')
prnt(' return %s;' % self._chained_list_constants[True])
@@ -677,15 +704,16 @@
def _generate_cpy_variable_collecttype(self, tp, name):
if isinstance(tp, model.ArrayType):
- self._do_collect_type(tp)
+ tp_ptr = model.PointerType(tp.item)
else:
tp_ptr = model.PointerType(tp)
- self._do_collect_type(tp_ptr)
+ self._do_collect_type(tp_ptr)
def _generate_cpy_variable_decl(self, tp, name):
if isinstance(tp, model.ArrayType):
tp_ptr = model.PointerType(tp.item)
- self._generate_cpy_const(False, name, tp, vartp=tp_ptr)
+ self._generate_cpy_const(False, name, tp, vartp=tp_ptr,
+ size_too = (tp.length == '...'))
else:
tp_ptr = model.PointerType(tp)
self._generate_cpy_const(False, name, tp_ptr, category='var')
@@ -694,11 +722,29 @@
_loading_cpy_variable = _loaded_noop
def _loaded_cpy_variable(self, tp, name, module, library):
+ value = getattr(library, name)
if isinstance(tp, model.ArrayType): # int a[5] is "constant" in the
- return # sense that "a=..." is forbidden
+ # sense that "a=..." is forbidden
+ if tp.length == '...':
+ assert isinstance(value, tuple)
+ (value, size) = value
+ BItemType = self.ffi._get_cached_btype(tp.item)
+ length, rest = divmod(size, self.ffi.sizeof(BItemType))
+ if rest != 0:
+ raise ffiplatform.VerificationError(
+ "bad size: %r does not seem to be an array of %s" %
+ (name, tp.item))
+ tp = tp.resolve_length(length)
+ # 'value' is a <cdata 'type *'> which we have to replace with
+ # a <cdata 'type[N]'> if the N is actually known
+ if tp.length is not None:
+ BArray = self.ffi._get_cached_btype(tp)
+ value = self.ffi.cast(BArray, value)
+ setattr(library, name, value)
+ return
# remove ptr=<cdata 'int *'> from the library instance, and replace
# it by a property on the class, which reads/writes into ptr[0].
- ptr = getattr(library, name)
+ ptr = value
delattr(library, name)
def getter(library):
return ptr[0]
@@ -711,12 +757,9 @@
def _generate_setup_custom(self):
prnt = self._prnt
- prnt('static PyObject *_cffi_setup_custom(PyObject *lib)')
+ prnt('static int _cffi_setup_custom(PyObject *lib)')
prnt('{')
- prnt(' if (%s < 0)' % self._chained_list_constants[True])
- prnt(' return NULL;')
- prnt(' Py_INCREF(Py_None);')
- prnt(' return Py_None;')
+ prnt(' return %s;' % self._chained_list_constants[True])
prnt('}')
cffimod_header = r'''
@@ -834,17 +877,20 @@
static void *_cffi_exports[_CFFI_NUM_EXPORTS];
static PyObject *_cffi_types, *_cffi_VerificationError;
-static PyObject *_cffi_setup_custom(PyObject *lib); /* forward */
+static int _cffi_setup_custom(PyObject *lib); /* forward */
static PyObject *_cffi_setup(PyObject *self, PyObject *args)
{
PyObject *library;
+ int was_alive = (_cffi_types != NULL);
if (!PyArg_ParseTuple(args, "OOO", &_cffi_types, &_cffi_VerificationError,
&library))
return NULL;
Py_INCREF(_cffi_types);
Py_INCREF(_cffi_VerificationError);
- return _cffi_setup_custom(library);
+ if (_cffi_setup_custom(library) < 0)
+ return NULL;
+ return PyBool_FromLong(was_alive);
}
static void _cffi_init(void)
diff --git a/lib_pypy/cffi/vengine_gen.py b/lib_pypy/cffi/vengine_gen.py
--- a/lib_pypy/cffi/vengine_gen.py
+++ b/lib_pypy/cffi/vengine_gen.py
@@ -20,15 +20,15 @@
# up in kwds['export_symbols'].
kwds.setdefault('export_symbols', self.export_symbols)
- def find_module(self, module_name, path, so_suffix):
- basename = module_name + so_suffix
- if path is None:
- path = sys.path
- for dirname in path:
- filename = os.path.join(dirname, basename)
- if os.path.isfile(filename):
- return filename
- return None
+ def find_module(self, module_name, path, so_suffixes):
+ for so_suffix in so_suffixes:
+ basename = module_name + so_suffix
+ if path is None:
+ path = sys.path
+ for dirname in path:
+ filename = os.path.join(dirname, basename)
+ if os.path.isfile(filename):
+ return filename
def collect_types(self):
pass # not needed in the generic engine
@@ -173,6 +173,7 @@
newfunction = self._load_constant(False, tp, name, module)
else:
indirections = []
+ base_tp = tp
if any(isinstance(typ, model.StructOrUnion) for typ in tp.args):
indirect_args = []
for i, typ in enumerate(tp.args):
@@ -186,16 +187,18 @@
wrappername = '_cffi_f_%s' % name
newfunction = module.load_function(BFunc, wrappername)
for i, typ in indirections:
- newfunction = self._make_struct_wrapper(newfunction, i, typ)
+ newfunction = self._make_struct_wrapper(newfunction, i, typ,
+ base_tp)
setattr(library, name, newfunction)
type(library)._cffi_dir.append(name)
- def _make_struct_wrapper(self, oldfunc, i, tp):
+ def _make_struct_wrapper(self, oldfunc, i, tp, base_tp):
backend = self.ffi._backend
BType = self.ffi._get_cached_btype(tp)
def newfunc(*args):
args = args[:i] + (backend.newp(BType, args[i]),) + args[i+1:]
return oldfunc(*args)
+ newfunc._cffi_base_type = base_tp
return newfunc
# ----------
@@ -252,11 +255,14 @@
prnt(' static ssize_t nums[] = {')
prnt(' sizeof(%s),' % cname)
prnt(' offsetof(struct _cffi_aligncheck, y),')
- for fname, _, fbitsize in tp.enumfields():
+ for fname, ftype, fbitsize in tp.enumfields():
if fbitsize >= 0:
continue # xxx ignore fbitsize for now
prnt(' offsetof(%s, %s),' % (cname, fname))
- prnt(' sizeof(((%s *)0)->%s),' % (cname, fname))
+ if isinstance(ftype, model.ArrayType) and ftype.length is None:
+ prnt(' 0, /* %s */' % ftype._get_c_name())
+ else:
+ prnt(' sizeof(((%s *)0)->%s),' % (cname, fname))
prnt(' -1')
prnt(' };')
prnt(' return nums[i];')
@@ -270,7 +276,7 @@
return # nothing to do with opaque structs
layoutfuncname = '_cffi_layout_%s_%s' % (prefix, name)
#
- BFunc = self.ffi.typeof("ssize_t(*)(ssize_t)")
+ BFunc = self.ffi._typeof_locked("ssize_t(*)(ssize_t)")[0]
function = module.load_function(BFunc, layoutfuncname)
layout = []
num = 0
@@ -279,7 +285,7 @@
if x < 0: break
layout.append(x)
num += 1
- if isinstance(tp, model.StructType) and tp.partial:
+ if isinstance(tp, model.StructOrUnion) and tp.partial:
# use the function()'s sizes and offsets to guide the
# layout of the struct
totalsize = layout[0]
@@ -316,9 +322,10 @@
continue # xxx ignore fbitsize for now
check(layout[i], ffi.offsetof(BStruct, fname),
"wrong offset for field %r" % (fname,))
- BField = ffi._get_cached_btype(ftype)
- check(layout[i+1], ffi.sizeof(BField),
- "wrong size for field %r" % (fname,))
+ if layout[i+1] != 0:
+ BField = ffi._get_cached_btype(ftype)
+ check(layout[i+1], ffi.sizeof(BField),
+ "wrong size for field %r" % (fname,))
i += 2
assert i == len(layout)
@@ -379,15 +386,17 @@
def _load_constant(self, is_int, tp, name, module):
funcname = '_cffi_const_%s' % name
if is_int:
- BFunc = self.ffi.typeof("int(*)(long long*)")
+ BType = self.ffi._typeof_locked("long long*")[0]
+ BFunc = self.ffi._typeof_locked("int(*)(long long*)")[0]
function = module.load_function(BFunc, funcname)
- p = self.ffi.new("long long*")
+ p = self.ffi.new(BType)
negative = function(p)
value = int(p[0])
if value < 0 and not negative:
- value += (1 << (8*self.ffi.sizeof("long long")))
+ BLongLong = self.ffi._typeof_locked("long long")[0]
+ value += (1 << (8*self.ffi.sizeof(BLongLong)))
else:
- BFunc = self.ffi.typeof(tp.get_c_name('(*)(void)', name))
+ BFunc = self.ffi._typeof_locked(tp.get_c_name('(*)(void)', name))[0]
function = module.load_function(BFunc, funcname)
value = function()
return value
@@ -413,11 +422,22 @@
prnt('int %s(char *out_error)' % funcname)
prnt('{')
for enumerator, enumvalue in zip(tp.enumerators, tp.enumvalues):
- prnt(' if (%s != %d) {' % (enumerator, enumvalue))
+ if enumvalue < 0:
+ prnt(' if ((%s) >= 0 || (long)(%s) != %dL) {' % (
+ enumerator, enumerator, enumvalue))
+ else:
+ prnt(' if ((%s) < 0 || (unsigned long)(%s) != %dUL) {' % (
+ enumerator, enumerator, enumvalue))
+ prnt(' char buf[64];')
+ prnt(' if ((%s) < 0)' % enumerator)
+ prnt(' snprintf(buf, 63, "%%ld", (long)(%s));' % enumerator)
+ prnt(' else')
+ prnt(' snprintf(buf, 63, "%%lu", (unsigned long)(%s));' %
+ enumerator)
prnt(' snprintf(out_error, 255,'
- '"%s has the real value %d, not %d",')
- prnt(' "%s", (int)%s, %d);' % (
- enumerator, enumerator, enumvalue))
+ ' "%s has the real value %s, not %s",')
+ prnt(' "%s", buf, "%d");' % (
+ enumerator, enumvalue))
prnt(' return -1;')
prnt(' }')
prnt(' return 0;')
@@ -431,10 +451,11 @@
tp.enumvalues = tuple(enumvalues)
tp.partial_resolved = True
else:
- BFunc = self.ffi.typeof("int(*)(char*)")
+ BType = self.ffi._typeof_locked("char[]")[0]
+ BFunc = self.ffi._typeof_locked("int(*)(char*)")[0]
funcname = '_cffi_e_%s_%s' % (prefix, name)
function = module.load_function(BFunc, funcname)
- p = self.ffi.new("char[]", 256)
+ p = self.ffi.new(BType, 256)
if function(p) < 0:
error = self.ffi.string(p)
if sys.version_info >= (3,):
@@ -465,6 +486,14 @@
def _generate_gen_variable_decl(self, tp, name):
if isinstance(tp, model.ArrayType):
+ if tp.length == '...':
+ prnt = self._prnt
+ funcname = '_cffi_sizeof_%s' % (name,)
+ self.export_symbols.append(funcname)
+ prnt("size_t %s(void)" % funcname)
+ prnt("{")
+ prnt(" return sizeof(%s);" % (name,))
+ prnt("}")
tp_ptr = model.PointerType(tp.item)
self._generate_gen_const(False, name, tp_ptr)
else:
@@ -476,6 +505,18 @@
def _loaded_gen_variable(self, tp, name, module, library):
if isinstance(tp, model.ArrayType): # int a[5] is "constant" in the
# sense that "a=..." is forbidden
+ if tp.length == '...':
+ funcname = '_cffi_sizeof_%s' % (name,)
+ BFunc = self.ffi._typeof_locked('size_t(*)(void)')[0]
+ function = module.load_function(BFunc, funcname)
+ size = function()
+ BItemType = self.ffi._get_cached_btype(tp.item)
+ length, rest = divmod(size, self.ffi.sizeof(BItemType))
+ if rest != 0:
+ raise ffiplatform.VerificationError(
+ "bad size: %r does not seem to be an array of %s" %
+ (name, tp.item))
+ tp = tp.resolve_length(length)
tp_ptr = model.PointerType(tp.item)
value = self._load_constant(False, tp_ptr, name, module)
# 'value' is a <cdata 'type *'> which we have to replace with
@@ -489,7 +530,7 @@
# remove ptr=<cdata 'int *'> from the library instance, and replace
# it by a property on the class, which reads/writes into ptr[0].
funcname = '_cffi_var_%s' % name
- BFunc = self.ffi.typeof(tp.get_c_name('*(*)(void)', name))
+ BFunc = self.ffi._typeof_locked(tp.get_c_name('*(*)(void)', name))[0]
function = module.load_function(BFunc, funcname)
ptr = function()
def getter(library):
diff --git a/lib_pypy/cffi/verifier.py b/lib_pypy/cffi/verifier.py
--- a/lib_pypy/cffi/verifier.py
+++ b/lib_pypy/cffi/verifier.py
@@ -31,7 +31,7 @@
k2 = k2.lstrip('0').rstrip('L')
modulename = '_cffi_%s_%s%s%s' % (tag, self._vengine._class_key,
k1, k2)
- suffix = _get_so_suffix()
+ suffix = _get_so_suffixes()[0]
self.tmpdir = tmpdir or _caller_dir_pycache()
self.sourcefilename = os.path.join(self.tmpdir, modulename + '.c')
self.modulefilename = os.path.join(self.tmpdir, modulename + suffix)
@@ -42,18 +42,21 @@
def write_source(self, file=None):
"""Write the C source code. It is produced in 'self.sourcefilename',
which can be tweaked beforehand."""
- if self._has_source and file is None:
- raise ffiplatform.VerificationError("source code already written")
- self._write_source(file)
+ with self.ffi._lock:
+ if self._has_source and file is None:
+ raise ffiplatform.VerificationError(
+ "source code already written")
+ self._write_source(file)
def compile_module(self):
"""Write the C source code (if not done already) and compile it.
This produces a dynamic link library in 'self.modulefilename'."""
- if self._has_module:
- raise ffiplatform.VerificationError("module already compiled")
- if not self._has_source:
- self._write_source()
- self._compile_module()
+ with self.ffi._lock:
+ if self._has_module:
+ raise ffiplatform.VerificationError("module already compiled")
+ if not self._has_source:
+ self._write_source()
+ self._compile_module()
def load_library(self):
"""Get a C module from this Verifier instance.
@@ -62,11 +65,14 @@
operations to the C module. If necessary, the C code is written
and compiled first.
"""
- if not self._has_module:
- self._locate_module()
+ with self.ffi._lock:
if not self._has_module:
- self.compile_module()
- return self._load_library()
+ self._locate_module()
+ if not self._has_module:
+ if not self._has_source:
+ self._write_source()
+ self._compile_module()
+ return self._load_library()
def get_module_name(self):
basename = os.path.basename(self.modulefilename)
@@ -81,7 +87,9 @@
def get_extension(self):
if not self._has_source:
- self._write_source()
+ with self.ffi._lock:
+ if not self._has_source:
+ self._write_source()
sourcename = ffiplatform.maybe_relative_path(self.sourcefilename)
modname = self.get_module_name()
return ffiplatform.get_extension(sourcename, modname, **self.kwds)
@@ -103,7 +111,7 @@
else:
path = None
filename = self._vengine.find_module(self.get_module_name(), path,
- _get_so_suffix())
+ _get_so_suffixes())
if filename is None:
return
self.modulefilename = filename
@@ -193,7 +201,7 @@
if keep_so:
suffix = '.c' # only remove .c files
else:
- suffix = _get_so_suffix().lower()
+ suffix = _get_so_suffixes()[0].lower()
for fn in filelist:
if fn.lower().startswith('_cffi_') and (
fn.lower().endswith(suffix) or fn.lower().endswith('.c')):
@@ -213,15 +221,20 @@
except OSError:
pass
-def _get_so_suffix():
+def _get_so_suffixes():
+ suffixes = []
for suffix, mode, type in imp.get_suffixes():
if type == imp.C_EXTENSION:
- return suffix
- # bah, no C_EXTENSION available. Occurs on pypy without cpyext
- if sys.platform == 'win32':
- return ".pyd"
- else:
- return ".so"
+ suffixes.append(suffix)
+
+ if not suffixes:
+ # bah, no C_EXTENSION available. Occurs on pypy without cpyext
+ if sys.platform == 'win32':
+ suffixes = [".pyd"]
+ else:
+ suffixes = [".so"]
+
+ return suffixes
def _ensure_dir(filename):
try:
diff --git a/lib_pypy/pyrepl/reader.py b/lib_pypy/pyrepl/reader.py
--- a/lib_pypy/pyrepl/reader.py
+++ b/lib_pypy/pyrepl/reader.py
@@ -19,11 +19,13 @@
# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
-import types
+import types, re
from pyrepl import unicodedata_
from pyrepl import commands
from pyrepl import input
+_r_csi_seq = re.compile(r"\033\[[ -@]*[A-~]")
+
def _make_unctrl_map():
uc_map = {}
for c in map(unichr, range(256)):
@@ -309,6 +311,10 @@
excluded from the length calculation. So also a copy of the prompt
is returned with these control characters removed. """
+ # The logic below also ignores the length of common escape
+ # sequences if they were not explicitly within \x01...\x02.
+ # They are CSI (or ANSI) sequences ( ESC [ ... LETTER )
+
out_prompt = ''
l = len(prompt)
pos = 0
@@ -321,9 +327,13 @@
break
# Found start and end brackets, subtract from string length
l = l - (e-s+1)
- out_prompt += prompt[pos:s] + prompt[s+1:e]
+ keep = prompt[pos:s]
+ l -= sum(map(len, _r_csi_seq.findall(keep)))
+ out_prompt += keep + prompt[s+1:e]
pos = e+1
- out_prompt += prompt[pos:]
+ keep = prompt[pos:]
+ l -= sum(map(len, _r_csi_seq.findall(keep)))
+ out_prompt += keep
return out_prompt, l
def bow(self, p=None):
diff --git a/pypy/doc/contributor.rst b/pypy/doc/contributor.rst
--- a/pypy/doc/contributor.rst
+++ b/pypy/doc/contributor.rst
@@ -17,24 +17,24 @@
David Schneider
Holger Krekel
Christian Tismer
+ Matti Picus
Hakan Ardo
Benjamin Peterson
- Matti Picus
Philip Jenvey
Anders Chrigstrom
Brian Kearns
+ Manuel Jacob
Eric van Riet Paap
+ Wim Lavrijsen
Richard Emslie
Alexander Schremmer
- Wim Lavrijsen
Dan Villiom Podlaski Christiansen
- Manuel Jacob
+ Ronan Lamy
Lukas Diekmann
Sven Hager
Anders Lehmann
Aurelien Campeas
Niklaus Haldimann
- Ronan Lamy
Camillo Bruni
Laura Creighton
Toon Verwaest
@@ -45,8 +45,10 @@
David Edelsohn
Anders Hammarquist
Jakub Gustak
+ Romain Guillebert
Guido Wesdorp
Lawrence Oluyede
+ Remi Meier
Bartosz Skowron
Daniel Roberts
Niko Matsakis
@@ -54,18 +56,17 @@
Ludovic Aubry
Alexander Hesse
Jacob Hallen
- Romain Guillebert
Jason Creighton
Alex Martelli
Michal Bendowski
Jan de Mooij
+ stian
Michael Foord
Stephan Diehl
Stefan Schwarzer
Valentino Volonghi
Tomek Meka
Patrick Maupin
- stian
Bob Ippolito
Bruno Gola
Jean-Paul Calderone
@@ -74,29 +75,33 @@
Simon Burton
Marius Gedminas
John Witulski
+ Konstantin Lopuhin
Greg Price
Dario Bertini
Mark Pearse
Simon Cross
- Konstantin Lopuhin
Andreas Stührk
Jean-Philippe St. Pierre
Guido van Rossum
Pavel Vinogradov
+ Paweł Piotr Przeradowski
Paul deGrandis
Ilya Osadchiy
Adrian Kuhn
Boris Feigin
+ Stefano Rivera
tav
+ Taavi Burns
Georg Brandl
Bert Freudenberg
Stian Andreassen
- Stefano Rivera
Wanja Saatkamp
Gerald Klix
Mike Blume
- Taavi Burns
Oscar Nierstrasz
+ Stefan H. Muller
+ Laurence Tratt
+ Rami Chowdhury
David Malcolm
Eugene Oden
Henry Mason
@@ -105,14 +110,15 @@
David Ripton
Dusty Phillips
Lukas Renggli
+ Edd Barrett
Guenter Jantzen
Tobias Oberstein
- Remi Meier
Ned Batchelder
Amit Regmi
Ben Young
Nicolas Chauvat
Andrew Durdin
+ Andrew Chambers
Michael Schneider
Nicholas Riley
Jason Chu
@@ -128,10 +134,13 @@
Olivier Dormond
Jared Grubb
Karl Bartel
+ Tobias Pape
Brian Dorsey
Victor Stinner
+ Andrews Medina
Stuart Williams
Jasper Schulz
+ Christian Hudon
Toby Watson
Antoine Pitrou
Aaron Iles
@@ -141,7 +150,6 @@
Neil Shepperd
Mikael Schönenberg
Elmo Mäntynen
- Tobias Pape
Jonathan David Riehl
Stanislaw Halik
Anders Qvist
@@ -153,19 +161,15 @@
Alexander Sedov
Corbin Simpson
Christopher Pope
- Laurence Tratt
- Guillebert Romain
Christian Tismer
Dan Stromberg
Stefano Parmesan
- Christian Hudon
Alexis Daboville
Jens-Uwe Mager
Carl Meyer
Karl Ramm
Pieter Zieschang
Gabriel
- Paweł Piotr Przeradowski
Andrew Dalke
Sylvain Thenault
Nathan Taylor
@@ -189,13 +193,15 @@
Martin Blais
Lene Wagner
Tomo Cocoa
- Andrews Medina
roberto at goyle
+ Yury V. Zaytsev
+ Anna Katrina Dominguez
William Leslie
Bobby Impollonia
timo at eistee.fritz.box
Andrew Thompson
Yusei Tahara
+ Ben Darnell
Roberto De Ioris
Juan Francisco Cantero Hurtado
Godefroid Chappelle
@@ -209,7 +215,7 @@
Akira Li
Gustavo Niemeyer
Stephan Busemann
- Anna Katrina Dominguez
+ Rafał Gałczyński
Christian Muirhead
James Lan
shoma hosaka
@@ -219,6 +225,7 @@
Dinu Gherman
Chris Lambacher
coolbutuseless at gmail.com
+ w31rd0
Jim Baker
Rodrigo Araújo
Armin Ronacher
@@ -234,12 +241,12 @@
Even Wiik Thomassen
jbs
soareschen
+ Mike Bayer
Flavio Percoco
Kristoffer Kleine
yasirs
Michael Chermside
Anna Ravencroft
- Andrew Chambers
Julien Phalip
Dan Loewenherz
diff --git a/pypy/doc/interpreter.rst b/pypy/doc/interpreter.rst
--- a/pypy/doc/interpreter.rst
+++ b/pypy/doc/interpreter.rst
@@ -137,7 +137,8 @@
control flow of a function (such as ``while`` and ``try`` constructs)
- a value stack where bytecode interpretation pulls object
- from and puts results on.
+ from and puts results on. (``locals_stack_w`` is actually a single
+ list containing both the local scope and the value stack.)
- a reference to the *globals* dictionary, containing
module-level name-value bindings
@@ -151,10 +152,7 @@
- the class ``PyFrame`` is defined in `pypy/interpreter/pyframe.py`_.
-- the file `pypy/interpreter/pyopcode.py`_ add support for all Python opcode.
-
-- nested scope support is added to the ``PyFrame`` class in
- `pypy/interpreter/nestedscope.py`_.
+- the file `pypy/interpreter/pyopcode.py`_ add support for all Python opcodes.
.. _Code:
@@ -184,12 +182,6 @@
* ``co_name`` name of the code object (often the function name)
* ``co_lnotab`` a helper table to compute the line-numbers corresponding to bytecodes
-In PyPy, code objects also have the responsibility of creating their Frame_ objects
-via the `'create_frame()`` method. With proper parser and compiler support this would
-allow to create custom Frame objects extending the execution of functions
-in various ways. The several Frame_ classes already utilize this flexibility
-in order to implement Generators and Nested Scopes.
-
.. _Function:
Function and Method classes
diff --git a/pypy/doc/tool/makecontributor.py b/pypy/doc/tool/makecontributor.py
--- a/pypy/doc/tool/makecontributor.py
+++ b/pypy/doc/tool/makecontributor.py
@@ -1,3 +1,5 @@
+# NOTE: run this script with LANG=en_US.UTF-8
+
import py
import sys
from collections import defaultdict
@@ -132,7 +134,7 @@
if show_numbers:
print '%5d %s' % (n, name)
else:
- print name
+ print ' ' + name
if __name__ == '__main__':
show_numbers = '-n' in sys.argv
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -91,6 +91,7 @@
.. branch: safe-win-mmap
.. branch: boolean-indexing-cleanup
.. branch: cpyext-best_base
+.. branch: cpyext-int
.. branch: fileops2
.. branch: nobold-backtrace
diff --git a/pypy/module/__pypy__/test/test_signal.py b/pypy/module/__pypy__/test/test_signal.py
--- a/pypy/module/__pypy__/test/test_signal.py
+++ b/pypy/module/__pypy__/test/test_signal.py
@@ -56,7 +56,7 @@
interrupted = []
print('--- start ---')
thread.start_new_thread(subthread, ())
- for j in range(10):
+ for j in range(30):
if len(done): break
print('.')
time.sleep(0.25)
diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -7,7 +7,7 @@
appleveldefs = {
}
interpleveldefs = {
- '__version__': 'space.wrap("0.7")',
+ '__version__': 'space.wrap("0.8")',
'load_library': 'libraryobj.load_library',
diff --git a/pypy/module/_cffi_backend/cdataobj.py b/pypy/module/_cffi_backend/cdataobj.py
--- a/pypy/module/_cffi_backend/cdataobj.py
+++ b/pypy/module/_cffi_backend/cdataobj.py
@@ -19,9 +19,9 @@
_cdata = lltype.nullptr(rffi.CCHARP.TO)
def __init__(self, space, cdata, ctype):
- from pypy.module._cffi_backend import ctypeprim
+ from pypy.module._cffi_backend import ctypeobj
assert lltype.typeOf(cdata) == rffi.CCHARP
- assert isinstance(ctype, ctypeprim.W_CType)
+ assert isinstance(ctype, ctypeobj.W_CType)
self.space = space
self._cdata = cdata # don't forget keepalive_until_here!
self.ctype = ctype
@@ -211,7 +211,21 @@
keepalive_until_here(w_value)
return
#
+ # A fast path for <char[]>[0:N] = "somestring".
+ from pypy.module._cffi_backend import ctypeprim
space = self.space
+ if (space.isinstance_w(w_value, space.w_str) and
+ isinstance(ctitem, ctypeprim.W_CTypePrimitiveChar)):
+ from rpython.rtyper.annlowlevel import llstr
+ from rpython.rtyper.lltypesystem.rstr import copy_string_to_raw
+ value = space.str_w(w_value)
+ if len(value) != length:
+ raise operationerrfmt(space.w_ValueError,
+ "need a string of length %d, got %d",
+ length, len(value))
+ copy_string_to_raw(llstr(value), cdata, 0, length)
+ return
+ #
w_iter = space.iter(w_value)
for i in range(length):
try:
@@ -245,19 +259,22 @@
space = self.space
if isinstance(w_other, W_CData):
from pypy.module._cffi_backend import ctypeptr, ctypearray
+ from pypy.module._cffi_backend import ctypevoid
ct = w_other.ctype
if isinstance(ct, ctypearray.W_CTypeArray):
ct = ct.ctptr
#
if (ct is not self.ctype or
not isinstance(ct, ctypeptr.W_CTypePointer) or
- ct.ctitem.size <= 0):
+ (ct.ctitem.size <= 0 and not ct.is_void_ptr)):
raise operationerrfmt(space.w_TypeError,
"cannot subtract cdata '%s' and cdata '%s'",
self.ctype.name, ct.name)
#
+ itemsize = ct.ctitem.size
+ if itemsize <= 0: itemsize = 1
diff = (rffi.cast(lltype.Signed, self._cdata) -
- rffi.cast(lltype.Signed, w_other._cdata)) // ct.ctitem.size
+ rffi.cast(lltype.Signed, w_other._cdata)) // itemsize
return space.wrap(diff)
#
return self._add_or_sub(w_other, -1)
@@ -441,6 +458,7 @@
__getitem__ = interp2app(W_CData.getitem),
__setitem__ = interp2app(W_CData.setitem),
__add__ = interp2app(W_CData.add),
+ __radd__ = interp2app(W_CData.add),
__sub__ = interp2app(W_CData.sub),
__getattr__ = interp2app(W_CData.getattr),
__setattr__ = interp2app(W_CData.setattr),
diff --git a/pypy/module/_cffi_backend/ctypearray.py b/pypy/module/_cffi_backend/ctypearray.py
--- a/pypy/module/_cffi_backend/ctypearray.py
+++ b/pypy/module/_cffi_backend/ctypearray.py
@@ -34,19 +34,8 @@
datasize = self.size
#
if datasize < 0:
- if (space.isinstance_w(w_init, space.w_list) or
- space.isinstance_w(w_init, space.w_tuple)):
- length = space.int_w(space.len(w_init))
- elif space.isinstance_w(w_init, space.w_basestring):
- # from a string, we add the null terminator
- length = space.int_w(space.len(w_init)) + 1
- else:
- length = space.getindex_w(w_init, space.w_OverflowError)
- if length < 0:
- raise OperationError(space.w_ValueError,
- space.wrap("negative array length"))
- w_init = space.w_None
- #
+ from pypy.module._cffi_backend import misc
+ w_init, length = misc.get_new_array_length(space, w_init)
try:
datasize = ovfcheck(length * self.ctitem.size)
except OverflowError:
diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py
--- a/pypy/module/_cffi_backend/ctypeptr.py
+++ b/pypy/module/_cffi_backend/ctypeptr.py
@@ -2,27 +2,25 @@
Pointers.
"""
-from pypy.interpreter.error import OperationError, operationerrfmt, wrap_oserror
-
from rpython.rlib import rposix
from rpython.rlib.objectmodel import keepalive_until_here
from rpython.rlib.rarithmetic import ovfcheck
+from rpython.rtyper.annlowlevel import llstr, llunicode
from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.rtyper.lltypesystem.rstr import copy_string_to_raw, copy_unicode_to_raw
+from pypy.interpreter.error import OperationError, operationerrfmt, wrap_oserror
from pypy.module._cffi_backend import cdataobj, misc, ctypeprim, ctypevoid
from pypy.module._cffi_backend.ctypeobj import W_CType
class W_CTypePtrOrArray(W_CType):
- _attrs_ = ['ctitem', 'can_cast_anything', 'is_struct_ptr',
- 'length']
- _immutable_fields_ = ['ctitem', 'can_cast_anything', 'is_struct_ptr',
- 'length']
+ _attrs_ = ['ctitem', 'can_cast_anything', 'length']
+ _immutable_fields_ = ['ctitem', 'can_cast_anything', 'length']
length = -1
def __init__(self, space, size, extra, extra_position, ctitem,
could_cast_anything=True):
- from pypy.module._cffi_backend.ctypestruct import W_CTypeStructOrUnion
name, name_position = ctitem.insert_name(extra, extra_position)
W_CType.__init__(self, space, size, name, name_position)
# this is the "underlying type":
@@ -31,7 +29,6 @@
# - for functions, it is the return type
self.ctitem = ctitem
self.can_cast_anything = could_cast_anything and ctitem.cast_anything
- self.is_struct_ptr = isinstance(ctitem, W_CTypeStructOrUnion)
def is_char_ptr_or_array(self):
return isinstance(self.ctitem, ctypeprim.W_CTypePrimitiveChar)
@@ -90,8 +87,7 @@
"initializer string is too long for '%s'"
" (got %d characters)",
self.name, n)
- for i in range(n):
- cdata[i] = s[i]
+ copy_string_to_raw(llstr(s), cdata, 0, n)
if n != self.length:
cdata[n] = '\x00'
elif isinstance(self.ctitem, ctypeprim.W_CTypePrimitiveUniChar):
@@ -105,8 +101,7 @@
" (got %d characters)",
self.name, n)
unichardata = rffi.cast(rffi.CWCHARP, cdata)
- for i in range(n):
- unichardata[i] = s[i]
+ copy_unicode_to_raw(llunicode(s), unichardata, 0, n)
if n != self.length:
unichardata[n] = u'\x00'
else:
@@ -157,7 +152,6 @@
return cdataobj.W_CData(self.space, ptrdata, self)
def convert_from_object(self, cdata, w_ob):
- space = self.space
if not isinstance(w_ob, cdataobj.W_CData):
raise self._convert_error("cdata pointer", w_ob)
other = w_ob.ctype
@@ -197,6 +191,7 @@
W_CTypePtrBase.__init__(self, space, size, extra, 2, ctitem)
def newp(self, w_init):
+ from pypy.module._cffi_backend.ctypestruct import W_CTypeStructOrUnion
space = self.space
ctitem = self.ctitem
datasize = ctitem.size
@@ -204,10 +199,15 @@
raise operationerrfmt(space.w_TypeError,
"cannot instantiate ctype '%s' of unknown size",
self.name)
- if self.is_struct_ptr:
+ if isinstance(ctitem, W_CTypeStructOrUnion):
# 'newp' on a struct-or-union pointer: in this case, we return
# a W_CDataPtrToStruct object which has a strong reference
# to a W_CDataNewOwning that really contains the structure.
+ #
+ if ctitem.with_var_array and not space.is_w(w_init, space.w_None):
+ datasize = ctitem.convert_struct_from_object(
+ lltype.nullptr(rffi.CCHARP.TO), w_init, datasize)
+ #
cdatastruct = cdataobj.W_CDataNewOwning(space, datasize, ctitem)
cdata = cdataobj.W_CDataPtrToStructOrUnion(space,
cdatastruct._cdata,
@@ -238,11 +238,15 @@
def add(self, cdata, i):
space = self.space
ctitem = self.ctitem
+ itemsize = ctitem.size
if ctitem.size < 0:
- raise operationerrfmt(space.w_TypeError,
+ if self.is_void_ptr:
+ itemsize = 1
+ else:
+ raise operationerrfmt(space.w_TypeError,
"ctype '%s' points to items of unknown size",
self.name)
- p = rffi.ptradd(cdata, i * self.ctitem.size)
+ p = rffi.ptradd(cdata, i * itemsize)
return cdataobj.W_CData(space, p, self)
def cast(self, w_ob):
@@ -298,7 +302,6 @@
def convert_argument_from_object(self, cdata, w_ob):
from pypy.module._cffi_backend.ctypefunc import set_mustfree_flag
- space = self.space
result = (not isinstance(w_ob, cdataobj.W_CData) and
self._prepare_pointer_call_argument(w_ob, cdata))
if result == 0:
@@ -320,7 +323,8 @@
space = self.space
ctype2 = cdata.ctype
if (isinstance(ctype2, W_CTypeStructOrUnion) or
- (isinstance(ctype2, W_CTypePtrOrArray) and ctype2.is_struct_ptr)):
+ (isinstance(ctype2, W_CTypePtrOrArray) and
+ isinstance(ctype2.ctitem, W_CTypeStructOrUnion))):
ptrdata = rffi.ptradd(cdata._cdata, offset)
return cdataobj.W_CData(space, ptrdata, self)
else:
diff --git a/pypy/module/_cffi_backend/ctypestruct.py b/pypy/module/_cffi_backend/ctypestruct.py
--- a/pypy/module/_cffi_backend/ctypestruct.py
+++ b/pypy/module/_cffi_backend/ctypestruct.py
@@ -9,7 +9,8 @@
from rpython.rlib import jit
from rpython.rlib.objectmodel import keepalive_until_here
from rpython.rlib.rarithmetic import r_uint, r_ulonglong, r_longlong, intmask
-from rpython.rtyper.lltypesystem import rffi
+from rpython.rlib.rarithmetic import ovfcheck
+from rpython.rtyper.lltypesystem import lltype, rffi
from pypy.module._cffi_backend import cdataobj, ctypeprim, misc
from pypy.module._cffi_backend.ctypeobj import W_CType
@@ -17,12 +18,13 @@
class W_CTypeStructOrUnion(W_CType):
_immutable_fields_ = ['alignment?', 'fields_list?', 'fields_dict?',
- 'custom_field_pos?']
+ 'custom_field_pos?', 'with_var_array?']
# fields added by complete_struct_or_union():
alignment = -1
fields_list = None
fields_dict = None
custom_field_pos = False
+ with_var_array = False
def __init__(self, space, name):
W_CType.__init__(self, space, -1, name, len(name))
@@ -90,12 +92,16 @@
pass
def convert_from_object(self, cdata, w_ob):
- space = self.space
- if self._copy_from_same(cdata, w_ob):
- return
+ if not self._copy_from_same(cdata, w_ob):
+ self.convert_struct_from_object(cdata, w_ob, optvarsize=-1)
+ @jit.look_inside_iff(
+ lambda self, cdata, w_ob, optvarsize: jit.isvirtual(w_ob)
+ )
+ def convert_struct_from_object(self, cdata, w_ob, optvarsize):
self._check_only_one_argument_for_union(w_ob)
+ space = self.space
if (space.isinstance_w(w_ob, space.w_list) or
space.isinstance_w(w_ob, space.w_tuple)):
lst_w = space.listview(w_ob)
@@ -104,7 +110,9 @@
"too many initializers for '%s' (got %d)",
self.name, len(lst_w))
for i in range(len(lst_w)):
- self.fields_list[i].write(cdata, lst_w[i])
+ optvarsize = self.fields_list[i].write_v(cdata, lst_w[i],
+ optvarsize)
+ return optvarsize
elif space.isinstance_w(w_ob, space.w_dict):
lst_w = space.fixedview(w_ob)
@@ -116,11 +124,16 @@
except KeyError:
space.raise_key_error(w_key)
assert 0
- cf.write(cdata, space.getitem(w_ob, w_key))
+ optvarsize = cf.write_v(cdata, space.getitem(w_ob, w_key),
+ optvarsize)
+ return optvarsize
else:
- raise self._convert_error("list or tuple or dict or struct-cdata",
- w_ob)
+ if optvarsize == -1:
+ msg = "list or tuple or dict or struct-cdata"
+ else:
+ msg = "list or tuple or dict"
+ raise self._convert_error(msg, w_ob)
@jit.elidable
def _getcfield_const(self, attr):
@@ -192,6 +205,37 @@
else:
self.ctype.convert_from_object(cdata, w_ob)
+ def write_v(self, cdata, w_ob, optvarsize):
+ # a special case for var-sized C99 arrays
+ from pypy.module._cffi_backend import ctypearray
+ ct = self.ctype
+ if isinstance(ct, ctypearray.W_CTypeArray) and ct.length < 0:
+ space = ct.space
+ w_ob, varsizelength = misc.get_new_array_length(space, w_ob)
+ if optvarsize != -1:
+ # in this mode, the only purpose of this function is to compute
+ # the real size of the structure from a var-sized C99 array
+ assert cdata == lltype.nullptr(rffi.CCHARP.TO)
+ itemsize = ct.ctitem.size
+ try:
+ varsize = ovfcheck(itemsize * varsizelength)
+ size = ovfcheck(self.offset + varsize)
+ except OverflowError:
+ raise OperationError(space.w_OverflowError,
+ space.wrap("array size would overflow a ssize_t"))
+ assert size >= 0
+ return max(size, optvarsize)
+ # if 'value' was only an integer, get_new_array_length() returns
+ # w_ob = space.w_None. Detect if this was the case,
+ # and if so, stop here, leaving the content uninitialized
+ # (it should be zero-initialized from somewhere else).
+ if space.is_w(w_ob, space.w_None):
+ return optvarsize
+ #
+ if optvarsize == -1:
+ self.write(cdata, w_ob)
+ return optvarsize
+
def convert_bitfield_to_object(self, cdata):
ctype = self.ctype
space = ctype.space
diff --git a/pypy/module/_cffi_backend/misc.py b/pypy/module/_cffi_backend/misc.py
--- a/pypy/module/_cffi_backend/misc.py
+++ b/pypy/module/_cffi_backend/misc.py
@@ -278,6 +278,22 @@
# ____________________________________________________________
+def get_new_array_length(space, w_value):
+ if (space.isinstance_w(w_value, space.w_list) or
+ space.isinstance_w(w_value, space.w_tuple)):
+ return (w_value, space.int_w(space.len(w_value)))
+ elif space.isinstance_w(w_value, space.w_basestring):
+ # from a string, we add the null terminator
+ return (w_value, space.int_w(space.len(w_value)) + 1)
+ else:
+ explicitlength = space.getindex_w(w_value, space.w_OverflowError)
+ if explicitlength < 0:
+ raise OperationError(space.w_ValueError,
+ space.wrap("negative array length"))
+ return (space.w_None, explicitlength)
+
+# ____________________________________________________________
+
@specialize.arg(0)
def _raw_memcopy_tp(TPP, source, dest):
# in its own function: LONGLONG may make the whole function jit-opaque
diff --git a/pypy/module/_cffi_backend/newtype.py b/pypy/module/_cffi_backend/newtype.py
--- a/pypy/module/_cffi_backend/newtype.py
+++ b/pypy/module/_cffi_backend/newtype.py
@@ -158,8 +158,10 @@
fields_list = []
fields_dict = {}
custom_field_pos = False
+ with_var_array = False
- for w_field in fields_w:
+ for i in range(len(fields_w)):
+ w_field = fields_w[i]
field_w = space.fixedview(w_field)
if not (2 <= len(field_w) <= 4):
raise OperationError(space.w_TypeError,
@@ -176,7 +178,11 @@
"duplicate field name '%s'", fname)
#
if ftype.size < 0:
- raise operationerrfmt(space.w_TypeError,
+ if (isinstance(ftype, ctypearray.W_CTypeArray) and fbitsize < 0
+ and (i == len(fields_w) - 1 or foffset != -1)):
+ with_var_array = True
+ else:
+ raise operationerrfmt(space.w_TypeError,
"field '%s.%s' has ctype '%s' of unknown size",
w_ctype.name, fname, ftype.name)
#
@@ -235,7 +241,8 @@
fields_list.append(fld)
fields_dict[fname] = fld
- boffset += ftype.size * 8
+ if ftype.size >= 0:
+ boffset += ftype.size * 8
prev_bitfield_size = 0
More information about the pypy-commit
mailing list