[pypy-commit] pypy py3.5: hg merge default
arigo
pypy.commits at gmail.com
Sat Feb 11 09:47:32 EST 2017
Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r90047:88164726e74b
Date: 2017-02-11 15:31 +0100
http://bitbucket.org/pypy/pypy/changeset/88164726e74b/
Log: hg merge default
diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py
--- a/lib_pypy/cffi/api.py
+++ b/lib_pypy/cffi/api.py
@@ -462,7 +462,12 @@
field or array item in the structure or array, recursively in
case of nested structures.
"""
- ctype = self._backend.typeof(cdata)
+ try:
+ ctype = self._backend.typeof(cdata)
+ except TypeError:
+ if '__addressof__' in type(cdata).__dict__:
+ return type(cdata).__addressof__(cdata, *fields_or_indexes)
+ raise
if fields_or_indexes:
ctype, offset = self._typeoffsetof(ctype, *fields_or_indexes)
else:
@@ -775,10 +780,7 @@
key = 'function ' + name
tp, _ = ffi._parser._declarations[key]
BType = ffi._get_cached_btype(tp)
- try:
- value = backendlib.load_function(BType, name)
- except KeyError as e:
- raise AttributeError('%s: %s' % (name, e))
+ value = backendlib.load_function(BType, name)
library.__dict__[name] = value
#
def accessor_variable(name):
@@ -791,6 +793,21 @@
lambda self: read_variable(BType, name),
lambda self, value: write_variable(BType, name, value)))
#
+ def addressof_var(name):
+ try:
+ return addr_variables[name]
+ except KeyError:
+ with ffi._lock:
+ if name not in addr_variables:
+ key = 'variable ' + name
+ tp, _ = ffi._parser._declarations[key]
+ BType = ffi._get_cached_btype(tp)
+ if BType.kind != 'array':
+ BType = model.pointer_cache(ffi, BType)
+ p = backendlib.load_function(BType, name)
+ addr_variables[name] = p
+ return addr_variables[name]
+ #
def accessor_constant(name):
raise NotImplementedError("non-integer constant '%s' cannot be "
"accessed from a dlopen() library" % (name,))
@@ -800,6 +817,7 @@
#
accessors = {}
accessors_version = [False]
+ addr_variables = {}
#
def update_accessors():
if accessors_version[0] is ffi._cdef_version:
@@ -850,6 +868,18 @@
with ffi._lock:
update_accessors()
return accessors.keys()
+ def __addressof__(self, name):
+ if name in library.__dict__:
+ return library.__dict__[name]
+ if name in FFILibrary.__dict__:
+ return addressof_var(name)
+ make_accessor(name)
+ if name in library.__dict__:
+ return library.__dict__[name]
+ if name in FFILibrary.__dict__:
+ return addressof_var(name)
+ raise AttributeError("cffi library has no function or "
+ "global variable named '%s'" % (name,))
#
if libname is not None:
try:
diff --git a/lib_pypy/greenlet.egg-info b/lib_pypy/greenlet.egg-info
--- a/lib_pypy/greenlet.egg-info
+++ b/lib_pypy/greenlet.egg-info
@@ -1,6 +1,6 @@
Metadata-Version: 1.0
Name: greenlet
-Version: 0.4.11
+Version: 0.4.12
Summary: Lightweight in-process concurrent programming
Home-page: https://github.com/python-greenlet/greenlet
Author: Ralf Schmitt (for CPython), PyPy team
diff --git a/lib_pypy/greenlet.py b/lib_pypy/greenlet.py
--- a/lib_pypy/greenlet.py
+++ b/lib_pypy/greenlet.py
@@ -2,7 +2,7 @@
import __pypy__
import _continuation
-__version__ = "0.4.11"
+__version__ = "0.4.12"
# ____________________________________________________________
# Exceptions
diff --git a/pypy/doc/project-ideas.rst b/pypy/doc/project-ideas.rst
--- a/pypy/doc/project-ideas.rst
+++ b/pypy/doc/project-ideas.rst
@@ -38,6 +38,21 @@
.. _mailing list: http://mail.python.org/mailman/listinfo/pypy-dev
+Explicit typing in RPython
+--------------------------
+
+RPython is mostly based around type inference, but there are many cases where
+specifying types explicitly is useful. We would like to be able to optionally
+specify the exact types of the arguments to any function. We already have
+solutions in that space, ``@rpython.rlib.objectmodel.enforceargs`` and
+``@rpython.rlib.signature.signature``, but they are inconvenient and limited.
+For instance, they do not easily allow to express the type "dict with ints as
+keys and lists of instances of Foo as values".
+
+Additionally, we would like to be able to specify the types of instance
+attributes. Unlike the function case, this is likely to require some
+refactoring of the annotator.
+
Make bytearray type fast
------------------------
@@ -176,28 +191,33 @@
* `hg`
-Optimising cpyext (CPython C-API compatibility layer)
------------------------------------------------------
-
-A lot of work has gone into PyPy's implementation of CPython's C-API over
-the last years to let it reach a practical level of compatibility, so that
-C extensions for CPython work on PyPy without major rewrites. However,
-there are still many edges and corner cases where it misbehaves.
-
-The objective of this project is to fix bugs in cpyext and to optimise
-several performance critical parts of it, such as the reference counting
-support and other heavily used C-API functions. The net result would be to
-have CPython extensions run much faster on PyPy than they currently do, or
-to make them work at all if they currently don't. A part of this work would
-be to get cpyext into a shape where it supports running Cython generated
-extensions.
-
======================================
Make more python modules pypy-friendly
======================================
-Work has been started on a few popular python packages. Here is a partial
-list of good work that needs to be finished:
+A lot of work has gone into PyPy's implementation of CPython's C-API, cpyext,
+over the last years to let it reach a practical level of compatibility, so that
+C extensions for CPython work on PyPy without major rewrites. However, there
+are still many edges and corner cases where it misbehaves.
+
+For any popular extension that does not already advertise full PyPy
+compatibility, it would thus be useful to take a close look at it in order to
+make it fully compatible with PyPy. The general process is something like:
+
+* Run the extension's tests on PyPy and look at the test failures.
+* Some of the failures may be solved by identifying cases where the extension
+ relies on undocumented or internal details of CPython, and rewriting the
+ relevant code to follow documented best practices. Open issues and send pull
+ requests as appropriate given the extension's development process.
+* Other failures may highlight incompatibilities between cpyext and CPython.
+ Please report them to us and try to fix them.
+* Run benchmarks, either provided by the extension developers or created by
+ you. Any case where PyPy is significantly slower than CPython is to be
+ considered a bug and solved as above.
+
+Alternatively, an approach we used to recommend was to rewrite C extensions
+using more pypy-friendly technologies, e.g. cffi. Here is a partial list of
+good work that needs to be finished:
**matplotlib** https://github.com/mattip/matplotlib
@@ -223,5 +243,3 @@
**pyopengl** https://bitbucket.org/duangle/pyopengl-cffi
Status: unknown
-
-
diff --git a/pypy/module/_cffi_backend/libraryobj.py b/pypy/module/_cffi_backend/libraryobj.py
--- a/pypy/module/_cffi_backend/libraryobj.py
+++ b/pypy/module/_cffi_backend/libraryobj.py
@@ -40,25 +40,22 @@
@unwrap_spec(w_ctype=W_CType, name=str)
def load_function(self, w_ctype, name):
- from pypy.module._cffi_backend import ctypefunc, ctypeptr, ctypevoid
+ from pypy.module._cffi_backend import ctypeptr, ctypearray
space = self.space
#
- ok = False
- if isinstance(w_ctype, ctypefunc.W_CTypeFunc):
- ok = True
- if (isinstance(w_ctype, ctypeptr.W_CTypePointer) and
- isinstance(w_ctype.ctitem, ctypevoid.W_CTypeVoid)):
- ok = True
- if not ok:
+ if not isinstance(w_ctype, ctypeptr.W_CTypePtrOrArray):
raise oefmt(space.w_TypeError,
- "function cdata expected, got '%s'", w_ctype.name)
+ "function or pointer or array cdata expected, got '%s'",
+ w_ctype.name)
#
try:
cdata = dlsym(self.handle, name)
except KeyError:
- raise oefmt(space.w_KeyError,
- "function '%s' not found in library '%s'",
+ raise oefmt(space.w_AttributeError,
+ "function/symbol '%s' not found in library '%s'",
name, self.name)
+ if isinstance(w_ctype, ctypearray.W_CTypeArray) and w_ctype.length < 0:
+ w_ctype = w_ctype.ctptr
return W_CData(space, rffi.cast(rffi.CCHARP, cdata), w_ctype)
@unwrap_spec(w_ctype=W_CType, name=str)
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -365,7 +365,7 @@
x = find_and_load_library(None)
BVoidP = new_pointer_type(new_void_type())
assert x.load_function(BVoidP, 'strcpy')
- py.test.raises(KeyError, x.load_function,
+ py.test.raises(AttributeError, x.load_function,
BVoidP, 'xxx_this_function_does_not_exist')
# the next one is from 'libm', not 'libc', but we assume
# that it is already loaded too, so it should work
diff --git a/pypy/module/itertools/interp_itertools.py b/pypy/module/itertools/interp_itertools.py
--- a/pypy/module/itertools/interp_itertools.py
+++ b/pypy/module/itertools/interp_itertools.py
@@ -670,14 +670,13 @@
self.space = space
self.saved_w = []
self.w_iterable = space.iter(w_iterable)
- self.index = 0
- self.exhausted = False
+ self.index = 0 # 0 during the first iteration; > 0 afterwards
def iter_w(self):
return self.space.wrap(self)
def next_w(self):
- if self.exhausted:
+ if self.index > 0:
if not self.saved_w:
raise OperationError(self.space.w_StopIteration, self.space.w_None)
try:
@@ -692,26 +691,27 @@
w_obj = self.space.next(self.w_iterable)
except OperationError as e:
if e.match(self.space, self.space.w_StopIteration):
- self.exhausted = True
+ self.index = 1
if not self.saved_w:
raise
- self.index = 1
w_obj = self.saved_w[0]
else:
raise
else:
- self.index += 1
self.saved_w.append(w_obj)
return w_obj
def descr_reduce(self, space):
+ # reduces differently than CPython 3.5. Unsure if it is a
+ # problem. To be on the safe side, keep three arguments for
+ # __setstate__; CPython takes two.
return space.newtuple([
space.type(self),
space.newtuple([self.w_iterable]),
space.newtuple([
space.newlist(self.saved_w),
space.wrap(self.index),
- space.wrap(self.exhausted),
+ space.wrap(self.index > 0),
]),
])
@@ -719,7 +719,7 @@
w_saved, w_index, w_exhausted = space.unpackiterable(w_state, 3)
self.saved_w = space.unpackiterable(w_saved)
self.index = space.int_w(w_index)
- self.exhausted = space.bool_w(w_exhausted)
+ # w_exhausted ignored
def W_Cycle___new__(space, w_subtype, w_iterable):
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
@@ -1227,6 +1227,26 @@
assert list(a)[:1000] + [0] * (len(a)-1000) == list(b)
f.close()
+ def test_ffi_buffer_comparisons(self):
+ ffi = FFI(backend=self.Backend())
+ ba = bytearray(range(100, 110))
+ assert ba == memoryview(ba) # justification for the following
+ a = ffi.new("uint8_t[]", list(ba))
+ c = ffi.new("uint8_t[]", [99] + list(ba))
+ try:
+ b_full = ffi.buffer(a)
+ b_short = ffi.buffer(a, 3)
+ b_mid = ffi.buffer(a, 6)
+ b_other = ffi.buffer(c, 6)
+ except NotImplementedError as e:
+ py.test.skip(str(e))
+ else:
+ content = b_full[:]
+ assert content == b_full == ba
+ assert b_other < b_short < b_mid < b_full
+ assert ba > b_mid > ba[0:2]
+ assert b_short != ba[1:4]
+
def test_array_in_struct(self):
ffi = FFI(backend=self.Backend())
ffi.cdef("struct foo_s { int len; short data[5]; };")
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py
@@ -193,9 +193,12 @@
setattr(s, name, value)
assert getattr(s, name) == value
raw1 = ffi.buffer(s)[:]
+ buff1 = ffi.buffer(s)
t = lib.try_with_value(fnames.index(name), value)
raw2 = ffi.buffer(t, len(raw1))[:]
assert raw1 == raw2
+ buff2 = ffi.buffer(t, len(buff1))
+ assert buff1 == buff2
def test_bitfield_basic(self):
self.check("int a; int b:9; int c:20; int y;", 8, 4, 12)
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py
@@ -283,3 +283,21 @@
assert ret.right == ownlib.right
assert ret.top == ownlib.top
assert ret.bottom == ownlib.bottom
+
+ def test_addressof_lib(self):
+ if self.module is None:
+ py.test.skip("fix the auto-generation of the tiny test lib")
+ if self.Backend is CTypesBackend:
+ py.test.skip("not implemented with the ctypes backend")
+ ffi = FFI(backend=self.Backend())
+ ffi.cdef("long left; int test_getting_errno(void);")
+ lib = ffi.dlopen(self.module)
+ lib.left = 123456
+ p = ffi.addressof(lib, "left")
+ assert ffi.typeof(p) == ffi.typeof("long *")
+ assert p[0] == 123456
+ p[0] += 1
+ assert lib.left == 123457
+ pfn = ffi.addressof(lib, "test_getting_errno")
+ assert ffi.typeof(pfn) == ffi.typeof("int(*)(void)")
+ assert pfn == lib.test_getting_errno
diff --git a/rpython/translator/c/src/stacklet/stacklet.c b/rpython/translator/c/src/stacklet/stacklet.c
--- a/rpython/translator/c/src/stacklet/stacklet.c
+++ b/rpython/translator/c/src/stacklet/stacklet.c
@@ -5,8 +5,8 @@
#include "src/stacklet/stacklet.h"
#include <stddef.h>
-#include <assert.h>
#include <string.h>
+#include <stdio.h>
/************************************************************
* platform specific code
@@ -34,7 +34,7 @@
/************************************************************/
struct stacklet_s {
- /* The portion of the real stack claimed by this paused tealet. */
+ /* The portion of the real stack claimed by this paused stacklet. */
char *stack_start; /* the "near" end of the stack */
char *stack_stop; /* the "far" end of the stack */
@@ -69,6 +69,19 @@
struct stacklet_s *g_target;
};
+#define _check(x) do { if (!(x)) _check_failed(#x); } while (0)
+
+static void _check_failed(const char *check)
+{
+ fprintf(stderr, "FATAL: stacklet: %s failed\n", check);
+ abort();
+}
+
+static void check_valid(struct stacklet_s *g)
+{
+ _check(g->stack_saved >= 0);
+}
+
/***************************************************************/
static void g_save(struct stacklet_s* g, char* stop
@@ -96,7 +109,8 @@
*/
ptrdiff_t sz1 = g->stack_saved;
ptrdiff_t sz2 = stop - g->stack_start;
- assert(stop <= g->stack_stop);
+ check_valid(g);
+ _check(stop <= g->stack_stop);
if (sz2 > sz1) {
char *c = (char *)(g + 1);
@@ -146,11 +160,13 @@
{
struct stacklet_s *current = thrd->g_stack_chain_head;
char *target_stop = g_target->stack_stop;
+ check_valid(g_target);
- /* save and unlink tealets that are completely within
+ /* save and unlink stacklets that are completely within
the area to clear. */
while (current != NULL && current->stack_stop <= target_stop) {
struct stacklet_s *prev = current->stack_prev;
+ check_valid(current);
current->stack_prev = NULL;
if (current != g_target) {
/* don't bother saving away g_target, because
@@ -222,14 +238,16 @@
struct stacklet_thread_s *thrd = (struct stacklet_thread_s *)rawthrd;
struct stacklet_s *g = thrd->g_target;
ptrdiff_t stack_saved = g->stack_saved;
+ check_valid(g);
- assert(new_stack_pointer == g->stack_start);
+ _check(new_stack_pointer == g->stack_start);
#if STACK_DIRECTION == 0
memcpy(g->stack_start, g+1, stack_saved);
#else
memcpy(g->stack_start - stack_saved, g+1, stack_saved);
#endif
thrd->g_current_stack_stop = g->stack_stop;
+ g->stack_saved = -13; /* debugging */
free(g);
return EMPTY_STACKLET_HANDLE;
}
@@ -250,10 +268,11 @@
result = run(thrd->g_source, run_arg);
/* Then switch to 'result'. */
+ check_valid(result);
thrd->g_target = result;
_stacklet_switchstack(g_destroy_state, g_restore_state, thrd);
- assert(!"stacklet: we should not return here");
+ _check_failed("we should not return here");
abort();
}
/* The second time it returns. */
@@ -287,7 +306,7 @@
stacklet_run_fn run, void *run_arg)
{
long stackmarker;
- assert((char *)NULL < (char *)&stackmarker);
+ _check((char *)NULL < (char *)&stackmarker);
if (thrd->g_current_stack_stop <= (char *)&stackmarker)
thrd->g_current_stack_stop = ((char *)&stackmarker) + 1;
@@ -300,6 +319,7 @@
{
long stackmarker;
stacklet_thread_handle thrd = target->stack_thrd;
+ check_valid(target);
if (thrd->g_current_stack_stop <= (char *)&stackmarker)
thrd->g_current_stack_stop = ((char *)&stackmarker) + 1;
@@ -310,6 +330,7 @@
void stacklet_destroy(stacklet_handle target)
{
+ check_valid(target);
if (target->stack_prev != NULL) {
/* 'target' appears to be in the chained list 'unsaved_stack',
so remove it from there. Note that if 'thrd' was already
@@ -319,12 +340,15 @@
we don't even read 'stack_thrd', already deallocated. */
stacklet_thread_handle thrd = target->stack_thrd;
struct stacklet_s **pp = &thrd->g_stack_chain_head;
- for (; *pp != NULL; pp = &(*pp)->stack_prev)
+ for (; *pp != NULL; pp = &(*pp)->stack_prev) {
+ check_valid(*pp);
if (*pp == target) {
*pp = target->stack_prev;
break;
}
+ }
}
+ target->stack_saved = -11; /* debugging */
free(target);
}
@@ -334,6 +358,7 @@
long delta;
if (context == NULL)
return ptr;
+ check_valid(context);
delta = p - context->stack_start;
if (((unsigned long)delta) < ((unsigned long)context->stack_saved)) {
/* a pointer to a saved away word */
@@ -345,8 +370,8 @@
/* out-of-stack pointer! it's only ok if we are the main stacklet
and we are reading past the end, because the main stacklet's
stack stop is not exactly known. */
- assert(delta >= 0);
- assert(((long)context->stack_stop) & 1);
+ _check(delta >= 0);
+ _check(((long)context->stack_stop) & 1);
}
return ptr;
}
More information about the pypy-commit
mailing list