From noreply at buildbot.pypy.org  Mon Jun  1 00:33:08 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Mon,  1 Jun 2015 00:33:08 +0200 (CEST)
Subject: [pypy-commit] pypy release-2.6.x: update release announcement to
	cffi 1.1
Message-ID: <20150531223308.D0C461C03CA@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: release-2.6.x
Changeset: r77734:a16eb184da5d
Date: 2015-06-01 01:01 +0300
http://bitbucket.org/pypy/pypy/changeset/a16eb184da5d/

Log:	update release announcement to cffi 1.1

diff --git a/pypy/doc/release-2.6.0.rst b/pypy/doc/release-2.6.0.rst
--- a/pypy/doc/release-2.6.0.rst
+++ b/pypy/doc/release-2.6.0.rst
@@ -3,7 +3,7 @@
 ========================
 
 We're pleased to announce PyPy 2.6.0, only two months after PyPy 2.5.1.
-We are particulary happy to update `cffi`_ to version 1.0, which makes the
+We are particulary happy to update `cffi`_ to version 1.1, which makes the
 popular ctypes-alternative even easier to use, and to support the new vmprof_
 statistical profiler.
 

From noreply at buildbot.pypy.org  Mon Jun  1 07:12:11 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Mon,  1 Jun 2015 07:12:11 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Adapt test to check that type.__eq__(a,
 b) returns the same as object.__eq__(a, b).
Message-ID: <20150601051211.8D1511C116B@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob <me at manueljacob.de>
Branch: py3k
Changeset: r77735:1e99d60c6c5b
Date: 2015-06-01 07:13 +0200
http://bitbucket.org/pypy/pypy/changeset/1e99d60c6c5b/

Log:	Adapt test to check that type.__eq__(a, b) returns the same as
	object.__eq__(a, b).

diff --git a/pypy/objspace/std/test/test_typeobject.py b/pypy/objspace/std/test/test_typeobject.py
--- a/pypy/objspace/std/test/test_typeobject.py
+++ b/pypy/objspace/std/test/test_typeobject.py
@@ -1255,5 +1255,5 @@
     def test_eq_returns_notimplemented(self):
         assert type.__eq__(int, 42) is NotImplemented
         assert type.__ne__(dict, 42) is NotImplemented
-        assert type.__eq__(int, int) is True
-        assert type.__eq__(int, dict) is False
+        assert type.__eq__(int, int) == True
+        assert type.__eq__(int, dict) is NotImplemented

From noreply at buildbot.pypy.org  Mon Jun  1 08:48:47 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  1 Jun 2015 08:48:47 +0200 (CEST)
Subject: [pypy-commit] cffi default: Tests for ffi.dlopen(None)
Message-ID: <20150601064847.BB3A41C0823@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2148:ffc0b54f762d
Date: 2015-06-01 08:48 +0200
http://bitbucket.org/cffi/cffi/changeset/ffc0b54f762d/

Log:	Tests for ffi.dlopen(None)

diff --git a/testing/cffi1/test_re_python.py b/testing/cffi1/test_re_python.py
--- a/testing/cffi1/test_re_python.py
+++ b/testing/cffi1/test_re_python.py
@@ -7,6 +7,7 @@
 
 def setup_module(mod):
     SRC = """
+    #include <string.h>
     #define FOOBAR (-42)
     static const int FOOBAZ = -43;
     #define BIGPOS 420000000000L
@@ -53,6 +54,7 @@
     struct foo_s;
     typedef struct bar_s { int x; signed char a[]; } bar_t;
     enum foo_e { AA, BB, CC };
+    int strlen(const char *);
     """)
     ffi.set_source('re_python_pysrc', None)
     ffi.emit_python_code(str(tmpdir.join('re_python_pysrc.py')))
@@ -81,10 +83,16 @@
 def test_function_with_varargs():
     import _cffi_backend
     from re_python_pysrc import ffi
-    lib = ffi.dlopen(extmod)
+    lib = ffi.dlopen(extmod, 0)
     assert lib.add43(45, ffi.cast("int", -5)) == 45
     assert type(lib.add43) is _cffi_backend.FFI.CData
 
+def test_dlopen_none():
+    import _cffi_backend
+    from re_python_pysrc import ffi
+    lib = ffi.dlopen(None)
+    assert lib.strlen(b"hello") == 5
+
 def test_dlclose():
     import _cffi_backend
     from re_python_pysrc import ffi

From noreply at buildbot.pypy.org  Mon Jun  1 08:48:48 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  1 Jun 2015 08:48:48 +0200 (CEST)
Subject: [pypy-commit] cffi default: Issue #203: add keyword arguments to
 ffi.string(), ffi.buffer(), 
Message-ID: <20150601064848.E81E51C0823@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2149:6571ca33dd6f
Date: 2015-06-01 08:49 +0200
http://bitbucket.org/cffi/cffi/changeset/6571ca33dd6f/

Log:	Issue #203: add keyword arguments to ffi.string(), ffi.buffer(),
	ffi.getwinerror()

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -5158,12 +5158,14 @@
     return PyText_FromStringAndSize(s, namelen + replacelen);
 }
 
-static PyObject *b_string(PyObject *self, PyObject *args)
+static PyObject *b_string(PyObject *self, PyObject *args, PyObject *kwds)
 {
     CDataObject *cd;
     Py_ssize_t maxlen = -1;
-    if (!PyArg_ParseTuple(args, "O!|n:string",
-                          &CData_Type, &cd, &maxlen))
+    static char *keywords[] = {"cdata", "maxlen", NULL};
+
+    if (!PyArg_ParseTupleAndKeywords(args, kwds, "O!|n:string", keywords,
+                                     &CData_Type, &cd, &maxlen))
         return NULL;
 
     if (cd->c_type->ct_itemdescr != NULL &&
@@ -5246,12 +5248,14 @@
     return NULL;
 }
 
-static PyObject *b_buffer(PyObject *self, PyObject *args)
+static PyObject *b_buffer(PyObject *self, PyObject *args, PyObject *kwds)
 {
     CDataObject *cd;
     Py_ssize_t size = -1;
-    if (!PyArg_ParseTuple(args, "O!|n:buffer",
-                          &CData_Type, &cd, &size))
+    static char *keywords[] = {"cdata", "size", NULL};
+
+    if (!PyArg_ParseTupleAndKeywords(args, kwds, "O!|n:buffer", keywords,
+                                     &CData_Type, &cd, &size))
         return NULL;
 
     if (cd->c_type->ct_flags & CT_POINTER) {
@@ -5796,15 +5800,15 @@
     {"typeoffsetof", b_typeoffsetof, METH_VARARGS},
     {"rawaddressof", b_rawaddressof, METH_VARARGS},
     {"getcname", b_getcname, METH_VARARGS},
-    {"string", b_string, METH_VARARGS},
-    {"buffer", b_buffer, METH_VARARGS},
+    {"string", (PyCFunction)b_string, METH_VARARGS | METH_KEYWORDS},
+    {"buffer", (PyCFunction)b_buffer, METH_VARARGS | METH_KEYWORDS},
     {"get_errno", b_get_errno, METH_NOARGS},
     {"set_errno", b_set_errno, METH_O},
     {"newp_handle", b_newp_handle, METH_VARARGS},
     {"from_handle", b_from_handle, METH_O},
     {"from_buffer", b_from_buffer, METH_VARARGS},
 #ifdef MS_WIN32
-    {"getwinerror", b_getwinerror, METH_VARARGS},
+    {"getwinerror", (PyCFunction)b_getwinerror, METH_VARARGS | METH_KEYWORDS},
 #endif
     {"_get_types", b__get_types, METH_NOARGS},
     {"_testfunc", b__testfunc, METH_VARARGS},
diff --git a/c/ffi_obj.c b/c/ffi_obj.c
--- a/c/ffi_obj.c
+++ b/c/ffi_obj.c
@@ -774,7 +774,7 @@
 static PyMethodDef ffi_methods[] = {
  {"addressof",  (PyCFunction)ffi_addressof,  METH_VARARGS, ffi_addressof_doc},
  {"alignof",    (PyCFunction)ffi_alignof,    METH_O,       ffi_alignof_doc},
- {"buffer",     (PyCFunction)ffi_buffer,     METH_VARARGS, ffi_buffer_doc},
+ {"buffer",     (PyCFunction)ffi_buffer,     METH_VKW,     ffi_buffer_doc},
  {"callback",   (PyCFunction)ffi_callback,   METH_VKW,     ffi_callback_doc},
  {"cast",       (PyCFunction)ffi_cast,       METH_VARARGS, ffi_cast_doc},
  {"dlclose",    (PyCFunction)ffi_dlclose,    METH_VARARGS, ffi_dlclose_doc},
@@ -784,14 +784,14 @@
  {"gc",         (PyCFunction)ffi_gc,         METH_VKW,     ffi_gc_doc},
  {"getctype",   (PyCFunction)ffi_getctype,   METH_VKW,     ffi_getctype_doc},
 #ifdef MS_WIN32
- {"getwinerror",(PyCFunction)ffi_getwinerror,METH_VARARGS, ffi_getwinerror_doc},
+ {"getwinerror",(PyCFunction)ffi_getwinerror,METH_VKW,     ffi_getwinerror_doc},
 #endif
  {"integer_const",(PyCFunction)ffi_int_const,METH_VKW,     ffi_int_const_doc},
  {"new",        (PyCFunction)ffi_new,        METH_VKW,     ffi_new_doc},
  {"new_handle", (PyCFunction)ffi_new_handle, METH_O,       ffi_new_handle_doc},
  {"offsetof",   (PyCFunction)ffi_offsetof,   METH_VARARGS, ffi_offsetof_doc},
  {"sizeof",     (PyCFunction)ffi_sizeof,     METH_O,       ffi_sizeof_doc},
- {"string",     (PyCFunction)ffi_string,     METH_VARARGS, ffi_string_doc},
+ {"string",     (PyCFunction)ffi_string,     METH_VKW,     ffi_string_doc},
  {"typeof",     (PyCFunction)ffi_typeof,     METH_O,       ffi_typeof_doc},
  {NULL}
 };
diff --git a/c/misc_win32.h b/c/misc_win32.h
--- a/c/misc_win32.h
+++ b/c/misc_win32.h
@@ -82,14 +82,15 @@
 }
 
 #if PY_MAJOR_VERSION >= 3
-static PyObject *b_getwinerror(PyObject *self, PyObject *args)
+static PyObject *b_getwinerror(PyObject *self, PyObject *args, PyObject *kwds)
 {
     int err = -1;
     int len;
     WCHAR *s_buf = NULL; /* Free via LocalFree */
     PyObject *v, *message;
+    static char *keywords[] = {"code", NULL};
 
-    if (!PyArg_ParseTuple(args, "|i", &err))
+    if (!PyArg_ParseTupleAndKeywords(args, kwds, "|i", keywords, &err))
         return NULL;
 
     if (err == -1) {
@@ -129,7 +130,7 @@
     return v;
 }
 #else
-static PyObject *b_getwinerror(PyObject *self, PyObject *args)
+static PyObject *b_getwinerror(PyObject *self, PyObject *args, PyObject *kwds)
 {
     int err = -1;
     int len;
@@ -137,8 +138,9 @@
     char *s_buf = NULL; /* Free via LocalFree */
     char s_small_buf[28]; /* Room for "Windows Error 0xFFFFFFFF" */
     PyObject *v;
+    static char *keywords[] = {"code", NULL};
 
-    if (!PyArg_ParseTuple(args, "|i", &err))
+    if (!PyArg_ParseTupleAndKeywords(args, kwds, "|i", keywords, &err))
         return NULL;
 
     if (err == -1) {
diff --git a/testing/cffi1/test_ffi_obj.py b/testing/cffi1/test_ffi_obj.py
--- a/testing/cffi1/test_ffi_obj.py
+++ b/testing/cffi1/test_ffi_obj.py
@@ -1,4 +1,4 @@
-import py
+import py, sys
 import _cffi_backend as _cffi1_backend
 
 
@@ -65,6 +65,7 @@
     ffi = _cffi1_backend.FFI()
     p = ffi.new("char[]", init=b"foobar\x00baz")
     assert ffi.string(p) == b"foobar"
+    assert ffi.string(cdata=p, maxlen=3) == b"foo"
 
 def test_ffi_errno():
     # xxx not really checking errno, just checking that we can read/write it
@@ -162,6 +163,7 @@
     ffi = _cffi1_backend.FFI()
     a = ffi.new("signed char[]", [5, 6, 7])
     assert ffi.buffer(a)[:] == b'\x05\x06\x07'
+    assert ffi.buffer(cdata=a, size=2)[:] == b'\x05\x06'
 
 def test_ffi_from_buffer():
     import array
@@ -178,3 +180,11 @@
     ffi = _cffi1_backend.FFI()
     assert isinstance(ffi.cast("int", 42), CData)
     assert isinstance(ffi.typeof("int"), CType)
+
+def test_ffi_getwinerror():
+    if sys.platform != "win32":
+        py.test.skip("for windows")
+    ffi = _cffi1_backend.FFI()
+    n = (1 << 29) + 42
+    code, message = ffi.getwinerror(code=n)
+    assert code == n

From noreply at buildbot.pypy.org  Mon Jun  1 09:13:37 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  1 Jun 2015 09:13:37 +0200 (CEST)
Subject: [pypy-commit] cffi default: Windows test fix,
 and documentation about ffi.dlopen()'s restriction
Message-ID: <20150601071337.ED7E71C0823@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2150:eb3b8aad6ecf
Date: 2015-06-01 09:14 +0200
http://bitbucket.org/cffi/cffi/changeset/eb3b8aad6ecf/

Log:	Windows test fix, and documentation about ffi.dlopen()'s restriction

diff --git a/doc/source/cdef.rst b/doc/source/cdef.rst
--- a/doc/source/cdef.rst
+++ b/doc/source/cdef.rst
@@ -278,6 +278,16 @@
 needed.  (Alternatively, the out-of-line FFIs have a method
 ``ffi.dlclose(lib)``.)
 
+Note: the old version of ``ffi.dlopen()`` from the in-line ABI mode
+tries to use ``ctypes.util.find_library()`` if it cannot directly find
+the library.  The newer out-of-line ``ffi.dlopen()`` no longer does it
+automatically; it simply passes the argument it receives to the
+underlying ``dlopen()`` or ``LoadLibrary()`` function.  If needed, it
+is up to you to use ``ctypes.util.find_library()`` or any other way to
+look for the library's filename.  This also means that
+``ffi.dlopen(None)`` no longer work on Windows; try instead
+``ffi.dlopen(ctypes.util.find_library('c'))``.
+
 
 ffi.set_source(): preparing out-of-line modules
 -----------------------------------------------
diff --git a/testing/cffi1/test_re_python.py b/testing/cffi1/test_re_python.py
--- a/testing/cffi1/test_re_python.py
+++ b/testing/cffi1/test_re_python.py
@@ -90,7 +90,11 @@
 def test_dlopen_none():
     import _cffi_backend
     from re_python_pysrc import ffi
-    lib = ffi.dlopen(None)
+    name = None
+    if sys.platform == 'win32':
+        import ctypes.util
+        name = ctypes.util.find_msvcrt()
+    lib = ffi.dlopen(name)
     assert lib.strlen(b"hello") == 5
 
 def test_dlclose():

From noreply at buildbot.pypy.org  Mon Jun  1 09:14:31 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Mon,  1 Jun 2015 09:14:31 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: distinct between input/output argument
 in vector type conversion
Message-ID: <20150601071431.B6DF31C0823@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r77736:cb7dddccc7f0
Date: 2015-06-01 09:14 +0200
http://bitbucket.org/pypy/pypy/changeset/cb7dddccc7f0/

Log:	distinct between input/output argument in vector type conversion
	call2 uses a list to track iterator and their states

diff --git a/pypy/module/micronumpy/loop.py b/pypy/module/micronumpy/loop.py
--- a/pypy/module/micronumpy/loop.py
+++ b/pypy/module/micronumpy/loop.py
@@ -15,7 +15,7 @@
 
 call2_driver = jit.JitDriver(
     name='numpy_call2',
-    greens=['shapelen', 'func', 'left_advance', 'right_advance', 'calc_dtype', 'res_dtype' ],
+    greens=['shapelen', 'func', 'left_iter_index', 'right_iter_index', 'calc_dtype', 'res_dtype' ],
     reds='auto', vectorize=True)
 
 def call2(space, shape, func, calc_dtype, res_dtype, w_lhs, w_rhs, out):
@@ -43,9 +43,12 @@
 
     # TODO handle __array_priorities__ and maybe flip the order
 
+    left_iter_index = 1
+    right_iter_index = 2
     if w_lhs.get_size() == 1:
         w_left = w_lhs.get_scalar_value().convert_to(space, calc_dtype)
         left_iter = left_state = None
+        left_iter_index = -1
     else:
         w_left = None
         left_iter, left_state = w_lhs.create_iter(shape)
@@ -54,6 +57,7 @@
     if w_rhs.get_size() == 1:
         w_right = w_rhs.get_scalar_value().convert_to(space, calc_dtype)
         right_iter = right_state = None
+        right_iter_index = -1
     else:
         w_right = None
         right_iter, right_state = w_rhs.create_iter(shape)
@@ -63,34 +67,34 @@
                                      w_instance=lhs_for_subtype)
     out_iter, out_state = out.create_iter(shape)
 
-    left_advance = True
-    right_advance = True
-    if left_iter and left_iter.matches_range(out_iter):
-        left_advance = False
-        left_state = out_state
-    if right_iter and right_iter.matches_range(out_iter):
-        right_advance = False
-        right_state = out_state
+    iter_list = [out_iter, left_iter, right_iter]
+    state_list = [out_state, left_state, right_state]
+
+    if left_iter_index > 0 and left_iter.matches_range(out_iter):
+        left_iter_index = 0
+    if right_iter_index > 0 and right_iter.matches_range(out_iter):
+        right_iter_index = 0
 
     shapelen = len(shape)
     while not out_iter.done(out_state):
-        call2_driver.jit_merge_point(shapelen=shapelen, left_advance=left_advance, right_advance=right_advance,
+        call2_driver.jit_merge_point(shapelen=shapelen, left_iter_index=left_iter_index,
+                                     right_iter_index=right_iter_index,
                                      func=func, calc_dtype=calc_dtype, res_dtype=res_dtype)
-        if left_iter:
-            w_left = left_iter.getitem(left_state).convert_to(space, calc_dtype)
-            if left_advance:
-                left_state = left_iter.next(left_state)
-        if right_iter:
-            w_right = right_iter.getitem(right_state).convert_to(space, calc_dtype)
-            if right_advance:
-                right_state = right_iter.next(right_state)
+        if left_iter_index > 0:
+            iter = iter_list[left_iter_index]
+            state = state_list[left_iter_index]
+            w_left = iter.getitem(state).convert_to(space, calc_dtype)
+            if left_iter_index == 1:
+                state_list[left_iter_index] = iter.next(state)
+        if right_iter_index > 0:
+            iter = iter_list[right_iter_index]
+            state = state_list[right_iter_index]
+            w_right = iter.getitem(state).convert_to(space, calc_dtype)
+            if right_iter_index == 2:
+                state_list[right_iter_index] = iter.next(state)
         out_iter.setitem(out_state, func(calc_dtype, w_left, w_right).convert_to(
             space, res_dtype))
-        out_state = out_iter.next(out_state)
-        if not left_advance:
-            left_state = out_state
-        if not right_advance:
-            right_state = out_state
+        state_list[0] = out_state = out_iter.next(out_state)
 
     return out
 
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_schedule.py b/rpython/jit/metainterp/optimizeopt/test/test_schedule.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_schedule.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_schedule.py
@@ -15,6 +15,7 @@
         ns = {
             'double': self.floatarraydescr,
             'float': self.singlefloatarraydescr,
+            'long': self.intarraydescr,
         }
         loop = opparse("        [p0,p1,p2,p3,p4,p5,i0,i1,i2,i3,i4,i5,f0,f1,f2,f3,f4,f5]\n" + source + \
                        "\n        jump(p0,p1,p2,p3,p4,p5,i0,i1,i2,i3,i4,i5,f0,f1,f2,f3,f4,f5)",
@@ -24,7 +25,7 @@
         return loop
 
     def pack(self, loop, l, r):
-        return [Node(op,i) for i,op in enumerate(loop.operations[l:r])]
+        return [Node(op,l+i) for i,op in enumerate(loop.operations[l:r])]
 
     def schedule(self, loop_orig, packs, vec_reg_size=16):
         loop = get_model(False).ExtendedTreeLoop("loop")
@@ -35,7 +36,7 @@
         vsd = VecScheduleData(vec_reg_size)
         for pack in packs:
             if len(pack) == 1:
-                ops.append(pack[0])
+                ops.append(pack[0].getoperation())
             else:
                 for op in vsd.as_vector_operation(Pack(pack)):
                     ops.append(op)
@@ -58,22 +59,42 @@
         loop2 = self.schedule(loop1, [pack1])
         loop3 = self.parse("""
         v1[i32#4] = vec_raw_load(p0, i0, 4, descr=float)
-        i14 = vec_raw_load(p0, i4, descr=float)
-        i15 = vec_raw_load(p0, i5, descr=float)
+        i14 = raw_load(p0, i4, descr=float)
+        i15 = raw_load(p0, i5, descr=float)
+        """)
+        self.assert_equal(loop2, loop3)
+
+    def test_int_to_float(self):
+        loop1 = self.parse("""
+        i10 = raw_load(p0, i0, descr=long)
+        i11 = raw_load(p0, i1, descr=long)
+        f10 = cast_int_to_float(i10)
+        f11 = cast_int_to_float(i11)
+        """)
+        pack1 = self.pack(loop1, 0, 2)
+        pack2 = self.pack(loop1, 2, 4)
+        print pack1
+        print pack2
+        loop2 = self.schedule(loop1, [pack1, pack2])
+        loop3 = self.parse("""
+        v1[i64#2] = vec_raw_load(p0, i0, 2, descr=long)
+        v2[i32#2] = vec_int_signext(v1[i64#2], 4)
+        v3[f64#2] = vec_cast_int_to_float(v2[i32#2])
         """)
         self.assert_equal(loop2, loop3)
 
     def test_cost_model_reject_only_load_vectorizable(self):
         loop1 = self.parse("""
-        f10 = raw_load(p0, i0, descr=double)
-        f11 = raw_load(p0, i1, descr=double)
-        i1 = int_add(1,1)
-        guard_true(i1) [f10]
+        f10 = raw_load(p0, i0, descr=long)
+        f11 = raw_load(p0, i1, descr=long)
+        guard_true(i0) [f10]
         guard_true(i1) [f11]
         """)
         try:
-            pack1 = self.pack(loop1, 0, 6)
-            loop2 = self.schedule(loop1, [pack1])
+            pack1 = self.pack(loop1, 0, 2)
+            pack2 = self.pack(loop1, 2, 3)
+            pack3 = self.pack(loop1, 3, 4)
+            loop2 = self.schedule(loop1, [pack1, pack2, pack3])
             py.test.fail("this loops should have bailed out")
         except NotAProfitableLoop:
             pass
diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/vectorize.py
@@ -789,44 +789,63 @@
         self.preamble_ops = None
         self.sched_data = None
         self.pack = None
+        self.input_type = None
+        self.output_type = None
 
     def is_vector_arg(self, i):
         if i < 0 or i >= len(self.arg_ptypes):
             return False
         return self.arg_ptypes[i] is not None
 
-    def pack_ptype(self, op):
+    def getsplitsize(self):
+        return self.input_type.getsize()
+
+    def determine_input_type(self, op):
         _, vbox = self.sched_data.getvector_of_box(op.getarg(0))
         if vbox:
             return PackType.of(vbox)
         else:
             raise RuntimeError("fatal: box %s is not in a vector box" % (op.getarg(0),))
 
+    def determine_output_type(self, op):
+        return self.determine_input_type(op)
+
     def as_vector_operation(self, pack, sched_data, oplist):
         self.sched_data = sched_data
         self.preamble_ops = oplist
         op0 = pack.operations[0].getoperation()
-        self.ptype = self.pack_ptype(op0)
+        self.input_type = self.determine_input_type(op0)
+        self.output_type = self.determine_output_type(op0)
 
         off = 0
         stride = self.split_pack(pack)
+        left = len(pack.operations)
         assert stride > 0
         while off < len(pack.operations):
+            if left < stride:
+                self.preamble_ops.append(pack.operations[off].getoperation())
+                off += 1
+                continue
             ops = pack.operations[off:off+stride]
             self.pack = Pack(ops)
             self.transform_pack(ops, off, stride)
             off += stride
+            left -= stride
 
         self.pack = None
         self.preamble_ops = None
         self.sched_data = None
-        self.ptype = None
+        self.input_type = None
+        self.output_type = None
 
     def split_pack(self, pack):
         pack_count = len(pack.operations)
         vec_reg_size = self.sched_data.vec_reg_size
-        if pack_count * self.ptype.getsize() > vec_reg_size:
-            return vec_reg_size // self.ptype.getsize()
+        bytes = pack_count * self.getsplitsize()
+        if bytes > vec_reg_size:
+            return vec_reg_size // self.getsplitsize()
+        if bytes < vec_reg_size:
+            return 1
         return pack_count
 
     def before_argument_transform(self, args):
@@ -838,11 +857,11 @@
         #
         self.before_argument_transform(args)
         #
-        result = op.result
         for i,arg in enumerate(args):
             if self.is_vector_arg(i):
                 args[i] = self.transform_argument(args[i], i, off)
         #
+        result = op.result
         result = self.transform_result(result, off)
         #
         vop = ResOperation(op.vector, args, result, op.getdescr())
@@ -860,31 +879,23 @@
         return vbox
 
     def new_result_vector_box(self):
-        size = self.ptype.getsize()
-        count = min(self.ptype.getcount(), len(self.pack.operations))
-        return BoxVector(self.ptype.gettype(), count, size, self.ptype.signed)
+        type = self.output_type.gettype()
+        size = self.output_type.getsize()
+        count = min(self.output_type.getcount(), len(self.pack.operations))
+        signed = self.output_type.signed
+        return BoxVector(type, count, size, signed)
 
     def transform_argument(self, arg, argidx, off):
         ops = self.pack.operations
         box_pos, vbox = self.sched_data.getvector_of_box(arg)
         if not vbox:
             # constant/variable expand this box
-            vbox = self.ptype.new_vector_box(len(ops))
+            vbox = self.input_type.new_vector_box(len(ops))
             vbox = self.expand_box_to_vector_box(vbox, ops, arg, argidx)
             box_pos = 0
 
-        enforced_type = self.ptype
-        # convert type f -> i, i -> f
-        # if enforced_type.gettype() != vbox.gettype():
-        #     raise NotImplementedError("cannot yet convert between types")
-
-        # convert size i64 -> i32, i32 -> i64, ...
-        if enforced_type.getsize() != vbox.getsize():
-            vbox = self.extend(vbox, self.ptype)
-
         # use the input as an indicator for the pack type
-        arg_ptype = PackType.of(vbox)
-        packable = self.sched_data.vec_reg_size // arg_ptype.getsize()
+        packable = self.sched_data.vec_reg_size // self.input_type.getsize()
         packed = vbox.item_count
         assert packed >= 0
         assert packable >= 0
@@ -894,21 +905,24 @@
             vbox = self._pack(vbox, packed, args, packable)
         elif packed > packable:
             # the argument has more items than the operation is able to process!
-            vbox = self.unpack(vbox, off, packable, arg_ptype)
+            vbox = self.unpack(vbox, off, packable, self.input_type)
         #
         if off != 0 and box_pos != 0:
             # The original box is at a position != 0 but it
             # is required to be at position 0. Unpack it!
-            vbox = self.unpack(vbox, off, len(ops), arg_ptype)
+            vbox = self.unpack(vbox, off, len(ops), self.input_type)
+        # convert type f -> i, i -> f
+        if self.input_type.gettype() != vbox.gettype():
+            raise NotImplementedError("cannot yet convert between types")
+        # convert size i64 -> i32, i32 -> i64, ...
+        if self.input_type.getsize() > 0 and \
+           self.input_type.getsize() != vbox.getsize():
+            vbox = self.extend(vbox, self.input_type)
         #
         return vbox
 
     def extend(self, vbox, newtype):
-        if vbox.item_count * vbox.item_size == self.sched_data.vec_reg_size:
-            return vbox
         assert vbox.gettype() == newtype.gettype()
-        assert (vbox.item_count * newtype.getsize()) == \
-               self.sched_data.vec_reg_size
         if vbox.gettype() == INT:
             return self.extend_int(vbox, newtype)
         else:
@@ -1025,6 +1039,12 @@
         self.to_size = outtype.getsize()
         OpToVectorOp.__init__(self, (intype, ), outtype)
 
+    def determine_input_type(self, op):
+        return self.arg_ptypes[0]
+
+    def determine_output_type(self, op):
+        return self.result_ptype
+
     def split_pack(self, pack):
         if self.from_size > self.to_size:
             # cast down
@@ -1037,12 +1057,14 @@
         return len(pack.operations)
 
     def new_result_vector_box(self):
+        type = self.output_type.gettype()
         size = self.to_size
-        count = self.ptype.getcount()
+        count = self.output_type.getcount()
         vec_reg_size = self.sched_data.vec_reg_size
         if count * size > vec_reg_size:
             count = vec_reg_size // size
-        return BoxVector(self.result_ptype.gettype(), count, size, self.ptype.signed)
+        signed = self.output_type.signed
+        return BoxVector(type, count, size, signed)
 
 class SignExtToVectorOp(OpToVectorOp):
     def __init__(self, intype, outtype):
@@ -1054,7 +1076,7 @@
         sizearg = op0.getarg(1)
         assert isinstance(sizearg, ConstInt)
         self.size = sizearg.value
-        if self.ptype.getsize() > self.size:
+        if self.input_type.getsize() > self.size:
             # cast down
             return OpToVectorOp.split_pack(self, pack)
         _, vbox = self.sched_data.getvector_of_box(op0.getarg(0))
@@ -1064,11 +1086,11 @@
         return vbox.getcount()
 
     def new_result_vector_box(self):
-        count = self.ptype.getcount()
+        count = self.input_type.getcount()
         vec_reg_size = self.sched_data.vec_reg_size
         if count * self.size > vec_reg_size:
             count = vec_reg_size // self.size
-        return BoxVector(self.result_ptype.gettype(), count, self.size, self.ptype.signed)
+        return BoxVector(self.result_ptype.gettype(), count, self.size, self.input_type.signed)
 
 PT_GENERIC = PackType(PackType.UNKNOWN_TYPE, -1, False)
 
@@ -1076,22 +1098,38 @@
     def __init__(self):
         OpToVectorOp.__init__(self, (), PT_GENERIC)
 
-    def pack_ptype(self, op):
+    def determine_input_type(self, op):
+        return None
+
+    def determine_output_type(self, op):
         return PackType.by_descr(op.getdescr(), self.sched_data.vec_reg_size)
 
     def before_argument_transform(self, args):
         args.append(ConstInt(len(self.pack.operations)))
 
+    def getsplitsize(self):
+        return self.output_type.getsize()
+
+    def new_result_vector_box(self):
+        type = self.output_type.gettype()
+        size = self.output_type.getsize()
+        count = len(self.pack.operations)
+        signed = self.output_type.signed
+        return BoxVector(type, count, size, signed)
+
 class StoreToVectorStore(OpToVectorOp):
     def __init__(self):
         OpToVectorOp.__init__(self, (None, None, PT_GENERIC), None)
         self.has_descr = True
 
-    def pack_ptype(self, op):
+    def determine_input_type(self, op):
         return PackType.by_descr(op.getdescr(), self.sched_data.vec_reg_size)
 
-PT_FLOAT = PackType(FLOAT, 4, False)
-PT_DOUBLE = PackType(FLOAT, 8, False)
+    def determine_output_type(self, op):
+        return None
+
+PT_FLOAT_2 = PackType(FLOAT, 4, False, 2)
+PT_DOUBLE_2 = PackType(FLOAT, 8, False, 2)
 PT_FLOAT_GENERIC = PackType(INT, -1, True)
 PT_INT64 = PackType(INT, 8, True)
 PT_INT32 = PackType(INT, 4, True)
@@ -1107,6 +1145,8 @@
 LOAD_TRANS = LoadToVectorLoad()
 STORE_TRANS = StoreToVectorStore()
 
+# note that the following definition is x86 machine
+# specific.
 ROP_ARG_RES_VECTOR = {
     rop.VEC_INT_ADD:     INT_OP_TO_VOP,
     rop.VEC_INT_SUB:     INT_OP_TO_VOP,
@@ -1130,10 +1170,10 @@
     rop.VEC_RAW_STORE:        STORE_TRANS,
     rop.VEC_SETARRAYITEM_RAW: STORE_TRANS,
 
-    rop.VEC_CAST_FLOAT_TO_SINGLEFLOAT: OpToVectorOpConv(PT_DOUBLE, PT_FLOAT),
-    rop.VEC_CAST_SINGLEFLOAT_TO_FLOAT: OpToVectorOpConv(PT_FLOAT, PT_DOUBLE),
-    rop.VEC_CAST_FLOAT_TO_INT: OpToVectorOpConv(PT_DOUBLE, PT_INT32),
-    rop.VEC_CAST_INT_TO_FLOAT: OpToVectorOpConv(PT_INT32, PT_DOUBLE),
+    rop.VEC_CAST_FLOAT_TO_SINGLEFLOAT: OpToVectorOpConv(PT_DOUBLE_2, PT_FLOAT_2),
+    rop.VEC_CAST_SINGLEFLOAT_TO_FLOAT: OpToVectorOpConv(PT_FLOAT_2, PT_DOUBLE_2),
+    rop.VEC_CAST_FLOAT_TO_INT: OpToVectorOpConv(PT_DOUBLE_2, PT_INT32),
+    rop.VEC_CAST_INT_TO_FLOAT: OpToVectorOpConv(PT_INT32, PT_DOUBLE_2),
 }
 
 class VecScheduleData(SchedulerData):
@@ -1274,7 +1314,6 @@
     def __init__(self, ops):
         self.operations = ops
         self.savings = 0
-        self.ptype = None
         for node in self.operations:
             node.pack = self
 
@@ -1288,13 +1327,6 @@
         leftmost = other.operations[0]
         return rightmost == leftmost
 
-    def size_in_bytes(self):
-        return self.ptype.get_byte_size() * len(self.operations)
-
-    def is_overloaded(self, vec_reg_byte_size):
-        size = self.size_in_bytes()
-        return size > vec_reg_byte_size
-
     def __repr__(self):
         return "Pack(%r)" % self.operations
 
diff --git a/rpython/jit/tool/oparser.py b/rpython/jit/tool/oparser.py
--- a/rpython/jit/tool/oparser.py
+++ b/rpython/jit/tool/oparser.py
@@ -123,13 +123,13 @@
             box = ts.BoxRef()
             _box_counter_more_than(self.model, elem[1:])
         elif elem.startswith('v'):
-            pattern = re.compile('.*\[(-?)(i|f)(\d+)#(\d+)\]')
+            pattern = re.compile('.*\[(u?)(i|f)(\d+)#(\d+)\]')
             match = pattern.match(elem)
             if match:
                 item_type = match.group(2)[0]
                 item_size = int(match.group(3)) // 8
                 item_count = int(match.group(4))
-                item_signed = match.group(1) == 's'
+                item_signed = not (match.group(1) == 'u')
                 box = self.model.BoxVector(item_type, item_count, item_size, item_signed)
                 lbracket = elem.find('[')
                 number = elem[1:lbracket]

From noreply at buildbot.pypy.org  Mon Jun  1 09:21:56 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  1 Jun 2015 09:21:56 +0200 (CEST)
Subject: [pypy-commit] cffi default: Update whatsnew
Message-ID: <20150601072156.232EC1C0F16@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2151:dc94c2f9fad5
Date: 2015-06-01 09:22 +0200
http://bitbucket.org/cffi/cffi/changeset/dc94c2f9fad5/

Log:	Update whatsnew

diff --git a/doc/source/cdef.rst b/doc/source/cdef.rst
--- a/doc/source/cdef.rst
+++ b/doc/source/cdef.rst
@@ -278,6 +278,8 @@
 needed.  (Alternatively, the out-of-line FFIs have a method
 ``ffi.dlclose(lib)``.)
 
+.. _dlopen-note:
+
 Note: the old version of ``ffi.dlopen()`` from the in-line ABI mode
 tries to use ``ctypes.util.find_library()`` if it cannot directly find
 the library.  The newer out-of-line ``ffi.dlopen()`` no longer does it
diff --git a/doc/source/whatsnew.rst b/doc/source/whatsnew.rst
--- a/doc/source/whatsnew.rst
+++ b/doc/source/whatsnew.rst
@@ -3,6 +3,19 @@
 ======================
 
 
+1.1.1
+=====
+
+* Out-of-line mode: ``ffi.string()``, ``ffi.buffer()`` and
+  ``ffi.getwinerror()`` didn't accept their arguments as keyword
+  arguments, unlike their in-line mode equivalent.  (It worked in PyPy.)
+
+* Out-of-line ABI mode: documented a restriction__ of ``ffi.dlopen()``
+  when compared to the in-line mode.
+
+.. __: cdef.html#dlopen-note
+
+
 1.1.0
 =====
 

From noreply at buildbot.pypy.org  Mon Jun  1 09:27:42 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Mon,  1 Jun 2015 09:27:42 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: reverted call2,
	list + index left the access in the trace
Message-ID: <20150601072742.4E6721C11F5@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r77737:c191d623bf18
Date: 2015-06-01 09:27 +0200
http://bitbucket.org/pypy/pypy/changeset/c191d623bf18/

Log:	reverted call2, list + index left the access in the trace

diff --git a/pypy/module/micronumpy/iterators.py b/pypy/module/micronumpy/iterators.py
--- a/pypy/module/micronumpy/iterators.py
+++ b/pypy/module/micronumpy/iterators.py
@@ -116,16 +116,6 @@
                 factors[ndim-i-1] = factors[ndim-i] * shape[ndim-i]
         self.factors = factors
 
-    def matches_range(self, other_iter):
-        assert isinstance(other_iter, ArrayIter)
-        return self.size == other_iter.size and \
-               self.contiguous == other_iter.contiguous and \
-               self.ndim_m1 == other_iter.ndim_m1 and \
-               self.shape_m1 == other_iter.shape_m1 and \
-               self.strides == other_iter.strides and \
-               self.factors == other_iter.factors and \
-               self.backstrides == other_iter.backstrides
-
     @jit.unroll_safe
     def reset(self, state=None, mutate=False):
         index = 0
diff --git a/pypy/module/micronumpy/loop.py b/pypy/module/micronumpy/loop.py
--- a/pypy/module/micronumpy/loop.py
+++ b/pypy/module/micronumpy/loop.py
@@ -15,7 +15,7 @@
 
 call2_driver = jit.JitDriver(
     name='numpy_call2',
-    greens=['shapelen', 'func', 'left_iter_index', 'right_iter_index', 'calc_dtype', 'res_dtype' ],
+    greens=['shapelen', 'func', 'calc_dtype', 'res_dtype'],
     reds='auto', vectorize=True)
 
 def call2(space, shape, func, calc_dtype, res_dtype, w_lhs, w_rhs, out):
@@ -43,12 +43,9 @@
 
     # TODO handle __array_priorities__ and maybe flip the order
 
-    left_iter_index = 1
-    right_iter_index = 2
     if w_lhs.get_size() == 1:
         w_left = w_lhs.get_scalar_value().convert_to(space, calc_dtype)
         left_iter = left_state = None
-        left_iter_index = -1
     else:
         w_left = None
         left_iter, left_state = w_lhs.create_iter(shape)
@@ -57,45 +54,28 @@
     if w_rhs.get_size() == 1:
         w_right = w_rhs.get_scalar_value().convert_to(space, calc_dtype)
         right_iter = right_state = None
-        right_iter_index = -1
     else:
         w_right = None
         right_iter, right_state = w_rhs.create_iter(shape)
         right_iter.track_index = False
+
     if out is None:
         out = W_NDimArray.from_shape(space, shape, res_dtype,
                                      w_instance=lhs_for_subtype)
     out_iter, out_state = out.create_iter(shape)
-
-    iter_list = [out_iter, left_iter, right_iter]
-    state_list = [out_state, left_state, right_state]
-
-    if left_iter_index > 0 and left_iter.matches_range(out_iter):
-        left_iter_index = 0
-    if right_iter_index > 0 and right_iter.matches_range(out_iter):
-        right_iter_index = 0
-
     shapelen = len(shape)
     while not out_iter.done(out_state):
-        call2_driver.jit_merge_point(shapelen=shapelen, left_iter_index=left_iter_index,
-                                     right_iter_index=right_iter_index,
-                                     func=func, calc_dtype=calc_dtype, res_dtype=res_dtype)
-        if left_iter_index > 0:
-            iter = iter_list[left_iter_index]
-            state = state_list[left_iter_index]
-            w_left = iter.getitem(state).convert_to(space, calc_dtype)
-            if left_iter_index == 1:
-                state_list[left_iter_index] = iter.next(state)
-        if right_iter_index > 0:
-            iter = iter_list[right_iter_index]
-            state = state_list[right_iter_index]
-            w_right = iter.getitem(state).convert_to(space, calc_dtype)
-            if right_iter_index == 2:
-                state_list[right_iter_index] = iter.next(state)
+        call2_driver.jit_merge_point(shapelen=shapelen, func=func,
+                                     calc_dtype=calc_dtype, res_dtype=res_dtype)
+        if left_iter:
+            w_left = left_iter.getitem(left_state).convert_to(space, calc_dtype)
+            left_state = left_iter.next(left_state)
+        if right_iter:
+            w_right = right_iter.getitem(right_state).convert_to(space, calc_dtype)
+            right_state = right_iter.next(right_state)
         out_iter.setitem(out_state, func(calc_dtype, w_left, w_right).convert_to(
             space, res_dtype))
-        state_list[0] = out_state = out_iter.next(out_state)
-
+        out_state = out_iter.next(out_state)
     return out
 
 call1_driver = jit.JitDriver(

From noreply at buildbot.pypy.org  Mon Jun  1 09:41:28 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Mon,  1 Jun 2015 09:41:28 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: started to add cost model,
 all enabled zjit tests passing now
Message-ID: <20150601074128.B67321C0262@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r77738:c322539e68de
Date: 2015-06-01 09:41 +0200
http://bitbucket.org/pypy/pypy/changeset/c322539e68de/

Log:	started to add cost model, all enabled zjit tests passing now

diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/vectorize.py
@@ -731,7 +731,7 @@
 class PackType(PrimitiveTypeMixin):
     UNKNOWN_TYPE = '-'
 
-    def __init__(self, type, size, signed, count=-1):
+    def __init__(self, type, size, signed, count=-1, scalar_cost=1, vector_cost=1):
         assert type in (FLOAT, INT, PackType.UNKNOWN_TYPE)
         self.type = type
         self.size = size
@@ -911,9 +911,6 @@
             # The original box is at a position != 0 but it
             # is required to be at position 0. Unpack it!
             vbox = self.unpack(vbox, off, len(ops), self.input_type)
-        # convert type f -> i, i -> f
-        if self.input_type.gettype() != vbox.gettype():
-            raise NotImplementedError("cannot yet convert between types")
         # convert size i64 -> i32, i32 -> i64, ...
         if self.input_type.getsize() > 0 and \
            self.input_type.getsize() != vbox.getsize():
@@ -1086,11 +1083,13 @@
         return vbox.getcount()
 
     def new_result_vector_box(self):
+        type = self.output_type.gettype()
         count = self.input_type.getcount()
         vec_reg_size = self.sched_data.vec_reg_size
         if count * self.size > vec_reg_size:
             count = vec_reg_size // self.size
-        return BoxVector(self.result_ptype.gettype(), count, self.size, self.input_type.signed)
+        signed = self.input_type.signed
+        return BoxVector(type, count, self.size, signed)
 
 PT_GENERIC = PackType(PackType.UNKNOWN_TYPE, -1, False)
 
@@ -1128,11 +1127,17 @@
     def determine_output_type(self, op):
         return None
 
+class CostModel(object):
+    pass
+
+class X86_CostModel(CostModel):
+    pass
+
 PT_FLOAT_2 = PackType(FLOAT, 4, False, 2)
 PT_DOUBLE_2 = PackType(FLOAT, 8, False, 2)
 PT_FLOAT_GENERIC = PackType(INT, -1, True)
 PT_INT64 = PackType(INT, 8, True)
-PT_INT32 = PackType(INT, 4, True)
+PT_INT32_2 = PackType(INT, 4, True, 2)
 PT_INT_GENERIC = PackType(INT, -1, True)
 PT_GENERIC = PackType(PackType.UNKNOWN_TYPE, -1, False)
 
@@ -1172,8 +1177,8 @@
 
     rop.VEC_CAST_FLOAT_TO_SINGLEFLOAT: OpToVectorOpConv(PT_DOUBLE_2, PT_FLOAT_2),
     rop.VEC_CAST_SINGLEFLOAT_TO_FLOAT: OpToVectorOpConv(PT_FLOAT_2, PT_DOUBLE_2),
-    rop.VEC_CAST_FLOAT_TO_INT: OpToVectorOpConv(PT_DOUBLE_2, PT_INT32),
-    rop.VEC_CAST_INT_TO_FLOAT: OpToVectorOpConv(PT_INT32, PT_DOUBLE_2),
+    rop.VEC_CAST_FLOAT_TO_INT: OpToVectorOpConv(PT_DOUBLE_2, PT_INT32_2),
+    rop.VEC_CAST_INT_TO_FLOAT: OpToVectorOpConv(PT_INT32_2, PT_DOUBLE_2),
 }
 
 class VecScheduleData(SchedulerData):
@@ -1286,7 +1291,6 @@
         for op in pack_j.operations[1:]:
             operations.append(op)
         self.packs[i] = pack = Pack(operations)
-        pack.ptype = pack_i.ptype
 
         # instead of deleting an item in the center of pack array,
         # the last element is assigned to position j and

From noreply at buildbot.pypy.org  Mon Jun  1 10:15:43 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Mon,  1 Jun 2015 10:15:43 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: rpython compliant,
	started to impl cost model
Message-ID: <20150601081543.2DC0A1C06F5@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r77739:443876d529b4
Date: 2015-06-01 10:15 +0200
http://bitbucket.org/pypy/pypy/changeset/443876d529b4/

Log:	rpython compliant, started to impl cost model

diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -2696,7 +2696,7 @@
                         # if source is a normal register (unpack)
                         assert count == 1
                         assert si == 0
-                        self.mc.move(X86_64_XMM_SCRATCH_REG, srcloc)
+                        self.mov(X86_64_XMM_SCRATCH_REG, srcloc)
                         src = X86_64_XMM_SCRATCH_REG.value
                     select = ((si & 0x3) << 6)|((ri & 0x3) << 4)
                     self.mc.INSERTPS_xxi(resloc.value, src, select)
diff --git a/rpython/jit/metainterp/optimizeopt/dependency.py b/rpython/jit/metainterp/optimizeopt/dependency.py
--- a/rpython/jit/metainterp/optimizeopt/dependency.py
+++ b/rpython/jit/metainterp/optimizeopt/dependency.py
@@ -540,7 +540,7 @@
                         last_guard = self.guards[-1]
                         last_guard.edge_to(node, failarg=True, label="guardorder")
                     for nonpure in tracker.non_pure:
-                        nonpure.edge_to(node, failarg=True)
+                        nonpure.edge_to(node, failarg=True, label="nonpure")
                     tracker.non_pure = []
                 self.guards.append(node)
             else:
@@ -690,7 +690,7 @@
             # before the last guard operation
             if len(self.guards) > 0:
                 last_guard = self.guards[-1]
-                last_guard.edge_to(node, "sideeffect")
+                last_guard.edge_to(node, label="sideeffect")
             # and the next guard instruction
             tracker.add_non_pure(node)
 
diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/vectorize.py
@@ -79,6 +79,7 @@
         self.early_exit_idx = -1
         self.sched_data = None
         self.tried_to_pack = False
+        self.costmodel = X86_CostModel()
 
     def propagate_all_forward(self, clear=True):
         self.clear_newoperations()
@@ -201,8 +202,8 @@
                 if copied_op.is_guard():
                     assert isinstance(copied_op, GuardResOp)
                     target_guard = copied_op
+                    # do not overwrite resume at loop header
                     if not isinstance(target_guard.getdescr(), ResumeAtLoopHeaderDescr):
-                        # do not overwrite resume at loop header
                         descr = invent_fail_descr_for_op(copied_op.getopnum(), self)
                         olddescr = copied_op.getdescr()
                         descr.copy_all_attributes_from(olddescr)
@@ -304,7 +305,7 @@
                 lnode = ldep.to
                 rnode = rdep.to
                 if lnode.is_before(rnode) and self.packset.can_be_packed(lnode, rnode):
-                    savings = self.packset.estimate_savings(lnode, rnode, pack, False)
+                    savings = self.costmodel.estimate_savings(lnode, rnode, pack, False)
                     if savings >= 0:
                         self.packset.add_pair(lnode, rnode)
 
@@ -319,7 +320,7 @@
                 if lnode.is_before(rnode) and \
                    self.packset.can_be_packed(lnode, rnode):
                     est_savings = \
-                        self.packset.estimate_savings(lnode, rnode, pack, True)
+                        self.costmodel.estimate_savings(lnode, rnode, pack, True)
                     if est_savings > savings:
                         savings = est_savings
                         candidate = (lnode, rnode)
@@ -476,6 +477,7 @@
             op.setarg(i, arg)
 
         if op.is_guard():
+            assert isinstance(op, GuardResOp)
             op.rd_snapshot = self.rename_rd_snapshot(op.rd_snapshot)
             self.rename_failargs(op)
 
@@ -558,14 +560,18 @@
             return self.cmp_op.boolinverse
 
     def inhert_attributes(self, other):
+        myop = self.op
+        otherop = other.op
+        assert isinstance(otherop, GuardResOp)
+        assert isinstance(myop, GuardResOp)
         self.stronger = True
         self.index = other.index
 
-        descr = self.op.getdescr()
+        descr = myop.getdescr()
         descr.copy_all_attributes_from(other.op.getdescr())
-        self.op.rd_frame_info_list = other.op.rd_frame_info_list
-        self.op.rd_snapshot = other.op.rd_snapshot
-        self.op.setfailargs(other.op.getfailargs())
+        myop.rd_frame_info_list = otherop.rd_frame_info_list
+        myop.rd_snapshot = otherop.rd_snapshot
+        myop.setfailargs(otherop.getfailargs())
 
     def compare(self, key1, key2):
         if isinstance(key1, Box):
@@ -722,11 +728,42 @@
         self.renamer.rename(op)
         self._newoperations.append(op)
 
-def must_unpack_result_to_exec(op, target_op):
-    # TODO either move to resop or util
-    if op.getoperation().vector != -1:
+class CostModel(object):
+    def estimate_savings(self, lnode, rnode, origin_pack, expand_forward):
+        """ Estimate the number of savings to add this pair.
+        Zero is the minimum value returned. This should take
+        into account the benefit of executing this instruction
+        as SIMD instruction.
+        """
+
+        lpacknode = origin_pack.left
+        if self.prohibit_packing(lpacknode.getoperation(), lnode.getoperation()):
+            return -1
+        rpacknode = origin_pack.right
+        if self.prohibit_packing(rpacknode.getoperation(), rnode.getoperation()):
+            return -1
+
+        return 0
+
+    def prohibit_packing(self, packed, inquestion):
+        """ Blocks the packing of some operations """
+        if inquestion.vector == -1:
+            return True
+        if packed.is_array_op():
+            if packed.getarg(1) == inquestion.result:
+                return True
         return False
-    return True
+
+    def must_unpack_result_to_exec(self, op, target_op):
+        # TODO either move to resop or util
+        if op.getoperation().vector != -1:
+            return False
+        return True
+
+class X86_CostModel(CostModel):
+
+    def savings(self, op, times):
+        return 0
 
 class PackType(PrimitiveTypeMixin):
     UNKNOWN_TYPE = '-'
@@ -1127,12 +1164,6 @@
     def determine_output_type(self, op):
         return None
 
-class CostModel(object):
-    pass
-
-class X86_CostModel(CostModel):
-    pass
-
 PT_FLOAT_2 = PackType(FLOAT, 4, False, 2)
 PT_DOUBLE_2 = PackType(FLOAT, 8, False, 2)
 PT_FLOAT_GENERIC = PackType(INT, -1, True)
@@ -1245,41 +1276,6 @@
                 return True
         return False
 
-    def estimate_savings(self, lnode, rnode, pack, expand_forward):
-        """ Estimate the number of savings to add this pair.
-        Zero is the minimum value returned. This should take
-        into account the benefit of executing this instruction
-        as SIMD instruction.
-        """
-        savings = -1
-
-        lpacknode = pack.left
-        if self.prohibit_packing(lpacknode.getoperation(), lnode.getoperation()):
-            return -1
-        rpacknode = pack.right
-        if self.prohibit_packing(rpacknode.getoperation(), rnode.getoperation()):
-            return -1
-
-        if not expand_forward:
-            if not must_unpack_result_to_exec(lpacknode, lnode) and \
-               not must_unpack_result_to_exec(rpacknode, rnode):
-                savings += 1
-        else:
-            if not must_unpack_result_to_exec(lpacknode, lnode) and \
-               not must_unpack_result_to_exec(rpacknode, rnode):
-                savings += 1
-
-        return savings
-
-    def prohibit_packing(self, packed, inquestion):
-        if inquestion.vector == -1:
-            return True
-        if packed.is_array_op():
-            if packed.getarg(1) == inquestion.result:
-                return True
-        return False
-
-
     def combine(self, i, j):
         """ combine two packs. it is assumed that the attribute self.packs
         is not iterated when calling this method. """

From noreply at buildbot.pypy.org  Mon Jun  1 12:12:07 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  1 Jun 2015 12:12:07 +0200 (CEST)
Subject: [pypy-commit] pypy.org extradoc: Put a link to vmprof.
Message-ID: <20150601101207.60C811C04BC@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r612:6aa41e28f863
Date: 2015-06-01 12:12 +0200
http://bitbucket.org/pypy/pypy.org/changeset/6aa41e28f863/

Log:	Put a link to vmprof.

diff --git a/performance.html b/performance.html
--- a/performance.html
+++ b/performance.html
@@ -72,17 +72,32 @@
 <div class="contents topic" id="contents">
 <p class="topic-title first">Contents</p>
 <ul class="simple">
-<li><a class="reference internal" href="#optimization-strategy" id="id1">Optimization strategy</a></li>
-<li><a class="reference internal" href="#micro-tuning-tips" id="id2">Micro-tuning tips</a></li>
-<li><a class="reference internal" href="#insider-s-point-of-view" id="id3">Insider's point of view</a></li>
+<li><a class="reference internal" href="#profiling-vmprof" id="id1">Profiling: vmprof</a></li>
+<li><a class="reference internal" href="#optimization-strategy" id="id2">Optimization strategy</a></li>
+<li><a class="reference internal" href="#micro-tuning-tips" id="id3">Micro-tuning tips</a></li>
+<li><a class="reference internal" href="#insider-s-point-of-view" id="id4">Insider's point of view</a></li>
 </ul>
 </div>
 <p>This document collects strategies, tactics and tricks for making your
 code run faster under PyPy.  Many of these are also useful hints for
 stock Python and other languages.  For contrast, we also describe some
 CPython (stock Python) optimizations that are not needed in PyPy.</p>
+<hr class="docutils" />
+<div class="section" id="profiling-vmprof">
+<span id="profiling"></span><span id="profiler"></span><h1><a class="toc-backref" href="#id1">Profiling: vmprof</a></h1>
+<p>As a general rule, when considering performance issues, follow these
+three points: first <em>measure</em> them (it is counter-productive to fight
+imaginary performance issues); then <em>profile</em> your code (it is useless
+to optimize the wrong parts).  Only optimize then.</p>
+<p>PyPy 2.6 introduced <a class="reference external" href="https://vmprof.readthedocs.org/">vmprof</a>, a very-low-overhead statistical profiler.
+The standard, non-statistical <tt class="docutils literal">cProfile</tt> is also supported, and can be
+enabled without turning off the JIT.  We do recommend vmprof anyway
+because turning on cProfile can distort the result (sometimes massively,
+though hopefully this should not be too common).</p>
+</div>
+<hr class="docutils" />
 <div class="section" id="optimization-strategy">
-<h1><a class="toc-backref" href="#id1">Optimization strategy</a></h1>
+<h1><a class="toc-backref" href="#id2">Optimization strategy</a></h1>
 <p>These suggestions apply to all computer languages.  They're here as
 reminders of things to try before any Python or PyPy-specific tweaking.</p>
 <div class="section" id="build-a-regression-test-suite">
@@ -95,7 +110,7 @@
 <div class="section" id="measure-don-t-guess">
 <h2>Measure, don't guess</h2>
 <p>Human beings are bad at guessing or intuiting where the hotspots in code are.
-Measure, don't guess; use a profiler to pin down the 20% of the
+Measure, don't guess; use a <a class="reference internal" href="#profiler">profiler</a> to pin down the 20% of the
 code where the code is spending 80% of its time, then speed-tune that.</p>
 <p>Measuring will save you a lot of effort wasted on tuning parts of the code
 that aren't actually bottlenecks.</p>
@@ -109,7 +124,7 @@
 bound (slow because of disk or network delays).</p>
 <p>Expect to get most of your gains from optimizing compute-bound code.
 It's usually (though not always) a sign that you're near the end of
-worthwhile tuning when profiling shows that the bulk of the
+worthwhile tuning when <a class="reference internal" href="#profiling">profiling</a> shows that the bulk of the
 application's time is spent on network and disk I/O.</p>
 </div>
 <div class="section" id="tune-your-algorithms-first">
@@ -160,8 +175,9 @@
 function of your regression test suite can be as a speed benchmark.</p>
 </div>
 </div>
+<hr class="docutils" />
 <div class="section" id="micro-tuning-tips">
-<h1><a class="toc-backref" href="#id2">Micro-tuning tips</a></h1>
+<h1><a class="toc-backref" href="#id3">Micro-tuning tips</a></h1>
 <p>These are in no particular order.</p>
 <div class="section" id="keep-it-simple">
 <h2>Keep it simple</h2>
@@ -270,8 +286,9 @@
 <p><em>(Thanks Eric S. Raymond for the text above)</em></p>
 </div>
 </div>
+<hr class="docutils" />
 <div class="section" id="insider-s-point-of-view">
-<h1><a class="toc-backref" href="#id3">Insider's point of view</a></h1>
+<h1><a class="toc-backref" href="#id4">Insider's point of view</a></h1>
 <p>This section describes performance issues from the point of view of
 insiders of the project; it should be particularly interesting if you
 plan to contribute in that area.</p>
diff --git a/source/performance.txt b/source/performance.txt
--- a/source/performance.txt
+++ b/source/performance.txt
@@ -11,6 +11,31 @@
 stock Python and other languages.  For contrast, we also describe some
 CPython (stock Python) optimizations that are not needed in PyPy.
 
+
+=================
+
+.. _profiler:
+.. _profiling:
+
+Profiling: vmprof
+=================
+
+As a general rule, when considering performance issues, follow these
+three points: first *measure* them (it is counter-productive to fight
+imaginary performance issues); then *profile* your code (it is useless
+to optimize the wrong parts).  Only optimize then.
+
+PyPy 2.6 introduced vmprof_, a very-low-overhead statistical profiler.
+The standard, non-statistical ``cProfile`` is also supported, and can be
+enabled without turning off the JIT.  We do recommend vmprof anyway
+because turning on cProfile can distort the result (sometimes massively,
+though hopefully this should not be too common).
+
+.. _vmprof: https://vmprof.readthedocs.org/
+
+
+=====================
+
 Optimization strategy
 =====================
 
@@ -29,7 +54,7 @@
 --------------------
 
 Human beings are bad at guessing or intuiting where the hotspots in code are.
-Measure, don't guess; use a profiler to pin down the 20% of the 
+Measure, don't guess; use a profiler_ to pin down the 20% of the 
 code where the code is spending 80% of its time, then speed-tune that.
 
 Measuring will save you a lot of effort wasted on tuning parts of the code
@@ -47,7 +72,7 @@
 
 Expect to get most of your gains from optimizing compute-bound code.
 It's usually (though not always) a sign that you're near the end of
-worthwhile tuning when profiling shows that the bulk of the
+worthwhile tuning when profiling_ shows that the bulk of the
 application's time is spent on network and disk I/O.
 
 Tune your algorithms first
@@ -107,6 +132,9 @@
 which takes us right back to "Measure, don't guess".  And another
 function of your regression test suite can be as a speed benchmark.
 
+
+=================
+
 Micro-tuning tips
 =================
 
@@ -239,6 +267,8 @@
 *(Thanks Eric S. Raymond for the text above)*
 
 
+=======================
+
 Insider's point of view
 =======================
 

From noreply at buildbot.pypy.org  Mon Jun  1 12:56:05 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Mon,  1 Jun 2015 12:56:05 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: costmodel impl extended
Message-ID: <20150601105605.294D31C06F5@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r77740:7ce427746614
Date: 2015-06-01 12:56 +0200
http://bitbucket.org/pypy/pypy/changeset/7ce427746614/

Log:	costmodel impl extended added tests for cost model extracted tests
	into another file

diff --git a/rpython/jit/metainterp/optimizeopt/dependency.py b/rpython/jit/metainterp/optimizeopt/dependency.py
--- a/rpython/jit/metainterp/optimizeopt/dependency.py
+++ b/rpython/jit/metainterp/optimizeopt/dependency.py
@@ -80,6 +80,7 @@
         self.adjacent_list_back = []
         self.memory_ref = None
         self.pack = None
+        self.pack_position = -1
         self.emitted = False
         self.schedule_position = -1
         self.priority = 0
@@ -962,12 +963,6 @@
             self.current_end.next_nonconst = idxvar
         self.current_end = idxvar
 
-    def is_adjacent_with_runtime_check(self, other, graph):
-        return self.next_nonconst is not None and \
-               self.next_nonconst is self.current_end and \
-               self.next_nonconst.opnum == rop.INT_ADD and \
-               self.next_nonconst.is_identity()
-
     def getvariable(self):
         return self.var
 
@@ -1086,15 +1081,6 @@
             return abs(self.index_var.diff(other.index_var)) - stride == 0
         return False
 
-    def is_adjacent_with_runtime_check(self, other, graph):
-        """there are many cases where the stride is variable
-           it is a priori not known if two unrolled memory accesses are
-           tightly packed"""
-        assert isinstance(other, MemoryRef)
-        if self.array == other.array and self.descr == other.descr:
-            return self.index_var.is_adjacent_with_runtime_check(other.index_var, graph)
-        return False
-
     def match(self, other):
         assert isinstance(other, MemoryRef)
         if self.array == other.array and self.descr == other.descr:
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_costmodel.py b/rpython/jit/metainterp/optimizeopt/test/test_costmodel.py
new file mode 100644
--- /dev/null
+++ b/rpython/jit/metainterp/optimizeopt/test/test_costmodel.py
@@ -0,0 +1,131 @@
+import py
+
+from rpython.jit.metainterp.history import TargetToken, JitCellToken, TreeLoop
+from rpython.jit.metainterp.optimizeopt.util import equaloplists
+from rpython.jit.metainterp.optimizeopt.vectorize import (VecScheduleData,
+        Pack, NotAProfitableLoop, VectorizingOptimizer)
+from rpython.jit.metainterp.optimizeopt.dependency import Node
+from rpython.jit.metainterp.optimizeopt.test.test_util import LLtypeMixin
+from rpython.jit.metainterp.optimizeopt.test.test_schedule import SchedulerBaseTest
+from rpython.jit.metainterp.optimizeopt.test.test_vectorize import (FakeMetaInterpStaticData,
+        FakeJitDriverStaticData)
+from rpython.jit.metainterp.resoperation import rop, ResOperation
+from rpython.jit.tool.oparser import parse as opparse
+from rpython.jit.tool.oparser_model import get_model
+
+class FakeMemoryRef(object):
+    def __init__(self, iv):
+        self.index_var = iv
+
+    def is_adjacent_to(self, other):
+        iv = self.index_var
+        ov = other.index_var
+        val = (int(str(ov.var)[1:]) - int(str(iv.var)[1:]))
+        print iv, ov, "adja?", val == 1
+        # i0 and i1 are adjacent
+        # i1 and i2 ...
+        # but not i0, i2
+        # ...
+        return val == 1
+
+class CostModelBaseTest(SchedulerBaseTest):
+    def savings(self, loop):
+        metainterp_sd = FakeMetaInterpStaticData(self.cpu)
+        jitdriver_sd = FakeJitDriverStaticData()
+        opt = VectorizingOptimizer(metainterp_sd, jitdriver_sd, loop, [])
+        opt.build_dependency_graph()
+        graph = opt.dependency_graph
+        for k,m in graph.memory_refs.items():
+            graph.memory_refs[k] = FakeMemoryRef(m.index_var)
+            print "memory ref", k, m
+        opt.find_adjacent_memory_refs()
+        opt.extend_packset()
+        opt.combine_packset()
+        for pack in opt.packset.packs:
+            print "apck:"
+            print '\n'.join([str(op.getoperation()) for op in pack.operations])
+            print
+        return opt.costmodel.calculate_savings(opt.packset)
+
+    def assert_operations_match(self, loop_a, loop_b):
+        assert equaloplists(loop_a.operations, loop_b.operations)
+
+    def test_load_2_unpack(self):
+        loop1 = self.parse("""
+        f10 = raw_load(p0, i0, descr=double)
+        f11 = raw_load(p0, i1, descr=double)
+        guard_true(i0) [f10]
+        guard_true(i1) [f11]
+        """)
+        # for double the costs are
+        # unpack index 1 savings: -2
+        # unpack index 0 savings: -1
+        savings = self.savings(loop1)
+        assert savings == -2
+
+    def test_load_4_unpack(self):
+        loop1 = self.parse("""
+        i10 = raw_load(p0, i0, descr=float)
+        i11 = raw_load(p0, i1, descr=float)
+        i12 = raw_load(p0, i2, descr=float)
+        i13 = raw_load(p0, i3, descr=float)
+        guard_true(i0) [i10]
+        guard_true(i1) [i11]
+        guard_true(i2) [i12]
+        guard_true(i3) [i13]
+        """)
+        savings = self.savings(loop1)
+        assert savings == -1
+
+    def test_load_2_unpack_1(self):
+        loop1 = self.parse("""
+        f10 = raw_load(p0, i0, descr=double)
+        f11 = raw_load(p0, i1, descr=double)
+        guard_true(i0) [f10]
+        """)
+        savings = self.savings(loop1)
+        assert savings == 0
+
+    def test_load_2_unpack_1_index1(self):
+        loop1 = self.parse("""
+        f10 = raw_load(p0, i0, descr=double)
+        f11 = raw_load(p0, i1, descr=double)
+        guard_true(i0) [f11]
+        """)
+        savings = self.savings(loop1)
+        assert savings == -1
+
+    def test_load_arith(self):
+        loop1 = self.parse("""
+        i10 = raw_load(p0, i0, descr=int)
+        i11 = raw_load(p0, i1, descr=int)
+        i12 = raw_load(p0, i2, descr=int)
+        i13 = raw_load(p0, i3, descr=int)
+        i15 = int_add(i10, 1)
+        i16 = int_add(i11, 1)
+        i17 = int_add(i12, 1)
+        i18 = int_add(i13, 1)
+        """)
+        savings = self.savings(loop1)
+        assert savings == 6
+
+    def test_load_arith_store(self):
+        loop1 = self.parse("""
+        i10 = raw_load(p0, i0, descr=int)
+        i11 = raw_load(p0, i1, descr=int)
+        i12 = raw_load(p0, i2, descr=int)
+        i13 = raw_load(p0, i3, descr=int)
+        i15 = int_add(i10, 1)
+        i16 = int_add(i11, 1)
+        i17 = int_add(i12, 1)
+        i18 = int_add(i13, 1)
+        raw_store(p1, i4, i15, descr=int)
+        raw_store(p1, i5, i16, descr=int)
+        raw_store(p1, i6, i17, descr=int)
+        raw_store(p1, i7, i18, descr=int)
+        """)
+        savings = self.savings(loop1)
+        assert savings == 6
+
+class Test(CostModelBaseTest, LLtypeMixin):
+    pass
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_schedule.py b/rpython/jit/metainterp/optimizeopt/test/test_schedule.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_schedule.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_schedule.py
@@ -1,31 +1,43 @@
 import py
 
+from rpython.jit.metainterp.history import TargetToken, JitCellToken, TreeLoop
 from rpython.jit.metainterp.optimizeopt.util import equaloplists
 from rpython.jit.metainterp.optimizeopt.vectorize import (VecScheduleData,
-        Pack, NotAProfitableLoop)
+        Pack, NotAProfitableLoop, VectorizingOptimizer)
 from rpython.jit.metainterp.optimizeopt.dependency import Node
 from rpython.jit.metainterp.optimizeopt.test.test_util import LLtypeMixin
 from rpython.jit.metainterp.optimizeopt.test.test_dependency import DependencyBaseTest
+from rpython.jit.metainterp.optimizeopt.test.test_vectorize import (FakeMetaInterpStaticData,
+        FakeJitDriverStaticData)
+from rpython.jit.metainterp.resoperation import rop, ResOperation
 from rpython.jit.tool.oparser import parse as opparse
 from rpython.jit.tool.oparser_model import get_model
 
 class SchedulerBaseTest(DependencyBaseTest):
 
-    def parse(self, source):
+    def parse(self, source, inc_label_jump=True):
         ns = {
             'double': self.floatarraydescr,
             'float': self.singlefloatarraydescr,
             'long': self.intarraydescr,
+            'int': self.int32arraydescr,
         }
-        loop = opparse("        [p0,p1,p2,p3,p4,p5,i0,i1,i2,i3,i4,i5,f0,f1,f2,f3,f4,f5]\n" + source + \
-                       "\n        jump(p0,p1,p2,p3,p4,p5,i0,i1,i2,i3,i4,i5,f0,f1,f2,f3,f4,f5)",
+        loop = opparse("        [p0,p1,p2,p3,p4,p5,i0,i1,i2,i3,i4,i5,i6,i7,i8,i9,f0,f1,f2,f3,f4,f5]\n" + source + \
+                       "\n        jump(p0,p1,p2,p3,p4,p5,i0,i1,i2,i3,i4,i5,i6,i7,i8,i9,f0,f1,f2,f3,f4,f5)",
                        cpu=self.cpu,
                        namespace=ns)
+        if inc_label_jump:
+            token = JitCellToken()
+            loop.operations = \
+                [ResOperation(rop.LABEL, loop.inputargs, None, descr=TargetToken(token))] + \
+                loop.operations
+            return loop
+
         del loop.operations[-1]
         return loop
 
     def pack(self, loop, l, r):
-        return [Node(op,l+i) for i,op in enumerate(loop.operations[l:r])]
+        return [Node(op,1+l+i) for i,op in enumerate(loop.operations[1+l:1+r])]
 
     def schedule(self, loop_orig, packs, vec_reg_size=16):
         loop = get_model(False).ExtendedTreeLoop("loop")
@@ -46,6 +58,7 @@
     def assert_operations_match(self, loop_a, loop_b):
         assert equaloplists(loop_a.operations, loop_b.operations)
 
+class Test(SchedulerBaseTest, LLtypeMixin):
     def test_schedule_split_load(self):
         loop1 = self.parse("""
         i10 = raw_load(p0, i0, descr=float)
@@ -61,7 +74,7 @@
         v1[i32#4] = vec_raw_load(p0, i0, 4, descr=float)
         i14 = raw_load(p0, i4, descr=float)
         i15 = raw_load(p0, i5, descr=float)
-        """)
+        """, False)
         self.assert_equal(loop2, loop3)
 
     def test_int_to_float(self):
@@ -73,31 +86,10 @@
         """)
         pack1 = self.pack(loop1, 0, 2)
         pack2 = self.pack(loop1, 2, 4)
-        print pack1
-        print pack2
         loop2 = self.schedule(loop1, [pack1, pack2])
         loop3 = self.parse("""
         v1[i64#2] = vec_raw_load(p0, i0, 2, descr=long)
         v2[i32#2] = vec_int_signext(v1[i64#2], 4)
         v3[f64#2] = vec_cast_int_to_float(v2[i32#2])
-        """)
+        """, False)
         self.assert_equal(loop2, loop3)
-
-    def test_cost_model_reject_only_load_vectorizable(self):
-        loop1 = self.parse("""
-        f10 = raw_load(p0, i0, descr=long)
-        f11 = raw_load(p0, i1, descr=long)
-        guard_true(i0) [f10]
-        guard_true(i1) [f11]
-        """)
-        try:
-            pack1 = self.pack(loop1, 0, 2)
-            pack2 = self.pack(loop1, 2, 3)
-            pack3 = self.pack(loop1, 3, 4)
-            loop2 = self.schedule(loop1, [pack1, pack2, pack3])
-            py.test.fail("this loops should have bailed out")
-        except NotAProfitableLoop:
-            pass
-
-class TestLLType(SchedulerBaseTest, LLtypeMixin):
-    pass
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_util.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py
@@ -155,6 +155,7 @@
     arraydescr = cpu.arraydescrof(lltype.GcArray(lltype.Signed))
     floatarraydescr = cpu.arraydescrof(lltype.GcArray(lltype.Float))
     intarraydescr = cpu.arraydescrof(lltype.GcArray(lltype.Signed))
+    int32arraydescr = cpu.arraydescrof(lltype.GcArray(rffi.INT))
     uintarraydescr = cpu.arraydescrof(lltype.GcArray(lltype.Unsigned))
     chararraydescr = cpu.arraydescrof(lltype.GcArray(lltype.Char))
     singlefloatarraydescr = cpu.arraydescrof(lltype.GcArray(lltype.SingleFloat))
diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/vectorize.py
@@ -112,6 +112,8 @@
         self.find_adjacent_memory_refs()
         self.extend_packset()
         self.combine_packset()
+        if not self.costmodel.profitable(self.packset):
+            raise NotAProfitableLoop()
         self.schedule()
 
         gso = GuardStrengthenOpt(self.dependency_graph.index_vars)
@@ -284,7 +286,7 @@
                 # that point forward:
                 if node_a.is_before(node_b):
                     if memref_a.is_adjacent_to(memref_b):
-                        if self.packset.can_be_packed(node_a, node_b):
+                        if self.packset.can_be_packed(node_a, node_b, None):
                             pair = Pair(node_a,node_b)
                             self.packset.packs.append(pair)
 
@@ -304,31 +306,21 @@
             for rdep in pack.right.depends():
                 lnode = ldep.to
                 rnode = rdep.to
-                if lnode.is_before(rnode) and self.packset.can_be_packed(lnode, rnode):
-                    savings = self.costmodel.estimate_savings(lnode, rnode, pack, False)
-                    if savings >= 0:
+                isomorph = isomorphic(lnode.getoperation(), rnode.getoperation())
+                if isomorph and lnode.is_before(rnode):
+                    if self.packset.can_be_packed(lnode, rnode, pack):
                         self.packset.add_pair(lnode, rnode)
 
     def follow_def_uses(self, pack):
         assert isinstance(pack, Pair)
-        savings = -1
-        candidate = (None,None)
         for ldep in pack.left.provides():
             for rdep in pack.right.provides():
                 lnode = ldep.to
                 rnode = rdep.to
-                if lnode.is_before(rnode) and \
-                   self.packset.can_be_packed(lnode, rnode):
-                    est_savings = \
-                        self.costmodel.estimate_savings(lnode, rnode, pack, True)
-                    if est_savings > savings:
-                        savings = est_savings
-                        candidate = (lnode, rnode)
-        #
-        if savings >= 0:
-            assert candidate[0] is not None
-            assert candidate[1] is not None
-            self.packset.add_pair(candidate[0], candidate[1])
+                isomorph = isomorphic(lnode.getoperation(), rnode.getoperation())
+                if isomorph and lnode.is_before(rnode):
+                    if self.packset.can_be_packed(lnode, rnode, pack):
+                        self.packset.add_pair(lnode, rnode)
 
     def combine_packset(self):
         if len(self.packset.packs) == 0:
@@ -729,41 +721,54 @@
         self._newoperations.append(op)
 
 class CostModel(object):
-    def estimate_savings(self, lnode, rnode, origin_pack, expand_forward):
-        """ Estimate the number of savings to add this pair.
-        Zero is the minimum value returned. This should take
-        into account the benefit of executing this instruction
-        as SIMD instruction.
-        """
+    def unpack_cost(self, index, op):
+        raise NotImplementedError
 
-        lpacknode = origin_pack.left
-        if self.prohibit_packing(lpacknode.getoperation(), lnode.getoperation()):
-            return -1
-        rpacknode = origin_pack.right
-        if self.prohibit_packing(rpacknode.getoperation(), rnode.getoperation()):
-            return -1
+    def savings_for_pack(self, opnum, times):
+        raise NotImplementedError
 
-        return 0
+    def savings_for_unpacking(self, node, index):
+        savings = 0
+        result = node.getoperation().result
+        print node.op, "[", index, "]===>"
+        for use in node.provides():
+            if use.to.pack is None and use.because_of(result):
+                savings -= self.unpack_cost(index, node.getoperation())
+                print "   - ", savings, use.to.op
+        return savings
 
-    def prohibit_packing(self, packed, inquestion):
-        """ Blocks the packing of some operations """
-        if inquestion.vector == -1:
-            return True
-        if packed.is_array_op():
-            if packed.getarg(1) == inquestion.result:
-                return True
-        return False
+    def calculate_savings(self, packset):
+        savings = 0
+        for pack in packset.packs:
+            savings += self.savings_for_pack(pack.opnum, pack.opcount())
+            print
+            print "pack", savings
+            op0 = pack.operations[0].getoperation()
+            if op0.result:
+                for i,node in enumerate(pack.operations):
+                    savings += self.savings_for_unpacking(node, i)
+                    print " +=> sss", savings
+        return savings
 
-    def must_unpack_result_to_exec(self, op, target_op):
-        # TODO either move to resop or util
-        if op.getoperation().vector != -1:
-            return False
-        return True
+    def profitable(self, packset):
+        return self.calculate_savings(packset) >= 0
 
 class X86_CostModel(CostModel):
 
-    def savings(self, op, times):
-        return 0
+    COST_BENEFIT = {
+    }
+
+    def savings_for_pack(self, opnum, times):
+        cost, benefit_factor = X86_CostModel.COST_BENEFIT.get(opnum, (1,1))
+        return benefit_factor * times - cost
+
+    def unpack_cost(self, index, op):
+        if op.getdescr():
+            if op.getdescr().is_array_of_floats():
+                if index == 1:
+                    return 2
+        return 1
+
 
 class PackType(PrimitiveTypeMixin):
     UNKNOWN_TYPE = '-'
@@ -1242,9 +1247,7 @@
         self.box_to_vbox[box] = (off, vector)
 
 def isomorphic(l_op, r_op):
-    """ Same instructions have the same operation name.
-    TODO what about parameters?
-    """
+    """ Subject of definition """
     if l_op.getopnum() == r_op.getopnum():
         return True
     return False
@@ -1266,13 +1269,34 @@
         p = Pair(l,r)
         self.packs.append(p)
 
-    def can_be_packed(self, lnode, rnode):
+    def can_be_packed(self, lnode, rnode, origin_pack):
         if isomorphic(lnode.getoperation(), rnode.getoperation()):
             if lnode.independent(rnode):
                 for pack in self.packs:
                     if pack.left == lnode or \
                        pack.right == rnode:
                         return False
+                if origin_pack is None:
+                    return True
+                return self.profitable_pack(lnode, rnode, origin_pack)
+        return False
+
+    def profitable_pack(self, lnode, rnode, origin_pack):
+        lpacknode = origin_pack.left
+        if self.prohibit_packing(lpacknode.getoperation(), lnode.getoperation()):
+            return False
+        rpacknode = origin_pack.right
+        if self.prohibit_packing(rpacknode.getoperation(), rnode.getoperation()):
+            return False
+
+        return True
+
+    def prohibit_packing(self, packed, inquestion):
+        """ Blocks the packing of some operations """
+        if inquestion.vector == -1:
+            return True
+        if packed.is_array_op():
+            if packed.getarg(1) == inquestion.result:
                 return True
         return False
 
@@ -1313,13 +1337,21 @@
     """
     def __init__(self, ops):
         self.operations = ops
-        self.savings = 0
-        for node in self.operations:
+        for i,node in enumerate(self.operations):
             node.pack = self
+            node.pack_position = i
+
+    def opcount(self):
+        return len(self.operations)
+
+    def opnum(self):
+        assert len(self.operations) > 0
+        return self.operations[0].getoperation().getopnum()
 
     def clear(self):
         for node in self.operations:
             node.pack = None
+            node.pack_position = -1
 
     def rightmost_match_leftmost(self, other):
         assert isinstance(other, Pack)

From noreply at buildbot.pypy.org  Mon Jun  1 15:47:52 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Mon,  1 Jun 2015 15:47:52 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: activated all but 3 zjit tests (pow,
 take missing), all others pass
Message-ID: <20150601134752.50C631C0262@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r77741:09b0ee52aaf2
Date: 2015-06-01 15:47 +0200
http://bitbucket.org/pypy/pypy/changeset/09b0ee52aaf2/

Log:	activated all but 3 zjit tests (pow, take missing), all others pass
	added a jit param vec_cost to still be able to test if internal
	errors occur

diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py
--- a/pypy/module/micronumpy/compile.py
+++ b/pypy/module/micronumpy/compile.py
@@ -539,15 +539,15 @@
                 w_rhs = IntObject(int(w_rhs.floatval))
             assert isinstance(w_lhs, W_NDimArray)
             w_res = w_lhs.descr_getitem(interp.space, w_rhs)
-            assert isinstance(w_rhs, IntObject)
-            if isinstance(w_res, boxes.W_Float64Box):
-                print "access", w_lhs, "[", w_rhs.intval, "] => ", float(w_res.value)
-            if isinstance(w_res, boxes.W_Float32Box):
-                print "access", w_lhs, "[", w_rhs.intval, "] => ", float(w_res.value)
-            if isinstance(w_res, boxes.W_Int64Box):
-                print "access", w_lhs, "[", w_rhs.intval, "] => ", int(w_res.value)
-            if isinstance(w_res, boxes.W_Int32Box):
-                print "access", w_lhs, "[", w_rhs.intval, "] => ", int(w_res.value)
+            if isinstance(w_rhs, IntObject):
+                if isinstance(w_res, boxes.W_Float64Box):
+                    print "access", w_lhs, "[", w_rhs.intval, "] => ", float(w_res.value)
+                if isinstance(w_res, boxes.W_Float32Box):
+                    print "access", w_lhs, "[", w_rhs.intval, "] => ", float(w_res.value)
+                if isinstance(w_res, boxes.W_Int64Box):
+                    print "access", w_lhs, "[", w_rhs.intval, "] => ", int(w_res.value)
+                if isinstance(w_res, boxes.W_Int32Box):
+                    print "access", w_lhs, "[", w_rhs.intval, "] => ", int(w_res.value)
         else:
             raise NotImplementedError
         if (not isinstance(w_res, W_NDimArray) and
diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -213,23 +213,6 @@
         assert int(result) == 7+1+8+1+11+2+12+2
         self.check_vectorized(2, 2)
 
-    def define_int_mul_array():
-        return """
-        a = astype(|30|, int)
-        b = astype(|30|, int)
-        c = a * b
-        x1 = c -> 7
-        x2 = c -> 8
-        x3 = c -> 11
-        x4 = c -> 12
-        x1 + x2 + x3 + x4
-        """
-    def test_int_mul_array(self):
-        py.test.skip("how to multiply quad word integers?")
-        result = self.run("int_mul_array")
-        assert int(result) == 7*7+8*8+11*11+12*12
-        self.check_vectorized(2, 2)
-
     def define_float_mul_array():
         return """
         a = astype(|30|, float)
@@ -278,6 +261,390 @@
         assert int(result) == 7*7+8*8+11*11+12*12
         self.check_vectorized(2, 2)
 
+    def define_sum():
+        return """
+        a = |30|
+        sum(a)
+        """
+
+    def test_sum(self):
+        result = self.run("sum")
+        assert result == sum(range(30))
+        # TODO impl reduce
+        self.check_vectorized(1, 0)
+
+    def define_cumsum():
+        return """
+        a = |30|
+        b = cumsum(a)
+        b -> 5
+        """
+
+    def test_cumsum(self):
+        result = self.run("cumsum")
+        assert result == 15
+        self.check_vectorized(1, 0)
+
+    def define_axissum():
+        return """
+        a = [[1, 2], [3, 4], [5, 6], [7, 8], [9, 10]]
+        b = sum(a,0)
+        b -> 1
+        """
+
+    def test_axissum(self):
+        result = self.run("axissum")
+        assert result == 30
+        # XXX note - the bridge here is fairly crucial and yet it's pretty
+        #            bogus. We need to improve the situation somehow.
+        self.check_vectorized(1, 0)
+
+    def define_reduce():
+        return """
+        a = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
+        sum(a)
+        """
+
+    def test_reduce_compile_only_once(self):
+        self.compile_graph()
+        reset_jit()
+        i = self.code_mapping['reduce']
+        # run it twice
+        retval = self.interp.eval_graph(self.graph, [i])
+        retval = self.interp.eval_graph(self.graph, [i])
+        # check that we got only one loop
+        assert len(get_stats().loops) == 1
+        # TODO imple reduce opt
+        self.check_vectorized(2, 0)
+
+    def test_reduce_axis_compile_only_once(self):
+        self.compile_graph()
+        reset_jit()
+        i = self.code_mapping['axissum']
+        # run it twice
+        retval = self.interp.eval_graph(self.graph, [i])
+        retval = self.interp.eval_graph(self.graph, [i])
+        # check that we got only one loop
+        assert len(get_stats().loops) == 1
+        # TODO imple reduce opt
+        self.check_vectorized(3, 0)
+
+    def define_prod():
+        return """
+        a = |30|
+        prod(a)
+        """
+
+    def test_prod(self):
+        result = self.run("prod")
+        expected = 1
+        for i in range(30):
+            expected *= i * 2
+        assert result == expected
+        self.check_trace_count(1)
+
+    def define_max():
+        return """
+        a = |30|
+        a[13] = 128.0
+        max(a)
+        """
+
+    def test_max(self):
+        result = self.run("max")
+        assert result == 128
+        self.check_vectorized(1, 0) # TODO reduce
+
+    def define_min():
+        return """
+        a = |30|
+        a[13] = -128
+        min(a)
+        """
+
+    def test_min(self):
+        result = self.run("min")
+        assert result == -128
+        self.check_vectorized(1, 0) # TODO reduce
+
+    def define_any():
+        return """
+        a = [0,0,0,0,0,0,0,1,0,0,0]
+        any(a)
+        """
+
+    def test_any(self):
+        result = self.run("any")
+        assert result == 1
+        self.check_vectorized(1, 1)
+
+    def define_all():
+        return """
+        a = [1,1,1,1,1,1,1,1]
+        all(a)
+        """
+
+    def test_all(self):
+        result = self.run("all")
+        assert result == 1
+        self.check_vectorized(1, 1)
+
+    def define_logical_xor_reduce():
+        return """
+        a = [1,1,1,1,1,1,1,1]
+        logical_xor_reduce(a)
+        """
+
+    def test_logical_xor_reduce(self):
+        result = self.run("logical_xor_reduce")
+        assert result == 0
+        self.check_vectorized(0, 0) # TODO reduce
+
+    def define_already_forced():
+        return """
+        a = |30|
+        b = a + 4.5
+        b -> 5 # forces
+        c = b * 8
+        c -> 5
+        """
+
+    def test_already_forced(self):
+        result = self.run("already_forced")
+        assert result == (5 + 4.5) * 8
+        self.check_vectorized(2, 2)
+
+    def define_ufunc():
+        return """
+        a = |30|
+        b = unegative(a)
+        b -> 3
+        """
+
+    def test_ufunc(self):
+        result = self.run("ufunc")
+        assert result == -3
+        self.check_vectorized(1, 1)
+
+    def define_specialization():
+        return """
+        a = |30|
+        b = a + a
+        c = unegative(b)
+        c -> 3
+        d = a * a
+        unegative(d)
+        d -> 3
+        d = a * a
+        unegative(d)
+        d -> 3
+        d = a * a
+        unegative(d)
+        d -> 3
+        d = a * a
+        unegative(d)
+        d -> 3
+        """
+
+    def test_specialization(self):
+        result = self.run("specialization")
+        assert result == (3*3)
+        self.check_vectorized(3, 3)
+
+    def define_multidim():
+        return """
+        a = [[1, 2], [3, 4], [5, 6], [7, 8], [9, 10]]
+        b = a + a
+        b -> 1 -> 1
+        """
+
+    def test_multidim(self):
+        result = self.run('multidim')
+        assert result == 8
+        self.check_vectorized(1, 1)
+
+    def define_broadcast():
+        return """
+        a = [[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12]]
+        b = [1, 2, 3, 4]
+        c = a + b
+        c -> 1 -> 2
+        """
+
+    def test_broadcast(self):
+        result = self.run("broadcast")
+        assert result == 10
+        self.check_vectorized(1, 0) # TODO check on broadcast
+
+    def define_setslice():
+        return """
+        a = |30|
+        b = |10|
+        b[1] = 5.5
+        a[0:30:3] = b
+        a -> 3
+        """
+
+    def test_setslice(self):
+        result = self.run("setslice")
+        assert result == 5.5
+        self.check_vectorized(1, 0) # TODO?
+
+    def define_virtual_slice():
+        return """
+        a = |30|
+        c = a + a
+        d = c -> 1:20
+        d -> 1
+        """
+
+    def test_virtual_slice(self):
+        result = self.run("virtual_slice")
+        assert result == 4
+        self.check_vectorized(1, 1)
+
+    def define_flat_iter():
+        return '''
+        a = |30|
+        b = flat(a)
+        c = b + a
+        c -> 3
+        '''
+
+    def test_flat_iter(self):
+        result = self.run("flat_iter")
+        assert result == 6
+        self.check_vectorized(1, 1)
+
+    def define_flat_getitem():
+        return '''
+        a = |30|
+        b = flat(a)
+        b -> 4: -> 6
+        '''
+
+    def test_flat_getitem(self):
+        result = self.run("flat_getitem")
+        assert result == 10.0
+        self.check_trace_count(1)
+        self.check_vectorized(0,0)
+
+    def define_flat_setitem():
+        return '''
+        a = |30|
+        b = flat(a)
+        b[4:] = a->:26
+        a -> 5
+        '''
+
+    def test_flat_setitem(self):
+        result = self.run("flat_setitem")
+        assert result == 1.0
+        self.check_trace_count(1)
+        self.check_vectorized(1,0) # TODO this can be improved
+
+    def define_dot():
+        return """
+        a = [[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12]]
+        b = [[0, 1, 2], [3, 4, 5], [6, 7, 8], [9, 10, 11]]
+        c = dot(a, b)
+        c -> 1 -> 2
+        """
+
+    def test_dot(self):
+        result = self.run("dot")
+        assert result == 184
+        self.check_trace_count(3)
+        self.check_vectorized(3,0)
+
+    def define_argsort():
+        return """
+        a = |30|
+        argsort(a)
+        a->6
+        """
+
+    def test_argsort(self):
+        result = self.run("argsort")
+        assert result == 6
+        self.check_trace_count(1)
+        self.check_vectorized(1,1) # vec. setslice
+
+    def define_where():
+        return """
+        a = [1, 0, 1, 0]
+        x = [1, 2, 3, 4]
+        y = [-10, -20, -30, -40]
+        r = where(a, x, y)
+        r -> 3
+        """
+
+    def test_where(self):
+        result = self.run("where")
+        assert result == -40
+        self.check_trace_count(1)
+        self.check_vectorized(1, 0) # TODO might be possible to vectorize
+
+    def define_searchsorted():
+        return """
+        a = [1, 4, 5, 6, 9]
+        b = |30| -> ::-1
+        c = searchsorted(a, b)
+        c -> -1
+        """
+
+    def test_searchsorted(self):
+        result = self.run("searchsorted")
+        assert result == 0
+        self.check_trace_count(6)
+        # TODO?
+
+    def define_int_mul_array():
+        return """
+        a = astype(|30|, int)
+        b = astype(|30|, int)
+        c = a * b
+        x1 = c -> 7
+        x2 = c -> 8
+        x3 = c -> 11
+        x4 = c -> 12
+        x1 + x2 + x3 + x4
+        """
+    def test_int_mul_array(self):
+        result = self.run("int_mul_array")
+        assert int(result) == 7*7+8*8+11*11+12*12
+        self.check_vectorized(2, 2)
+
+    def define_slice():
+        return """
+        a = |30|
+        b = a -> ::3
+        c = b + b
+        c -> 3
+        """
+
+    def test_slice(self):
+        result = self.run("slice")
+        assert result == 18
+        self.check_trace_count(1)
+        self.check_vectorized(1,1)
+
+    def define_multidim_slice():
+        return """
+        a = [[1, 2, 3, 4], [3, 4, 5, 6], [5, 6, 7, 8], [7, 8, 9, 10], [9, 10, 11, 12], [11, 12, 13, 14], [13, 14, 15, 16], [16, 17, 18, 19]]
+        b = a -> ::2
+        c = b + b
+        c -> 1 -> 1
+        """
+
+    def test_multidim_slice(self):
+        result = self.run('multidim_slice')
+        assert result == 12
+        self.check_trace_count(2)
+        self.check_vectorized(1,0) # TODO?
+
+    # NOT WORKING
+
     def define_pow():
         return """
         a = |30| ** 2
@@ -304,245 +671,6 @@
         assert result == 15 ** 2
         self.check_trace_count(4)  # extra one for the astype
 
-    def define_sum():
-        return """
-        a = |30|
-        sum(a)
-        """
-
-    def test_sum(self):
-        result = self.run("sum")
-        assert result == sum(range(30))
-        self.check_trace_count(1)
-
-    def define_cumsum():
-        return """
-        a = |30|
-        b = cumsum(a)
-        b -> 5
-        """
-
-    def test_cumsum(self):
-        result = self.run("cumsum")
-        assert result == 15
-        self.check_trace_count(1)
-
-    def define_axissum():
-        return """
-        a = [[1, 2], [3, 4], [5, 6], [7, 8], [9, 10]]
-        b = sum(a,0)
-        b -> 1
-        """
-
-    def test_axissum(self):
-        result = self.run("axissum")
-        assert result == 30
-        # XXX note - the bridge here is fairly crucial and yet it's pretty
-        #            bogus. We need to improve the situation somehow.
-
-    def define_reduce():
-        return """
-        a = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
-        sum(a)
-        """
-
-    def test_reduce_compile_only_once(self):
-        self.compile_graph()
-        reset_jit()
-        i = self.code_mapping['reduce']
-        # run it twice
-        retval = self.interp.eval_graph(self.graph, [i])
-        retval = self.interp.eval_graph(self.graph, [i])
-        # check that we got only one loop
-        assert len(get_stats().loops) == 1
-
-    def test_reduce_axis_compile_only_once(self):
-        self.compile_graph()
-        reset_jit()
-        i = self.code_mapping['axissum']
-        # run it twice
-        retval = self.interp.eval_graph(self.graph, [i])
-        retval = self.interp.eval_graph(self.graph, [i])
-        # check that we got only one loop
-        assert len(get_stats().loops) == 1
-
-    def define_prod():
-        return """
-        a = |30|
-        prod(a)
-        """
-
-    def test_prod(self):
-        result = self.run("prod")
-        expected = 1
-        for i in range(30):
-            expected *= i * 2
-        assert result == expected
-        self.check_trace_count(1)
-
-    def define_max():
-        return """
-        a = |30|
-        a[13] = 128.0
-        max(a)
-        """
-
-    def test_max(self):
-        result = self.run("max")
-        assert result == 128
-        # TODO self.check_trace_count(3)
-
-    def define_min():
-        return """
-        a = |30|
-        a[13] = -128
-        min(a)
-        """
-
-    def test_min(self):
-        result = self.run("min")
-        assert result == -128
-        #self.check_trace_count(1)
-
-    def define_any():
-        return """
-        a = [0,0,0,0,0,0,0,1,0,0,0]
-        any(a)
-        """
-
-    def test_any(self):
-        result = self.run("any")
-        assert result == 1
-        self.check_trace_count(1)
-
-    def define_all():
-        return """
-        a = [1,1,1,1,1,1,1,1]
-        all(a)
-        """
-
-    def test_all(self):
-        result = self.run("all")
-        assert result == 1
-        self.check_trace_count(1)
-
-    def define_logical_xor_reduce():
-        return """
-        a = [1,1,1,1,1,1,1,1]
-        logical_xor_reduce(a)
-        """
-
-    def test_logical_xor_reduce(self):
-        result = self.run("logical_xor_reduce")
-        assert result == 0
-        self.check_trace_count(2)
-        # XXX fix this
-        #self.check_simple_loop({
-        #    'cast_float_to_int': 1,
-        #    'getfield_gc': 2,
-        #    'getfield_gc_pure': 11,
-        #    'guard_class': 1,
-        #    'guard_false': 1,
-        #    'guard_not_invalidated': 1,
-        #    'guard_true': 5,
-        #    'int_add': 2,
-        #    'int_and': 1,
-        #    'int_ge': 1,
-        #    'int_is_true': 2,
-        #    'jump': 1,
-        #    'new_with_vtable': 1,
-        #    'raw_load': 1,
-        #    'setfield_gc': 4,
-        #})
-
-    def define_already_forced():
-        return """
-        a = |30|
-        b = a + 4.5
-        b -> 5 # forces
-        c = b * 8
-        c -> 5
-        """
-
-    def test_already_forced(self):
-        #py.test.skip('TODO')
-        result = self.run("already_forced")
-        assert result == (5 + 4.5) * 8
-        # This is the sum of the ops for both loops, however if you remove the
-        # optimization then you end up with 2 float_adds, so we can still be
-        # sure it was optimized correctly.
-        #py.test.skip("too fragile")
-        #self.check_resops({'raw_store': 4, 'getfield_gc': 22,
-        #                   'getarrayitem_gc': 4, 'getarrayitem_gc_pure': 2,
-        #                   'getfield_gc_pure': 8,
-        #                   'guard_class': 8, 'int_add': 8, 'float_mul': 2,
-        #                   'jump': 2, 'int_ge': 4,
-        #                   'raw_load': 4, 'float_add': 2,
-        #                   'guard_false': 4, 'arraylen_gc': 2, 'same_as': 2})
-
-    def define_ufunc():
-        return """
-        a = |30|
-        b = unegative(a)
-        b -> 3
-        """
-
-    def test_ufunc(self):
-        result = self.run("ufunc")
-        assert result == -3
-
-    def define_specialization():
-        return """
-        a = |30|
-        b = a + a
-        c = unegative(b)
-        c -> 3
-        d = a * a
-        unegative(d)
-        d -> 3
-        d = a * a
-        unegative(d)
-        d -> 3
-        d = a * a
-        unegative(d)
-        d -> 3
-        d = a * a
-        unegative(d)
-        d -> 3
-        """
-
-    def test_specialization(self):
-        result = self.run("specialization")
-        # TODO
-        assert result == (3*3)
-        #py.test.skip("don't run for now")
-        # This is 3, not 2 because there is a bridge for the exit.
-        #self.check_trace_count(3)
-
-    def define_slice():
-        return """
-        a = |30|
-        b = a -> ::3
-        c = b + b
-        c -> 3
-        """
-
-    def test_slice(self):
-        py.test.skip("slice not impl in compile.py")
-        result = self.run("slice")
-        assert result == 18
-        self.check_trace_count(1)
-        #self.check_simple_loop({
-        #    'arraylen_gc': 2,
-        #    'float_add': 1,
-        #    'guard_false': 1,
-        #    'guard_not_invalidated': 1,
-        #    'int_add': 4,
-        #    'int_ge': 1,
-        #    'jump': 1,
-        #    'raw_load': 2,
-        #    'raw_store': 1,
-        #})
 
     def define_take():
         return """
@@ -552,366 +680,6 @@
         """
 
     def test_take(self):
-        py.test.skip("not impl")
+        py.test.skip("key error get item?")
         result = self.run("take")
         assert result == 3
-
-    def define_multidim():
-        return """
-        a = [[1, 2], [3, 4], [5, 6], [7, 8], [9, 10]]
-        b = a + a
-        b -> 1 -> 1
-        """
-
-    def test_multidim(self):
-        result = self.run('multidim')
-        assert result == 8
-        # int_add might be 1 here if we try slightly harder with
-        # reusing indexes or some optimization
-        self.check_trace_count(1)
-        #self.check_simple_loop({
-        #    'float_add': 1,
-        #    'guard_false': 1,
-        #    'guard_not_invalidated': 1,
-        #    'int_add': 4,
-        #    'int_ge': 1,
-        #    'jump': 1,
-        #    'raw_load': 2,
-        #    'raw_store': 1,
-        #})
-
-    def define_multidim_slice():
-        return """
-        a = [[1, 2, 3, 4], [3, 4, 5, 6], [5, 6, 7, 8], [7, 8, 9, 10], [9, 10, 11, 12], [11, 12, 13, 14], [13, 14, 15, 16], [16, 17, 18, 19]]
-        b = a -> ::2
-        c = b + b
-        c -> 1 -> 1
-        """
-
-    def test_multidim_slice(self):
-        py.test.skip("seems to be a problem in compile.py")
-        result = self.run('multidim_slice')
-        assert result == 12
-        # XXX the bridge here is scary. Hopefully jit-targets will fix that,
-        #     otherwise it looks kind of good
-        self.check_trace_count(2)
-        #self.check_simple_loop({
-        #    'float_add': 1,
-        #    'getarrayitem_gc': 2,
-        #    'guard_false': 1,
-        #    'guard_not_invalidated': 1,
-        #    'guard_true': 2,
-        #    'int_add': 6,
-        #    'int_ge': 1,
-        #    'int_lt': 2,
-        #    'jump': 1,
-        #    'raw_load': 2,
-        #    'raw_store': 1,
-        #    'setarrayitem_gc': 2,
-        #})
-        #self.check_resops({
-        #    'float_add': 3,
-        #    'getarrayitem_gc': 7,
-        #    'getarrayitem_gc_pure': 14,
-        #    'getfield_gc': 6,
-        #    'getfield_gc_pure': 63,
-        #    'guard_class': 5,
-        #    'guard_false': 19,
-        #    'guard_nonnull': 6,
-        #    'guard_nonnull_class': 1,
-        #    'guard_not_invalidated': 3,
-        #    'guard_true': 16,
-        #    'guard_value': 3,
-        #    'int_add': 24,
-        #    'int_ge': 4,
-        #    'int_is_true': 5,
-        #    'int_is_zero': 4,
-        #    'int_le': 5,
-        #    'int_lt': 7,
-        #    'int_sub': 2,
-        #    'jump': 2,
-        #    'raw_load': 5,
-        #    'raw_store': 3,
-        #    'setarrayitem_gc': 8,
-        #})
-
-    def define_broadcast():
-        return """
-        a = [[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12]]
-        b = [1, 2, 3, 4]
-        c = a + b
-        c -> 1 -> 2
-        """
-
-    def test_broadcast(self):
-        result = self.run("broadcast")
-        assert result == 10
-        #self.check_trace_count(2)
-        #self.check_simple_loop({
-        #    'float_add': 1,
-        #    'getarrayitem_gc': 1,
-        #    'guard_false': 1,
-        #    'guard_not_invalidated': 1,
-        #    'guard_true': 1,
-        #    'int_add': 5,
-        #    'int_ge': 1,
-        #    'int_lt': 1,
-        #    'jump': 1,
-        #    'raw_load': 2,
-        #    'raw_store': 1,
-        #    'setarrayitem_gc': 1,
-        #})
-        #self.check_resops({
-        #    'float_add': 2,
-        #    'getarrayitem_gc': 2,
-        #    'getarrayitem_gc_pure': 2,
-        #    'getfield_gc': 6,
-        #    'getfield_gc_pure': 30,
-        #    'guard_class': 3,
-        #    'guard_false': 7,
-        #    'guard_nonnull': 2,
-        #    'guard_not_invalidated': 2,
-        #    'guard_true': 8,
-        #    'int_add': 11,
-        #    'int_ge': 2,
-        #    'int_is_true': 3,
-        #    'int_is_zero': 1,
-        #    'int_le': 1,
-        #    'int_lt': 2,
-        #    'jump': 1,
-        #    'raw_load': 4,
-        #    'raw_store': 2,
-        #    'setarrayitem_gc': 2,
-        #})
-
-    def define_setslice():
-        return """
-        a = |30|
-        b = |10|
-        b[1] = 5.5
-        a[0:30:3] = b
-        a -> 3
-        """
-
-    def test_setslice(self):
-        result = self.run("setslice")
-        assert result == 5.5
-        self.check_trace_count(1)
-        #self.check_simple_loop({
-        #    'arraylen_gc': 1,
-        #    'guard_false': 1,
-        #    'guard_not_invalidated': 1,
-        #    'int_add': 3,
-        #    'int_ge': 1,
-        #    'jump': 1,
-        #    'raw_load': 1,
-        #    'raw_store': 1,
-        #})
-
-    def define_virtual_slice():
-        return """
-        a = |30|
-        c = a + a
-        d = c -> 1:20
-        d -> 1
-        """
-
-    def test_virtual_slice(self):
-        py.test.skip('TODO')
-        result = self.run("virtual_slice")
-        assert result == 4
-        py.test.skip("don't run for now")
-        self.check_trace_count(1)
-        self.check_simple_loop({'raw_load': 2, 'float_add': 1,
-                                'raw_store': 1, 'int_add': 1,
-                                'int_ge': 1, 'guard_false': 1, 'jump': 1,
-                                'arraylen_gc': 1})
-
-    def define_flat_iter():
-        return '''
-        a = |30|
-        b = flat(a)
-        c = b + a
-        c -> 3
-        '''
-
-    def test_flat_iter(self):
-        py.test.skip('TODO')
-        result = self.run("flat_iter")
-        assert result == 6
-        self.check_trace_count(1)
-        self.check_simple_loop({
-            'float_add': 1,
-            'guard_false': 1,
-            'guard_not_invalidated': 1,
-            'int_add': 4,
-            'int_ge': 1,
-            'jump': 1,
-            'raw_load': 2,
-            'raw_store': 1,
-        })
-
-    def define_flat_getitem():
-        return '''
-        a = |30|
-        b = flat(a)
-        b -> 4: -> 6
-        '''
-
-    def test_flat_getitem(self):
-        py.test.skip('TODO')
-        result = self.run("flat_getitem")
-        assert result == 10.0
-        self.check_trace_count(1)
-        self.check_simple_loop({
-            'guard_false': 1,
-            'int_add': 4,
-            'int_ge': 1,
-            'int_mul': 1,
-            'jump': 1,
-            'raw_load': 1,
-            'raw_store': 1,
-        })
-
-    def define_flat_setitem():
-        return '''
-        a = |30|
-        b = flat(a)
-        b[4:] = a->:26
-        a -> 5
-        '''
-
-    def test_flat_setitem(self):
-        py.test.skip('TODO')
-        result = self.run("flat_setitem")
-        assert result == 1.0
-        self.check_trace_count(1)
-        self.check_simple_loop({
-            'guard_false': 1,
-            'guard_not_invalidated': 1,
-            'guard_true': 1,
-            'int_add': 4,
-            'int_ge': 1,
-            'int_gt': 1,
-            'int_mul': 1,
-            'int_sub': 1,
-            'jump': 1,
-            'raw_load': 1,
-            'raw_store': 1,
-        })
-
-    def define_dot():
-        return """
-        a = [[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12]]
-        b = [[0, 1, 2], [3, 4, 5], [6, 7, 8], [9, 10, 11]]
-        c = dot(a, b)
-        c -> 1 -> 2
-        """
-
-    def test_dot(self):
-        py.test.skip('TODO')
-        result = self.run("dot")
-        assert result == 184
-        self.check_trace_count(3)
-        self.check_simple_loop({
-            'float_add': 1,
-            'float_mul': 1,
-            'guard_not_invalidated': 1,
-            'guard_true': 1,
-            'int_add': 3,
-            'int_lt': 1,
-            'jump': 1,
-            'raw_load': 2,
-        })
-        self.check_resops({
-            'float_add': 2,
-            'float_mul': 2,
-            'getarrayitem_gc': 4,
-            'getarrayitem_gc_pure': 9,
-            'getfield_gc': 7,
-            'getfield_gc_pure': 42,
-            'guard_class': 4,
-            'guard_false': 15,
-            'guard_not_invalidated': 2,
-            'guard_true': 14,
-            'int_add': 17,
-            'int_ge': 4,
-            'int_is_true': 3,
-            'int_is_zero': 2,
-            'int_le': 5,
-            'int_lt': 8,
-            'int_sub': 3,
-            'jump': 3,
-            'new_with_vtable': 7,
-            'raw_load': 6,
-            'raw_store': 1,
-            'same_as': 2,
-            'setarrayitem_gc': 7,
-            'setfield_gc': 22,
-        })
-
-    def define_argsort():
-        return """
-        a = |30|
-        argsort(a)
-        a->6
-        """
-
-    def test_argsort(self):
-        py.test.skip('TODO')
-        result = self.run("argsort")
-        assert result == 6
-
-    def define_where():
-        return """
-        a = [1, 0, 1, 0]
-        x = [1, 2, 3, 4]
-        y = [-10, -20, -30, -40]
-        r = where(a, x, y)
-        r -> 3
-        """
-
-    def test_where(self):
-        py.test.skip('TODO')
-        result = self.run("where")
-        assert result == -40
-        self.check_trace_count(1)
-        self.check_simple_loop({
-            'float_ne': 1,
-            'guard_false': 1,
-            'guard_not_invalidated': 1,
-            'guard_true': 1,
-            'int_add': 5,
-            'int_ge': 1,
-            'jump': 1,
-            'raw_load': 2,
-            'raw_store': 1,
-        })
-
-    def define_searchsorted():
-        return """
-        a = [1, 4, 5, 6, 9]
-        b = |30| -> ::-1
-        c = searchsorted(a, b)
-        c -> -1
-        """
-
-    def test_searchsorted(self):
-        py.test.skip('TODO')
-        result = self.run("searchsorted")
-        assert result == 0
-        self.check_trace_count(6)
-        self.check_simple_loop({
-            'float_lt': 1,
-            'guard_false': 2,
-            'guard_not_invalidated': 1,
-            'guard_true': 2,
-            'int_add': 3,
-            'int_ge': 1,
-            'int_lt': 2,
-            'int_mul': 1,
-            'int_rshift': 1,
-            'int_sub': 1,
-            'jump': 1,
-            'raw_load': 1,
-        })
diff --git a/rpython/jit/metainterp/optimizeopt/__init__.py b/rpython/jit/metainterp/optimizeopt/__init__.py
--- a/rpython/jit/metainterp/optimizeopt/__init__.py
+++ b/rpython/jit/metainterp/optimizeopt/__init__.py
@@ -71,8 +71,9 @@
             if not export_state and \
                 ((warmstate.vectorize and jitdriver_sd.vectorize) \
                  or warmstate.vectorize_user):
-                optimize_vector(metainterp_sd, jitdriver_sd, loop, optimizations,
-                                inline_short_preamble, start_state)
+                optimize_vector(metainterp_sd, jitdriver_sd, loop,
+                                optimizations, inline_short_preamble,
+                                start_state, warmstate.vec_cost)
             else:
                 return optimize_unroll(metainterp_sd, jitdriver_sd, loop,
                                        optimizations, inline_short_preamble,
diff --git a/rpython/jit/metainterp/optimizeopt/dependency.py b/rpython/jit/metainterp/optimizeopt/dependency.py
--- a/rpython/jit/metainterp/optimizeopt/dependency.py
+++ b/rpython/jit/metainterp/optimizeopt/dependency.py
@@ -126,7 +126,6 @@
 
     def edge_to(self, to, arg=None, failarg=False, label=None):
         if self is to:
-            #debug_print "debug: tried to put edge from: ", self.op, "to:", to.op
             return
         dep = self.depends_on(to)
         if not dep:
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_costmodel.py b/rpython/jit/metainterp/optimizeopt/test/test_costmodel.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_costmodel.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_costmodel.py
@@ -21,7 +21,6 @@
         iv = self.index_var
         ov = other.index_var
         val = (int(str(ov.var)[1:]) - int(str(iv.var)[1:]))
-        print iv, ov, "adja?", val == 1
         # i0 and i1 are adjacent
         # i1 and i2 ...
         # but not i0, i2
@@ -37,13 +36,12 @@
         graph = opt.dependency_graph
         for k,m in graph.memory_refs.items():
             graph.memory_refs[k] = FakeMemoryRef(m.index_var)
-            print "memory ref", k, m
         opt.find_adjacent_memory_refs()
         opt.extend_packset()
         opt.combine_packset()
         for pack in opt.packset.packs:
-            print "apck:"
-            print '\n'.join([str(op.getoperation()) for op in pack.operations])
+            print "pack: \n   ",
+            print '\n    '.join([str(op.getoperation()) for op in pack.operations])
             print
         return opt.costmodel.calculate_savings(opt.packset)
 
@@ -111,21 +109,17 @@
 
     def test_load_arith_store(self):
         loop1 = self.parse("""
-        i10 = raw_load(p0, i0, descr=int)
-        i11 = raw_load(p0, i1, descr=int)
-        i12 = raw_load(p0, i2, descr=int)
-        i13 = raw_load(p0, i3, descr=int)
-        i15 = int_add(i10, 1)
-        i16 = int_add(i11, 1)
-        i17 = int_add(i12, 1)
-        i18 = int_add(i13, 1)
-        raw_store(p1, i4, i15, descr=int)
-        raw_store(p1, i5, i16, descr=int)
-        raw_store(p1, i6, i17, descr=int)
-        raw_store(p1, i7, i18, descr=int)
+        f10 = raw_load(p0, i0, descr=double)
+        f11 = raw_load(p0, i1, descr=double)
+        i20 = cast_float_to_int(f10)
+        i21 = cast_float_to_int(f11)
+        i30 = int_signext(i20, 4)
+        i31 = int_signext(i21, 4)
+        raw_store(p0, i3, i30, descr=int)
+        raw_store(p0, i4, i31, descr=int)
         """)
         savings = self.savings(loop1)
-        assert savings == 6
+        assert savings == 1
 
 class Test(CostModelBaseTest, LLtypeMixin):
     pass
diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/vectorize.py
@@ -40,7 +40,7 @@
                 print ""
 
 def optimize_vector(metainterp_sd, jitdriver_sd, loop, optimizations,
-                    inline_short_preamble, start_state):
+                    inline_short_preamble, start_state, cost_threshold):
     optimize_unroll(metainterp_sd, jitdriver_sd, loop, optimizations,
                     inline_short_preamble, start_state, False)
     orig_ops = loop.operations
@@ -48,13 +48,16 @@
         debug_start("vec-opt-loop")
         metainterp_sd.logger_noopt.log_loop(loop.inputargs, loop.operations, -2, None, None, "pre vectorize")
         metainterp_sd.profiler.count(Counters.OPT_VECTORIZE_TRY)
-        opt = VectorizingOptimizer(metainterp_sd, jitdriver_sd, loop, optimizations)
+        opt = VectorizingOptimizer(metainterp_sd, jitdriver_sd, loop, cost_threshold)
         opt.propagate_all_forward()
         metainterp_sd.profiler.count(Counters.OPT_VECTORIZED)
         metainterp_sd.logger_noopt.log_loop(loop.inputargs, loop.operations, -2, None, None, "post vectorize")
     except NotAVectorizeableLoop:
         # vectorization is not possible
         loop.operations = orig_ops
+    except NotAProfitableLoop:
+        # cost model says to skip this loop
+        loop.operations = orig_ops
     except Exception as e:
         loop.operations = orig_ops
         debug_print("failed to vectorize loop. THIS IS A FATAL ERROR!")
@@ -70,8 +73,8 @@
 class VectorizingOptimizer(Optimizer):
     """ Try to unroll the loop and find instructions to group """
 
-    def __init__(self, metainterp_sd, jitdriver_sd, loop, optimizations):
-        Optimizer.__init__(self, metainterp_sd, jitdriver_sd, loop, optimizations)
+    def __init__(self, metainterp_sd, jitdriver_sd, loop, cost_threshold=0):
+        Optimizer.__init__(self, metainterp_sd, jitdriver_sd, loop, [])
         self.dependency_graph = None
         self.packset = None
         self.unroll_count = 0
@@ -79,13 +82,16 @@
         self.early_exit_idx = -1
         self.sched_data = None
         self.tried_to_pack = False
-        self.costmodel = X86_CostModel()
+        self.costmodel = X86_CostModel(cost_threshold)
 
     def propagate_all_forward(self, clear=True):
         self.clear_newoperations()
         label = self.loop.operations[0]
         jump = self.loop.operations[-1]
-        if jump.getopnum() not in (rop.LABEL, rop.JUMP):
+        if jump.getopnum() not in (rop.LABEL, rop.JUMP) or \
+           label.getopnum() != rop.LABEL:
+            raise NotAVectorizeableLoop()
+        if jump.numargs() != label.numargs():
             raise NotAVectorizeableLoop()
 
         self.linear_find_smallest_type(self.loop)
@@ -721,6 +727,9 @@
         self._newoperations.append(op)
 
 class CostModel(object):
+    def __init__(self, threshold):
+        self.threshold = threshold
+
     def unpack_cost(self, index, op):
         raise NotImplementedError
 
@@ -730,28 +739,23 @@
     def savings_for_unpacking(self, node, index):
         savings = 0
         result = node.getoperation().result
-        print node.op, "[", index, "]===>"
         for use in node.provides():
             if use.to.pack is None and use.because_of(result):
                 savings -= self.unpack_cost(index, node.getoperation())
-                print "   - ", savings, use.to.op
         return savings
 
     def calculate_savings(self, packset):
         savings = 0
         for pack in packset.packs:
             savings += self.savings_for_pack(pack.opnum, pack.opcount())
-            print
-            print "pack", savings
             op0 = pack.operations[0].getoperation()
             if op0.result:
                 for i,node in enumerate(pack.operations):
                     savings += self.savings_for_unpacking(node, i)
-                    print " +=> sss", savings
         return savings
 
     def profitable(self, packset):
-        return self.calculate_savings(packset) >= 0
+        return self.calculate_savings(packset) >= self.threshold
 
 class X86_CostModel(CostModel):
 
diff --git a/rpython/jit/metainterp/warmspot.py b/rpython/jit/metainterp/warmspot.py
--- a/rpython/jit/metainterp/warmspot.py
+++ b/rpython/jit/metainterp/warmspot.py
@@ -74,7 +74,7 @@
                     function_threshold=4,
                     enable_opts=ALL_OPTS_NAMES, max_retrace_guards=15, 
                     max_unroll_recursion=7, vectorize=0, vectorize_user=0,
-                    **kwds):
+                    vec_cost=0, **kwds):
     from rpython.config.config import ConfigError
     translator = interp.typer.annotator.translator
     try:
@@ -99,6 +99,7 @@
         jd.warmstate.set_param_max_unroll_recursion(max_unroll_recursion)
         jd.warmstate.set_param_vectorize(vectorize)
         jd.warmstate.set_param_vectorize_user(vectorize_user)
+        jd.warmstate.set_param_vec_cost(vec_cost)
     warmrunnerdesc.finish()
     if graph_and_interp_only:
         return interp, graph
diff --git a/rpython/jit/metainterp/warmstate.py b/rpython/jit/metainterp/warmstate.py
--- a/rpython/jit/metainterp/warmstate.py
+++ b/rpython/jit/metainterp/warmstate.py
@@ -303,6 +303,9 @@
     def set_param_vectorize_user(self, value):
         self.vectorize_user = bool(value)
 
+    def set_param_vec_cost(self, value):
+        self.vec_cost = bool(value)
+
     def disable_noninlinable_function(self, greenkey):
         cell = self.JitCell.ensure_jit_cell_at_key(greenkey)
         cell.flags |= JC_DONT_TRACE_HERE
diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py
--- a/rpython/rlib/jit.py
+++ b/rpython/rlib/jit.py
@@ -554,7 +554,8 @@
     'max_unroll_recursion': 'how many levels deep to unroll a recursive function',
     'vectorize': 'turn on the vectorization optimization (vecopt). requires sse4.1',
     'vectorize_user': 'turn on the vecopt for the python user program. requires sse4.1',
-    }
+    'vec_cost': 'threshold which traces to vectorize.',
+}
 
 PARAMETERS = {'threshold': 1039, # just above 1024, prime
               'function_threshold': 1619, # slightly more than one above, also prime
@@ -570,6 +571,7 @@
               'max_unroll_recursion': 7,
               'vectorize': 0,
               'vectorize_user': 0,
+              'vec_cost': 0,
               }
 unroll_parameters = unrolling_iterable(PARAMETERS.items())
 

From noreply at buildbot.pypy.org  Mon Jun  1 15:52:59 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Mon,  1 Jun 2015 15:52:59 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: removed unused dict,
 lead to rpy error (no values included)
Message-ID: <20150601135259.4CCAA1C0262@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r77742:9492770f0741
Date: 2015-06-01 15:52 +0200
http://bitbucket.org/pypy/pypy/changeset/9492770f0741/

Log:	removed unused dict, lead to rpy error (no values included)

diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/vectorize.py
@@ -759,11 +759,8 @@
 
 class X86_CostModel(CostModel):
 
-    COST_BENEFIT = {
-    }
-
     def savings_for_pack(self, opnum, times):
-        cost, benefit_factor = X86_CostModel.COST_BENEFIT.get(opnum, (1,1))
+        cost, benefit_factor = (1,1) # TODO custom values for different ops
         return benefit_factor * times - cost
 
     def unpack_cost(self, index, op):

From noreply at buildbot.pypy.org  Mon Jun  1 16:46:52 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  1 Jun 2015 16:46:52 +0200 (CEST)
Subject: [pypy-commit] stmgc c8-locking: hg merge default
Message-ID: <20150601144652.884981C04BC@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: c8-locking
Changeset: r1783:1f946043b7b4
Date: 2015-05-30 17:58 +0200
http://bitbucket.org/pypy/stmgc/changeset/1f946043b7b4/

Log:	hg merge default

diff too long, truncating to 2000 out of 2193 lines

diff --git a/c7/gdb/gdb_stm.py b/c7/gdb/gdb_stm.py
--- a/c7/gdb/gdb_stm.py
+++ b/c7/gdb/gdb_stm.py
@@ -25,7 +25,13 @@
 def gdb_function(func):
     class Func(gdb.Function):
         __doc__ = func.__doc__
-        invoke = staticmethod(func)
+        def invoke(self, *args, **kwds):
+            try:
+                return func(*args, **kwds)
+            except:
+                import traceback
+                traceback.print_exc()
+                raise
     Func(func.__name__)
 
 # -------------------------------------------------------
diff --git a/c7/stmgc.h b/c7/stmgc.h
--- a/c7/stmgc.h
+++ b/c7/stmgc.h
@@ -340,8 +340,6 @@
    returns: it jumps back to the stm_start_transaction(). */
 void stm_abort_transaction(void) __attribute__((noreturn));
 
-/* Turn the current transaction inevitable.
-   The stm_become_inevitable() itself may still abort. */
 #ifdef STM_NO_AUTOMATIC_SETJMP
 int stm_is_inevitable(void);
 #else
@@ -349,6 +347,10 @@
     return !rewind_jmp_armed(&STM_SEGMENT->running_thread->rjthread);
 }
 #endif
+
+/* Turn the current transaction inevitable.
+   stm_become_inevitable() itself may still abort the transaction instead
+   of returning. */
 static inline void stm_become_inevitable(stm_thread_local_t *tl,
                                          const char* msg) {
     assert(STM_SEGMENT->running_thread == tl);
diff --git a/c8/stm/core.c b/c8/stm/core.c
--- a/c8/stm/core.c
+++ b/c8/stm/core.c
@@ -280,8 +280,14 @@
         struct stm_undo_s *end = undo + cl->written_count;
         for (; undo < end; undo++) {
             if (undo->type == TYPE_POSITION_MARKER) {
-                fprintf(stderr, "    marker %p %lu\n",
-                        undo->marker_object, undo->marker_odd_number);
+                if (undo->type2 == TYPE_MODIFIED_HASHTABLE) {
+                    fprintf(stderr, "    hashtable %p\n",
+                            undo->modif_hashtable);
+                }
+                else {
+                    fprintf(stderr, "    marker %p %lu\n",
+                            undo->marker_object, undo->marker_odd_number);
+                }
                 continue;
             }
             fprintf(stderr, "    obj %p, size %d, ofs %lu: ", undo->object,
@@ -302,7 +308,7 @@
 
 static void reset_modified_from_backup_copies(int segment_num);  /* forward */
 
-static bool _stm_validate()
+static bool _stm_validate(void)
 {
     /* returns true if we reached a valid state, or false if
        we need to abort now */
@@ -383,21 +389,40 @@
                     struct stm_undo_s *undo = cl->written;
                     struct stm_undo_s *end = cl->written + cl->written_count;
                     for (; undo < end; undo++) {
-                        if (undo->type == TYPE_POSITION_MARKER)
+                        object_t *obj;
+
+                        if (undo->type != TYPE_POSITION_MARKER) {
+                            /* common case: 'undo->object' was written to
+                               in this past commit, so we must check that
+                               it was not read by us. */
+                            obj = undo->object;
+                        }
+                        else if (undo->type2 != TYPE_MODIFIED_HASHTABLE)
                             continue;
-                        if (_stm_was_read(undo->object)) {
-                            /* first reset all modified objects from the backup
-                               copies as soon as the first conflict is detected;
-                               then we will proceed below to update our segment from
-                               the old (but unmodified) version to the newer version.
-                            */
-                            reset_modified_from_backup_copies(my_segnum);
-                            timing_write_read_contention(cl->written, undo);
-                            needs_abort = true;
+                        else {
+                            /* the previous stm_undo_s is about a written
+                               'entry' object, which belongs to the hashtable
+                               given now.  Check that we haven't read the
+                               hashtable (via stm_hashtable_list()). */
+                            obj = undo->modif_hashtable;
+                        }
 
-                            dprintf(("_stm_validate() failed for obj %p\n", undo->object));
-                            break;
-                        }
+                        if (LIKELY(!_stm_was_read(obj)))
+                            continue;
+
+                        /* conflict! */
+                        dprintf(("_stm_validate() failed for obj %p\n", obj));
+
+                        /* first reset all modified objects from the backup
+                           copies as soon as the first conflict is detected;
+                           then we will proceed below to update our segment
+                           from the old (but unmodified) version to the newer
+                           version.
+                        */
+                        reset_modified_from_backup_copies(my_segnum);
+                        timing_write_read_contention(cl->written, undo);
+                        needs_abort = true;
+                        break;
                     }
                 }
 
diff --git a/c8/stm/core.h b/c8/stm/core.h
--- a/c8/stm/core.h
+++ b/c8/stm/core.h
@@ -190,9 +190,15 @@
         uintptr_t marker_odd_number; /* the odd number part of the marker */
         object_t *marker_object;     /* the object part of the marker */
     };
+    struct {
+        intptr_t type1;             /* TYPE_POSITION_MARKER (again) */
+        intptr_t type2;             /* TYPE_MODIFIED_HASHTABLE */
+        object_t *modif_hashtable;  /* modified entry is previous stm_undo_s */
+    };
   };
 };
 #define TYPE_POSITION_MARKER    (-1)
+#define TYPE_MODIFIED_HASHTABLE (-2)
 #define SLICE_OFFSET(slice)  ((slice) >> 16)
 #define SLICE_SIZE(slice)    ((int)((slice) & 0xFFFF))
 #define NEW_SLICE(offset, size) (((uint64_t)(offset)) << 16 | (size))
@@ -251,6 +257,14 @@
     return stm_object_pages + segment_num * (NB_PAGES * 4096UL);
 }
 
+static inline long get_num_segment_containing_address(char *addr)
+{
+    uintptr_t delta = addr - stm_object_pages;
+    uintptr_t result = delta / (NB_PAGES * 4096UL);
+    assert(result < NB_SEGMENTS);
+    return result;
+}
+
 static inline
 struct stm_segment_info_s *get_segment(long segment_num) {
     return (struct stm_segment_info_s *)REAL_ADDRESS(
@@ -283,7 +297,18 @@
 static void synchronize_objects_flush(void);
 
 static void _signal_handler(int sig, siginfo_t *siginfo, void *context);
-static bool _stm_validate();
+static bool _stm_validate(void);
+
+static inline bool was_read_remote(char *base, object_t *obj)
+{
+    uint8_t other_transaction_read_version =
+        ((struct stm_segment_info_s *)REAL_ADDRESS(base, STM_PSEGMENT))
+            ->transaction_read_version;
+    uint8_t rm = ((struct stm_read_marker_s *)
+                  (base + (((uintptr_t)obj) >> 4)))->rm;
+    assert(rm <= other_transaction_read_version);
+    return rm == other_transaction_read_version;
+}
 
 static inline void _duck(void) {
     /* put a call to _duck() between two instructions that set 0 into
@@ -304,7 +329,7 @@
     spinlock_release(get_priv_segment(segnum)->privatization_lock);
 }
 
-static inline bool all_privatization_locks_acquired()
+static inline bool all_privatization_locks_acquired(void)
 {
 #ifndef NDEBUG
     long l;
@@ -318,7 +343,7 @@
 #endif
 }
 
-static inline void acquire_all_privatization_locks()
+static inline void acquire_all_privatization_locks(void)
 {
     /* XXX: don't do for the sharing seg0 */
     long l;
@@ -327,7 +352,7 @@
     }
 }
 
-static inline void release_all_privatization_locks()
+static inline void release_all_privatization_locks(void)
 {
     long l;
     for (l = NB_SEGMENTS-1; l >= 0; l--) {
diff --git a/c8/stm/gcpage.c b/c8/stm/gcpage.c
--- a/c8/stm/gcpage.c
+++ b/c8/stm/gcpage.c
@@ -130,6 +130,58 @@
     return o;
 }
 
+static void _fill_preexisting_slice(long segnum, char *dest,
+                                    const char *src, uintptr_t size)
+{
+    uintptr_t np = dest - get_segment_base(segnum);
+    if (get_page_status_in(segnum, np / 4096) != PAGE_NO_ACCESS)
+        memcpy(dest, src, size);
+}
+
+object_t *stm_allocate_preexisting(ssize_t size_rounded_up,
+                                   const char *initial_data)
+{
+    stm_char *np = allocate_outside_nursery_large(size_rounded_up);
+    uintptr_t nobj = (uintptr_t)np;
+    dprintf(("allocate_preexisting: %p\n", (object_t *)nobj));
+
+    char *nobj_seg0 = stm_object_pages + nobj;
+    memcpy(nobj_seg0, initial_data, size_rounded_up);
+    ((struct object_s *)nobj_seg0)->stm_flags = GCFLAG_WRITE_BARRIER;
+
+    acquire_privatization_lock(STM_SEGMENT->segment_num);
+    DEBUG_EXPECT_SEGFAULT(false);
+
+    long j;
+    for (j = 1; j < NB_SEGMENTS; j++) {
+        const char *src = nobj_seg0;
+        char *dest = get_segment_base(j) + nobj;
+        char *end = dest + size_rounded_up;
+
+        while (((uintptr_t)dest) / 4096 != ((uintptr_t)end - 1) / 4096) {
+            uintptr_t count = 4096 - (((uintptr_t)dest) & 4095);
+            _fill_preexisting_slice(j, dest, src, count);
+            src += count;
+            dest += count;
+        }
+        _fill_preexisting_slice(j, dest, src, end - dest);
+
+#ifdef STM_TESTS
+        /* can't really enable this check outside tests, because there is
+           a change that the transaction_state changes in parallel */
+        if (get_priv_segment(j)->transaction_state != TS_NONE) {
+            assert(!was_read_remote(get_segment_base(j), (object_t *)nobj));
+        }
+#endif
+    }
+
+    DEBUG_EXPECT_SEGFAULT(true);
+    release_privatization_lock(STM_SEGMENT->segment_num);
+
+    write_fence();     /* make sure 'nobj' is fully initialized from
+                          all threads here */
+    return (object_t *)nobj;
+}
 
 /************************************************************/
 
@@ -249,6 +301,8 @@
 }
 
 
+#define TRACE_FOR_MAJOR_COLLECTION  (&mark_record_trace)
+
 static void mark_and_trace(
     object_t *obj,
     char *segment_base, /* to trace obj in */
@@ -408,7 +462,8 @@
         struct stm_undo_s *modified = (struct stm_undo_s *)lst->items;
         struct stm_undo_s *end = (struct stm_undo_s *)(lst->items + lst->count);
         for (; modified < end; modified++) {
-            if (modified->type == TYPE_POSITION_MARKER)
+            if (modified->type == TYPE_POSITION_MARKER &&
+                    modified->type2 != TYPE_MODIFIED_HASHTABLE)
                 mark_visit_possibly_new_object(modified->marker_object, pseg);
         }
     }
@@ -541,6 +596,31 @@
                 list_set_item(lst, n, list_pop_item(lst));
             }
         }
+
+        /* Remove from 'modified_old_objects' all old hashtables that die */
+        {
+            lst = pseg->modified_old_objects;
+            uintptr_t j, k = 0, limit = list_count(lst);
+            for (j = 0; j < limit; j += 3) {
+                uintptr_t e0 = list_item(lst, j + 0);
+                uintptr_t e1 = list_item(lst, j + 1);
+                uintptr_t e2 = list_item(lst, j + 2);
+                if (e0 == TYPE_POSITION_MARKER &&
+                    e1 == TYPE_MODIFIED_HASHTABLE &&
+                    !mark_visited_test((object_t *)e2)) {
+                    /* hashtable object dies */
+                }
+                else {
+                    if (j != k) {
+                        list_set_item(lst, k + 0, e0);
+                        list_set_item(lst, k + 1, e1);
+                        list_set_item(lst, k + 2, e2);
+                    }
+                    k += 3;
+                }
+            }
+            lst->count = k;
+        }
     }
 #pragma pop_macro("STM_SEGMENT")
 #pragma pop_macro("STM_PSEGMENT")
@@ -601,7 +681,7 @@
     _stm_smallmalloc_sweep();
 }
 
-static void clean_up_commit_log_entries()
+static void clean_up_commit_log_entries(void)
 {
     struct stm_commit_log_entry_s *cl, *next;
 
diff --git a/c8/stm/hashtable.c b/c8/stm/hashtable.c
new file mode 100644
--- /dev/null
+++ b/c8/stm/hashtable.c
@@ -0,0 +1,538 @@
+/*
+Design of stmgc's "hashtable" objects
+=====================================
+
+A "hashtable" is theoretically a lazily-filled array of objects of
+length 2**64.  Initially it is full of NULLs.  It's obviously
+implemented as a dictionary in which NULL objects are not needed.
+
+A real dictionary can be implemented on top of it, by using the index
+`hash(key)` in the hashtable, and storing a list of `(key, value)`
+pairs at that index (usually only one, unless there is a hash
+collision).
+
+The main operations on a hashtable are reading or writing an object at a
+given index.  It also supports fetching the list of non-NULL entries.
+
+There are two markers for every index (a read and a write marker).
+This is unlike regular arrays, which have only two markers in total.
+
+Additionally, we use the read marker for the hashtable object itself
+to mean "we have read the complete list of keys".  This plays the role
+of a "global" read marker: when any thread adds a new key/value object
+to the hashtable, this new object's read marker is initialized with a
+copy of the "global" read marker --- in all segments.
+
+
+Implementation
+--------------
+
+First idea: have the hashtable in raw memory, pointing to "entry"
+objects (which are regular, GC- and STM-managed objects).  The entry
+objects themselves point to the user-specified objects.  The entry
+objects hold the read/write markers.  Every entry object, once
+created, stays around.  It is only removed by the next major GC if it
+points to NULL and its read/write markers are not set in any
+currently-running transaction.
+
+References
+----------
+
+Inspired by: http://ppl.stanford.edu/papers/podc011-bronson.pdf
+*/
+
+
+uint32_t stm_hashtable_entry_userdata;
+
+
+#define INITIAL_HASHTABLE_SIZE   8
+#define PERTURB_SHIFT            5
+#define RESIZING_LOCK            0
+
+typedef struct {
+    uintptr_t mask;
+
+    /* 'resize_counter' start at an odd value, and is decremented (by
+       6) for every new item put in 'items'.  When it crosses 0, we
+       instead allocate a bigger table and change 'resize_counter' to
+       be a regular pointer to it (which is then even).  The whole
+       structure is immutable then.
+
+       The field 'resize_counter' also works as a write lock: changes
+       go via the intermediate value RESIZING_LOCK (0).
+    */
+    uintptr_t resize_counter;
+
+    stm_hashtable_entry_t *items[INITIAL_HASHTABLE_SIZE];
+} stm_hashtable_table_t;
+
+#define IS_EVEN(p) (((p) & 1) == 0)
+
+struct stm_hashtable_s {
+    stm_hashtable_table_t *table;
+    stm_hashtable_table_t initial_table;
+    uint64_t additions;
+};
+
+
+static inline void init_table(stm_hashtable_table_t *table, uintptr_t itemcount)
+{
+    table->mask = itemcount - 1;
+    table->resize_counter = itemcount * 4 + 1;
+    memset(table->items, 0, itemcount * sizeof(stm_hashtable_entry_t *));
+}
+
+stm_hashtable_t *stm_hashtable_create(void)
+{
+    stm_hashtable_t *hashtable = malloc(sizeof(stm_hashtable_t));
+    assert(hashtable);
+    hashtable->table = &hashtable->initial_table;
+    hashtable->additions = 0;
+    init_table(&hashtable->initial_table, INITIAL_HASHTABLE_SIZE);
+    return hashtable;
+}
+
+void stm_hashtable_free(stm_hashtable_t *hashtable)
+{
+    uintptr_t rc = hashtable->initial_table.resize_counter;
+    free(hashtable);
+    while (IS_EVEN(rc)) {
+        assert(rc != RESIZING_LOCK);
+
+        stm_hashtable_table_t *table = (stm_hashtable_table_t *)rc;
+        rc = table->resize_counter;
+        free(table);
+    }
+}
+
+static bool _stm_was_read_by_anybody(object_t *obj)
+{
+    /* can only be safely called during major GC, when all other threads
+       are suspended */
+    assert(_has_mutex());
+
+    long i;
+    for (i = 1; i < NB_SEGMENTS; i++) {
+        if (get_priv_segment(i)->transaction_state == TS_NONE)
+            continue;
+        if (was_read_remote(get_segment_base(i), obj))
+            return true;
+    }
+    return false;
+}
+
+#define VOLATILE_HASHTABLE(p)    ((volatile stm_hashtable_t *)(p))
+#define VOLATILE_TABLE(p)  ((volatile stm_hashtable_table_t *)(p))
+
+static void _insert_clean(stm_hashtable_table_t *table,
+                          stm_hashtable_entry_t *entry,
+                          uintptr_t index)
+{
+    uintptr_t mask = table->mask;
+    uintptr_t i = index & mask;
+    if (table->items[i] == NULL) {
+        table->items[i] = entry;
+        return;
+    }
+
+    uintptr_t perturb = index;
+    while (1) {
+        i = (i << 2) + i + perturb + 1;
+        i &= mask;
+        if (table->items[i] == NULL) {
+            table->items[i] = entry;
+            return;
+        }
+
+        perturb >>= PERTURB_SHIFT;
+    }
+}
+
+static void _stm_rehash_hashtable(stm_hashtable_t *hashtable,
+                                  uintptr_t biggercount,
+                                  char *segment_base)
+{
+    dprintf(("rehash %p to size %ld, segment_base=%p\n",
+             hashtable, biggercount, segment_base));
+
+    size_t size = (offsetof(stm_hashtable_table_t, items)
+                   + biggercount * sizeof(stm_hashtable_entry_t *));
+    stm_hashtable_table_t *biggertable = malloc(size);
+    assert(biggertable);   // XXX
+
+    stm_hashtable_table_t *table = hashtable->table;
+    table->resize_counter = (uintptr_t)biggertable;
+    /* ^^^ this unlocks the table by writing a non-zero value to
+       table->resize_counter, but the new value is a pointer to the
+       new bigger table, so IS_EVEN() is still true */
+    assert(IS_EVEN(table->resize_counter));
+
+    init_table(biggertable, biggercount);
+
+    uintptr_t j, mask = table->mask;
+    uintptr_t rc = biggertable->resize_counter;
+    for (j = 0; j <= mask; j++) {
+        stm_hashtable_entry_t *entry = table->items[j];
+        if (entry == NULL)
+            continue;
+        if (segment_base != NULL) {
+            /* -> compaction during major GC */
+            if (((struct stm_hashtable_entry_s *)
+                       REAL_ADDRESS(segment_base, entry))->object == NULL &&
+                   !_stm_was_read_by_anybody((object_t *)entry)) {
+                dprintf(("  removing dead %p\n", entry));
+                continue;
+            }
+        }
+
+        uintptr_t eindex;
+        if (segment_base == NULL)
+            eindex = entry->index;   /* read from STM_SEGMENT */
+        else
+            eindex = ((struct stm_hashtable_entry_s *)
+                       REAL_ADDRESS(segment_base, entry))->index;
+
+        dprintf(("  insert_clean %p at index=%ld\n",
+                 entry, eindex));
+        _insert_clean(biggertable, entry, eindex);
+        assert(rc > 6);
+        rc -= 6;
+    }
+    biggertable->resize_counter = rc;
+
+    write_fence();   /* make sure that 'biggertable' is valid here,
+                        and make sure 'table->resize_counter' is updated
+                        ('table' must be immutable from now on). */
+    VOLATILE_HASHTABLE(hashtable)->table = biggertable;
+}
+
+stm_hashtable_entry_t *stm_hashtable_lookup(object_t *hashtableobj,
+                                            stm_hashtable_t *hashtable,
+                                            uintptr_t index)
+{
+    stm_hashtable_table_t *table;
+    uintptr_t mask;
+    uintptr_t i;
+    stm_hashtable_entry_t *entry;
+
+ restart:
+    /* classical dict lookup logic */
+    table = VOLATILE_HASHTABLE(hashtable)->table;
+    mask = table->mask;      /* read-only field */
+    i = index & mask;
+    entry = VOLATILE_TABLE(table)->items[i];
+    if (entry != NULL) {
+        if (entry->index == index)
+            return entry;           /* found at the first try */
+
+        uintptr_t perturb = index;
+        while (1) {
+            i = (i << 2) + i + perturb + 1;
+            i &= mask;
+            entry = VOLATILE_TABLE(table)->items[i];
+            if (entry != NULL) {
+                if (entry->index == index)
+                    return entry;    /* found */
+            }
+            else
+                break;
+            perturb >>= PERTURB_SHIFT;
+        }
+    }
+    /* here, we didn't find the 'entry' with the correct index.  Note
+       that even if the same 'table' is modified or resized by other
+       threads concurrently, any new item found from a race condition
+       would anyway contain NULL in the present segment (ensured by
+       the first write_fence() below).  If the 'table' grows an entry
+       just after we checked above, then we go ahead and lock the
+       table; but after we get the lock, we will notice the new entry
+       (ensured by the second write_fence() below) and restart the
+       whole process.
+     */
+
+    uintptr_t rc = VOLATILE_TABLE(table)->resize_counter;
+
+    /* if rc is RESIZING_LOCK (which is 0, so even), a concurrent thread
+       is writing to the hashtable.  Or, if rc is another even number, it is
+       actually a pointer to the next version of the table, installed
+       just now.  In both cases, this thread must simply spin loop.
+    */
+    if (IS_EVEN(rc)) {
+        spin_loop();
+        goto restart;
+    }
+    /* in the other cases, we need to grab the RESIZING_LOCK.
+     */
+    if (!__sync_bool_compare_and_swap(&table->resize_counter,
+                                      rc, RESIZING_LOCK)) {
+        goto restart;
+    }
+    /* we now have the lock.  The only table with a non-even value of
+       'resize_counter' should be the last one in the chain, so if we
+       succeeded in locking it, check this. */
+    assert(table == hashtable->table);
+
+    /* Check that 'table->items[i]' is still NULL,
+       i.e. hasn't been populated under our feet.
+    */
+    if (table->items[i] != NULL) {
+        table->resize_counter = rc;    /* unlock */
+        goto restart;
+    }
+    /* if rc is greater than 6, there is enough room for a new
+       item in the current table.
+    */
+    if (rc > 6) {
+        /* we can only enter here once!  If we allocate stuff, we may
+           run the GC, and so 'hashtableobj' might move afterwards. */
+        if (_is_in_nursery(hashtableobj)) {
+            /* this also means that the hashtable is from this
+               transaction and not visible to other segments yet, so
+               the new entry can be nursery-allocated. */
+            entry = (stm_hashtable_entry_t *)
+                stm_allocate(sizeof(stm_hashtable_entry_t));
+            entry->userdata = stm_hashtable_entry_userdata;
+            entry->index = index;
+            entry->object = NULL;
+        }
+        else {
+            /* for a non-nursery 'hashtableobj', we pretend that the
+               'entry' object we're about to return was already
+               existing all along, with NULL in all segments.  If the
+               caller of this function is going to modify the 'object'
+               field, it will call stm_write(entry) first, which will
+               correctly schedule 'entry' for write propagation.  We
+               do that even if 'hashtableobj' was created by the
+               running transaction: the new 'entry' object is created
+               as if it was older than the transaction.
+
+               Note the following difference: if 'hashtableobj' is
+               still in the nursery (case above), the 'entry' object
+               is also allocated from the nursery, and after a minor
+               collection it ages as an old-but-created-by-the-
+               current-transaction object.  We could try to emulate
+               this here, or to create young 'entry' objects, but
+               doing either of these would require careful
+               synchronization with other pieces of the code that may
+               change.
+            */
+            struct stm_hashtable_entry_s initial = {
+                .userdata = stm_hashtable_entry_userdata,
+                .index = index,
+                .object = NULL
+            };
+            entry = (stm_hashtable_entry_t *)
+                stm_allocate_preexisting(sizeof(stm_hashtable_entry_t),
+                                         (char *)&initial.header);
+            hashtable->additions++;
+        }
+        table->items[i] = entry;
+        write_fence();     /* make sure 'table->items' is written here */
+        VOLATILE_TABLE(table)->resize_counter = rc - 6;    /* unlock */
+        return entry;
+    }
+    else {
+        /* if rc is smaller than 6, we must allocate a new bigger table.
+         */
+        uintptr_t biggercount = table->mask + 1;
+        if (biggercount < 50000)
+            biggercount *= 4;
+        else
+            biggercount *= 2;
+        _stm_rehash_hashtable(hashtable, biggercount, /*segment_base=*/NULL);
+        goto restart;
+    }
+}
+
+object_t *stm_hashtable_read(object_t *hobj, stm_hashtable_t *hashtable,
+                             uintptr_t key)
+{
+    stm_hashtable_entry_t *e = stm_hashtable_lookup(hobj, hashtable, key);
+    stm_read((object_t *)e);
+    return e->object;
+}
+
+void stm_hashtable_write_entry(object_t *hobj, stm_hashtable_entry_t *entry,
+                               object_t *nvalue)
+{
+    if (_STM_WRITE_CHECK_SLOWPATH((object_t *)entry)) {
+
+        stm_write((object_t *)entry);
+
+        uintptr_t i = list_count(STM_PSEGMENT->modified_old_objects);
+        if (i > 0 && list_item(STM_PSEGMENT->modified_old_objects, i - 3)
+                     == (uintptr_t)entry) {
+            /* The stm_write() above recorded a write to 'entry'.  Here,
+               we add another stm_undo_s to modified_old_objects with
+               TYPE_MODIFIED_HASHTABLE.  It is ignored everywhere except
+               in _stm_validate().
+
+               The goal is that this TYPE_MODIFIED_HASHTABLE ends up in
+               the commit log's 'cl_written' array.  Later, another
+               transaction validating that log will check two things:
+
+               - the regular stm_undo_s entry put by stm_write() above
+                 will make the other transaction check that it didn't
+                 read the same 'entry' object;
+
+                 - the TYPE_MODIFIED_HASHTABLE entry we're adding now
+                   will make the other transaction check that it didn't
+                   do any stm_hashtable_list() on the complete hashtable.
+            */
+            STM_PSEGMENT->modified_old_objects = list_append3(
+                STM_PSEGMENT->modified_old_objects,
+                TYPE_POSITION_MARKER,      /* type1 */
+                TYPE_MODIFIED_HASHTABLE,   /* type2 */
+                (uintptr_t)hobj);          /* modif_hashtable */
+        }
+    }
+    entry->object = nvalue;
+}
+
+void stm_hashtable_write(object_t *hobj, stm_hashtable_t *hashtable,
+                         uintptr_t key, object_t *nvalue,
+                         stm_thread_local_t *tl)
+{
+    STM_PUSH_ROOT(*tl, nvalue);
+    STM_PUSH_ROOT(*tl, hobj);
+    stm_hashtable_entry_t *e = stm_hashtable_lookup(hobj, hashtable, key);
+    STM_POP_ROOT(*tl, hobj);
+    STM_POP_ROOT(*tl, nvalue);
+    stm_hashtable_write_entry(hobj, e, nvalue);
+}
+
+long stm_hashtable_length_upper_bound(stm_hashtable_t *hashtable)
+{
+    stm_hashtable_table_t *table;
+    uintptr_t rc;
+
+ restart:
+    table = VOLATILE_HASHTABLE(hashtable)->table;
+    rc = VOLATILE_TABLE(table)->resize_counter;
+    if (IS_EVEN(rc)) {
+        spin_loop();
+        goto restart;
+    }
+
+    uintptr_t initial_rc = (table->mask + 1) * 4 + 1;
+    uintptr_t num_entries_times_6 = initial_rc - rc;
+    return num_entries_times_6 / 6;
+}
+
+long stm_hashtable_list(object_t *hobj, stm_hashtable_t *hashtable,
+                        stm_hashtable_entry_t **results)
+{
+    /* Set the read marker.  It will be left as long as we're running
+       the same transaction.
+    */
+    stm_read(hobj);
+
+    /* Get the table.  No synchronization is needed: we may miss some
+       entries that are being added, but they would contain NULL in
+       this segment anyway. */
+    stm_hashtable_table_t *table = VOLATILE_HASHTABLE(hashtable)->table;
+
+    /* Read all entries, check which ones are not NULL, count them,
+       and optionally list them in 'results'.
+    */
+    uintptr_t i, mask = table->mask;
+    stm_hashtable_entry_t *entry;
+    long nresult = 0;
+
+    if (results != NULL) {
+        /* collect the results in the provided list */
+        for (i = 0; i <= mask; i++) {
+            entry = VOLATILE_TABLE(table)->items[i];
+            if (entry != NULL) {
+                stm_read((object_t *)entry);
+                if (entry->object != NULL)
+                    results[nresult++] = entry;
+            }
+        }
+    }
+    else {
+        /* don't collect, just get the exact number of results */
+        for (i = 0; i <= mask; i++) {
+            entry = VOLATILE_TABLE(table)->items[i];
+            if (entry != NULL) {
+                stm_read((object_t *)entry);
+                if (entry->object != NULL)
+                    nresult++;
+            }
+        }
+    }
+    return nresult;
+}
+
+static void _stm_compact_hashtable(struct object_s *hobj,
+                                   stm_hashtable_t *hashtable)
+{
+    stm_hashtable_table_t *table = hashtable->table;
+    uintptr_t rc = table->resize_counter;
+    assert(!IS_EVEN(rc));
+
+    if (hashtable->additions * 4 > table->mask) {
+        hashtable->additions = 0;
+
+        /* If 'hobj' was created in some current transaction, i.e. if it is
+           now an overflow object, then we have the risk that some of its
+           entry objects were not created with stm_allocate_preexisting().
+           In that situation, a valid workaround is to read all entry
+           objects in the segment of the running transaction.  Otherwise,
+           the base case is to read them all from segment zero.
+        */
+        long segnum = get_num_segment_containing_address((char *)hobj);
+        if (!IS_OVERFLOW_OBJ(get_priv_segment(segnum), hobj))
+            segnum = 0;
+
+        uintptr_t initial_rc = (table->mask + 1) * 4 + 1;
+        uintptr_t num_entries_times_6 = initial_rc - rc;
+        uintptr_t count = INITIAL_HASHTABLE_SIZE;
+        while (count * 4 < num_entries_times_6)
+            count *= 2;
+        /* sanity-check: 'num_entries_times_6 < initial_rc', and so 'count'
+           can never grow larger than the current table size. */
+        assert(count <= table->mask + 1);
+
+        dprintf(("compact with %ld items:\n", num_entries_times_6 / 6));
+        _stm_rehash_hashtable(hashtable, count, get_segment_base(segnum));
+    }
+
+    table = hashtable->table;
+    assert(!IS_EVEN(table->resize_counter));
+
+    if (table != &hashtable->initial_table) {
+        uintptr_t rc = hashtable->initial_table.resize_counter;
+        while (1) {
+            assert(IS_EVEN(rc));
+            assert(rc != RESIZING_LOCK);
+
+            stm_hashtable_table_t *old_table = (stm_hashtable_table_t *)rc;
+            if (old_table == table)
+                break;
+            rc = old_table->resize_counter;
+            free(old_table);
+        }
+        hashtable->initial_table.resize_counter = (uintptr_t)table;
+        assert(IS_EVEN(hashtable->initial_table.resize_counter));
+    }
+}
+
+void stm_hashtable_tracefn(struct object_s *hobj, stm_hashtable_t *hashtable,
+                           void trace(object_t **))
+{
+    if (trace == TRACE_FOR_MAJOR_COLLECTION)
+        _stm_compact_hashtable(hobj, hashtable);
+
+    stm_hashtable_table_t *table;
+    table = VOLATILE_HASHTABLE(hashtable)->table;
+
+    uintptr_t j, mask = table->mask;
+    for (j = 0; j <= mask; j++) {
+        stm_hashtable_entry_t *volatile *pentry;
+        pentry = &VOLATILE_TABLE(table)->items[j];
+        if (*pentry != NULL) {
+            trace((object_t **)pentry);
+        }
+    }
+}
diff --git a/c8/stm/marker.c b/c8/stm/marker.c
--- a/c8/stm/marker.c
+++ b/c8/stm/marker.c
@@ -42,7 +42,8 @@
     */
     while (contention != start) {
         --contention;
-        if (contention->type == TYPE_POSITION_MARKER) {
+        if (contention->type == TYPE_POSITION_MARKER &&
+            contention->type2 != TYPE_MODIFIED_HASHTABLE) {
             out_marker->odd_number = contention->marker_odd_number;
             out_marker->object = contention->marker_object;
             return;
@@ -69,6 +70,9 @@
             return;    /* already up-to-date */
     }
 
+    /* -2 is not odd */
+    assert(marker.odd_number != (uintptr_t)TYPE_MODIFIED_HASHTABLE);
+
     STM_PSEGMENT->position_markers_last = list_count(list);
     STM_PSEGMENT->modified_old_objects = list_append3(
         list,
diff --git a/c8/stm/misc.c b/c8/stm/misc.c
--- a/c8/stm/misc.c
+++ b/c8/stm/misc.c
@@ -31,10 +31,7 @@
 
 bool _stm_was_read(object_t *obj)
 {
-    uint8_t rm = ((struct stm_read_marker_s *)
-                  (STM_SEGMENT->segment_base + (((uintptr_t)obj) >> 4)))->rm;
-    assert(rm <= STM_SEGMENT->transaction_read_version);
-    return rm == STM_SEGMENT->transaction_read_version;
+    return was_read_remote(STM_SEGMENT->segment_base, obj);
 }
 
 bool _stm_was_written(object_t *obj)
@@ -47,7 +44,7 @@
     return obj->stm_flags & _STM_GCFLAG_CARDS_SET;
 }
 
-long _stm_count_cl_entries()
+long _stm_count_cl_entries(void)
 {
     struct stm_commit_log_entry_s *cl = &commit_log_root;
 
@@ -118,7 +115,7 @@
     return cards[get_index_to_card_index(idx)].rm;
 }
 
-uint8_t _stm_get_transaction_read_version()
+uint8_t _stm_get_transaction_read_version(void)
 {
     return STM_SEGMENT->transaction_read_version;
 }
@@ -127,7 +124,7 @@
 
 static struct stm_commit_log_entry_s *_last_cl_entry;
 static long _last_cl_entry_index;
-void _stm_start_enum_last_cl_entry()
+void _stm_start_enum_last_cl_entry(void)
 {
     _last_cl_entry = &commit_log_root;
     struct stm_commit_log_entry_s *cl = &commit_log_root;
@@ -138,7 +135,7 @@
     _last_cl_entry_index = 0;
 }
 
-object_t *_stm_next_last_cl_entry()
+object_t *_stm_next_last_cl_entry(void)
 {
     if (_last_cl_entry == &commit_log_root)
         return NULL;
@@ -153,7 +150,7 @@
 }
 
 
-void _stm_smallmalloc_sweep_test()
+void _stm_smallmalloc_sweep_test(void)
 {
     acquire_all_privatization_locks();
     _stm_smallmalloc_sweep();
diff --git a/c8/stm/nursery.c b/c8/stm/nursery.c
--- a/c8/stm/nursery.c
+++ b/c8/stm/nursery.c
@@ -424,6 +424,7 @@
     struct stm_undo_s *end = (struct stm_undo_s *)(list->items + list->count);
 
     for (; undo < end; undo++) {
+        /* this logic also works if type2 == TYPE_MODIFIED_HASHTABLE */
         if (undo->type == TYPE_POSITION_MARKER)
             minor_trace_if_young(&undo->marker_object);
     }
diff --git a/c8/stm/pages.h b/c8/stm/pages.h
--- a/c8/stm/pages.h
+++ b/c8/stm/pages.h
@@ -62,7 +62,11 @@
 
 static inline bool get_page_status_in(long segnum, uintptr_t pagenum)
 {
-    /* reading page status requires "read"-lock: */
+    /* reading page status requires "read"-lock, which is defined as
+       "any segment has the privatization_lock".  This is enough to
+       prevent the "write"-lock from being acquired by somebody else
+       (defined as "_all_ segments have the privatization_lock").
+    */
     assert(STM_PSEGMENT->privatization_lock);
 
     OPT_ASSERT(segnum < 8 * sizeof(struct page_shared_s));
diff --git a/c8/stm/setup.c b/c8/stm/setup.c
--- a/c8/stm/setup.c
+++ b/c8/stm/setup.c
@@ -250,8 +250,6 @@
     set_gs_register(get_segment_base(num + 1));
     s_mutex_unlock();
 
-    DEBUG_EXPECT_SEGFAULT(true);
-
     if (num == 0) {
         dprintf(("STM_GC_NURSERY: %d\n", STM_GC_NURSERY));
         dprintf(("NB_PAGES: %d\n", NB_PAGES));
diff --git a/c8/stm/setup.h b/c8/stm/setup.h
--- a/c8/stm/setup.h
+++ b/c8/stm/setup.h
@@ -3,8 +3,8 @@
 static pthread_t *_get_cpth(stm_thread_local_t *);
 
 #ifndef NDEBUG
-static __thread long _stm_segfault_expected = 0;
-#define DEBUG_EXPECT_SEGFAULT(v) do {if (v) _stm_segfault_expected++; else _stm_segfault_expected--;} while (0)
+static __thread long _stm_segfault_expected = 1;
+#define DEBUG_EXPECT_SEGFAULT(v) do {if (v) _stm_segfault_expected++; else _stm_segfault_expected--; assert(_stm_segfault_expected <= 1);} while (0)
 #else
 #define DEBUG_EXPECT_SEGFAULT(v) {}
 #endif
diff --git a/c8/stmgc.c b/c8/stmgc.c
--- a/c8/stmgc.c
+++ b/c8/stmgc.c
@@ -39,3 +39,4 @@
 #include "stm/prof.c"
 #include "stm/rewind_setjmp.c"
 #include "stm/finalizer.c"
+#include "stm/hashtable.c"
diff --git a/c8/stmgc.h b/c8/stmgc.h
--- a/c8/stmgc.h
+++ b/c8/stmgc.h
@@ -57,13 +57,16 @@
 typedef struct stm_thread_local_s {
     /* rewind_setjmp's interface */
     rewind_jmp_thread rjthread;
+    /* every thread should handle the shadow stack itself */
     struct stm_shadowentry_s *shadowstack, *shadowstack_base;
-
     /* a generic optional thread-local object */
     object_t *thread_local_obj;
-
+    /* in case this thread runs a transaction that aborts,
+       the following raw region of memory is cleared. */
     char *mem_clear_on_abort;
     size_t mem_bytes_to_clear_on_abort;
+    /* after an abort, some details about the abort are stored there.
+       (this field is not modified on a successful commit) */
     long last_abort__bytes_in_nursery;
     /* the next fields are handled internally by the library */
     int associated_segment_num;
@@ -73,34 +76,22 @@
     void *creating_pthread[2];
 } stm_thread_local_t;
 
-#ifndef _STM_NURSERY_ZEROED
-#define _STM_NURSERY_ZEROED               0
-#endif
 
-#define _STM_GCFLAG_WRITE_BARRIER      0x01
-#define _STM_FAST_ALLOC           (66*1024)
-#define _STM_NSE_SIGNAL_ABORT             1
-#define _STM_NSE_SIGNAL_MAX               2
-
-#define _STM_CARD_MARKED 1      /* should always be 1... */
-#define _STM_GCFLAG_CARDS_SET          0x8
-#define _STM_CARD_BITS                 5   /* must be 5/6/7 for the pypy jit */
-#define _STM_CARD_SIZE                 (1 << _STM_CARD_BITS)
-#define _STM_MIN_CARD_COUNT            17
-#define _STM_MIN_CARD_OBJ_SIZE         (_STM_CARD_SIZE * _STM_MIN_CARD_COUNT)
-
+/* this should use llvm's coldcc calling convention,
+   but it's not exposed to C code so far */
 void _stm_write_slowpath(object_t *);
 void _stm_write_slowpath_card(object_t *, uintptr_t);
 object_t *_stm_allocate_slowpath(ssize_t);
 object_t *_stm_allocate_external(ssize_t);
 void _stm_become_inevitable(const char*);
-void _stm_collectable_safe_point();
+void _stm_collectable_safe_point(void);
 
+/* for tests, but also used in duhton: */
 object_t *_stm_allocate_old(ssize_t size_rounded_up);
 char *_stm_real_address(object_t *o);
 #ifdef STM_TESTS
 #include <stdbool.h>
-uint8_t _stm_get_transaction_read_version();
+uint8_t _stm_get_transaction_read_version(void);
 uint8_t _stm_get_card_value(object_t *obj, long idx);
 bool _stm_was_read(object_t *obj);
 bool _stm_was_written(object_t *obj);
@@ -137,14 +128,32 @@
 long _stm_count_objects_pointing_to_nursery(void);
 object_t *_stm_enum_modified_old_objects(long index);
 object_t *_stm_enum_objects_pointing_to_nursery(long index);
-object_t *_stm_next_last_cl_entry();
-void _stm_start_enum_last_cl_entry();
-long _stm_count_cl_entries();
+object_t *_stm_next_last_cl_entry(void);
+void _stm_start_enum_last_cl_entry(void);
+long _stm_count_cl_entries(void);
 long _stm_count_old_objects_with_cards_set(void);
 object_t *_stm_enum_old_objects_with_cards_set(long index);
 uint64_t _stm_total_allocated(void);
 #endif
 
+
+#ifndef _STM_NURSERY_ZEROED
+#define _STM_NURSERY_ZEROED               0
+#endif
+
+#define _STM_GCFLAG_WRITE_BARRIER      0x01
+#define _STM_FAST_ALLOC           (66*1024)
+#define _STM_NSE_SIGNAL_ABORT             1
+#define _STM_NSE_SIGNAL_MAX               2
+
+#define _STM_CARD_MARKED 1      /* should always be 1... */
+#define _STM_GCFLAG_CARDS_SET          0x8
+#define _STM_CARD_BITS                 5   /* must be 5/6/7 for the pypy jit */
+#define _STM_CARD_SIZE                 (1 << _STM_CARD_BITS)
+#define _STM_MIN_CARD_COUNT            17
+#define _STM_MIN_CARD_OBJ_SIZE         (_STM_CARD_SIZE * _STM_MIN_CARD_COUNT)
+
+
 /* ==================== HELPERS ==================== */
 #ifdef NDEBUG
 #define OPT_ASSERT(cond) do { if (!(cond)) __builtin_unreachable(); } while (0)
@@ -165,30 +174,32 @@
 */
 #define STM_NB_SEGMENTS    4
 
+/* Structure of objects
+   --------------------
 
+   Objects manipulated by the user program, and managed by this library,
+   must start with a "struct object_s" field.  Pointers to any user object
+   must use the "TLPREFIX struct foo *" type --- don't forget TLPREFIX.
+   The best is to use typedefs like above.
+
+   The object_s part contains some fields reserved for the STM library.
+   Right now this is only four bytes.
+*/
 struct object_s {
     uint32_t stm_flags;            /* reserved for the STM library */
 };
 
-extern ssize_t stmcb_size_rounded_up(struct object_s *);
-void stmcb_trace(struct object_s *obj, void visit(object_t **));
-/* a special trace-callback that is only called for the marked
-   ranges of indices (using stm_write_card(o, index)) */
-extern void stmcb_trace_cards(struct object_s *, void (object_t **),
-                              uintptr_t start, uintptr_t stop);
-/* this function will be called on objects that support cards.
-   It returns the base_offset (in bytes) inside the object from
-   where the indices start, and item_size (in bytes) for the size of
-   one item */
-extern void stmcb_get_card_base_itemsize(struct object_s *,
-                                         uintptr_t offset_itemsize[2]);
-/* returns whether this object supports cards. we will only call
-   stmcb_get_card_base_itemsize on objs that do so. */
-extern long stmcb_obj_supports_cards(struct object_s *);
 
-
-
-
+/* The read barrier must be called whenever the object 'obj' is read.
+   It is not required to call it before reading: it can be delayed for a
+   bit, but we must still be in the same "scope": no allocation, no
+   transaction commit, nothing that can potentially collect or do a safe
+   point (like stm_write() on a different object).  Also, if we might
+   have finished the transaction and started the next one, then
+   stm_read() needs to be called again.  It can be omitted if
+   stm_write() is called, or immediately after getting the object from
+   stm_allocate(), as long as the rules above are respected.
+*/
 __attribute__((always_inline))
 static inline void stm_read(object_t *obj)
 {
@@ -196,19 +207,34 @@
         STM_SEGMENT->transaction_read_version;
 }
 
+#define _STM_WRITE_CHECK_SLOWPATH(obj)  \
+    UNLIKELY(((obj)->stm_flags & _STM_GCFLAG_WRITE_BARRIER) != 0)
+
+/* The write barrier must be called *before* doing any change to the
+   object 'obj'.  If we might have finished the transaction and started
+   the next one, then stm_write() needs to be called again.  It is not
+   necessary to call it immediately after stm_allocate().
+*/
 __attribute__((always_inline))
 static inline void stm_write(object_t *obj)
 {
-    if (UNLIKELY((obj->stm_flags & _STM_GCFLAG_WRITE_BARRIER) != 0))
+    if (_STM_WRITE_CHECK_SLOWPATH(obj))
         _stm_write_slowpath(obj);
 }
 
-
+/* The following is a GC-optimized barrier that works on the granularity
+   of CARD_SIZE.  It can be used on any array object, but it is only
+   useful with those that were internally marked with GCFLAG_HAS_CARDS.
+   It has the same purpose as stm_write() for TM and allows write-access
+   to a part of an object/array.
+   'index' is the array-item-based position within the object, which
+   is measured in units returned by stmcb_get_card_base_itemsize().
+*/
 __attribute__((always_inline))
 static inline void stm_write_card(object_t *obj, uintptr_t index)
 {
     /* if GCFLAG_WRITE_BARRIER is set, then don't do anything more. */
-    if (UNLIKELY((obj->stm_flags & _STM_GCFLAG_WRITE_BARRIER) != 0)) {
+    if (_STM_WRITE_CHECK_SLOWPATH(obj)) {
 
         /* GCFLAG_WRITE_BARRIER is not set.  This might be because
            it's the first time we see a given small array; or it might
@@ -242,7 +268,34 @@
     }
 }
 
+/* Must be provided by the user of this library.
+   The "size rounded up" must be a multiple of 8 and at least 16.
+   "Tracing" an object means enumerating all GC references in it,
+   by invoking the callback passed as argument.
+*/
+extern ssize_t stmcb_size_rounded_up(struct object_s *);
+void stmcb_trace(struct object_s *obj, void visit(object_t **));
+/* a special trace-callback that is only called for the marked
+   ranges of indices (using stm_write_card(o, index)) */
+extern void stmcb_trace_cards(struct object_s *, void (object_t **),
+                              uintptr_t start, uintptr_t stop);
+/* this function will be called on objects that support cards.
+   It returns the base_offset (in bytes) inside the object from
+   where the indices start, and item_size (in bytes) for the size of
+   one item */
+extern void stmcb_get_card_base_itemsize(struct object_s *,
+                                         uintptr_t offset_itemsize[2]);
+/* returns whether this object supports cards. we will only call
+   stmcb_get_card_base_itemsize on objs that do so. */
+extern long stmcb_obj_supports_cards(struct object_s *);
 
+
+
+
+/* Allocate an object of the given size, which must be a multiple
+   of 8 and at least 16.  In the fast-path, this is inlined to just
+   a few assembler instructions.
+*/
 __attribute__((always_inline))
 static inline object_t *stm_allocate(ssize_t size_rounded_up)
 {
@@ -264,21 +317,48 @@
     return (object_t *)p;
 }
 
-
+/* Allocate a weakref object. Weakref objects have a
+   reference to an object at the byte-offset
+       stmcb_size_rounded_up(obj) - sizeof(void*)
+   You must assign the reference before the next collection may happen.
+   After that, you must not mutate the reference anymore. However,
+   it can become NULL after any GC if the reference dies during that
+   collection.
+   NOTE: For performance, we assume stmcb_size_rounded_up(weakref)==16
+*/
 object_t *stm_allocate_weakref(ssize_t size_rounded_up);
 
 
+/* stm_setup() needs to be called once at the beginning of the program.
+   stm_teardown() can be called at the end, but that's not necessary
+   and rather meant for tests.
+ */
 void stm_setup(void);
 void stm_teardown(void);
 
+/* The size of each shadow stack, in number of entries.
+   Must be big enough to accomodate all STM_PUSH_ROOTs! */
 #define STM_SHADOW_STACK_DEPTH   163840
+
+/* Push and pop roots from/to the shadow stack. Only allowed inside
+   transaction. */
 #define STM_PUSH_ROOT(tl, p)   ((tl).shadowstack++->ss = (object_t *)(p))
 #define STM_POP_ROOT(tl, p)    ((p) = (typeof(p))((--(tl).shadowstack)->ss))
 #define STM_POP_ROOT_RET(tl)   ((--(tl).shadowstack)->ss)
 
+/* Every thread needs to have a corresponding stm_thread_local_t
+   structure.  It may be a "__thread" global variable or something else.
+   Use the following functions at the start and at the end of a thread.
+   The user of this library needs to maintain the two shadowstack fields;
+   at any call to stm_allocate(), these fields should point to a range
+   of memory that can be walked in order to find the stack roots.
+*/
 void stm_register_thread_local(stm_thread_local_t *tl);
 void stm_unregister_thread_local(stm_thread_local_t *tl);
 
+/* At some key places, like the entry point of the thread and in the
+   function with the interpreter's dispatch loop, you need to declare
+   a local variable of type 'rewind_jmp_buf' and call these macros. */
 #define stm_rewind_jmp_enterprepframe(tl, rjbuf)                        \
     rewind_jmp_enterprepframe(&(tl)->rjthread, rjbuf, (tl)->shadowstack)
 #define stm_rewind_jmp_enterframe(tl, rjbuf)       \
@@ -300,37 +380,23 @@
     rewind_jmp_enum_shadowstack(&(tl)->rjthread, callback)
 
 
+/* Starting and ending transactions.  stm_read(), stm_write() and
+   stm_allocate() should only be called from within a transaction.
+   The stm_start_transaction() call returns the number of times it
+   returned, starting at 0.  If it is > 0, then the transaction was
+   aborted and restarted this number of times. */
 long stm_start_transaction(stm_thread_local_t *tl);
 void stm_start_inevitable_transaction(stm_thread_local_t *tl);
-
 void stm_commit_transaction(void);
 
 /* Temporary fix?  Call this outside a transaction.  If there is an
    inevitable transaction running somewhere else, wait until it finishes. */
 void stm_wait_for_current_inevitable_transaction(void);
 
+/* Abort the currently running transaction.  This function never
+   returns: it jumps back to the stm_start_transaction(). */
 void stm_abort_transaction(void) __attribute__((noreturn));
 
-void stm_collect(long level);
-
-long stm_identityhash(object_t *obj);
-long stm_id(object_t *obj);
-void stm_set_prebuilt_identityhash(object_t *obj, long hash);
-
-long stm_can_move(object_t *obj);
-
-object_t *stm_setup_prebuilt(object_t *);
-object_t *stm_setup_prebuilt_weakref(object_t *);
-
-long stm_call_on_abort(stm_thread_local_t *, void *key, void callback(void *));
-long stm_call_on_commit(stm_thread_local_t *, void *key, void callback(void *));
-
-static inline void stm_safe_point(void) {
-    if (STM_SEGMENT->nursery_end <= _STM_NSE_SIGNAL_MAX)
-        _stm_collectable_safe_point();
-}
-
-
 #ifdef STM_NO_AUTOMATIC_SETJMP
 int stm_is_inevitable(void);
 #else
@@ -338,6 +404,10 @@
     return !rewind_jmp_armed(&STM_SEGMENT->running_thread->rjthread);
 }
 #endif
+
+/* Turn the current transaction inevitable.
+   stm_become_inevitable() itself may still abort the transaction instead
+   of returning. */
 static inline void stm_become_inevitable(stm_thread_local_t *tl,
                                          const char* msg) {
     assert(STM_SEGMENT->running_thread == tl);
@@ -345,7 +415,64 @@
         _stm_become_inevitable(msg);
 }
 
+/* Forces a safe-point if needed.  Normally not needed: this is
+   automatic if you call stm_allocate(). */
+static inline void stm_safe_point(void) {
+    if (STM_SEGMENT->nursery_end <= _STM_NSE_SIGNAL_MAX)
+        _stm_collectable_safe_point();
+}
+
+/* Forces a collection. */
+void stm_collect(long level);
+
+
+/* Prepare an immortal "prebuilt" object managed by the GC.  Takes a
+   pointer to an 'object_t', which should not actually be a GC-managed
+   structure but a real static structure.  Returns the equivalent
+   GC-managed pointer.  Works by copying it into the GC pages, following
+   and fixing all pointers it contains, by doing stm_setup_prebuilt() on
+   each of them recursively.  (Note that this will leave garbage in the
+   static structure, but it should never be used anyway.) */
+object_t *stm_setup_prebuilt(object_t *);
+/* The same, if the prebuilt object is actually a weakref. */
+object_t *stm_setup_prebuilt_weakref(object_t *);
+
+/* Hash, id.  The id is just the address of the object (of the address
+   where it *will* be after the next minor collection).  The hash is the
+   same, mangled -- except on prebuilt objects, where it can be
+   controlled for each prebuilt object individually.  (Useful uor PyPy) */
+long stm_identityhash(object_t *obj);
+long stm_id(object_t *obj);
+void stm_set_prebuilt_identityhash(object_t *obj, long hash);
+
+/* Returns 1 if the object can still move (it's in the nursery), or 0
+   otherwise.  After a minor collection no object can move any more. */
+long stm_can_move(object_t *obj);
+
+/* If the current transaction aborts later, invoke 'callback(key)'.  If
+   the current transaction commits, then the callback is forgotten.  You
+   can only register one callback per key.  You can call
+   'stm_call_on_abort(key, NULL)' to cancel an existing callback
+   (returns 0 if there was no existing callback to cancel).
+   Note: 'key' must be aligned to a multiple of 8 bytes. */
+long stm_call_on_abort(stm_thread_local_t *, void *key, void callback(void *));
+/* If the current transaction commits later, invoke 'callback(key)'.  If
+   the current transaction aborts, then the callback is forgotten.  Same
+   restrictions as stm_call_on_abort().  If the transaction is or becomes
+   inevitable, 'callback(key)' is called immediately. */
+long stm_call_on_commit(stm_thread_local_t *, void *key, void callback(void *));
+
+
+/* Similar to stm_become_inevitable(), but additionally suspend all
+   other threads.  A very heavy-handed way to make sure that no other
+   transaction is running concurrently.  Avoid as much as possible.
+   Other transactions will continue running only after this transaction
+   commits.  (xxx deprecated and may be removed) */
 void stm_become_globally_unique_transaction(stm_thread_local_t *tl, const char *msg);
+
+/* Moves the transaction forward in time by validating the read and
+   write set with all commits that happened since the last validation
+   (explicit or implicit). */
 void stm_validate(void);
 
 /* Temporarily stop all the other threads, by waiting until they
@@ -404,8 +531,8 @@
 /* The markers pushed in the shadowstack are an odd number followed by a
    regular object pointer. */
 typedef struct {
-    uintptr_t odd_number;
-    object_t *object;
+    uintptr_t odd_number;  /* marker odd number, or 0 if marker is missing */
+    object_t *object;      /* marker object, or NULL if marker is missing */
 } stm_loc_marker_t;
 extern void (*stmcb_timing_event)(stm_thread_local_t *tl, /* the local thread */
                                   enum stm_event_e event,
@@ -479,6 +606,38 @@
 /* dummies for now: */
 static inline void stm_flush_timing(stm_thread_local_t *tl, int verbose) {}
 
+
+/* Hashtables.  Keys are 64-bit unsigned integers, values are
+   'object_t *'.  Note that the type 'stm_hashtable_t' is not an
+   object type at all; you need to allocate and free it explicitly.
+   If you want to embed the hashtable inside an 'object_t' you
+   probably need a light finalizer to do the freeing. */
+typedef struct stm_hashtable_s stm_hashtable_t;
+typedef TLPREFIX struct stm_hashtable_entry_s stm_hashtable_entry_t;
+
+stm_hashtable_t *stm_hashtable_create(void);
+void stm_hashtable_free(stm_hashtable_t *);
+stm_hashtable_entry_t *stm_hashtable_lookup(object_t *, stm_hashtable_t *,
+                                            uintptr_t key);
+object_t *stm_hashtable_read(object_t *, stm_hashtable_t *, uintptr_t key);
+void stm_hashtable_write(object_t *, stm_hashtable_t *, uintptr_t key,
+                         object_t *nvalue, stm_thread_local_t *);
+void stm_hashtable_write_entry(object_t *hobj, stm_hashtable_entry_t *entry,
+                               object_t *nvalue);
+long stm_hashtable_length_upper_bound(stm_hashtable_t *);
+long stm_hashtable_list(object_t *, stm_hashtable_t *,
+                        stm_hashtable_entry_t **results);
+extern uint32_t stm_hashtable_entry_userdata;
+void stm_hashtable_tracefn(struct object_s *, stm_hashtable_t *,
+                           void (object_t **));
+
+struct stm_hashtable_entry_s {
+    struct object_s header;
+    uint32_t userdata;
+    uintptr_t index;
+    object_t *object;
+};
+
 /* ==================== END ==================== */
 
 static void (*stmcb_expand_marker)(char *segment_base, uintptr_t odd_number,
diff --git a/c8/test/support.py b/c8/test/support.py
--- a/c8/test/support.py
+++ b/c8/test/support.py
@@ -197,6 +197,26 @@
 void stm_enable_light_finalizer(object_t *);
 
 void (*stmcb_finalizer)(object_t *);
+
+typedef struct stm_hashtable_s stm_hashtable_t;
+typedef ... stm_hashtable_entry_t;
+stm_hashtable_t *stm_hashtable_create(void);
+void stm_hashtable_free(stm_hashtable_t *);
+bool _check_hashtable_read(object_t *, stm_hashtable_t *, uintptr_t key);
+object_t *hashtable_read_result;
+bool _check_hashtable_write(object_t *, stm_hashtable_t *, uintptr_t key,
+                            object_t *nvalue, stm_thread_local_t *tl);
+long stm_hashtable_length_upper_bound(stm_hashtable_t *);
+long stm_hashtable_list(object_t *, stm_hashtable_t *,
+                        stm_hashtable_entry_t **results);
+uint32_t stm_hashtable_entry_userdata;
+void stm_hashtable_tracefn(struct object_s *, stm_hashtable_t *,
+                           void trace(object_t **));
+
+void _set_hashtable(object_t *obj, stm_hashtable_t *h);
+stm_hashtable_t *_get_hashtable(object_t *obj);
+uintptr_t _get_entry_index(stm_hashtable_entry_t *entry);
+object_t *_get_entry_object(stm_hashtable_entry_t *entry);
 """)
 
 
@@ -299,6 +319,19 @@
     CHECKED(stm_validate());
 }
 
+object_t *hashtable_read_result;
+
+bool _check_hashtable_read(object_t *hobj, stm_hashtable_t *h, uintptr_t key)
+{
+    CHECKED(hashtable_read_result = stm_hashtable_read(hobj, h, key));
+}
+
+bool _check_hashtable_write(object_t *hobj, stm_hashtable_t *h, uintptr_t key,
+                            object_t *nvalue, stm_thread_local_t *tl)
+{
+    CHECKED(stm_hashtable_write(hobj, h, key, nvalue, tl));
+}
+
 #undef CHECKED
 
 
@@ -326,6 +359,32 @@
     return *WEAKREF_PTR(obj, size);
 }
 
+void _set_hashtable(object_t *obj, stm_hashtable_t *h)
+{
+    stm_char *field_addr = ((stm_char*)obj);
+    field_addr += SIZEOF_MYOBJ; /* header */
+    *(stm_hashtable_t *TLPREFIX *)field_addr = h;
+}
+
+stm_hashtable_t *_get_hashtable(object_t *obj)
+{
+    stm_char *field_addr = ((stm_char*)obj);
+    field_addr += SIZEOF_MYOBJ; /* header */
+    return *(stm_hashtable_t *TLPREFIX *)field_addr;
+}
+
+uintptr_t _get_entry_index(stm_hashtable_entry_t *entry)
+{
+    stm_read((object_t *)entry);
+    return entry->index;
+}
+
+object_t *_get_entry_object(stm_hashtable_entry_t *entry)
+{
+    stm_read((object_t *)entry);
+    return entry->object;
+}
+
 void _set_ptr(object_t *obj, int n, object_t *v)
 {
     long nrefs = (long)((myobj_t*)obj)->type_id - 421420;
@@ -351,11 +410,17 @@
 }
 
 
-
 ssize_t stmcb_size_rounded_up(struct object_s *obj)
 {
     struct myobj_s *myobj = (struct myobj_s*)obj;
+    assert(myobj->type_id != 0);
     if (myobj->type_id < 421420) {
+        if (myobj->type_id == 421419) {    /* hashtable */
+            return sizeof(struct myobj_s) + 1 * sizeof(void*);
+        }
+        if (myobj->type_id == 421418) {    /* hashtable entry */
+            return sizeof(struct stm_hashtable_entry_s);
+        }
         /* basic case: tid equals 42 plus the size of the object */
         assert(myobj->type_id >= 42 + sizeof(struct myobj_s));
         assert((myobj->type_id - 42) >= 16);
@@ -371,11 +436,21 @@
     }
 }
 
-
 void stmcb_trace(struct object_s *obj, void visit(object_t **))
 {
     int i;
     struct myobj_s *myobj = (struct myobj_s*)obj;
+    if (myobj->type_id == 421419) {
+        /* hashtable */
+        stm_hashtable_t *h = *((stm_hashtable_t **)(myobj + 1));
+        stm_hashtable_tracefn(obj, h, visit);
+        return;
+    }
+    if (myobj->type_id == 421418) {
+        /* hashtable entry */
+        object_t **ref = &((struct stm_hashtable_entry_s *)myobj)->object;
+        visit(ref);
+    }
     if (myobj->type_id < 421420) {
         /* basic case: no references */
         return;
@@ -396,6 +471,7 @@
 {
     int i;
     struct myobj_s *myobj = (struct myobj_s*)obj;
+    assert(myobj->type_id != 0);
     assert(myobj->type_id != 421419);
     assert(myobj->type_id != 421418);
     if (myobj->type_id < 421420) {
@@ -413,6 +489,9 @@
                                   uintptr_t offset_itemsize[2])
 {
     struct myobj_s *myobj = (struct myobj_s*)obj;
+    assert(myobj->type_id != 0);
+    assert(myobj->type_id != 421419);
+    assert(myobj->type_id != 421418);
     if (myobj->type_id < 421420) {
         offset_itemsize[0] = SIZEOF_MYOBJ;
         offset_itemsize[1] = 1;
@@ -468,6 +547,7 @@
 CARD_CLEAR = 0
 CARD_MARKED = lib._STM_CARD_MARKED
 CARD_MARKED_OLD = lib._stm_get_transaction_read_version
+lib.stm_hashtable_entry_userdata = 421418
 
 
 class Conflict(Exception):
@@ -530,6 +610,18 @@
     lib._set_type_id(o, tid)
     return o
 
+def stm_allocate_hashtable():
+    o = lib.stm_allocate(16)
+    tid = 421419
+    lib._set_type_id(o, tid)
+    h = lib.stm_hashtable_create()
+    lib._set_hashtable(o, h)
+    return o
+
+def get_hashtable(o):
+    assert lib._get_type_id(o) == 421419
+    return lib._get_hashtable(o)
+
 def stm_get_weakref(o):
     return lib._get_weakref(o)
 
diff --git a/c8/test/test_hashtable.py b/c8/test/test_hashtable.py
new file mode 100644
--- /dev/null
+++ b/c8/test/test_hashtable.py
@@ -0,0 +1,561 @@
+from support import *
+import random
+import py, sys
+
+
+def htget(o, key):
+    h = get_hashtable(o)
+    res = lib._check_hashtable_read(o, h, key)
+    if res:
+        raise Conflict
+    return lib.hashtable_read_result
+
+def htset(o, key, nvalue, tl):
+    h = get_hashtable(o)
+    res = lib._check_hashtable_write(o, h, key, nvalue, tl)
+    if res:
+        raise Conflict
+
+def ht_length_upper_bound(o):
+    h = get_hashtable(o)
+    return lib.stm_hashtable_length_upper_bound(h)
+
+def htitems(o):
+    h = get_hashtable(o)
+    upper_bound = lib.stm_hashtable_length_upper_bound(h)
+    entries = ffi.new("stm_hashtable_entry_t *[]", upper_bound)
+    count = lib.stm_hashtable_list(o, h, entries)
+    assert count <= upper_bound
+    return [(lib._get_entry_index(entries[i]),
+             lib._get_entry_object(entries[i])) for i in range(count)]
+
+def htlen(o):
+    h = get_hashtable(o)
+    count = lib.stm_hashtable_list(o, h, ffi.NULL)
+    return count
+
+
+class BaseTestHashtable(BaseTest):
+
+    def setup_method(self, meth):
+        BaseTest.setup_method(self, meth)
+        #
+        @ffi.callback("void(object_t *)")
+        def light_finalizer(obj):
+            print 'light_finalizer:', obj
+            try:
+                assert lib._get_type_id(obj) == 421419
+                self.seen_hashtables -= 1
+            except:
+                self.errors.append(sys.exc_info()[2])
+                raise
+
+        lib.stmcb_light_finalizer = light_finalizer
+        self._light_finalizer_keepalive = light_finalizer
+        self.seen_hashtables = 0
+        self.errors = []
+
+    def teardown_method(self, meth):
+        BaseTest.teardown_method(self, meth)
+        lib.stmcb_light_finalizer = ffi.NULL
+        assert self.errors == []
+        assert self.seen_hashtables == 0
+
+    def allocate_hashtable(self):
+        h = stm_allocate_hashtable()
+        lib.stm_enable_light_finalizer(h)
+        self.seen_hashtables += 1
+        return h
+
+
+class TestHashtable(BaseTestHashtable):
+
+    def test_empty(self):
+        self.start_transaction()
+        h = self.allocate_hashtable()
+        for i in range(100):
+            index = random.randrange(0, 1<<64)
+            got = htget(h, index)
+            assert got == ffi.NULL
+
+    def test_set_value(self):
+        self.start_transaction()
+        tl0 = self.tls[self.current_thread]
+        h = self.allocate_hashtable()
+        lp1 = stm_allocate(16)
+        htset(h, 12345678901, lp1, tl0)
+        assert htget(h, 12345678901) == lp1
+        for i in range(64):
+            index = 12345678901 ^ (1 << i)
+            assert htget(h, index) == ffi.NULL
+        assert htget(h, 12345678901) == lp1
+
+    def test_no_conflict(self):
+        lp1 = stm_allocate_old(16)
+        lp2 = stm_allocate_old(16)
+        #
+        self.start_transaction()
+        tl0 = self.tls[self.current_thread]
+        h = self.allocate_hashtable()
+        self.push_root(h)
+        stm_set_char(lp1, 'A')
+        htset(h, 1234, lp1, tl0)
+        self.commit_transaction()
+        #
+        self.start_transaction()
+        h = self.pop_root()
+        stm_set_char(lp2, 'B')
+        self.switch(1)
+        self.start_transaction()
+        self.switch(0)
+        htset(h, 9991234, lp2, tl0)
+        #
+        self.switch(1)
+        lp1b = htget(h, 1234)
+        assert lp1b != ffi.NULL
+        assert stm_get_char(lp1b) == 'A'
+        assert lp1b == lp1
+        self.commit_transaction()
+        #
+        self.switch(0)
+        assert htget(h, 9991234) == lp2
+        assert stm_get_char(lp2) == 'B'
+        assert htget(h, 1234) == lp1
+        htset(h, 1234, ffi.NULL, tl0)
+        self.commit_transaction()
+        #
+        self.start_transaction()
+        stm_major_collect()       # to get rid of the hashtable object
+
+    def test_conflict(self):
+        lp1 = stm_allocate_old(16)
+        lp2 = stm_allocate_old(16)
+        #
+        self.start_transaction()
+        h = self.allocate_hashtable()
+        self.push_root(h)
+        self.commit_transaction()
+        #
+        self.start_transaction()
+        h = self.pop_root()
+        self.push_root(h)
+        tl0 = self.tls[self.current_thread]
+        htset(h, 1234, lp1, tl0)
+        #
+        self.switch(1)
+        self.start_transaction()
+        tl1 = self.tls[self.current_thread]
+        htset(h, 1234, lp2, tl1)
+        #
+        self.switch(0)
+        self.commit_transaction()
+        #
+        py.test.raises(Conflict, self.switch, 1)
+        #
+        self.switch(0)
+        self.start_transaction()
+        self.pop_root()
+        stm_major_collect()       # to get rid of the hashtable object
+        self.commit_transaction()
+
+    def test_keepalive_minor(self):
+        self.start_transaction()
+        h = self.allocate_hashtable()
+        self.push_root(h)
+        lp1 = stm_allocate(16)
+        stm_set_char(lp1, 'N')
+        tl0 = self.tls[self.current_thread]
+        htset(h, 1234, lp1, tl0)
+        stm_minor_collect()
+        h = self.pop_root()
+        lp1b = htget(h, 1234)
+        assert lp1b != ffi.NULL
+        assert stm_get_char(lp1b) == 'N'
+        assert lp1b != lp1
+
+    def test_keepalive_major(self):
+        lp1 = stm_allocate_old(16)
+        #
+        self.start_transaction()
+        h = self.allocate_hashtable()
+        self.push_root(h)
+        stm_set_char(lp1, 'N')
+        tl0 = self.tls[self.current_thread]
+        htset(h, 1234, lp1, tl0)
+        self.commit_transaction()
+        #
+        self.start_transaction()
+        stm_major_collect()
+        h = self.pop_root()
+        lp1b = htget(h, 1234)
+        assert lp1b == lp1
+        assert stm_get_char(lp1b) == 'N'
+        #
+        stm_major_collect()       # to get rid of the hashtable object
+        self.commit_transaction()
+
+    def test_minor_collect_bug1(self):
+        self.start_transaction()
+        lp1 = stm_allocate(32)
+        self.push_root(lp1)
+        h = self.allocate_hashtable()
+        self.push_root(h)
+        stm_minor_collect()
+        h = self.pop_root()
+        lp1 = self.pop_root()
+        print 'h', h                       # 0xa040010
+        print 'lp1', lp1                   # 0xa040040
+        tl0 = self.tls[self.current_thread]
+        htset(h, 1, lp1, tl0)
+        self.commit_transaction()
+        #
+        self.start_transaction()
+        assert htget(h, 1) == lp1
+        stm_major_collect()       # to get rid of the hashtable object
+
+    def test_minor_collect_bug1_different_thread(self):
+        self.start_transaction()
+        lp1 = stm_allocate(32)
+        self.push_root(lp1)
+        h = self.allocate_hashtable()
+        self.push_root(h)
+        stm_minor_collect()
+        h = self.pop_root()
+        lp1 = self.pop_root()
+        print 'h', h                       # 0xa040010
+        print 'lp1', lp1                   # 0xa040040
+        tl0 = self.tls[self.current_thread]
+        htset(h, 1, lp1, tl0)
+        self.commit_transaction()
+        #
+        self.switch(1)            # in a different thread
+        self.start_transaction()
+        assert htget(h, 1) == lp1
+        stm_major_collect()       # to get rid of the hashtable object
+
+    def test_major_collect_bug2(self):
+        self.start_transaction()
+        lp1 = stm_allocate(24)
+        self.push_root(lp1)
+        self.commit_transaction()
+        lp1 = self.pop_root()
+        #
+        self.switch(1)
+        self.start_transaction()
+        stm_write(lp1)            # force this page to be shared
+        #
+        self.switch(0)
+        self.start_transaction()
+        h = self.allocate_hashtable()
+        tl0 = self.tls[self.current_thread]
+        htset(h, 10, stm_allocate(32), tl0)
+        htset(h, 11, stm_allocate(32), tl0)
+        htset(h, 12, stm_allocate(32), tl0)
+        self.push_root(h)
+        #
+        self.switch(1)            # in a different thread
+        stm_major_collect()       # force a _stm_rehash_hashtable()
+        #
+        self.switch(0)            # back to the original thread
+        h = self.pop_root()
+        assert htget(h, 10) != ffi.NULL
+        assert htget(h, 11) != ffi.NULL
+        assert htget(h, 12) != ffi.NULL
+
+    def test_list_1(self):
+        self.start_transaction()
+        h = self.allocate_hashtable()
+        tl0 = self.tls[self.current_thread]
+        for i in range(32):
+            assert ht_length_upper_bound(h) == i
+            htset(h, 19 ^ i, stm_allocate(32), tl0)
+        assert ht_length_upper_bound(h) == 32
+
+    def test_list_2(self):
+        self.start_transaction()
+        h = self.allocate_hashtable()
+        tl0 = self.tls[self.current_thread]
+        expected = []
+        for i in range(29):
+            lp = stm_allocate(32)
+            htset(h, 19 ^ i, lp, tl0)
+            expected.append((19 ^ i, lp))
+        lst = htitems(h)
+        assert len(lst) == 29
+        assert sorted(lst) == sorted(expected)
+
+    def test_list_3(self):
+        self.start_transaction()
+        h = self.allocate_hashtable()
+        tl0 = self.tls[self.current_thread]
+        for i in range(29):
+            htset(h, 19 ^ i, stm_allocate(32), tl0)
+        assert htlen(h) == 29
+
+    def test_len_conflicts_with_additions(self):
+        self.start_transaction()
+        h = self.allocate_hashtable()
+        self.push_root(h)
+        self.commit_transaction()
+        #
+        self.start_transaction()
+        h = self.pop_root()
+        self.push_root(h)
+        tl0 = self.tls[self.current_thread]
+        htset(h, 10, stm_allocate(32), tl0)
+        #
+        self.switch(1)
+        self.start_transaction()
+        assert htlen(h) == 0
+        #
+        self.switch(0)
+        self.commit_transaction()
+        #
+        py.test.raises(Conflict, self.switch, 1)
+        #
+        self.switch(0)
+        self.start_transaction()
+        self.pop_root()
+        stm_major_collect()       # to get rid of the hashtable object
+        self.commit_transaction()
+
+    def test_grow_without_conflict(self):
+        self.start_transaction()
+        h = self.allocate_hashtable()
+        self.push_root(h)
+        self.commit_transaction()
+        h = self.pop_root()
+        self.push_root(h)
+        #
+        STEPS = 50
+        for i in range(STEPS):
+            self.switch(1)
+            self.start_transaction()
+            tl0 = self.tls[self.current_thread]
+            htset(h, i + STEPS, stm_allocate(32), tl0)
+            #
+            self.switch(0)
+            self.start_transaction()
+            tl0 = self.tls[self.current_thread]
+            htset(h, i, stm_allocate(24), tl0)
+            #
+            self.switch(1)
+            self.commit_transaction()
+            #
+            self.switch(0)
+            self.commit_transaction()
+        #
+        self.pop_root()
+        self.start_transaction()
+        stm_major_collect()       # to get rid of the hashtable object
+
+
+class TestRandomHashtable(BaseTestHashtable):
+
+    def setup_method(self, meth):
+        BaseTestHashtable.setup_method(self, meth)
+        self.values = []
+        self.mirror = None
+        self.roots = []
+        self.other_thread = ([], [])
+
+    def push_roots(self):
+        assert self.roots is None
+        self.roots = []
+        for k, hitems in self.mirror.items():
+            assert lib._get_type_id(k) == 421419
+            for key, value in hitems.items():
+                assert lib._get_type_id(value) < 1000

From noreply at buildbot.pypy.org  Mon Jun  1 16:46:53 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  1 Jun 2015 16:46:53 +0200 (CEST)
Subject: [pypy-commit] stmgc c8-locking: Turn 'modification_lock' into one
 real POSIX read-write lock per
Message-ID: <20150601144653.A63C81C04BC@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: c8-locking
Changeset: r1784:f0d995d5609d
Date: 2015-06-01 16:47 +0200
http://bitbucket.org/pypy/stmgc/changeset/f0d995d5609d/

Log:	Turn 'modification_lock' into one real POSIX read-write lock per
	segment.

diff --git a/c8/LOCKS b/c8/LOCKS
--- a/c8/LOCKS
+++ b/c8/LOCKS
@@ -41,6 +41,9 @@
 any more.
 
 
+--- UPDATE: modification_lock is now done with pthread_rwlock_xxx().
+
+
 
 privatization_lock
 ==================
diff --git a/c8/stm/core.c b/c8/stm/core.c
--- a/c8/stm/core.c
+++ b/c8/stm/core.c
@@ -50,8 +50,8 @@
     char *src_segment_base = (from_segnum >= 0 ? get_segment_base(from_segnum)
                                                : NULL);
 
-    assert(IMPLY(from_segnum >= 0, get_priv_segment(from_segnum)->modification_lock));
-    assert(STM_PSEGMENT->modification_lock);
+    assert(IMPLY(from_segnum >= 0, modification_lock_check_rdlock(from_segnum)));
+    assert(modification_lock_check_wrlock(STM_SEGMENT->segment_num));
 
     long my_segnum = STM_SEGMENT->segment_num;
     DEBUG_EXPECT_SEGFAULT(false);
@@ -131,7 +131,7 @@
                            struct stm_commit_log_entry_s *from,
                            struct stm_commit_log_entry_s *to)
 {
-    assert(STM_PSEGMENT->modification_lock);
+    assert(modification_lock_check_wrlock(STM_SEGMENT->segment_num));
     assert(from->rev_num >= to->rev_num);
     /* walk BACKWARDS the commit log and update the page 'pagenum',
        initially at revision 'from', until we reach the revision 'to'. */
@@ -199,8 +199,8 @@
 
     /* before copying anything, acquire modification locks from our and
        the other segment */
-    uint64_t to_lock = (1UL << copy_from_segnum)| (1UL << my_segnum);
-    acquire_modification_lock_set(to_lock);
+    uint64_t to_lock = (1UL << copy_from_segnum);
+    acquire_modification_lock_set(to_lock, my_segnum);
     pagecopy(get_virtual_page(my_segnum, pagenum),
              get_virtual_page(copy_from_segnum, pagenum));
 
@@ -223,7 +223,7 @@
     if (src_version->rev_num > target_version->rev_num)
         go_to_the_past(pagenum, src_version, target_version);
 
-    release_modification_lock_set(to_lock);
+    release_modification_lock_set(to_lock, my_segnum);
     release_all_privatization_locks();
 }
 
@@ -357,7 +357,7 @@
         }
 
         /* Find the set of segments we need to copy from and lock them: */
-        uint64_t segments_to_lock = 1UL << my_segnum;
+        uint64_t segments_to_lock = 0;
         cl = first_cl;
         while ((next_cl = cl->next) != NULL) {
             if (next_cl == INEV_RUNNING) {
@@ -375,8 +375,8 @@
 
         /* HERE */
 
-        acquire_privatization_lock(STM_SEGMENT->segment_num);
-        acquire_modification_lock_set(segments_to_lock);
+        acquire_privatization_lock(my_segnum);
+        acquire_modification_lock_set(segments_to_lock, my_segnum);
 
 
         /* import objects from first_cl to last_cl: */
@@ -466,8 +466,8 @@
         }
 
         /* done with modifications */
-        release_modification_lock_set(segments_to_lock);
-        release_privatization_lock(STM_SEGMENT->segment_num);
+        release_modification_lock_set(segments_to_lock, my_segnum);
+        release_privatization_lock(my_segnum);
     }
 
     return !needs_abort;
@@ -545,7 +545,7 @@
                time" as the attach to commit log. Otherwise, another thread may
                see the new CL entry, import it, look for backup copies in this
                segment and find the old backup copies! */
-            acquire_modification_lock(STM_SEGMENT->segment_num);
+            acquire_modification_lock_wr(STM_SEGMENT->segment_num);
         }
 
         /* try to attach to commit log: */
@@ -559,7 +559,7 @@
         }
 
         if (is_commit) {
-            release_modification_lock(STM_SEGMENT->segment_num);
+            release_modification_lock_wr(STM_SEGMENT->segment_num);
             /* XXX: unfortunately, if we failed to attach our CL entry,
                we have to re-add the WB_EXECUTED flags before we try to
                validate again because of said condition (s.a) */
@@ -596,7 +596,7 @@
 
         list_clear(STM_PSEGMENT->modified_old_objects);
         STM_PSEGMENT->last_commit_log_entry = new;
-        release_modification_lock(STM_SEGMENT->segment_num);
+        release_modification_lock_wr(STM_SEGMENT->segment_num);
     }
 }
 
@@ -692,7 +692,7 @@
         increment_total_allocated(slice_sz);
         memcpy(bk_slice, realobj + slice_off, slice_sz);
 
-        acquire_modification_lock(STM_SEGMENT->segment_num);
+        acquire_modification_lock_wr(STM_SEGMENT->segment_num);
         /* !! follows layout of "struct stm_undo_s" !! */
         STM_PSEGMENT->modified_old_objects = list_append3(
             STM_PSEGMENT->modified_old_objects,
@@ -700,7 +700,7 @@
             (uintptr_t)bk_slice,  /* bk_addr */
             NEW_SLICE(slice_off, slice_sz));
         dprintf(("> append slice %p, off=%lu, sz=%lu\n", bk_slice, slice_off, slice_sz));
-        release_modification_lock(STM_SEGMENT->segment_num);
+        release_modification_lock_wr(STM_SEGMENT->segment_num);
 
         slice_off += slice_sz;
     }
@@ -1347,7 +1347,7 @@
 #pragma push_macro("STM_SEGMENT")
 #undef STM_PSEGMENT
 #undef STM_SEGMENT
-    assert(get_priv_segment(segment_num)->modification_lock);
+    assert(modification_lock_check_wrlock(segment_num));
 
     struct stm_priv_segment_info_s *pseg = get_priv_segment(segment_num);
     struct list_s *list = pseg->modified_old_objects;
@@ -1409,9 +1409,9 @@
             _reset_object_cards(pseg, item, CARD_CLEAR, false, false);
         });
 
-    acquire_modification_lock(segment_num);
+    acquire_modification_lock_wr(segment_num);
     reset_modified_from_backup_copies(segment_num);
-    release_modification_lock(segment_num);
+    release_modification_lock_wr(segment_num);
     _verify_cards_cleared_in_all_lists(pseg);
 
     stm_thread_local_t *tl = pseg->pub.running_thread;
diff --git a/c8/stm/core.h b/c8/stm/core.h
--- a/c8/stm/core.h
+++ b/c8/stm/core.h
@@ -74,11 +74,6 @@
 struct stm_priv_segment_info_s {
     struct stm_segment_info_s pub;
 
-    /* lock protecting from concurrent modification of
-       'modified_old_objects', page-revision-changes, ...
-       Always acquired in global order of segments to avoid deadlocks. */
-    uint8_t modification_lock;
-
     /* All the old objects (older than the current transaction) that
        the current transaction attempts to modify.  This is used to
        track the STM status: these are old objects that where written
@@ -359,53 +354,3 @@
         release_privatization_lock(l);
     }
 }
-
-
-
-/* Modification locks are used to prevent copying from a segment
-   where either the revision of some pages is inconsistent with the
-   rest, or the modified_old_objects list is being modified (bk_copys).
-
-   Lock ordering: acquire privatization lock around acquiring a set
-   of modification locks!
-*/
-
-static inline void acquire_modification_lock(int segnum)
-{
-    spinlock_acquire(get_priv_segment(segnum)->modification_lock);
-}
-
-static inline void release_modification_lock(int segnum)
-{
-    spinlock_release(get_priv_segment(segnum)->modification_lock);
-}
-
-static inline void acquire_modification_lock_set(uint64_t seg_set)
-{
-    assert(NB_SEGMENTS <= 64);
-    OPT_ASSERT(seg_set < (1 << NB_SEGMENTS));
-
-    /* acquire locks in global order */
-    int i;
-    for (i = 0; i < NB_SEGMENTS; i++) {
-        if ((seg_set & (1 << i)) == 0)
-            continue;
-
-        spinlock_acquire(get_priv_segment(i)->modification_lock);
-    }
-}
-
-static inline void release_modification_lock_set(uint64_t seg_set)
-{
-    assert(NB_SEGMENTS <= 64);
-    OPT_ASSERT(seg_set < (1 << NB_SEGMENTS));
-
-    int i;
-    for (i = 0; i < NB_SEGMENTS; i++) {
-        if ((seg_set & (1 << i)) == 0)
-            continue;
-
-        assert(get_priv_segment(i)->modification_lock);
-        spinlock_release(get_priv_segment(i)->modification_lock);
-    }
-}
diff --git a/c8/stm/forksupport.c b/c8/stm/forksupport.c
--- a/c8/stm/forksupport.c
+++ b/c8/stm/forksupport.c
@@ -120,6 +120,9 @@
        just release these locks early */
     s_mutex_unlock();
 
+    /* Re-init these locks; might be needed after a fork() */
+    setup_modification_locks();
+
 
     /* Unregister all other stm_thread_local_t, mostly as a way to free
        the memory used by the shadowstacks
diff --git a/c8/stm/locks.h b/c8/stm/locks.h
new file mode 100644
--- /dev/null
+++ b/c8/stm/locks.h
@@ -0,0 +1,124 @@
+
+/* Modification locks protect from concurrent modification of
+   'modified_old_objects', page-revision-changes, ...
+
+   Modification locks are used to prevent copying from a segment
+   where either the revision of some pages is inconsistent with the
+   rest, or the modified_old_objects list is being modified (bk_copys).
+
+   Lock ordering: acquire privatization lock around acquiring a set
+   of modification locks!
+*/
+
+typedef struct {
+    pthread_rwlock_t lock;
+#ifndef NDEBUG
+    volatile bool write_locked;
+#endif
+} modification_lock_t __attribute__((aligned(64)));
+
+static modification_lock_t _modlocks[NB_SEGMENTS - 1];
+
+
+static void setup_modification_locks(void)
+{
+    int i;
+    for (i = 1; i < NB_SEGMENTS; i++) {
+        if (pthread_rwlock_init(&_modlocks[i - 1].lock, NULL) != 0)
+            stm_fatalerror("pthread_rwlock_init: %m");
+    }
+}
+
+static void teardown_modification_locks(void)
+{
+    int i;
+    for (i = 1; i < NB_SEGMENTS; i++)
+        pthread_rwlock_destroy(&_modlocks[i - 1].lock);
+    memset(_modlocks, 0, sizeof(_modlocks));
+}
+
+
+static inline void acquire_modification_lock_wr(int segnum)
+{
+    if (UNLIKELY(pthread_rwlock_wrlock(&_modlocks[segnum - 1].lock) != 0))
+        stm_fatalerror("pthread_rwlock_wrlock: %m");
+#ifndef NDEBUG
+    assert(!_modlocks[segnum - 1].write_locked);
+    _modlocks[segnum - 1].write_locked = true;
+#endif
+}
+
+static inline void release_modification_lock_wr(int segnum)
+{
+#ifndef NDEBUG
+    assert(_modlocks[segnum - 1].write_locked);
+    _modlocks[segnum - 1].write_locked = false;
+#endif
+    if (UNLIKELY(pthread_rwlock_unlock(&_modlocks[segnum - 1].lock) != 0))
+        stm_fatalerror("pthread_rwlock_unlock(wr): %m");
+}
+
+static void acquire_modification_lock_set(uint64_t readset, int write)
+{
+    /* acquire the modification lock in 'read' mode for all segments
+       in 'readset', plus the modification lock in 'write' mode for
+       the segment number 'write'.
+    */
+    assert(NB_SEGMENTS <= 64);
+    OPT_ASSERT(readset < (1 << NB_SEGMENTS));
+    assert((readset & 1) == 0);       /* segment numbers normally start at 1 */
+    assert(0 <= write && write < NB_SEGMENTS);     /* use 0 to mean "nobody" */
+
+    /* acquire locks in global order */
+    readset |= (1UL << write);
+    int i;
+    for (i = 1; i < NB_SEGMENTS; i++) {
+        if ((readset & (1UL << i)) == 0)
+            continue;
+        if (i == write) {
+            acquire_modification_lock_wr(write);
+        }
+        else {
+            if (UNLIKELY(pthread_rwlock_rdlock(&_modlocks[i - 1].lock) != 0))
+                stm_fatalerror("pthread_rwlock_rdlock: %m");
+        }
+    }
+}
+
+static void release_modification_lock_set(uint64_t readset, int write)
+{
+    assert(NB_SEGMENTS <= 64);
+    OPT_ASSERT(readset < (1 << NB_SEGMENTS));
+
+    /* release lock order does not matter; prefer early release of
+       the write lock */
+    if (write > 0) {
+        release_modification_lock_wr(write);
+        readset &= ~(1UL << write);
+    }
+    int i;
+    for (i = 1; i < NB_SEGMENTS; i++) {
+        if ((readset & (1UL << i)) == 0)
+            continue;
+        if (UNLIKELY(pthread_rwlock_unlock(&_modlocks[i - 1].lock) != 0))
+            stm_fatalerror("pthread_rwlock_unlock(rd): %m");
+    }
+}
+
+#ifndef NDEBUG
+static bool modification_lock_check_rdlock(int segnum)
+{
+    assert(segnum > 0);
+    if (_modlocks[segnum - 1].write_locked)
+        return false;
+    if (pthread_rwlock_trywrlock(&_modlocks[segnum - 1].lock) == 0) {
+        pthread_rwlock_unlock(&_modlocks[segnum - 1].lock);
+        return false;
+    }
+    return true;
+}
+static bool modification_lock_check_wrlock(int segnum)
+{
+    return segnum == 0 || _modlocks[segnum - 1].write_locked;
+}
+#endif
diff --git a/c8/stm/setup.c b/c8/stm/setup.c
--- a/c8/stm/setup.c
+++ b/c8/stm/setup.c
@@ -127,6 +127,7 @@
        private range of addresses.
     */
 
+    setup_modification_locks();
     setup_sync();
     setup_nursery();
     setup_gcpage();
@@ -174,6 +175,7 @@
     teardown_gcpage();
     teardown_smallmalloc();
     teardown_pages();
+    teardown_modification_locks();
 }
 
 static void _shadowstack_trap_page(char *start, int prot)
diff --git a/c8/stmgc.c b/c8/stmgc.c
--- a/c8/stmgc.c
+++ b/c8/stmgc.c
@@ -17,6 +17,7 @@
 #include "stm/marker.h"
 #include "stm/rewind_setjmp.h"
 #include "stm/finalizer.h"
+#include "stm/locks.h"
 
 #include "stm/misc.c"
 #include "stm/list.c"

From noreply at buildbot.pypy.org  Mon Jun  1 16:54:15 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Mon,  1 Jun 2015 16:54:15 +0200 (CEST)
Subject: [pypy-commit] pypy.org extradoc: update for 2.6.0 release
Message-ID: <20150601145415.4A2EF1C04BC@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: extradoc
Changeset: r613:732d0624508d
Date: 2015-06-01 01:12 +0300
http://bitbucket.org/pypy/pypy.org/changeset/732d0624508d/

Log:	update for 2.6.0 release

diff --git a/download.html b/download.html
--- a/download.html
+++ b/download.html
@@ -74,7 +74,7 @@
 performance improvements.</p>
 <p>We provide binaries for x86 and ARM Linux, Mac OS/X and Windows for:</p>
 <ul class="simple">
-<li>the Python2.7 compatible release &#8212; <strong>PyPy 2.5.1</strong> &#8212; (<a class="reference external" href="http://doc.pypy.org/en/latest/release-2.5.1.html">what's new in PyPy 2.5.1?</a>)</li>
+<li>the Python2.7 compatible release &#8212; <strong>PyPy 2.6.0</strong> &#8212; (<a class="reference external" href="http://doc.pypy.org/en/latest/release-2.6.0.html">what's new in PyPy 2.6.0?</a>)</li>
 <li>the Python3.2.5 compatible release &#8212; <strong>PyPy3 2.4.0</strong> &#8212; (<a class="reference external" href="http://doc.pypy.org/en/latest/release-pypy3-2.4.0.html">what's new in PyPy3 2.4.0?</a>).</li>
 <li>the Python2.7 Software Transactional Memory special release &#8212; <strong>PyPy-STM 2.5.1</strong> (Linux x86-64 only)</li>
 </ul>
@@ -113,18 +113,18 @@
 degrees of being up-to-date.</li>
 </ul>
 </div>
-<div class="section" id="python2-7-compatible-pypy-2-5-1">
-<h1>Python2.7 compatible PyPy 2.5.1</h1>
+<div class="section" id="python2-7-compatible-pypy-2-6-0">
+<h1>Python2.7 compatible PyPy 2.6.0</h1>
 <ul class="simple">
-<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy-2.5.1-linux.tar.bz2">Linux x86 binary (32bit, tar.bz2 built on Ubuntu 12.04 - 14.04)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
-<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy-2.5.1-linux64.tar.bz2">Linux x86-64 binary (64bit, tar.bz2 built on Ubuntu 12.04 - 14.04)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
-<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy-2.5.1-linux-armhf-raspbian.tar.bz2">ARM Hardfloat Linux binary (ARMHF/gnueabihf, tar.bz2, Raspbian)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
-<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy-2.5.1-linux-armhf-raring.tar.bz2">ARM Hardfloat Linux binary (ARMHF/gnueabihf, tar.bz2, Ubuntu Raring)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
-<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy-2.5.1-linux-armel.tar.bz2">ARM Softfloat Linux binary (ARMEL/gnueabi, tar.bz2,  Ubuntu Precise)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
-<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy-2.5.1-osx64.tar.bz2">Mac OS/X binary (64bit)</a></li>
-<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy-2.5.1-win32.zip">Windows binary (32bit)</a> (you might need the VS 2008 runtime library
+<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy-2.6.0-linux.tar.bz2">Linux x86 binary (32bit, tar.bz2 built on Ubuntu 12.04 - 14.04)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
+<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy-2.6.0-linux64.tar.bz2">Linux x86-64 binary (64bit, tar.bz2 built on Ubuntu 12.04 - 14.04)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
+<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy-2.6.0-linux-armhf-raspbian.tar.bz2">ARM Hardfloat Linux binary (ARMHF/gnueabihf, tar.bz2, Raspbian)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
+<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy-2.6.0-linux-armhf-raring.tar.bz2">ARM Hardfloat Linux binary (ARMHF/gnueabihf, tar.bz2, Ubuntu Raring)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
+<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy-2.6.0-linux-armel.tar.bz2">ARM Softfloat Linux binary (ARMEL/gnueabi, tar.bz2,  Ubuntu Precise)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
+<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy-2.6.0-osx64.tar.bz2">Mac OS/X binary (64bit)</a></li>
+<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy-2.6.0-win32.zip">Windows binary (32bit)</a> (you might need the VS 2008 runtime library
 installer <a class="reference external" href="http://www.microsoft.com/en-us/download/details.aspx?id=5582">vcredist_x86.exe</a>.)</li>
-<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy-2.5.1-src.tar.bz2">Source (tar.bz2)</a>; <a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy-2.5.1-src.zip">Source (zip)</a>.  See below for more about the sources.</li>
+<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy-2.6.0-src.tar.bz2">Source (tar.bz2)</a>; <a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy-2.6.0-src.zip">Source (zip)</a>.  See below for more about the sources.</li>
 <li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads">All our downloads,</a> including previous versions.  We also have a
 <a class="reference external" href="http://cobra.cs.uni-duesseldorf.de/~buildmaster/mirror/">mirror</a>, but please use only if you have troubles accessing the links above</li>
 </ul>
@@ -190,7 +190,7 @@
 uncompressed, they run in-place.  For now you can uncompress them
 either somewhere in your home directory or, say, in <tt class="docutils literal">/opt</tt>, and
 if you want, put a symlink from somewhere like
-<tt class="docutils literal">/usr/local/bin/pypy</tt> to <tt class="docutils literal"><span class="pre">/path/to/pypy-2.5.1/bin/pypy</span></tt>.  Do
+<tt class="docutils literal">/usr/local/bin/pypy</tt> to <tt class="docutils literal"><span class="pre">/path/to/pypy-2.6.0/bin/pypy</span></tt>.  Do
 not move or copy the executable <tt class="docutils literal">pypy</tt> outside the tree &ndash; put
 a symlink to it, otherwise it will not find its libraries.</p>
 </div>
@@ -232,7 +232,7 @@
 <li><p class="first">Get the source code.  The following packages contain the source at
 the same revision as the above binaries:</p>
 <ul class="simple">
-<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy-2.5.1-src.tar.bz2">pypy-2.5.1-src.tar.bz2</a> (sources)</li>
+<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy-2.6.0-src.tar.bz2">pypy-2.6.0-src.tar.bz2</a> (sources)</li>
 </ul>
 <p>Or you can checkout the current trunk using <a class="reference external" href="http://mercurial.selenic.com/">Mercurial</a> (the trunk
 usually works and is of course more up-to-date):</p>
@@ -326,17 +326,17 @@
 <div class="section" id="checksums">
 <h1>Checksums</h1>
 <p>Here are the checksums for each of the downloads</p>
-<p>pypy-2.5.1 md5:</p>
+<p>pypy-2.6.0 md5:</p>
 <pre class="literal-block">
-b3cc9f8a419f9f89c3fac34b39e92e0a  pypy-2.5.1-linux64.tar.bz2
-fe663500fb87d251ebf02917d25dca23  pypy-2.5.1-linux-armel.tar.bz2
-7a0f845baec7a6ccfb57a66f0e7980e9  pypy-2.5.1-linux-armhf-raring.tar.bz2
-27ad5e2ca3b0abd00be74ee707ef9e53  pypy-2.5.1-linux-armhf-raspbian.tar.bz2
-ca07245e27417034a786365947022eb5  pypy-2.5.1-linux.tar.bz2
-c26e06f3de54fdaaaf1830fb7ca99b70  pypy-2.5.1-osx64.tar.bz2
-de4da75efe3e3b1325861c8883504fdc  pypy-2.5.1-src.tar.bz2
-3b309573ea7ec0835dc922a5940a4bdc  pypy-2.5.1-src.zip
-99a77a5610f6a4941ea310d01933e71f  pypy-2.5.1-win32.zip
+7ea431ab25737462e23a65b9c3819de3  pypy-2.6.0-linux64.tar.bz2
+edec421b668d945c3922bb6a543b58c8  pypy-2.6.0-linux-armel.tar.bz2
+884f1e49fb130c0b8216795d7017025a  pypy-2.6.0-linux-armhf-raring.tar.bz2
+f6e7a7ffc775150994319c2d6f932c41  pypy-2.6.0-linux-armhf-raspbian.tar.bz2
+802f5122a691718dddcece43687cb2cf  pypy-2.6.0-linux.tar.bz2
+63d49e5ead794e6a83c9d87b577d806d  pypy-2.6.0-osx64.tar.bz2
+b09ab96f50ab3021d020e321f210e4c0  pypy-2.6.0-src.tar.bz2
+81297e691d861adb0c89e8a94ef44e8b  pypy-2.6.0-src.zip
+6a1e5451e98a19027333368280b465e1  pypy-2.6.0-win32.zip
 </pre>
 <p>pypy3-2.4.0 md5:</p>
 <pre class="literal-block">
@@ -355,17 +355,17 @@
 2c9f0054f3b93a6473f10be35277825a  pypy-1.8-sandbox-linux64.tar.bz2
 009c970b5fa75754ae4c32a5d108a8d4  pypy-1.8-sandbox-linux.tar.bz2
 </pre>
-<p>pypy-2.5.1 sha1:</p>
+<p>pypy-2.6.0 sha1:</p>
 <pre class="literal-block">
-e598559cdf819707d8f89b31605118385323d8e4  pypy-2.5.1-linux64.tar.bz2
-747d75b36960788692a64a652b6397b2ccdda227  pypy-2.5.1-linux-armel.tar.bz2
-f142697aadc5c7dbe9099331668b2f3f851f51a2  pypy-2.5.1-linux-armhf-raring.tar.bz2
-110bd34f0a648dc0b4e3bd80d72ce953276ea54f  pypy-2.5.1-linux-armhf-raspbian.tar.bz2
-97962ccaa3d7eecff95d71abea3514491563d59f  pypy-2.5.1-linux.tar.bz2
-1daf39a6fafa757c7a96189b21dac40071db2284  pypy-2.5.1-osx64.tar.bz2
-e642ad3968e40399cf1989e7b6c70860a5675a65  pypy-2.5.1-src.tar.bz2
-df0ef936ba0e689e3ed9729f1f0569b91d8e0088  pypy-2.5.1-src.zip
-4af985fad28e4eb7d7400c7475acee65ddf3ebcc  pypy-2.5.1-win32.zip
+03374692eac05b5402b8fb16be9284efda5a0440  pypy-2.6.0-linux64.tar.bz2
+a3029480d3da45793b4a754ef86fca76f5aa0664  pypy-2.6.0-linux-armel.tar.bz2
+075864a8a8148c9439f8f1b59690d946d7c44ce8  pypy-2.6.0-linux-armhf-raring.tar.bz2
+0fa9a25781659e2b1d40482af315f5b7e73d7473  pypy-2.6.0-linux-armhf-raspbian.tar.bz2
+792db4424bf1654ee66f6dc7bdddc89746cef3f4  pypy-2.6.0-linux.tar.bz2
+fb1da056f355a19181b1a4e13699119a92807ccc  pypy-2.6.0-osx64.tar.bz2
+d6580ed01c0b963ef3735e810bc750b8d44f11f2  pypy-2.6.0-src.tar.bz2
+b768f87e8db6432bff3970bbb7a664c412fb4e1c  pypy-2.6.0-src.zip
+a4e212172f9656021d69af4baddc24f7139dde8c  pypy-2.6.0-win32.zip
 </pre>
 <p>pypy3-2.4.0 sha1:</p>
 <pre class="literal-block">
diff --git a/source/download.txt b/source/download.txt
--- a/source/download.txt
+++ b/source/download.txt
@@ -14,12 +14,12 @@
 
 We provide binaries for x86 and ARM Linux, Mac OS/X and Windows for:
 
-* the Python2.7 compatible release &#8212; **PyPy 2.5.1** &#8212; (`what's new in PyPy 2.5.1?`_)
+* the Python2.7 compatible release &#8212; **PyPy 2.6.0** &#8212; (`what's new in PyPy 2.6.0?`_)
 * the Python3.2.5 compatible release &#8212; **PyPy3 2.4.0** &#8212; (`what's new in PyPy3 2.4.0?`_).
 
 * the Python2.7 Software Transactional Memory special release &#8212; **PyPy-STM 2.5.1** (Linux x86-64 only)
 
-.. _what's new in PyPy 2.5.1?: http://doc.pypy.org/en/latest/release-2.5.1.html
+.. _what's new in PyPy 2.6.0?: http://doc.pypy.org/en/latest/release-2.6.0.html
 .. _what's new in PyPy3 2.4.0?: http://doc.pypy.org/en/latest/release-pypy3-2.4.0.html
 
 
@@ -73,7 +73,7 @@
 .. _`portable Linux binaries`: https://github.com/squeaky-pl/portable-pypy#portable-pypy-distribution-for-linux
 
 
-Python2.7 compatible PyPy 2.5.1
+Python2.7 compatible PyPy 2.6.0
 -----------------------------------
 
 * `Linux x86 binary (32bit, tar.bz2 built on Ubuntu 12.04 - 14.04)`__ (see ``[1]`` below)
@@ -88,15 +88,15 @@
 * `All our downloads,`__ including previous versions.  We also have a
   mirror_, but please use only if you have troubles accessing the links above
 
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.5.1-linux.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.5.1-linux64.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.5.1-linux-armhf-raspbian.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.5.1-linux-armhf-raring.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.5.1-linux-armel.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.5.1-osx64.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.5.1-win32.zip
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.5.1-src.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.5.1-src.zip
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.6.0-linux.tar.bz2
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.6.0-linux64.tar.bz2
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.6.0-linux-armhf-raspbian.tar.bz2
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.6.0-linux-armhf-raring.tar.bz2
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.6.0-linux-armel.tar.bz2
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.6.0-osx64.tar.bz2
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.6.0-win32.zip
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.6.0-src.tar.bz2
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.6.0-src.zip
 .. _`vcredist_x86.exe`: http://www.microsoft.com/en-us/download/details.aspx?id=5582
 .. __: https://bitbucket.org/pypy/pypy/downloads
 .. _mirror: http://cobra.cs.uni-duesseldorf.de/~buildmaster/mirror/
@@ -190,7 +190,7 @@
 uncompressed, they run in-place.  For now you can uncompress them
 either somewhere in your home directory or, say, in ``/opt``, and
 if you want, put a symlink from somewhere like
-``/usr/local/bin/pypy`` to ``/path/to/pypy-2.5.1/bin/pypy``.  Do
+``/usr/local/bin/pypy`` to ``/path/to/pypy-2.6.0/bin/pypy``.  Do
 not move or copy the executable ``pypy`` outside the tree --- put
 a symlink to it, otherwise it will not find its libraries.
 
@@ -246,9 +246,9 @@
 1. Get the source code.  The following packages contain the source at
    the same revision as the above binaries:
 
-   * `pypy-2.5.1-src.tar.bz2`__ (sources)
+   * `pypy-2.6.0-src.tar.bz2`__ (sources)
 
-   .. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.5.1-src.tar.bz2
+   .. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.6.0-src.tar.bz2
 
    Or you can checkout the current trunk using Mercurial_ (the trunk
    usually works and is of course more up-to-date)::
@@ -353,17 +353,17 @@
 
 Here are the checksums for each of the downloads
 
-pypy-2.5.1 md5::
+pypy-2.6.0 md5::
 
-   b3cc9f8a419f9f89c3fac34b39e92e0a  pypy-2.5.1-linux64.tar.bz2
-   fe663500fb87d251ebf02917d25dca23  pypy-2.5.1-linux-armel.tar.bz2
-   7a0f845baec7a6ccfb57a66f0e7980e9  pypy-2.5.1-linux-armhf-raring.tar.bz2
-   27ad5e2ca3b0abd00be74ee707ef9e53  pypy-2.5.1-linux-armhf-raspbian.tar.bz2
-   ca07245e27417034a786365947022eb5  pypy-2.5.1-linux.tar.bz2
-   c26e06f3de54fdaaaf1830fb7ca99b70  pypy-2.5.1-osx64.tar.bz2
-   de4da75efe3e3b1325861c8883504fdc  pypy-2.5.1-src.tar.bz2
-   3b309573ea7ec0835dc922a5940a4bdc  pypy-2.5.1-src.zip
-   99a77a5610f6a4941ea310d01933e71f  pypy-2.5.1-win32.zip
+    7ea431ab25737462e23a65b9c3819de3  pypy-2.6.0-linux64.tar.bz2
+    edec421b668d945c3922bb6a543b58c8  pypy-2.6.0-linux-armel.tar.bz2
+    884f1e49fb130c0b8216795d7017025a  pypy-2.6.0-linux-armhf-raring.tar.bz2
+    f6e7a7ffc775150994319c2d6f932c41  pypy-2.6.0-linux-armhf-raspbian.tar.bz2
+    802f5122a691718dddcece43687cb2cf  pypy-2.6.0-linux.tar.bz2
+    63d49e5ead794e6a83c9d87b577d806d  pypy-2.6.0-osx64.tar.bz2
+    b09ab96f50ab3021d020e321f210e4c0  pypy-2.6.0-src.tar.bz2
+    81297e691d861adb0c89e8a94ef44e8b  pypy-2.6.0-src.zip
+    6a1e5451e98a19027333368280b465e1  pypy-2.6.0-win32.zip
 
 pypy3-2.4.0 md5::
 
@@ -383,17 +383,17 @@
    2c9f0054f3b93a6473f10be35277825a  pypy-1.8-sandbox-linux64.tar.bz2
    009c970b5fa75754ae4c32a5d108a8d4  pypy-1.8-sandbox-linux.tar.bz2
 
-pypy-2.5.1 sha1::
+pypy-2.6.0 sha1::
 
-   e598559cdf819707d8f89b31605118385323d8e4  pypy-2.5.1-linux64.tar.bz2
-   747d75b36960788692a64a652b6397b2ccdda227  pypy-2.5.1-linux-armel.tar.bz2
-   f142697aadc5c7dbe9099331668b2f3f851f51a2  pypy-2.5.1-linux-armhf-raring.tar.bz2
-   110bd34f0a648dc0b4e3bd80d72ce953276ea54f  pypy-2.5.1-linux-armhf-raspbian.tar.bz2
-   97962ccaa3d7eecff95d71abea3514491563d59f  pypy-2.5.1-linux.tar.bz2
-   1daf39a6fafa757c7a96189b21dac40071db2284  pypy-2.5.1-osx64.tar.bz2
-   e642ad3968e40399cf1989e7b6c70860a5675a65  pypy-2.5.1-src.tar.bz2
-   df0ef936ba0e689e3ed9729f1f0569b91d8e0088  pypy-2.5.1-src.zip
-   4af985fad28e4eb7d7400c7475acee65ddf3ebcc  pypy-2.5.1-win32.zip
+    03374692eac05b5402b8fb16be9284efda5a0440  pypy-2.6.0-linux64.tar.bz2
+    a3029480d3da45793b4a754ef86fca76f5aa0664  pypy-2.6.0-linux-armel.tar.bz2
+    075864a8a8148c9439f8f1b59690d946d7c44ce8  pypy-2.6.0-linux-armhf-raring.tar.bz2
+    0fa9a25781659e2b1d40482af315f5b7e73d7473  pypy-2.6.0-linux-armhf-raspbian.tar.bz2
+    792db4424bf1654ee66f6dc7bdddc89746cef3f4  pypy-2.6.0-linux.tar.bz2
+    fb1da056f355a19181b1a4e13699119a92807ccc  pypy-2.6.0-osx64.tar.bz2
+    d6580ed01c0b963ef3735e810bc750b8d44f11f2  pypy-2.6.0-src.tar.bz2
+    b768f87e8db6432bff3970bbb7a664c412fb4e1c  pypy-2.6.0-src.zip
+    a4e212172f9656021d69af4baddc24f7139dde8c  pypy-2.6.0-win32.zip
 
 pypy3-2.4.0 sha1::
 

From noreply at buildbot.pypy.org  Mon Jun  1 16:59:11 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  1 Jun 2015 16:59:11 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c8: import stmgc/f0d995d5609d,
	branch c8-locking
Message-ID: <20150601145911.E257A1C071B@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c8
Changeset: r77743:a3a33fcdc546
Date: 2015-06-01 15:58 +0100
http://bitbucket.org/pypy/pypy/changeset/a3a33fcdc546/

Log:	import stmgc/f0d995d5609d, branch c8-locking

diff --git a/rpython/translator/stm/src_stm/revision b/rpython/translator/stm/src_stm/revision
--- a/rpython/translator/stm/src_stm/revision
+++ b/rpython/translator/stm/src_stm/revision
@@ -1,1 +1,1 @@
-e55658d12179
+f0d995d5609d
diff --git a/rpython/translator/stm/src_stm/stm/core.c b/rpython/translator/stm/src_stm/stm/core.c
--- a/rpython/translator/stm/src_stm/stm/core.c
+++ b/rpython/translator/stm/src_stm/stm/core.c
@@ -50,8 +50,8 @@
     char *src_segment_base = (from_segnum >= 0 ? get_segment_base(from_segnum)
                                                : NULL);
 
-    assert(IMPLY(from_segnum >= 0, get_priv_segment(from_segnum)->modification_lock));
-    assert(STM_PSEGMENT->modification_lock);
+    assert(IMPLY(from_segnum >= 0, modification_lock_check_rdlock(from_segnum)));
+    assert(modification_lock_check_wrlock(STM_SEGMENT->segment_num));
 
     long my_segnum = STM_SEGMENT->segment_num;
     DEBUG_EXPECT_SEGFAULT(false);
@@ -131,7 +131,7 @@
                            struct stm_commit_log_entry_s *from,
                            struct stm_commit_log_entry_s *to)
 {
-    assert(STM_PSEGMENT->modification_lock);
+    assert(modification_lock_check_wrlock(STM_SEGMENT->segment_num));
     assert(from->rev_num >= to->rev_num);
     /* walk BACKWARDS the commit log and update the page 'pagenum',
        initially at revision 'from', until we reach the revision 'to'. */
@@ -199,8 +199,8 @@
 
     /* before copying anything, acquire modification locks from our and
        the other segment */
-    uint64_t to_lock = (1UL << copy_from_segnum)| (1UL << my_segnum);
-    acquire_modification_lock_set(to_lock);
+    uint64_t to_lock = (1UL << copy_from_segnum);
+    acquire_modification_lock_set(to_lock, my_segnum);
     pagecopy(get_virtual_page(my_segnum, pagenum),
              get_virtual_page(copy_from_segnum, pagenum));
 
@@ -223,7 +223,7 @@
     if (src_version->rev_num > target_version->rev_num)
         go_to_the_past(pagenum, src_version, target_version);
 
-    release_modification_lock_set(to_lock);
+    release_modification_lock_set(to_lock, my_segnum);
     release_all_privatization_locks();
 }
 
@@ -308,7 +308,7 @@
 
 static void reset_modified_from_backup_copies(int segment_num);  /* forward */
 
-static bool _stm_validate()
+static bool _stm_validate(void)
 {
     /* returns true if we reached a valid state, or false if
        we need to abort now */
@@ -357,7 +357,7 @@
         }
 
         /* Find the set of segments we need to copy from and lock them: */
-        uint64_t segments_to_lock = 1UL << my_segnum;
+        uint64_t segments_to_lock = 0;
         cl = first_cl;
         while ((next_cl = cl->next) != NULL) {
             if (next_cl == INEV_RUNNING) {
@@ -375,8 +375,8 @@
 
         /* HERE */
 
-        acquire_privatization_lock(STM_SEGMENT->segment_num);
-        acquire_modification_lock_set(segments_to_lock);
+        acquire_privatization_lock(my_segnum);
+        acquire_modification_lock_set(segments_to_lock, my_segnum);
 
 
         /* import objects from first_cl to last_cl: */
@@ -466,8 +466,8 @@
         }
 
         /* done with modifications */
-        release_modification_lock_set(segments_to_lock);
-        release_privatization_lock(STM_SEGMENT->segment_num);
+        release_modification_lock_set(segments_to_lock, my_segnum);
+        release_privatization_lock(my_segnum);
     }
 
     return !needs_abort;
@@ -545,7 +545,7 @@
                time" as the attach to commit log. Otherwise, another thread may
                see the new CL entry, import it, look for backup copies in this
                segment and find the old backup copies! */
-            acquire_modification_lock(STM_SEGMENT->segment_num);
+            acquire_modification_lock_wr(STM_SEGMENT->segment_num);
         }
 
         /* try to attach to commit log: */
@@ -559,7 +559,7 @@
         }
 
         if (is_commit) {
-            release_modification_lock(STM_SEGMENT->segment_num);
+            release_modification_lock_wr(STM_SEGMENT->segment_num);
             /* XXX: unfortunately, if we failed to attach our CL entry,
                we have to re-add the WB_EXECUTED flags before we try to
                validate again because of said condition (s.a) */
@@ -596,7 +596,7 @@
 
         list_clear(STM_PSEGMENT->modified_old_objects);
         STM_PSEGMENT->last_commit_log_entry = new;
-        release_modification_lock(STM_SEGMENT->segment_num);
+        release_modification_lock_wr(STM_SEGMENT->segment_num);
     }
 }
 
@@ -692,7 +692,7 @@
         increment_total_allocated(slice_sz);
         memcpy(bk_slice, realobj + slice_off, slice_sz);
 
-        acquire_modification_lock(STM_SEGMENT->segment_num);
+        acquire_modification_lock_wr(STM_SEGMENT->segment_num);
         /* !! follows layout of "struct stm_undo_s" !! */
         STM_PSEGMENT->modified_old_objects = list_append3(
             STM_PSEGMENT->modified_old_objects,
@@ -700,7 +700,7 @@
             (uintptr_t)bk_slice,  /* bk_addr */
             NEW_SLICE(slice_off, slice_sz));
         dprintf(("> append slice %p, off=%lu, sz=%lu\n", bk_slice, slice_off, slice_sz));
-        release_modification_lock(STM_SEGMENT->segment_num);
+        release_modification_lock_wr(STM_SEGMENT->segment_num);
 
         slice_off += slice_sz;
     }
@@ -896,6 +896,8 @@
 
 static void touch_all_pages_of_obj(object_t *obj, size_t obj_size)
 {
+    /* XXX should it be simpler, just really trying to read a dummy
+       byte in each page? */
     int my_segnum = STM_SEGMENT->segment_num;
     uintptr_t end_page, first_page = ((uintptr_t)obj) / 4096UL;
 
@@ -1345,7 +1347,7 @@
 #pragma push_macro("STM_SEGMENT")
 #undef STM_PSEGMENT
 #undef STM_SEGMENT
-    assert(get_priv_segment(segment_num)->modification_lock);
+    assert(modification_lock_check_wrlock(segment_num));
 
     struct stm_priv_segment_info_s *pseg = get_priv_segment(segment_num);
     struct list_s *list = pseg->modified_old_objects;
@@ -1407,9 +1409,9 @@
             _reset_object_cards(pseg, item, CARD_CLEAR, false, false);
         });
 
-    acquire_modification_lock(segment_num);
+    acquire_modification_lock_wr(segment_num);
     reset_modified_from_backup_copies(segment_num);
-    release_modification_lock(segment_num);
+    release_modification_lock_wr(segment_num);
     _verify_cards_cleared_in_all_lists(pseg);
 
     stm_thread_local_t *tl = pseg->pub.running_thread;
diff --git a/rpython/translator/stm/src_stm/stm/core.h b/rpython/translator/stm/src_stm/stm/core.h
--- a/rpython/translator/stm/src_stm/stm/core.h
+++ b/rpython/translator/stm/src_stm/stm/core.h
@@ -74,11 +74,6 @@
 struct stm_priv_segment_info_s {
     struct stm_segment_info_s pub;
 
-    /* lock protecting from concurrent modification of
-       'modified_old_objects', page-revision-changes, ...
-       Always acquired in global order of segments to avoid deadlocks. */
-    uint8_t modification_lock;
-
     /* All the old objects (older than the current transaction) that
        the current transaction attempts to modify.  This is used to
        track the STM status: these are old objects that where written
@@ -297,7 +292,7 @@
 static void synchronize_objects_flush(void);
 
 static void _signal_handler(int sig, siginfo_t *siginfo, void *context);
-static bool _stm_validate();
+static bool _stm_validate(void);
 
 static inline bool was_read_remote(char *base, object_t *obj)
 {
@@ -329,7 +324,7 @@
     spinlock_release(get_priv_segment(segnum)->privatization_lock);
 }
 
-static inline bool all_privatization_locks_acquired()
+static inline bool all_privatization_locks_acquired(void)
 {
 #ifndef NDEBUG
     long l;
@@ -343,7 +338,7 @@
 #endif
 }
 
-static inline void acquire_all_privatization_locks()
+static inline void acquire_all_privatization_locks(void)
 {
     /* XXX: don't do for the sharing seg0 */
     long l;
@@ -352,60 +347,10 @@
     }
 }
 
-static inline void release_all_privatization_locks()
+static inline void release_all_privatization_locks(void)
 {
     long l;
     for (l = NB_SEGMENTS-1; l >= 0; l--) {
         release_privatization_lock(l);
     }
 }
-
-
-
-/* Modification locks are used to prevent copying from a segment
-   where either the revision of some pages is inconsistent with the
-   rest, or the modified_old_objects list is being modified (bk_copys).
-
-   Lock ordering: acquire privatization lock around acquiring a set
-   of modification locks!
-*/
-
-static inline void acquire_modification_lock(int segnum)
-{
-    spinlock_acquire(get_priv_segment(segnum)->modification_lock);
-}
-
-static inline void release_modification_lock(int segnum)
-{
-    spinlock_release(get_priv_segment(segnum)->modification_lock);
-}
-
-static inline void acquire_modification_lock_set(uint64_t seg_set)
-{
-    assert(NB_SEGMENTS <= 64);
-    OPT_ASSERT(seg_set < (1 << NB_SEGMENTS));
-
-    /* acquire locks in global order */
-    int i;
-    for (i = 0; i < NB_SEGMENTS; i++) {
-        if ((seg_set & (1 << i)) == 0)
-            continue;
-
-        spinlock_acquire(get_priv_segment(i)->modification_lock);
-    }
-}
-
-static inline void release_modification_lock_set(uint64_t seg_set)
-{
-    assert(NB_SEGMENTS <= 64);
-    OPT_ASSERT(seg_set < (1 << NB_SEGMENTS));
-
-    int i;
-    for (i = 0; i < NB_SEGMENTS; i++) {
-        if ((seg_set & (1 << i)) == 0)
-            continue;
-
-        assert(get_priv_segment(i)->modification_lock);
-        spinlock_release(get_priv_segment(i)->modification_lock);
-    }
-}
diff --git a/rpython/translator/stm/src_stm/stm/forksupport.c b/rpython/translator/stm/src_stm/stm/forksupport.c
--- a/rpython/translator/stm/src_stm/stm/forksupport.c
+++ b/rpython/translator/stm/src_stm/stm/forksupport.c
@@ -120,6 +120,9 @@
        just release these locks early */
     s_mutex_unlock();
 
+    /* Re-init these locks; might be needed after a fork() */
+    setup_modification_locks();
+
 
     /* Unregister all other stm_thread_local_t, mostly as a way to free
        the memory used by the shadowstacks
diff --git a/rpython/translator/stm/src_stm/stm/gcpage.c b/rpython/translator/stm/src_stm/stm/gcpage.c
--- a/rpython/translator/stm/src_stm/stm/gcpage.c
+++ b/rpython/translator/stm/src_stm/stm/gcpage.c
@@ -681,7 +681,7 @@
     _stm_smallmalloc_sweep();
 }
 
-static void clean_up_commit_log_entries()
+static void clean_up_commit_log_entries(void)
 {
     struct stm_commit_log_entry_s *cl, *next;
 
diff --git a/rpython/translator/stm/src_stm/stm/locks.h b/rpython/translator/stm/src_stm/stm/locks.h
new file mode 100644
--- /dev/null
+++ b/rpython/translator/stm/src_stm/stm/locks.h
@@ -0,0 +1,124 @@
+/* Imported by rpython/translator/stm/import_stmgc.py */
+/* Modification locks protect from concurrent modification of
+   'modified_old_objects', page-revision-changes, ...
+
+   Modification locks are used to prevent copying from a segment
+   where either the revision of some pages is inconsistent with the
+   rest, or the modified_old_objects list is being modified (bk_copys).
+
+   Lock ordering: acquire privatization lock around acquiring a set
+   of modification locks!
+*/
+
+typedef struct {
+    pthread_rwlock_t lock;
+#ifndef NDEBUG
+    volatile bool write_locked;
+#endif
+} modification_lock_t __attribute__((aligned(64)));
+
+static modification_lock_t _modlocks[NB_SEGMENTS - 1];
+
+
+static void setup_modification_locks(void)
+{
+    int i;
+    for (i = 1; i < NB_SEGMENTS; i++) {
+        if (pthread_rwlock_init(&_modlocks[i - 1].lock, NULL) != 0)
+            stm_fatalerror("pthread_rwlock_init: %m");
+    }
+}
+
+static void teardown_modification_locks(void)
+{
+    int i;
+    for (i = 1; i < NB_SEGMENTS; i++)
+        pthread_rwlock_destroy(&_modlocks[i - 1].lock);
+    memset(_modlocks, 0, sizeof(_modlocks));
+}
+
+
+static inline void acquire_modification_lock_wr(int segnum)
+{
+    if (UNLIKELY(pthread_rwlock_wrlock(&_modlocks[segnum - 1].lock) != 0))
+        stm_fatalerror("pthread_rwlock_wrlock: %m");
+#ifndef NDEBUG
+    assert(!_modlocks[segnum - 1].write_locked);
+    _modlocks[segnum - 1].write_locked = true;
+#endif
+}
+
+static inline void release_modification_lock_wr(int segnum)
+{
+#ifndef NDEBUG
+    assert(_modlocks[segnum - 1].write_locked);
+    _modlocks[segnum - 1].write_locked = false;
+#endif
+    if (UNLIKELY(pthread_rwlock_unlock(&_modlocks[segnum - 1].lock) != 0))
+        stm_fatalerror("pthread_rwlock_unlock(wr): %m");
+}
+
+static void acquire_modification_lock_set(uint64_t readset, int write)
+{
+    /* acquire the modification lock in 'read' mode for all segments
+       in 'readset', plus the modification lock in 'write' mode for
+       the segment number 'write'.
+    */
+    assert(NB_SEGMENTS <= 64);
+    OPT_ASSERT(readset < (1 << NB_SEGMENTS));
+    assert((readset & 1) == 0);       /* segment numbers normally start at 1 */
+    assert(0 <= write && write < NB_SEGMENTS);     /* use 0 to mean "nobody" */
+
+    /* acquire locks in global order */
+    readset |= (1UL << write);
+    int i;
+    for (i = 1; i < NB_SEGMENTS; i++) {
+        if ((readset & (1UL << i)) == 0)
+            continue;
+        if (i == write) {
+            acquire_modification_lock_wr(write);
+        }
+        else {
+            if (UNLIKELY(pthread_rwlock_rdlock(&_modlocks[i - 1].lock) != 0))
+                stm_fatalerror("pthread_rwlock_rdlock: %m");
+        }
+    }
+}
+
+static void release_modification_lock_set(uint64_t readset, int write)
+{
+    assert(NB_SEGMENTS <= 64);
+    OPT_ASSERT(readset < (1 << NB_SEGMENTS));
+
+    /* release lock order does not matter; prefer early release of
+       the write lock */
+    if (write > 0) {
+        release_modification_lock_wr(write);
+        readset &= ~(1UL << write);
+    }
+    int i;
+    for (i = 1; i < NB_SEGMENTS; i++) {
+        if ((readset & (1UL << i)) == 0)
+            continue;
+        if (UNLIKELY(pthread_rwlock_unlock(&_modlocks[i - 1].lock) != 0))
+            stm_fatalerror("pthread_rwlock_unlock(rd): %m");
+    }
+}
+
+#ifndef NDEBUG
+static bool modification_lock_check_rdlock(int segnum)
+{
+    assert(segnum > 0);
+    if (_modlocks[segnum - 1].write_locked)
+        return false;
+    if (pthread_rwlock_trywrlock(&_modlocks[segnum - 1].lock) == 0) {
+        pthread_rwlock_unlock(&_modlocks[segnum - 1].lock);
+        return false;
+    }
+    return true;
+}
+static bool modification_lock_check_wrlock(int segnum)
+{
+    return segnum == 0 || _modlocks[segnum - 1].write_locked;
+}
+#endif
diff --git a/rpython/translator/stm/src_stm/stm/misc.c b/rpython/translator/stm/src_stm/stm/misc.c
--- a/rpython/translator/stm/src_stm/stm/misc.c
+++ b/rpython/translator/stm/src_stm/stm/misc.c
@@ -44,7 +44,7 @@
     return obj->stm_flags & _STM_GCFLAG_CARDS_SET;
 }
 
-long _stm_count_cl_entries()
+long _stm_count_cl_entries(void)
 {
     struct stm_commit_log_entry_s *cl = &commit_log_root;
 
@@ -115,7 +115,7 @@
     return cards[get_index_to_card_index(idx)].rm;
 }
 
-uint8_t _stm_get_transaction_read_version()
+uint8_t _stm_get_transaction_read_version(void)
 {
     return STM_SEGMENT->transaction_read_version;
 }
@@ -124,7 +124,7 @@
 
 static struct stm_commit_log_entry_s *_last_cl_entry;
 static long _last_cl_entry_index;
-void _stm_start_enum_last_cl_entry()
+void _stm_start_enum_last_cl_entry(void)
 {
     _last_cl_entry = &commit_log_root;
     struct stm_commit_log_entry_s *cl = &commit_log_root;
@@ -135,7 +135,7 @@
     _last_cl_entry_index = 0;
 }
 
-object_t *_stm_next_last_cl_entry()
+object_t *_stm_next_last_cl_entry(void)
 {
     if (_last_cl_entry == &commit_log_root)
         return NULL;
@@ -150,7 +150,7 @@
 }
 
 
-void _stm_smallmalloc_sweep_test()
+void _stm_smallmalloc_sweep_test(void)
 {
     acquire_all_privatization_locks();
     _stm_smallmalloc_sweep();
diff --git a/rpython/translator/stm/src_stm/stm/setup.c b/rpython/translator/stm/src_stm/stm/setup.c
--- a/rpython/translator/stm/src_stm/stm/setup.c
+++ b/rpython/translator/stm/src_stm/stm/setup.c
@@ -127,6 +127,7 @@
        private range of addresses.
     */
 
+    setup_modification_locks();
     setup_sync();
     setup_nursery();
     setup_gcpage();
@@ -174,6 +175,7 @@
     teardown_gcpage();
     teardown_smallmalloc();
     teardown_pages();
+    teardown_modification_locks();
 }
 
 static void _shadowstack_trap_page(char *start, int prot)
diff --git a/rpython/translator/stm/src_stm/stmgc.c b/rpython/translator/stm/src_stm/stmgc.c
--- a/rpython/translator/stm/src_stm/stmgc.c
+++ b/rpython/translator/stm/src_stm/stmgc.c
@@ -18,6 +18,7 @@
 #include "stm/marker.h"
 #include "stm/rewind_setjmp.h"
 #include "stm/finalizer.h"
+#include "stm/locks.h"
 #include "stm/misc.c"
 #include "stm/list.c"
 #include "stm/smallmalloc.c"
diff --git a/rpython/translator/stm/src_stm/stmgc.h b/rpython/translator/stm/src_stm/stmgc.h
--- a/rpython/translator/stm/src_stm/stmgc.h
+++ b/rpython/translator/stm/src_stm/stmgc.h
@@ -57,13 +57,16 @@
 typedef struct stm_thread_local_s {
     /* rewind_setjmp's interface */
     rewind_jmp_thread rjthread;
+    /* every thread should handle the shadow stack itself */
     struct stm_shadowentry_s *shadowstack, *shadowstack_base;
-
     /* a generic optional thread-local object */
     object_t *thread_local_obj;
-
+    /* in case this thread runs a transaction that aborts,
+       the following raw region of memory is cleared. */
     char *mem_clear_on_abort;
     size_t mem_bytes_to_clear_on_abort;
+    /* after an abort, some details about the abort are stored there.
+       (this field is not modified on a successful commit) */
     long last_abort__bytes_in_nursery;
     /* the next fields are handled internally by the library */
     int associated_segment_num;
@@ -73,34 +76,22 @@
     void *creating_pthread[2];
 } stm_thread_local_t;
 
-#ifndef _STM_NURSERY_ZEROED
-#define _STM_NURSERY_ZEROED               0
-#endif
 
-#define _STM_GCFLAG_WRITE_BARRIER      0x01
-#define _STM_FAST_ALLOC           (66*1024)
-#define _STM_NSE_SIGNAL_ABORT             1
-#define _STM_NSE_SIGNAL_MAX               2
-
-#define _STM_CARD_MARKED 1      /* should always be 1... */
-#define _STM_GCFLAG_CARDS_SET          0x8
-#define _STM_CARD_BITS                 5   /* must be 5/6/7 for the pypy jit */
-#define _STM_CARD_SIZE                 (1 << _STM_CARD_BITS)
-#define _STM_MIN_CARD_COUNT            17
-#define _STM_MIN_CARD_OBJ_SIZE         (_STM_CARD_SIZE * _STM_MIN_CARD_COUNT)
-
+/* this should use llvm's coldcc calling convention,
+   but it's not exposed to C code so far */
 void _stm_write_slowpath(object_t *);
 void _stm_write_slowpath_card(object_t *, uintptr_t);
 object_t *_stm_allocate_slowpath(ssize_t);
 object_t *_stm_allocate_external(ssize_t);
 void _stm_become_inevitable(const char*);
-void _stm_collectable_safe_point();
+void _stm_collectable_safe_point(void);
 
+/* for tests, but also used in duhton: */
 object_t *_stm_allocate_old(ssize_t size_rounded_up);
 char *_stm_real_address(object_t *o);
 #ifdef STM_TESTS
 #include <stdbool.h>
-uint8_t _stm_get_transaction_read_version();
+uint8_t _stm_get_transaction_read_version(void);
 uint8_t _stm_get_card_value(object_t *obj, long idx);
 bool _stm_was_read(object_t *obj);
 bool _stm_was_written(object_t *obj);
@@ -137,14 +128,32 @@
 long _stm_count_objects_pointing_to_nursery(void);
 object_t *_stm_enum_modified_old_objects(long index);
 object_t *_stm_enum_objects_pointing_to_nursery(long index);
-object_t *_stm_next_last_cl_entry();
-void _stm_start_enum_last_cl_entry();
-long _stm_count_cl_entries();
+object_t *_stm_next_last_cl_entry(void);
+void _stm_start_enum_last_cl_entry(void);
+long _stm_count_cl_entries(void);
 long _stm_count_old_objects_with_cards_set(void);
 object_t *_stm_enum_old_objects_with_cards_set(long index);
 uint64_t _stm_total_allocated(void);
 #endif
 
+
+#ifndef _STM_NURSERY_ZEROED
+#define _STM_NURSERY_ZEROED               0
+#endif
+
+#define _STM_GCFLAG_WRITE_BARRIER      0x01
+#define _STM_FAST_ALLOC           (66*1024)
+#define _STM_NSE_SIGNAL_ABORT             1
+#define _STM_NSE_SIGNAL_MAX               2
+
+#define _STM_CARD_MARKED 1      /* should always be 1... */
+#define _STM_GCFLAG_CARDS_SET          0x8
+#define _STM_CARD_BITS                 5   /* must be 5/6/7 for the pypy jit */
+#define _STM_CARD_SIZE                 (1 << _STM_CARD_BITS)
+#define _STM_MIN_CARD_COUNT            17
+#define _STM_MIN_CARD_OBJ_SIZE         (_STM_CARD_SIZE * _STM_MIN_CARD_COUNT)
+
+
 /* ==================== HELPERS ==================== */
 #ifdef NDEBUG
 #define OPT_ASSERT(cond) do { if (!(cond)) __builtin_unreachable(); } while (0)
@@ -165,30 +174,32 @@
 */
 #define STM_NB_SEGMENTS    4
 
+/* Structure of objects
+   --------------------
 
+   Objects manipulated by the user program, and managed by this library,
+   must start with a "struct object_s" field.  Pointers to any user object
+   must use the "TLPREFIX struct foo *" type --- don't forget TLPREFIX.
+   The best is to use typedefs like above.
+
+   The object_s part contains some fields reserved for the STM library.
+   Right now this is only four bytes.
+*/
 struct object_s {
     uint32_t stm_flags;            /* reserved for the STM library */
 };
 
-extern ssize_t stmcb_size_rounded_up(struct object_s *);
-void stmcb_trace(struct object_s *obj, void visit(object_t **));
-/* a special trace-callback that is only called for the marked
-   ranges of indices (using stm_write_card(o, index)) */
-extern void stmcb_trace_cards(struct object_s *, void (object_t **),
-                              uintptr_t start, uintptr_t stop);
-/* this function will be called on objects that support cards.
-   It returns the base_offset (in bytes) inside the object from
-   where the indices start, and item_size (in bytes) for the size of
-   one item */
-extern void stmcb_get_card_base_itemsize(struct object_s *,
-                                         uintptr_t offset_itemsize[2]);
-/* returns whether this object supports cards. we will only call
-   stmcb_get_card_base_itemsize on objs that do so. */
-extern long stmcb_obj_supports_cards(struct object_s *);
 
-
-
-
+/* The read barrier must be called whenever the object 'obj' is read.
+   It is not required to call it before reading: it can be delayed for a
+   bit, but we must still be in the same "scope": no allocation, no
+   transaction commit, nothing that can potentially collect or do a safe
+   point (like stm_write() on a different object).  Also, if we might
+   have finished the transaction and started the next one, then
+   stm_read() needs to be called again.  It can be omitted if
+   stm_write() is called, or immediately after getting the object from
+   stm_allocate(), as long as the rules above are respected.
+*/
 __attribute__((always_inline))
 static inline void stm_read(object_t *obj)
 {
@@ -199,6 +210,11 @@
 #define _STM_WRITE_CHECK_SLOWPATH(obj)  \
     UNLIKELY(((obj)->stm_flags & _STM_GCFLAG_WRITE_BARRIER) != 0)
 
+/* The write barrier must be called *before* doing any change to the
+   object 'obj'.  If we might have finished the transaction and started
+   the next one, then stm_write() needs to be called again.  It is not
+   necessary to call it immediately after stm_allocate().
+*/
 __attribute__((always_inline))
 static inline void stm_write(object_t *obj)
 {
@@ -206,7 +222,14 @@
         _stm_write_slowpath(obj);
 }
 
-
+/* The following is a GC-optimized barrier that works on the granularity
+   of CARD_SIZE.  It can be used on any array object, but it is only
+   useful with those that were internally marked with GCFLAG_HAS_CARDS.
+   It has the same purpose as stm_write() for TM and allows write-access
+   to a part of an object/array.
+   'index' is the array-item-based position within the object, which
+   is measured in units returned by stmcb_get_card_base_itemsize().
+*/
 __attribute__((always_inline))
 static inline void stm_write_card(object_t *obj, uintptr_t index)
 {
@@ -245,7 +268,34 @@
     }
 }
 
+/* Must be provided by the user of this library.
+   The "size rounded up" must be a multiple of 8 and at least 16.
+   "Tracing" an object means enumerating all GC references in it,
+   by invoking the callback passed as argument.
+*/
+extern ssize_t stmcb_size_rounded_up(struct object_s *);
+void stmcb_trace(struct object_s *obj, void visit(object_t **));
+/* a special trace-callback that is only called for the marked
+   ranges of indices (using stm_write_card(o, index)) */
+extern void stmcb_trace_cards(struct object_s *, void (object_t **),
+                              uintptr_t start, uintptr_t stop);
+/* this function will be called on objects that support cards.
+   It returns the base_offset (in bytes) inside the object from
+   where the indices start, and item_size (in bytes) for the size of
+   one item */
+extern void stmcb_get_card_base_itemsize(struct object_s *,
+                                         uintptr_t offset_itemsize[2]);
+/* returns whether this object supports cards. we will only call
+   stmcb_get_card_base_itemsize on objs that do so. */
+extern long stmcb_obj_supports_cards(struct object_s *);
 
+
+
+
+/* Allocate an object of the given size, which must be a multiple
+   of 8 and at least 16.  In the fast-path, this is inlined to just
+   a few assembler instructions.
+*/
 __attribute__((always_inline))
 static inline object_t *stm_allocate(ssize_t size_rounded_up)
 {
@@ -267,21 +317,48 @@
     return (object_t *)p;
 }
 
-
+/* Allocate a weakref object. Weakref objects have a
+   reference to an object at the byte-offset
+       stmcb_size_rounded_up(obj) - sizeof(void*)
+   You must assign the reference before the next collection may happen.
+   After that, you must not mutate the reference anymore. However,
+   it can become NULL after any GC if the reference dies during that
+   collection.
+   NOTE: For performance, we assume stmcb_size_rounded_up(weakref)==16
+*/
 object_t *stm_allocate_weakref(ssize_t size_rounded_up);
 
 
+/* stm_setup() needs to be called once at the beginning of the program.
+   stm_teardown() can be called at the end, but that's not necessary
+   and rather meant for tests.
+ */
 void stm_setup(void);
 void stm_teardown(void);
 
+/* The size of each shadow stack, in number of entries.
+   Must be big enough to accomodate all STM_PUSH_ROOTs! */
 #define STM_SHADOW_STACK_DEPTH   163840
+
+/* Push and pop roots from/to the shadow stack. Only allowed inside
+   transaction. */
 #define STM_PUSH_ROOT(tl, p)   ((tl).shadowstack++->ss = (object_t *)(p))
 #define STM_POP_ROOT(tl, p)    ((p) = (typeof(p))((--(tl).shadowstack)->ss))
 #define STM_POP_ROOT_RET(tl)   ((--(tl).shadowstack)->ss)
 
+/* Every thread needs to have a corresponding stm_thread_local_t
+   structure.  It may be a "__thread" global variable or something else.
+   Use the following functions at the start and at the end of a thread.
+   The user of this library needs to maintain the two shadowstack fields;
+   at any call to stm_allocate(), these fields should point to a range
+   of memory that can be walked in order to find the stack roots.
+*/
 void stm_register_thread_local(stm_thread_local_t *tl);
 void stm_unregister_thread_local(stm_thread_local_t *tl);
 
+/* At some key places, like the entry point of the thread and in the
+   function with the interpreter's dispatch loop, you need to declare
+   a local variable of type 'rewind_jmp_buf' and call these macros. */
 #define stm_rewind_jmp_enterprepframe(tl, rjbuf)                        \
     rewind_jmp_enterprepframe(&(tl)->rjthread, rjbuf, (tl)->shadowstack)
 #define stm_rewind_jmp_enterframe(tl, rjbuf)       \
@@ -303,37 +380,23 @@
     rewind_jmp_enum_shadowstack(&(tl)->rjthread, callback)
 
 
+/* Starting and ending transactions.  stm_read(), stm_write() and
+   stm_allocate() should only be called from within a transaction.
+   The stm_start_transaction() call returns the number of times it
+   returned, starting at 0.  If it is > 0, then the transaction was
+   aborted and restarted this number of times. */
 long stm_start_transaction(stm_thread_local_t *tl);
 void stm_start_inevitable_transaction(stm_thread_local_t *tl);
-
 void stm_commit_transaction(void);
 
 /* Temporary fix?  Call this outside a transaction.  If there is an
    inevitable transaction running somewhere else, wait until it finishes. */
 void stm_wait_for_current_inevitable_transaction(void);
 
+/* Abort the currently running transaction.  This function never
+   returns: it jumps back to the stm_start_transaction(). */
 void stm_abort_transaction(void) __attribute__((noreturn));
 
-void stm_collect(long level);
-
-long stm_identityhash(object_t *obj);
-long stm_id(object_t *obj);
-void stm_set_prebuilt_identityhash(object_t *obj, long hash);
-
-long stm_can_move(object_t *obj);
-
-object_t *stm_setup_prebuilt(object_t *);
-object_t *stm_setup_prebuilt_weakref(object_t *);
-
-long stm_call_on_abort(stm_thread_local_t *, void *key, void callback(void *));
-long stm_call_on_commit(stm_thread_local_t *, void *key, void callback(void *));
-
-static inline void stm_safe_point(void) {
-    if (STM_SEGMENT->nursery_end <= _STM_NSE_SIGNAL_MAX)
-        _stm_collectable_safe_point();
-}
-
-
 #ifdef STM_NO_AUTOMATIC_SETJMP
 int stm_is_inevitable(void);
 #else
@@ -341,6 +404,10 @@
     return !rewind_jmp_armed(&STM_SEGMENT->running_thread->rjthread);
 }
 #endif
+
+/* Turn the current transaction inevitable.
+   stm_become_inevitable() itself may still abort the transaction instead
+   of returning. */
 static inline void stm_become_inevitable(stm_thread_local_t *tl,
                                          const char* msg) {
     assert(STM_SEGMENT->running_thread == tl);
@@ -348,7 +415,64 @@
         _stm_become_inevitable(msg);
 }
 
+/* Forces a safe-point if needed.  Normally not needed: this is
+   automatic if you call stm_allocate(). */
+static inline void stm_safe_point(void) {
+    if (STM_SEGMENT->nursery_end <= _STM_NSE_SIGNAL_MAX)
+        _stm_collectable_safe_point();
+}
+
+/* Forces a collection. */
+void stm_collect(long level);
+
+
+/* Prepare an immortal "prebuilt" object managed by the GC.  Takes a
+   pointer to an 'object_t', which should not actually be a GC-managed
+   structure but a real static structure.  Returns the equivalent
+   GC-managed pointer.  Works by copying it into the GC pages, following
+   and fixing all pointers it contains, by doing stm_setup_prebuilt() on
+   each of them recursively.  (Note that this will leave garbage in the
+   static structure, but it should never be used anyway.) */
+object_t *stm_setup_prebuilt(object_t *);
+/* The same, if the prebuilt object is actually a weakref. */
+object_t *stm_setup_prebuilt_weakref(object_t *);
+
+/* Hash, id.  The id is just the address of the object (of the address
+   where it *will* be after the next minor collection).  The hash is the
+   same, mangled -- except on prebuilt objects, where it can be
+   controlled for each prebuilt object individually.  (Useful uor PyPy) */
+long stm_identityhash(object_t *obj);
+long stm_id(object_t *obj);
+void stm_set_prebuilt_identityhash(object_t *obj, long hash);
+
+/* Returns 1 if the object can still move (it's in the nursery), or 0
+   otherwise.  After a minor collection no object can move any more. */
+long stm_can_move(object_t *obj);
+
+/* If the current transaction aborts later, invoke 'callback(key)'.  If
+   the current transaction commits, then the callback is forgotten.  You
+   can only register one callback per key.  You can call
+   'stm_call_on_abort(key, NULL)' to cancel an existing callback
+   (returns 0 if there was no existing callback to cancel).
+   Note: 'key' must be aligned to a multiple of 8 bytes. */
+long stm_call_on_abort(stm_thread_local_t *, void *key, void callback(void *));
+/* If the current transaction commits later, invoke 'callback(key)'.  If
+   the current transaction aborts, then the callback is forgotten.  Same
+   restrictions as stm_call_on_abort().  If the transaction is or becomes
+   inevitable, 'callback(key)' is called immediately. */
+long stm_call_on_commit(stm_thread_local_t *, void *key, void callback(void *));
+
+
+/* Similar to stm_become_inevitable(), but additionally suspend all
+   other threads.  A very heavy-handed way to make sure that no other
+   transaction is running concurrently.  Avoid as much as possible.
+   Other transactions will continue running only after this transaction
+   commits.  (xxx deprecated and may be removed) */
 void stm_become_globally_unique_transaction(stm_thread_local_t *tl, const char *msg);
+
+/* Moves the transaction forward in time by validating the read and
+   write set with all commits that happened since the last validation
+   (explicit or implicit). */
 void stm_validate(void);
 
 /* Temporarily stop all the other threads, by waiting until they
@@ -407,8 +531,8 @@
 /* The markers pushed in the shadowstack are an odd number followed by a
    regular object pointer. */
 typedef struct {
-    uintptr_t odd_number;
-    object_t *object;
+    uintptr_t odd_number;  /* marker odd number, or 0 if marker is missing */
+    object_t *object;      /* marker object, or NULL if marker is missing */
 } stm_loc_marker_t;
 extern void (*stmcb_timing_event)(stm_thread_local_t *tl, /* the local thread */
                                   enum stm_event_e event,

From noreply at buildbot.pypy.org  Mon Jun  1 17:00:25 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Mon,  1 Jun 2015 17:00:25 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge release-2.6.x into default for
	pypy/doc changes
Message-ID: <20150601150025.DBF941C071B@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: 
Changeset: r77744:657eccac8467
Date: 2015-06-01 17:57 +0300
http://bitbucket.org/pypy/pypy/changeset/657eccac8467/

Log:	merge release-2.6.x into default for pypy/doc changes

diff --git a/pypy/doc/coding-guide.rst b/pypy/doc/coding-guide.rst
--- a/pypy/doc/coding-guide.rst
+++ b/pypy/doc/coding-guide.rst
@@ -385,8 +385,9 @@
 namespace.
 
 Sometimes it is necessary to really write some functions in C (or whatever
-target language). See :ref:`rffi <rpython:rffi>` details.
+target language). See rffi_ details.
 
+.. _rffi: https://rpython.readthedocs.org/en/latest/rffi.html
 
 application level definitions
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst
--- a/pypy/doc/index-of-release-notes.rst
+++ b/pypy/doc/index-of-release-notes.rst
@@ -6,6 +6,7 @@
 
 .. toctree::
 
+   release-2.6.0.rst
    release-2.5.1.rst
    release-2.5.0.rst
    release-2.4.0.rst
diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst
--- a/pypy/doc/index-of-whatsnew.rst
+++ b/pypy/doc/index-of-whatsnew.rst
@@ -7,6 +7,7 @@
 .. toctree::
 
    whatsnew-head.rst
+   whatsnew-2.6.0.rst
    whatsnew-2.5.1.rst
    whatsnew-2.5.0.rst
    whatsnew-2.4.0.rst
diff --git a/pypy/doc/release-2.6.0.rst b/pypy/doc/release-2.6.0.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/release-2.6.0.rst
@@ -0,0 +1,128 @@
+========================
+PyPy 2.6.0 - Cameo Charm
+========================
+
+We're pleased to announce PyPy 2.6.0, only two months after PyPy 2.5.1.
+We are particulary happy to update `cffi`_ to version 1.1, which makes the
+popular ctypes-alternative even easier to use, and to support the new vmprof_
+statistical profiler.
+
+You can download the PyPy 2.6.0 release here:
+
+    http://pypy.org/download.html
+
+We would like to thank our donors for the continued support of the PyPy
+project, and for those who donate to our three sub-projects, as well as our
+volunteers and contributors.  
+
+Thanks also to Yury V. Zaytsev and David Wilson who recently started
+running nightly builds on Windows and MacOSX buildbots.
+
+We've shown quite a bit of progress, but we're slowly running out of funds.
+Please consider donating more, or even better convince your employer to donate,
+so we can finish those projects! The three sub-projects are:
+
+* `Py3k`_ (supporting Python 3.x): We have released a Python 3.2.5 compatible version
+  we call PyPy3 2.4.0, and are working toward a Python 3.3 compatible version
+
+* `STM`_ (software transactional memory): We have released a first working version,
+  and continue to try out new promising paths of achieving a fast multithreaded Python
+
+* `NumPy`_ which requires installation of our fork of upstream numpy,
+  available `on bitbucket`_
+
+.. _`cffi`: https://cffi.readthedocs.org
+.. _`Py3k`: http://pypy.org/py3donate.html
+.. _`STM`: http://pypy.org/tmdonate2.html
+.. _`NumPy`: http://pypy.org/numpydonate.html
+.. _`on bitbucket`: https://www.bitbucket.org/pypy/numpy
+
+We would also like to encourage new people to join the project. PyPy has many
+layers and we need help with all of them: `PyPy`_ and `Rpython`_ documentation
+improvements, tweaking popular `modules`_ to run on pypy, or general `help`_ with making
+Rpython's JIT even better. Nine new people contributed since the last release,
+you too could be one of them.
+
+.. _`PyPy`: http://doc.pypy.org 
+.. _`Rpython`: https://rpython.readthedocs.org
+.. _`modules`: http://doc.pypy.org/en/latest/project-ideas.html#make-more-python-modules-pypy-friendly
+.. _`help`: http://doc.pypy.org/en/latest/project-ideas.html
+
+What is PyPy?
+=============
+
+PyPy is a very compliant Python interpreter, almost a drop-in replacement for
+CPython 2.7. It's fast (`pypy and cpython 2.7.x`_ performance comparison)
+due to its integrated tracing JIT compiler.
+
+This release supports **x86** machines on most common operating systems
+(Linux 32/64, Mac OS X 64, Windows, OpenBSD_, freebsd_),
+as well as newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux.
+
+While we support 32 bit python on Windows, work on the native Windows 64
+bit python is still stalling, we would welcome a volunteer 
+to `handle that`_. We also welcome developers with other operating systems or
+`dynamic languages`_ to see what RPython can do for them.
+
+.. _`pypy and cpython 2.7.x`: http://speed.pypy.org
+.. _OpenBSD: http://cvsweb.openbsd.org/cgi-bin/cvsweb/ports/lang/pypy
+.. _freebsd: https://svnweb.freebsd.org/ports/head/lang/pypy/
+.. _`handle that`: http://doc.pypy.org/en/latest/windows.html#what-is-missing-for-a-full-64-bit-translation
+.. _`dynamic languages`: http://pypyjs.org
+
+Highlights 
+===========
+
+* Python compatibility:
+
+  * Improve support for TLS 1.1 and 1.2
+
+  * Windows downloads now package a pypyw.exe in addition to pypy.exe
+
+  * Support for the PYTHONOPTIMIZE environment variable (impacting builtin's
+    __debug__ property)
+
+  * Issues reported with our previous release were resolved_ after reports from users on
+    our issue tracker at https://bitbucket.org/pypy/pypy/issues or on IRC at
+    #pypy.
+
+* New features:
+
+  * Add preliminary support for a new lightweight statistical profiler
+    `vmprof`_, which has been designed to accomodate profiling JITted code
+
+* Numpy:
+
+  * Support for ``object`` dtype via a garbage collector hook
+
+  * Support for .can_cast and .min_scalar_type as well as beginning
+    a refactoring of the internal casting rules 
+
+  * Better support for subtypes, via the __array_interface__,
+    __array_priority__, and __array_wrap__ methods (still a work-in-progress)
+
+  * Better support for ndarray.flags
+
+* Performance improvements:
+
+  * Slight improvement in frame sizes, improving some benchmarks
+
+  * Internal refactoring and cleanups leading to improved JIT performance
+
+  * Improved IO performance of ``zlib`` and ``bz2`` modules
+
+  * We continue to improve the JIT's optimizations. Our benchmark suite is now
+    over 7 times faster than cpython
+
+.. _`vmprof`: https://vmprof.readthedocs.org
+.. _resolved: https://doc.pypy.org/en/latest/whatsnew-2.6.0.html
+
+Please try it out and let us know what you think. We welcome
+success stories, `experiments`_,  or `benchmarks`_, we know you are using PyPy, please tell us about it!
+
+Cheers
+
+The PyPy Team
+
+.. _`experiments`: https://morepypy.blogspot.com/2015/02/experiments-in-pyrlang-with-rpython.html
+.. _`benchmarks`: https://mithrandi.net/blog/2015/03/axiom-benchmark-results-on-pypy-2-5-0
diff --git a/pypy/doc/whatsnew-2.6.0.rst b/pypy/doc/whatsnew-2.6.0.rst
--- a/pypy/doc/whatsnew-2.6.0.rst
+++ b/pypy/doc/whatsnew-2.6.0.rst
@@ -131,3 +131,11 @@
 
 branch fold-arith-ops
 remove multiple adds on add chains ("1 + 1 + 1 + ...")
+
+.. branch: fix-result-types
+
+branch fix-result-types:
+* Refactor dtype casting and promotion rules for consistency and compatibility
+with CNumPy.
+* Refactor ufunc creation.
+* Implement np.promote_types().
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -3,12 +3,6 @@
 =======================
 
 .. this is a revision shortly after release-2.6.0
-.. startrev: 2ac87a870acf562301840cace411e34c1b96589c
+.. startrev: 91904d5c5188
 
-.. branch: fix-result-types
 
-branch fix-result-types:
-* Refactor dtype casting and promotion rules for consistency and compatibility
-with CNumPy.
-* Refactor ufunc creation.
-* Implement np.promote_types().
diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h
--- a/pypy/module/cpyext/include/patchlevel.h
+++ b/pypy/module/cpyext/include/patchlevel.h
@@ -29,7 +29,7 @@
 #define PY_VERSION		"2.7.9"
 
 /* PyPy version as a string */
-#define PYPY_VERSION "2.7.0-alpha0"
+#define PYPY_VERSION "2.6.0"
 
 /* Subversion Revision number of this file (not of the repository).
  * Empty since Mercurial migration. */
diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py
--- a/pypy/module/sys/version.py
+++ b/pypy/module/sys/version.py
@@ -10,7 +10,7 @@
 #XXX # sync CPYTHON_VERSION with patchlevel.h, package.py
 CPYTHON_API_VERSION        = 1013   #XXX # sync with include/modsupport.h
 
-PYPY_VERSION               = (2, 7, 0, "alpha", 0)    #XXX # sync patchlevel.h
+PYPY_VERSION               = (2, 6, 0, "final", 0)    #XXX # sync patchlevel.h
 
 if platform.name == 'msvc':
     COMPILER_INFO = 'MSC v.%d 32 bit' % (platform.version * 10 + 600)

From noreply at buildbot.pypy.org  Mon Jun  1 17:00:27 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Mon,  1 Jun 2015 17:00:27 +0200 (CEST)
Subject: [pypy-commit] pypy default: fix merge
Message-ID: <20150601150027.2971C1C071B@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: 
Changeset: r77745:f394f184314b
Date: 2015-06-01 18:00 +0300
http://bitbucket.org/pypy/pypy/changeset/f394f184314b/

Log:	fix merge

diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h
--- a/pypy/module/cpyext/include/patchlevel.h
+++ b/pypy/module/cpyext/include/patchlevel.h
@@ -29,7 +29,7 @@
 #define PY_VERSION		"2.7.9"
 
 /* PyPy version as a string */
-#define PYPY_VERSION "2.6.0"
+#define PYPY_VERSION "2.7.0-alpha0"
 
 /* Subversion Revision number of this file (not of the repository).
  * Empty since Mercurial migration. */
diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py
--- a/pypy/module/sys/version.py
+++ b/pypy/module/sys/version.py
@@ -10,7 +10,7 @@
 #XXX # sync CPYTHON_VERSION with patchlevel.h, package.py
 CPYTHON_API_VERSION        = 1013   #XXX # sync with include/modsupport.h
 
-PYPY_VERSION               = (2, 6, 0, "final", 0)    #XXX # sync patchlevel.h
+PYPY_VERSION               = (2, 7, 0, "alpha", 0)    #XXX # sync patchlevel.h
 
 if platform.name == 'msvc':
     COMPILER_INFO = 'MSC v.%d 32 bit' % (platform.version * 10 + 600)

From noreply at buildbot.pypy.org  Mon Jun  1 17:08:37 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  1 Jun 2015 17:08:37 +0200 (CEST)
Subject: [pypy-commit] pypy default: Small tweaks
Message-ID: <20150601150837.AD9FC1C071B@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r77746:b0eb8c2581e0
Date: 2015-06-01 17:08 +0200
http://bitbucket.org/pypy/pypy/changeset/b0eb8c2581e0/

Log:	Small tweaks

diff --git a/pypy/doc/release-2.6.0.rst b/pypy/doc/release-2.6.0.rst
--- a/pypy/doc/release-2.6.0.rst
+++ b/pypy/doc/release-2.6.0.rst
@@ -38,13 +38,13 @@
 .. _`on bitbucket`: https://www.bitbucket.org/pypy/numpy
 
 We would also like to encourage new people to join the project. PyPy has many
-layers and we need help with all of them: `PyPy`_ and `Rpython`_ documentation
+layers and we need help with all of them: `PyPy`_ and `RPython`_ documentation
 improvements, tweaking popular `modules`_ to run on pypy, or general `help`_ with making
-Rpython's JIT even better. Nine new people contributed since the last release,
+RPython's JIT even better. Nine new people contributed since the last release,
 you too could be one of them.
 
 .. _`PyPy`: http://doc.pypy.org 
-.. _`Rpython`: https://rpython.readthedocs.org
+.. _`RPython`: https://rpython.readthedocs.org
 .. _`modules`: http://doc.pypy.org/en/latest/project-ideas.html#make-more-python-modules-pypy-friendly
 .. _`help`: http://doc.pypy.org/en/latest/project-ideas.html
 
@@ -115,7 +115,7 @@
     over 7 times faster than cpython
 
 .. _`vmprof`: https://vmprof.readthedocs.org
-.. _resolved: https://doc.pypy.org/en/latest/whatsnew-2.6.0.html
+.. _resolved: http://doc.pypy.org/en/latest/whatsnew-2.6.0.html
 
 Please try it out and let us know what you think. We welcome
 success stories, `experiments`_,  or `benchmarks`_, we know you are using PyPy, please tell us about it!
diff --git a/pypy/doc/whatsnew-2.6.0.rst b/pypy/doc/whatsnew-2.6.0.rst
--- a/pypy/doc/whatsnew-2.6.0.rst
+++ b/pypy/doc/whatsnew-2.6.0.rst
@@ -1,6 +1,6 @@
-=======================
-What's new in PyPy 2.5+
-=======================
+========================
+What's new in PyPy 2.6.0
+========================
 
 .. this is a revision shortly after release-2.5.1
 .. startrev: cb01edcb59414d9d93056e54ed060673d24e67c1

From noreply at buildbot.pypy.org  Mon Jun  1 17:12:02 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  1 Jun 2015 17:12:02 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c8: hg merge stmgc-c7
Message-ID: <20150601151202.B4EA71C0262@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c8
Changeset: r77747:b0f74e631c15
Date: 2015-06-01 16:12 +0100
http://bitbucket.org/pypy/pypy/changeset/b0f74e631c15/

Log:	hg merge stmgc-c7

diff --git a/lib_pypy/pypy_test/test_transaction.py b/lib_pypy/pypy_test/test_transaction.py
--- a/lib_pypy/pypy_test/test_transaction.py
+++ b/lib_pypy/pypy_test/test_transaction.py
@@ -47,6 +47,19 @@
             print lsts
         assert lsts == (range(10),) * 5, lsts
 
+def test_add_generator():
+    lst = []
+    def do_stuff(n):
+        for i in range(n):
+            lst.append(i)
+            yield
+    tq = transaction.TransactionQueue()
+    tq.add_generator(do_stuff(10))
+    tq.run()
+    if VERBOSE:
+        print lst
+    assert lst == range(10), lst
+
 def test_raise():
     class FooError(Exception):
         pass
diff --git a/lib_pypy/transaction.py b/lib_pypy/transaction.py
--- a/lib_pypy/transaction.py
+++ b/lib_pypy/transaction.py
@@ -128,6 +128,19 @@
         #   thread-local list.
         self._pending.append((f, args, kwds))
 
+    def add_generator(self, generator_iterator):
+        """Register N new transactions to be done by a generator-iterator
+        object.  Each 'yield' marks the limit of transactions.
+        """
+        def transact_until_yield():
+            # Ask for the next item in this transaction.  If we get it,
+            # then the 'for' loop below adds this function again and
+            # returns.
+            for ignored_yielded_value in generator_iterator:
+                self.add(transact_until_yield)
+                return
+        self.add(transact_until_yield)
+
     def run(self, nb_segments=0):
         """Run all transactions, and all transactions started by these
         ones, recursively, until the queue is empty.  If one transaction
diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py
--- a/pypy/tool/release/package.py
+++ b/pypy/tool/release/package.py
@@ -65,6 +65,9 @@
 add --without-{0} option to skip packaging binary CFFI extension.""".format(module)
             raise MissingDependenciesError(module)
 
+def pypy_runs(pypy_c):
+    return subprocess.call([str(pypy_c), '-c', 'pass']) == 0
+
 def create_package(basedir, options):
     retval = 0
     name = options.name
@@ -87,6 +90,8 @@
             ' Please compile pypy first, using translate.py,'
             ' or check that you gave the correct path'
             ' with --override_pypy_c' % pypy_c)
+    if not pypy_runs(pypy_c):
+        raise OSError("Running %r failed!" % (str(pypy_c),))
     if not options.no_cffi:
         try:
             create_cffi_import_libraries(pypy_c, options)
@@ -100,8 +105,22 @@
     libpypy_name = 'libpypy-c.so' if not sys.platform.startswith('darwin') else 'libpypy-c.dylib'
     libpypy_c = pypy_c.new(basename=libpypy_name)
     if libpypy_c.check():
+        # check that this libpypy_c is really needed
+        os.rename(str(libpypy_c), str(libpypy_c) + '~')
+        try:
+            if pypy_runs(pypy_c):
+                raise Exception("It seems that %r runs without needing %r.  "
+                                "Please check and remove the latter" %
+                                (str(pypy_c), str(libpypy_c)))
+        finally:
+            os.rename(str(libpypy_c) + '~', str(libpypy_c))
         binaries.append((libpypy_c, libpypy_name))
     #
+    # PyPy-STM only
+    p = basedir.join('pypy', 'stm', 'print_stm_log.py')
+    assert p.check(), "this version should be run in the stm branch"
+    binaries.append((p, p.basename))
+    #
     builddir = options.builddir
     pypydir = builddir.ensure(name, dir=True)
     includedir = basedir.join('include')

From noreply at buildbot.pypy.org  Mon Jun  1 17:25:34 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  1 Jun 2015 17:25:34 +0200 (CEST)
Subject: [pypy-commit] stmgc default: hg merge c8-locking
Message-ID: <20150601152534.F3B8C1C0823@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r1785:5cea4f9c70af
Date: 2015-06-01 17:26 +0200
http://bitbucket.org/pypy/stmgc/changeset/5cea4f9c70af/

Log:	hg merge c8-locking

diff --git a/c8/LOCKS b/c8/LOCKS
new file mode 100644
--- /dev/null
+++ b/c8/LOCKS
@@ -0,0 +1,93 @@
+
+
+main lock-free operation
+========================
+
+The main lock-free operation is at commit time: the compare-and-swap
+that attaches a new log entry after 'last_commit_log_entry'.
+
+
+
+modification_lock
+=================
+
+one per segment.
+
+acquired on segment N when we want to read or write the segment N's
+copy of 'modified_old_objects', the backup copies, etc.
+
+an important user is _stm_validate(): it locks the current segment,
+and all the other segments out of which it is going to read data
+
+could be improved, because _stm_validate() writes into the current
+segment but only reads the other ones.  So far it mostly serializes
+calls to _stm_validate(): if we have two of them starting at roughly
+the same time, they need both to acquire the modification_lock of at
+least the segment that did the most recent commit --- even though it
+could proceed in parallel if they could both realize that they only
+want to read from that same segment.
+
+same, handle_segfault_in_page() acquires two modification_locks: the
+current segment (which needs to be written to), and the
+'copy_from_segnum' (which only needs to be read from).
+
+the current segment modification_lock is also acquired briefly
+whenever we change our segment's 'modified_old_objects'.
+
+_validate_and_attach() needs to have its own segment's
+modification_lock *around* the compare-and-swap, so that
+_stm_validate() sees either the commit not done and the backup copies
+still in modified_old_objects, or the commit done and no backup copies
+any more.
+
+
+--- UPDATE: modification_lock is now done with pthread_rwlock_xxx().
+
+
+
+privatization_lock
+==================
+
+one per segment.  Works like a single "reader-writer" lock: each
+segment acquires either only its copy ("reader") or all of them
+("writer").
+
+"Reader" status is needed to call get_page_status_in().
+"Writer" status is needed to call set_page_status_in/page_mark_(in)accessible.
+
+Essential "writers":
+- handle_segfault_in_page(), but it only writes the status for the current seg
+
+Essential "readers":
+- _stm_validate()
+- push_large_overflow_objects_to_other_segments()
+- nursery.c calling synchronize_object_enqueue()
+
+
+
+mutex and conditions
+====================
+
+There is also one global mutex and a few condition codes.  It's
+unclear if these are still the best solution.
+
+The mutex is acquired in stm_start_transaction() and in
+stm_commit_transaction().  The main purpose is to wait for or signal
+the C_SEGMENT_FREE condition code.
+
+The C_AT_SAFE_POINT and C_REQUEST_REMOVED condition codes are used by
+synchronize_all_threads().  That's used only in rare cases, for
+example because we want to start a major collection.
+
+The mutex also needs to be acquired for rewind_longjmp's setjmp() and
+longjmp() equivalent.
+
+
+
+usleep loop
+===========
+
+core.c: wait_for_other_inevitable()
+sync.c: stm_wait_for_current_inevitable_transaction()
+
+Must be fixed!
diff --git a/c8/stm/core.c b/c8/stm/core.c
--- a/c8/stm/core.c
+++ b/c8/stm/core.c
@@ -50,8 +50,8 @@
     char *src_segment_base = (from_segnum >= 0 ? get_segment_base(from_segnum)
                                                : NULL);
 
-    assert(IMPLY(from_segnum >= 0, get_priv_segment(from_segnum)->modification_lock));
-    assert(STM_PSEGMENT->modification_lock);
+    assert(IMPLY(from_segnum >= 0, modification_lock_check_rdlock(from_segnum)));
+    assert(modification_lock_check_wrlock(STM_SEGMENT->segment_num));
 
     long my_segnum = STM_SEGMENT->segment_num;
     DEBUG_EXPECT_SEGFAULT(false);
@@ -131,7 +131,7 @@
                            struct stm_commit_log_entry_s *from,
                            struct stm_commit_log_entry_s *to)
 {
-    assert(STM_PSEGMENT->modification_lock);
+    assert(modification_lock_check_wrlock(STM_SEGMENT->segment_num));
     assert(from->rev_num >= to->rev_num);
     /* walk BACKWARDS the commit log and update the page 'pagenum',
        initially at revision 'from', until we reach the revision 'to'. */
@@ -199,8 +199,8 @@
 
     /* before copying anything, acquire modification locks from our and
        the other segment */
-    uint64_t to_lock = (1UL << copy_from_segnum)| (1UL << my_segnum);
-    acquire_modification_lock_set(to_lock);
+    uint64_t to_lock = (1UL << copy_from_segnum);
+    acquire_modification_lock_set(to_lock, my_segnum);
     pagecopy(get_virtual_page(my_segnum, pagenum),
              get_virtual_page(copy_from_segnum, pagenum));
 
@@ -223,7 +223,7 @@
     if (src_version->rev_num > target_version->rev_num)
         go_to_the_past(pagenum, src_version, target_version);
 
-    release_modification_lock_set(to_lock);
+    release_modification_lock_set(to_lock, my_segnum);
     release_all_privatization_locks();
 }
 
@@ -357,7 +357,7 @@
         }
 
         /* Find the set of segments we need to copy from and lock them: */
-        uint64_t segments_to_lock = 1UL << my_segnum;
+        uint64_t segments_to_lock = 0;
         cl = first_cl;
         while ((next_cl = cl->next) != NULL) {
             if (next_cl == INEV_RUNNING) {
@@ -375,8 +375,8 @@
 
         /* HERE */
 
-        acquire_privatization_lock(STM_SEGMENT->segment_num);
-        acquire_modification_lock_set(segments_to_lock);
+        acquire_privatization_lock(my_segnum);
+        acquire_modification_lock_set(segments_to_lock, my_segnum);
 
 
         /* import objects from first_cl to last_cl: */
@@ -466,8 +466,8 @@
         }
 
         /* done with modifications */
-        release_modification_lock_set(segments_to_lock);
-        release_privatization_lock(STM_SEGMENT->segment_num);
+        release_modification_lock_set(segments_to_lock, my_segnum);
+        release_privatization_lock(my_segnum);
     }
 
     return !needs_abort;
@@ -545,7 +545,7 @@
                time" as the attach to commit log. Otherwise, another thread may
                see the new CL entry, import it, look for backup copies in this
                segment and find the old backup copies! */
-            acquire_modification_lock(STM_SEGMENT->segment_num);
+            acquire_modification_lock_wr(STM_SEGMENT->segment_num);
         }
 
         /* try to attach to commit log: */
@@ -559,7 +559,7 @@
         }
 
         if (is_commit) {
-            release_modification_lock(STM_SEGMENT->segment_num);
+            release_modification_lock_wr(STM_SEGMENT->segment_num);
             /* XXX: unfortunately, if we failed to attach our CL entry,
                we have to re-add the WB_EXECUTED flags before we try to
                validate again because of said condition (s.a) */
@@ -596,7 +596,7 @@
 
         list_clear(STM_PSEGMENT->modified_old_objects);
         STM_PSEGMENT->last_commit_log_entry = new;
-        release_modification_lock(STM_SEGMENT->segment_num);
+        release_modification_lock_wr(STM_SEGMENT->segment_num);
     }
 }
 
@@ -692,7 +692,7 @@
         increment_total_allocated(slice_sz);
         memcpy(bk_slice, realobj + slice_off, slice_sz);
 
-        acquire_modification_lock(STM_SEGMENT->segment_num);
+        acquire_modification_lock_wr(STM_SEGMENT->segment_num);
         /* !! follows layout of "struct stm_undo_s" !! */
         STM_PSEGMENT->modified_old_objects = list_append3(
             STM_PSEGMENT->modified_old_objects,
@@ -700,7 +700,7 @@
             (uintptr_t)bk_slice,  /* bk_addr */
             NEW_SLICE(slice_off, slice_sz));
         dprintf(("> append slice %p, off=%lu, sz=%lu\n", bk_slice, slice_off, slice_sz));
-        release_modification_lock(STM_SEGMENT->segment_num);
+        release_modification_lock_wr(STM_SEGMENT->segment_num);
 
         slice_off += slice_sz;
     }
@@ -896,6 +896,8 @@
 
 static void touch_all_pages_of_obj(object_t *obj, size_t obj_size)
 {
+    /* XXX should it be simpler, just really trying to read a dummy
+       byte in each page? */
     int my_segnum = STM_SEGMENT->segment_num;
     uintptr_t end_page, first_page = ((uintptr_t)obj) / 4096UL;
 
@@ -1345,7 +1347,7 @@
 #pragma push_macro("STM_SEGMENT")
 #undef STM_PSEGMENT
 #undef STM_SEGMENT
-    assert(get_priv_segment(segment_num)->modification_lock);
+    assert(modification_lock_check_wrlock(segment_num));
 
     struct stm_priv_segment_info_s *pseg = get_priv_segment(segment_num);
     struct list_s *list = pseg->modified_old_objects;
@@ -1407,9 +1409,9 @@
             _reset_object_cards(pseg, item, CARD_CLEAR, false, false);
         });
 
-    acquire_modification_lock(segment_num);
+    acquire_modification_lock_wr(segment_num);
     reset_modified_from_backup_copies(segment_num);
-    release_modification_lock(segment_num);
+    release_modification_lock_wr(segment_num);
     _verify_cards_cleared_in_all_lists(pseg);
 
     stm_thread_local_t *tl = pseg->pub.running_thread;
diff --git a/c8/stm/core.h b/c8/stm/core.h
--- a/c8/stm/core.h
+++ b/c8/stm/core.h
@@ -74,11 +74,6 @@
 struct stm_priv_segment_info_s {
     struct stm_segment_info_s pub;
 
-    /* lock protecting from concurrent modification of
-       'modified_old_objects', page-revision-changes, ...
-       Always acquired in global order of segments to avoid deadlocks. */
-    uint8_t modification_lock;
-
     /* All the old objects (older than the current transaction) that
        the current transaction attempts to modify.  This is used to
        track the STM status: these are old objects that where written
@@ -359,53 +354,3 @@
         release_privatization_lock(l);
     }
 }
-
-
-
-/* Modification locks are used to prevent copying from a segment
-   where either the revision of some pages is inconsistent with the
-   rest, or the modified_old_objects list is being modified (bk_copys).
-
-   Lock ordering: acquire privatization lock around acquiring a set
-   of modification locks!
-*/
-
-static inline void acquire_modification_lock(int segnum)
-{
-    spinlock_acquire(get_priv_segment(segnum)->modification_lock);
-}
-
-static inline void release_modification_lock(int segnum)
-{
-    spinlock_release(get_priv_segment(segnum)->modification_lock);
-}
-
-static inline void acquire_modification_lock_set(uint64_t seg_set)
-{
-    assert(NB_SEGMENTS <= 64);
-    OPT_ASSERT(seg_set < (1 << NB_SEGMENTS));
-
-    /* acquire locks in global order */
-    int i;
-    for (i = 0; i < NB_SEGMENTS; i++) {
-        if ((seg_set & (1 << i)) == 0)
-            continue;
-
-        spinlock_acquire(get_priv_segment(i)->modification_lock);
-    }
-}
-
-static inline void release_modification_lock_set(uint64_t seg_set)
-{
-    assert(NB_SEGMENTS <= 64);
-    OPT_ASSERT(seg_set < (1 << NB_SEGMENTS));
-
-    int i;
-    for (i = 0; i < NB_SEGMENTS; i++) {
-        if ((seg_set & (1 << i)) == 0)
-            continue;
-
-        assert(get_priv_segment(i)->modification_lock);
-        spinlock_release(get_priv_segment(i)->modification_lock);
-    }
-}
diff --git a/c8/stm/forksupport.c b/c8/stm/forksupport.c
--- a/c8/stm/forksupport.c
+++ b/c8/stm/forksupport.c
@@ -120,6 +120,9 @@
        just release these locks early */
     s_mutex_unlock();
 
+    /* Re-init these locks; might be needed after a fork() */
+    setup_modification_locks();
+
 
     /* Unregister all other stm_thread_local_t, mostly as a way to free
        the memory used by the shadowstacks
diff --git a/c8/stm/locks.h b/c8/stm/locks.h
new file mode 100644
--- /dev/null
+++ b/c8/stm/locks.h
@@ -0,0 +1,124 @@
+
+/* Modification locks protect from concurrent modification of
+   'modified_old_objects', page-revision-changes, ...
+
+   Modification locks are used to prevent copying from a segment
+   where either the revision of some pages is inconsistent with the
+   rest, or the modified_old_objects list is being modified (bk_copys).
+
+   Lock ordering: acquire privatization lock around acquiring a set
+   of modification locks!
+*/
+
+typedef struct {
+    pthread_rwlock_t lock;
+#ifndef NDEBUG
+    volatile bool write_locked;
+#endif
+} modification_lock_t __attribute__((aligned(64)));
+
+static modification_lock_t _modlocks[NB_SEGMENTS - 1];
+
+
+static void setup_modification_locks(void)
+{
+    int i;
+    for (i = 1; i < NB_SEGMENTS; i++) {
+        if (pthread_rwlock_init(&_modlocks[i - 1].lock, NULL) != 0)
+            stm_fatalerror("pthread_rwlock_init: %m");
+    }
+}
+
+static void teardown_modification_locks(void)
+{
+    int i;
+    for (i = 1; i < NB_SEGMENTS; i++)
+        pthread_rwlock_destroy(&_modlocks[i - 1].lock);
+    memset(_modlocks, 0, sizeof(_modlocks));
+}
+
+
+static inline void acquire_modification_lock_wr(int segnum)
+{
+    if (UNLIKELY(pthread_rwlock_wrlock(&_modlocks[segnum - 1].lock) != 0))
+        stm_fatalerror("pthread_rwlock_wrlock: %m");
+#ifndef NDEBUG
+    assert(!_modlocks[segnum - 1].write_locked);
+    _modlocks[segnum - 1].write_locked = true;
+#endif
+}
+
+static inline void release_modification_lock_wr(int segnum)
+{
+#ifndef NDEBUG
+    assert(_modlocks[segnum - 1].write_locked);
+    _modlocks[segnum - 1].write_locked = false;
+#endif
+    if (UNLIKELY(pthread_rwlock_unlock(&_modlocks[segnum - 1].lock) != 0))
+        stm_fatalerror("pthread_rwlock_unlock(wr): %m");
+}
+
+static void acquire_modification_lock_set(uint64_t readset, int write)
+{
+    /* acquire the modification lock in 'read' mode for all segments
+       in 'readset', plus the modification lock in 'write' mode for
+       the segment number 'write'.
+    */
+    assert(NB_SEGMENTS <= 64);
+    OPT_ASSERT(readset < (1 << NB_SEGMENTS));
+    assert((readset & 1) == 0);       /* segment numbers normally start at 1 */
+    assert(0 <= write && write < NB_SEGMENTS);     /* use 0 to mean "nobody" */
+
+    /* acquire locks in global order */
+    readset |= (1UL << write);
+    int i;
+    for (i = 1; i < NB_SEGMENTS; i++) {
+        if ((readset & (1UL << i)) == 0)
+            continue;
+        if (i == write) {
+            acquire_modification_lock_wr(write);
+        }
+        else {
+            if (UNLIKELY(pthread_rwlock_rdlock(&_modlocks[i - 1].lock) != 0))
+                stm_fatalerror("pthread_rwlock_rdlock: %m");
+        }
+    }
+}
+
+static void release_modification_lock_set(uint64_t readset, int write)
+{
+    assert(NB_SEGMENTS <= 64);
+    OPT_ASSERT(readset < (1 << NB_SEGMENTS));
+
+    /* release lock order does not matter; prefer early release of
+       the write lock */
+    if (write > 0) {
+        release_modification_lock_wr(write);
+        readset &= ~(1UL << write);
+    }
+    int i;
+    for (i = 1; i < NB_SEGMENTS; i++) {
+        if ((readset & (1UL << i)) == 0)
+            continue;
+        if (UNLIKELY(pthread_rwlock_unlock(&_modlocks[i - 1].lock) != 0))
+            stm_fatalerror("pthread_rwlock_unlock(rd): %m");
+    }
+}
+
+#ifndef NDEBUG
+static bool modification_lock_check_rdlock(int segnum)
+{
+    assert(segnum > 0);
+    if (_modlocks[segnum - 1].write_locked)
+        return false;
+    if (pthread_rwlock_trywrlock(&_modlocks[segnum - 1].lock) == 0) {
+        pthread_rwlock_unlock(&_modlocks[segnum - 1].lock);
+        return false;
+    }
+    return true;
+}
+static bool modification_lock_check_wrlock(int segnum)
+{
+    return segnum == 0 || _modlocks[segnum - 1].write_locked;
+}
+#endif
diff --git a/c8/stm/setup.c b/c8/stm/setup.c
--- a/c8/stm/setup.c
+++ b/c8/stm/setup.c
@@ -127,6 +127,7 @@
        private range of addresses.
     */
 
+    setup_modification_locks();
     setup_sync();
     setup_nursery();
     setup_gcpage();
@@ -174,6 +175,7 @@
     teardown_gcpage();
     teardown_smallmalloc();
     teardown_pages();
+    teardown_modification_locks();
 }
 
 static void _shadowstack_trap_page(char *start, int prot)
diff --git a/c8/stmgc.c b/c8/stmgc.c
--- a/c8/stmgc.c
+++ b/c8/stmgc.c
@@ -17,6 +17,7 @@
 #include "stm/marker.h"
 #include "stm/rewind_setjmp.h"
 #include "stm/finalizer.h"
+#include "stm/locks.h"
 
 #include "stm/misc.c"
 #include "stm/list.c"

From noreply at buildbot.pypy.org  Mon Jun  1 18:02:47 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Mon,  1 Jun 2015 18:02:47 +0200 (CEST)
Subject: [pypy-commit] jitviewer default: Added tag pypy-2.6.0 for changeset
	3a0152b4ac6b
Message-ID: <20150601160247.729B61C0F16@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: 
Changeset: r271:3b5a42cf8039
Date: 2015-06-01 19:03 +0300
http://bitbucket.org/pypy/jitviewer/changeset/3b5a42cf8039/

Log:	Added tag pypy-2.6.0 for changeset 3a0152b4ac6b

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -4,3 +4,4 @@
 62ad3e746dacc21c8e5dff2a37738659e1b61b7a pypy-2.4
 62ad3e746dacc21c8e5dff2a37738659e1b61b7a pypy-2.3
 ec561fb900e02df04e47b11c413f4a8449cbbb3a pypy-2.5
+3a0152b4ac6b8f930c493ef357fc5e9d8f4b91b7 pypy-2.6.0

From noreply at buildbot.pypy.org  Mon Jun  1 18:22:52 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  1 Jun 2015 18:22:52 +0200 (CEST)
Subject: [pypy-commit] cffi default: Issue #205: Python 2.6 compat: "import"
 accept files that don't finish
Message-ID: <20150601162252.38E1A1C04BC@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2152:161ecb5a7186
Date: 2015-06-01 18:23 +0200
http://bitbucket.org/cffi/cffi/changeset/161ecb5a7186/

Log:	Issue #205: Python 2.6 compat: "import" accept files that don't
	finish with a newline, but "compile()" doesn't.

diff --git a/cffi/setuptools_ext.py b/cffi/setuptools_ext.py
--- a/cffi/setuptools_ext.py
+++ b/cffi/setuptools_ext.py
@@ -18,7 +18,9 @@
     # __init__.py files may already try to import the file that
     # we are generating.
     with open(filename) as f:
-        code = compile(f.read(), filename, 'exec')
+        src = f.read()
+        src += '\n'      # Python 2.6 compatibility
+        code = compile(src, filename, 'exec')
     exec(code, glob, glob)
 
 

From noreply at buildbot.pypy.org  Mon Jun  1 18:28:30 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  1 Jun 2015 18:28:30 +0200 (CEST)
Subject: [pypy-commit] cffi default: Move these two lines outside the "with"
Message-ID: <20150601162830.2865F1C04BC@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2153:13a8f80fab66
Date: 2015-06-01 18:29 +0200
http://bitbucket.org/cffi/cffi/changeset/13a8f80fab66/

Log:	Move these two lines outside the "with"

diff --git a/cffi/setuptools_ext.py b/cffi/setuptools_ext.py
--- a/cffi/setuptools_ext.py
+++ b/cffi/setuptools_ext.py
@@ -19,8 +19,8 @@
     # we are generating.
     with open(filename) as f:
         src = f.read()
-        src += '\n'      # Python 2.6 compatibility
-        code = compile(src, filename, 'exec')
+    src += '\n'      # Python 2.6 compatibility
+    code = compile(src, filename, 'exec')
     exec(code, glob, glob)
 
 

From noreply at buildbot.pypy.org  Mon Jun  1 19:24:53 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  1 Jun 2015 19:24:53 +0200 (CEST)
Subject: [pypy-commit] cffi default: Issue #204: Spinkle "--no-user-cfg" in
 this test to avoid getting
Message-ID: <20150601172453.E3F821C04BC@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2154:3b9f80a475ee
Date: 2015-06-01 19:25 +0200
http://bitbucket.org/cffi/cffi/changeset/3b9f80a475ee/

Log:	Issue #204: Spinkle "--no-user-cfg" in this test to avoid getting
	confusion from ~/.pydistutils.cfg.

diff --git a/testing/cffi1/test_zdist.py b/testing/cffi1/test_zdist.py
--- a/testing/cffi1/test_zdist.py
+++ b/testing/cffi1/test_zdist.py
@@ -44,7 +44,8 @@
             import setuptools
         except ImportError:
             py.test.skip("setuptools not found")
-        subprocess.check_call([self.executable, 'setup.py', 'egg_info'],
+        subprocess.check_call([self.executable, 'setup.py', '--no-user-cfg',
+                               'egg_info'],
                               cwd=self.rootdir)
         TestDist._setuptools_ready = True
 
@@ -242,7 +243,7 @@
     @chdir_to_tmp
     def test_distutils_api_1(self):
         self._make_distutils_api()
-        self.run(["setup.py", "build"])
+        self.run(["setup.py", "--no-user-cfg", "build"])
         self.check_produced_files({'setup.py': None,
                                    'build': '?',
                                    'src': {'pack1': {'__init__.py': None}}})
@@ -250,7 +251,7 @@
     @chdir_to_tmp
     def test_distutils_api_2(self):
         self._make_distutils_api()
-        self.run(["setup.py", "build_ext", "-i"])
+        self.run(["setup.py", "--no-user-cfg", "build_ext", "-i"])
         self.check_produced_files({'setup.py': None,
                                    'build': '?',
                                    'src': {'pack1': {'__init__.py': None,
@@ -281,7 +282,7 @@
     @chdir_to_tmp
     def test_setuptools_abi_1(self):
         self._make_setuptools_abi()
-        self.run(["setup.py", "build"])
+        self.run(["setup.py", "--no-user-cfg", "build"])
         self.check_produced_files({'setup.py': None,
                                    'build': '?',
                                    'src0': {'pack2': {'__init__.py': None,
@@ -290,7 +291,7 @@
     @chdir_to_tmp
     def test_setuptools_abi_2(self):
         self._make_setuptools_abi()
-        self.run(["setup.py", "build_ext", "-i"])
+        self.run(["setup.py", "--no-user-cfg", "build_ext", "-i"])
         self.check_produced_files({'setup.py': None,
                                    'src0': {'pack2': {'__init__.py': None,
                                                       '_build.py': None,
@@ -321,7 +322,7 @@
     @chdir_to_tmp
     def test_setuptools_api_1(self):
         self._make_setuptools_api()
-        self.run(["setup.py", "build"])
+        self.run(["setup.py", "--no-user-cfg", "build"])
         self.check_produced_files({'setup.py': None,
                                    'build': '?',
                                    'src1': {'pack3': {'__init__.py': None,
@@ -330,7 +331,7 @@
     @chdir_to_tmp
     def test_setuptools_api_2(self):
         self._make_setuptools_api()
-        self.run(["setup.py", "build_ext", "-i"])
+        self.run(["setup.py", "--no-user-cfg", "build_ext", "-i"])
         self.check_produced_files({'setup.py': None,
                                    'build': '?',
                                    'src1': {'pack3': {'__init__.py': None,

From noreply at buildbot.pypy.org  Mon Jun  1 20:19:55 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  1 Jun 2015 20:19:55 +0200 (CEST)
Subject: [pypy-commit] cffi default: Backed out changeset 3b9f80a475ee: not
 supported on python 2.6
Message-ID: <20150601181955.96C021C04BC@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2155:feea0af4a450
Date: 2015-06-01 20:07 +0200
http://bitbucket.org/cffi/cffi/changeset/feea0af4a450/

Log:	Backed out changeset 3b9f80a475ee: not supported on python 2.6

diff --git a/testing/cffi1/test_zdist.py b/testing/cffi1/test_zdist.py
--- a/testing/cffi1/test_zdist.py
+++ b/testing/cffi1/test_zdist.py
@@ -44,8 +44,7 @@
             import setuptools
         except ImportError:
             py.test.skip("setuptools not found")
-        subprocess.check_call([self.executable, 'setup.py', '--no-user-cfg',
-                               'egg_info'],
+        subprocess.check_call([self.executable, 'setup.py', 'egg_info'],
                               cwd=self.rootdir)
         TestDist._setuptools_ready = True
 
@@ -243,7 +242,7 @@
     @chdir_to_tmp
     def test_distutils_api_1(self):
         self._make_distutils_api()
-        self.run(["setup.py", "--no-user-cfg", "build"])
+        self.run(["setup.py", "build"])
         self.check_produced_files({'setup.py': None,
                                    'build': '?',
                                    'src': {'pack1': {'__init__.py': None}}})
@@ -251,7 +250,7 @@
     @chdir_to_tmp
     def test_distutils_api_2(self):
         self._make_distutils_api()
-        self.run(["setup.py", "--no-user-cfg", "build_ext", "-i"])
+        self.run(["setup.py", "build_ext", "-i"])
         self.check_produced_files({'setup.py': None,
                                    'build': '?',
                                    'src': {'pack1': {'__init__.py': None,
@@ -282,7 +281,7 @@
     @chdir_to_tmp
     def test_setuptools_abi_1(self):
         self._make_setuptools_abi()
-        self.run(["setup.py", "--no-user-cfg", "build"])
+        self.run(["setup.py", "build"])
         self.check_produced_files({'setup.py': None,
                                    'build': '?',
                                    'src0': {'pack2': {'__init__.py': None,
@@ -291,7 +290,7 @@
     @chdir_to_tmp
     def test_setuptools_abi_2(self):
         self._make_setuptools_abi()
-        self.run(["setup.py", "--no-user-cfg", "build_ext", "-i"])
+        self.run(["setup.py", "build_ext", "-i"])
         self.check_produced_files({'setup.py': None,
                                    'src0': {'pack2': {'__init__.py': None,
                                                       '_build.py': None,
@@ -322,7 +321,7 @@
     @chdir_to_tmp
     def test_setuptools_api_1(self):
         self._make_setuptools_api()
-        self.run(["setup.py", "--no-user-cfg", "build"])
+        self.run(["setup.py", "build"])
         self.check_produced_files({'setup.py': None,
                                    'build': '?',
                                    'src1': {'pack3': {'__init__.py': None,
@@ -331,7 +330,7 @@
     @chdir_to_tmp
     def test_setuptools_api_2(self):
         self._make_setuptools_api()
-        self.run(["setup.py", "--no-user-cfg", "build_ext", "-i"])
+        self.run(["setup.py", "build_ext", "-i"])
         self.check_produced_files({'setup.py': None,
                                    'build': '?',
                                    'src1': {'pack3': {'__init__.py': None,

From noreply at buildbot.pypy.org  Mon Jun  1 20:19:56 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  1 Jun 2015 20:19:56 +0200 (CEST)
Subject: [pypy-commit] cffi default: Issue #204: second try
Message-ID: <20150601181956.AA1121C04BC@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2156:34d5fd98bc84
Date: 2015-06-01 20:20 +0200
http://bitbucket.org/cffi/cffi/changeset/34d5fd98bc84/

Log:	Issue #204: second try

diff --git a/testing/cffi1/test_zdist.py b/testing/cffi1/test_zdist.py
--- a/testing/cffi1/test_zdist.py
+++ b/testing/cffi1/test_zdist.py
@@ -29,13 +29,17 @@
         if hasattr(self, 'saved_cwd'):
             os.chdir(self.saved_cwd)
 
-    def run(self, args):
+    def run(self, args, cwd=None):
         env = os.environ.copy()
-        newpath = self.rootdir
-        if 'PYTHONPATH' in env:
-            newpath += os.pathsep + env['PYTHONPATH']
-        env['PYTHONPATH'] = newpath
-        subprocess.check_call([self.executable] + args, env=env)
+        # a horrible hack to prevent distutils from finding ~/.pydistutils.cfg
+        # (there is the --no-user-cfg option, but not in Python 2.6...)
+        env['HOME'] = '/this/path/does/not/exist'
+        if cwd is None:
+            newpath = self.rootdir
+            if 'PYTHONPATH' in env:
+                newpath += os.pathsep + env['PYTHONPATH']
+            env['PYTHONPATH'] = newpath
+        subprocess.check_call([self.executable] + args, cwd=cwd, env=env)
 
     def _prepare_setuptools(self):
         if hasattr(TestDist, '_setuptools_ready'):
@@ -44,8 +48,7 @@
             import setuptools
         except ImportError:
             py.test.skip("setuptools not found")
-        subprocess.check_call([self.executable, 'setup.py', 'egg_info'],
-                              cwd=self.rootdir)
+        self.run(['setup.py', 'egg_info'], cwd=self.rootdir)
         TestDist._setuptools_ready = True
 
     def check_produced_files(self, content, curdir=None):

From noreply at buildbot.pypy.org  Mon Jun  1 21:22:42 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Mon,  1 Jun 2015 21:22:42 +0200 (CEST)
Subject: [pypy-commit] pypy use_min_scalar: correct handling of scalars for
 simple binary ufuncs
Message-ID: <20150601192242.BC6C21C04BC@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: use_min_scalar
Changeset: r77748:5b71a45fc55b
Date: 2015-06-01 20:22 +0100
http://bitbucket.org/pypy/pypy/changeset/5b71a45fc55b/

Log:	correct handling of scalars for simple binary ufuncs

diff --git a/pypy/module/micronumpy/ufuncs.py b/pypy/module/micronumpy/ufuncs.py
--- a/pypy/module/micronumpy/ufuncs.py
+++ b/pypy/module/micronumpy/ufuncs.py
@@ -640,7 +640,14 @@
             return w_val.w_obj
         return w_val
 
-    def _find_specialization(self, space, l_dtype, r_dtype, out, casting):
+    def _find_specialization(self, space, l_dtype, r_dtype, out, casting,
+                             w_arg1, w_arg2):
+        if (self.are_common_types(l_dtype, r_dtype) and
+                w_arg1 is not None and w_arg2 is not None):
+            if not w_arg1.is_scalar() and w_arg2.is_scalar():
+                r_dtype = l_dtype
+            elif w_arg1.is_scalar() and not w_arg2.is_scalar():
+                l_dtype = r_dtype
         if (not self.allow_bool and (l_dtype.is_bool() or
                                          r_dtype.is_bool()) or
                 not self.allow_complex and (l_dtype.is_complex() or
@@ -657,17 +664,17 @@
 
     def find_specialization(self, space, l_dtype, r_dtype, out, casting,
                             w_arg1=None, w_arg2=None):
-        if (self.are_common_types(l_dtype, r_dtype) and
-                w_arg1 is not None and w_arg2 is not None):
-            if not w_arg1.is_scalar() and w_arg2.is_scalar():
-                r_dtype = l_dtype
-            elif w_arg1.is_scalar() and not w_arg2.is_scalar():
-                l_dtype = r_dtype
         if self.simple_binary:
             if out is None and not (l_dtype.is_object() or r_dtype.is_object()):
-                dtype = promote_types(space, l_dtype, r_dtype)
+                if w_arg1 is not None and w_arg2 is not None:
+                    w_arg1 = convert_to_array(space, w_arg1)
+                    w_arg2 = convert_to_array(space, w_arg2)
+                    dtype = find_result_type(space, [w_arg1, w_arg2], [])
+                else:
+                    dtype = promote_types(space, l_dtype, r_dtype)
                 return dtype, dtype, self.func
-        return self._find_specialization(space, l_dtype, r_dtype, out, casting)
+        return self._find_specialization(
+            space, l_dtype, r_dtype, out, casting, w_arg1, w_arg2)
 
     def find_binop_type(self, space, dtype):
         """Find a valid dtype signature of the form xx->x"""

From noreply at buildbot.pypy.org  Tue Jun  2 08:11:43 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue,  2 Jun 2015 08:11:43 +0200 (CEST)
Subject: [pypy-commit] pypy.org extradoc: update the values
Message-ID: <20150602061143.639DF1C0262@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r614:3ca2cf7a87a7
Date: 2015-06-02 08:12 +0200
http://bitbucket.org/pypy/pypy.org/changeset/3ca2cf7a87a7/

Log:	update the values

diff --git a/don1.html b/don1.html
--- a/don1.html
+++ b/don1.html
@@ -15,7 +15,7 @@
 </script>
 
    <!-- Income:PyPy:Donations:Py3k / 0.95 -->
-   $59426 of $105000 (56.6%)
+   $59469 of $105000 (56.6%)
    <div id="progressbar">
    </div>
 
diff --git a/don3.html b/don3.html
--- a/don3.html
+++ b/don3.html
@@ -9,13 +9,13 @@
 <script>
   $(function() {
     $("#progressbar").progressbar({
-      value: 86.7
+      value: 86.9
    });
   });
 </script>
 
    <!-- Income:PyPy:Donations:Numpy / 0.95 -->
-   $52029 of $60000 (86.7%)
+   $52155 of $60000 (86.9%)
    <div id="progressbar">
    </div>
 
diff --git a/don4.html b/don4.html
--- a/don4.html
+++ b/don4.html
@@ -9,7 +9,7 @@
 <script>
   $(function() {
     $("#progressbar").progressbar({
-      value: 36.1
+      value: 36.3
    });
   });
 </script>
@@ -17,7 +17,7 @@
    2nd call:
    <!-- Income:PyPy:Donations:Transactional Memory from April 8, 2014,
    divided by 0.9 -->
-   $28897 of $80000 (36.1%)
+   $29007 of $80000 (36.3%)
    <div id="progressbar">
    </div>
 

From noreply at buildbot.pypy.org  Tue Jun  2 08:43:47 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue,  2 Jun 2015 08:43:47 +0200 (CEST)
Subject: [pypy-commit] pypy.org extradoc: Update the link to Ubuntu PyPy ---
 go to the search page, which lists
Message-ID: <20150602064347.8E43D1C024E@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r615:8a210421cca7
Date: 2015-06-02 08:44 +0200
http://bitbucket.org/pypy/pypy.org/changeset/8a210421cca7/

Log:	Update the link to Ubuntu PyPy --- go to the search page, which
	lists various versions for the different Ubuntu releases.

diff --git a/download.html b/download.html
--- a/download.html
+++ b/download.html
@@ -108,7 +108,7 @@
 <ul class="simple">
 <li>use Squeaky's <a class="reference external" href="https://github.com/squeaky-pl/portable-pypy#portable-pypy-distribution-for-linux">portable Linux binaries</a>.</li>
 <li>or download PyPy from your release vendor (usually an outdated
-version): <a class="reference external" href="http://packages.ubuntu.com/raring/pypy">Ubuntu</a> (<a class="reference external" href="https://launchpad.net/~pypy/+archive/ppa">PPA</a>), <a class="reference external" href="http://packages.debian.org/sid/pypy">Debian</a>, <a class="reference external" href="https://github.com/mxcl/homebrew/blob/master/Library/Formula/pypy.rb">Homebrew</a>, MacPorts,
+version): <a class="reference external" href="http://packages.ubuntu.com/search?keywords=pypy&amp;searchon=names">Ubuntu</a> (<a class="reference external" href="https://launchpad.net/~pypy/+archive/ppa">PPA</a>), <a class="reference external" href="http://packages.debian.org/sid/pypy">Debian</a>, <a class="reference external" href="https://github.com/mxcl/homebrew/blob/master/Library/Formula/pypy.rb">Homebrew</a>, MacPorts,
 <a class="reference external" href="http://fedoraproject.org/wiki/Features/PyPyStack">Fedora</a>, <a class="reference external" href="http://packages.gentoo.org/package/dev-python/pypy">Gentoo</a> and <a class="reference external" href="https://wiki.archlinux.org/index.php/PyPy">Arch</a> are known to package PyPy, with various
 degrees of being up-to-date.</li>
 </ul>
diff --git a/source/download.txt b/source/download.txt
--- a/source/download.txt
+++ b/source/download.txt
@@ -63,7 +63,7 @@
   `Fedora`_, `Gentoo`_ and `Arch`_ are known to package PyPy, with various
   degrees of being up-to-date.
 
-.. _`Ubuntu`: http://packages.ubuntu.com/raring/pypy
+.. _`Ubuntu`: http://packages.ubuntu.com/search?keywords=pypy&searchon=names
 .. _`PPA`: https://launchpad.net/~pypy/+archive/ppa
 .. _`Debian`: http://packages.debian.org/sid/pypy
 .. _`Fedora`: http://fedoraproject.org/wiki/Features/PyPyStack

From noreply at buildbot.pypy.org  Tue Jun  2 09:58:51 2015
From: noreply at buildbot.pypy.org (Raemi)
Date: Tue,  2 Jun 2015 09:58:51 +0200 (CEST)
Subject: [pypy-commit] stmgc c8-locking: update todo list
Message-ID: <20150602075851.DDB051C024E@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <remi.meier at inf.ethz.ch>
Branch: c8-locking
Changeset: r1786:bc85b4567b48
Date: 2015-06-02 09:59 +0200
http://bitbucket.org/pypy/stmgc/changeset/bc85b4567b48/

Log:	update todo list

diff --git a/c8/TODO b/c8/TODO
--- a/c8/TODO
+++ b/c8/TODO
@@ -1,9 +1,6 @@
 
 - improve sync of small objs on commit (see FLAG_SYNC_LARGE in nursery.c)
 
-- non-zeroed nursery:
-  read-the-docs benchmark shows 8% time spent in memset of throw_away_nursery
-
 - reshare pages:
   make seg0 MAP_SHARED in order to re-share private pages during major GC
 
@@ -28,3 +25,9 @@
 
 - avoid __builtin_frame_address(0) in precisely the performance-critical
   functions like the interpreter main loop
+
+
+---------------------------
+DONE:
+- non-zeroed nursery:
+  read-the-docs benchmark shows 8% time spent in memset of throw_away_nursery

From noreply at buildbot.pypy.org  Tue Jun  2 10:00:12 2015
From: noreply at buildbot.pypy.org (Raemi)
Date: Tue,  2 Jun 2015 10:00:12 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Merge with c8-locking
Message-ID: <20150602080012.E29031C034D@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <remi.meier at inf.ethz.ch>
Branch: 
Changeset: r1787:cca95721cc80
Date: 2015-06-02 10:00 +0200
http://bitbucket.org/pypy/stmgc/changeset/cca95721cc80/

Log:	Merge with c8-locking

diff --git a/c8/TODO b/c8/TODO
--- a/c8/TODO
+++ b/c8/TODO
@@ -1,9 +1,6 @@
 
 - improve sync of small objs on commit (see FLAG_SYNC_LARGE in nursery.c)
 
-- non-zeroed nursery:
-  read-the-docs benchmark shows 8% time spent in memset of throw_away_nursery
-
 - reshare pages:
   make seg0 MAP_SHARED in order to re-share private pages during major GC
 
@@ -28,3 +25,9 @@
 
 - avoid __builtin_frame_address(0) in precisely the performance-critical
   functions like the interpreter main loop
+
+
+---------------------------
+DONE:
+- non-zeroed nursery:
+  read-the-docs benchmark shows 8% time spent in memset of throw_away_nursery

From noreply at buildbot.pypy.org  Tue Jun  2 10:15:27 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue,  2 Jun 2015 10:15:27 +0200 (CEST)
Subject: [pypy-commit] pypy default: Mention the modern
 multiple-list-types-friendly variant and not at all
Message-ID: <20150602081527.13A4C1C0FD4@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r77749:6bfb2b6982a7
Date: 2015-06-02 10:15 +0200
http://bitbucket.org/pypy/pypy/changeset/6bfb2b6982a7/

Log:	Mention the modern multiple-list-types-friendly variant and not at
	all the prebuilt TimSort class.

diff --git a/rpython/doc/rlib.rst b/rpython/doc/rlib.rst
--- a/rpython/doc/rlib.rst
+++ b/rpython/doc/rlib.rst
@@ -15,14 +15,17 @@
 listsort
 --------
 
-The :source:`rpython/rlib/listsort.py` module contains an implementation of the timsort sorting algorithm
-(the sort method of lists is not RPython). To use it, subclass from the
-``listsort.TimSort`` class and override the ``lt`` method to change the
-comparison behaviour. The constructor of ``TimSort`` takes a list as an
-argument, which will be sorted in place when the ``sort`` method of the
-``TimSort`` instance is called. **Warning:** currently only one type of list can
-be sorted using the ``listsort`` module in one program, otherwise the annotator
-will be confused.
+The :source:`rpython/rlib/listsort.py` module contains an implementation
+of the timsort sorting algorithm (the sort method of lists is not
+RPython). To use it, make (globally) one class by calling ``MySort =
+listsort.make_timsort_class(lt=my_comparison_func)``.  There are also
+other optional arguments, but usually you give with ``lt=...`` a
+function that compares two objects from your lists.  You need one class
+per "type" of list and per comparison function.
+
+The constructor of ``MySort`` takes a list as an argument, which will be
+sorted in place when the ``sort`` method of the ``MySort`` instance is
+called.
 
 
 nonconst

From noreply at buildbot.pypy.org  Tue Jun  2 10:25:02 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue,  2 Jun 2015 10:25:02 +0200 (CEST)
Subject: [pypy-commit] pypy default: Quickly mention rsre,
 and note that the `parsing' module's regexps are
Message-ID: <20150602082502.7EF281C0FD4@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r77750:bf714a418f74
Date: 2015-06-02 10:25 +0200
http://bitbucket.org/pypy/pypy/changeset/bf714a418f74/

Log:	Quickly mention rsre, and note that the `parsing' module's regexps
	are different.

diff --git a/rpython/doc/rlib.rst b/rpython/doc/rlib.rst
--- a/rpython/doc/rlib.rst
+++ b/rpython/doc/rlib.rst
@@ -157,6 +157,20 @@
 produce a loop in the resulting flow graph but will unroll the loop instead.
 
 
+rsre
+----
+
+The implementation of regular expressions we use for PyPy.  Note that it
+is hard to reuse in other languages: in Python, regular expressions are
+first compiled into a bytecode format by pure Python code from the
+standard library.  This lower-level module only understands this
+bytecode format.  Without a complete Python interpreter you can't
+translate the regexp syntax to the bytecode format.  (There are hacks
+for limited use cases where you have only static regexps: they can be
+precompiled during translation.  Alternatively, you could imagine
+executing a Python subprocess just to translate a regexp at runtime...)
+
+
 parsing
 -------
 
@@ -177,7 +191,8 @@
 -------------------
 
 The regular expression syntax is mostly a subset of the syntax of the `re`_
-module. By default, non-special characters match themselves. If you concatenate
+module.  *Note: this is different from rlib.rsre.*
+By default, non-special characters match themselves. If you concatenate
 regular expressions the result will match the concatenation of strings matched
 by the single regular expressions.
 

From noreply at buildbot.pypy.org  Tue Jun  2 10:49:09 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Tue,  2 Jun 2015 10:49:09 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: add a workaround
Message-ID: <20150602084909.398971C024E@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77751:5c6831ad7336
Date: 2015-06-02 10:28 +0200
http://bitbucket.org/pypy/pypy/changeset/5c6831ad7336/

Log:	add a workaround

diff --git a/rpython/jit/metainterp/optimizeopt/intutils.py b/rpython/jit/metainterp/optimizeopt/intutils.py
--- a/rpython/jit/metainterp/optimizeopt/intutils.py
+++ b/rpython/jit/metainterp/optimizeopt/intutils.py
@@ -204,6 +204,10 @@
             return IntUnbounded()
 
     def contains(self, val):
+        if not isinstance(val, int):
+            if ((not self.has_lower or self.lower == MININT) and
+                not self.has_upper or self.upper == MAXINT):
+                return True # workaround for address as int
         if self.has_lower and val < self.lower:
             return False
         if self.has_upper and val > self.upper:

From noreply at buildbot.pypy.org  Tue Jun  2 10:49:10 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Tue,  2 Jun 2015 10:49:10 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: fix RECORD_KNOWN_CLASS
Message-ID: <20150602084910.6CF711C024E@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77752:0308c512080c
Date: 2015-06-02 10:31 +0200
http://bitbucket.org/pypy/pypy/changeset/0308c512080c/

Log:	fix RECORD_KNOWN_CLASS

diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py
--- a/rpython/jit/metainterp/optimizeopt/rewrite.py
+++ b/rpython/jit/metainterp/optimizeopt/rewrite.py
@@ -345,14 +345,15 @@
         self.optimize_guard(op, CONST_0)
 
     def optimize_RECORD_KNOWN_CLASS(self, op):
-        value = self.getvalue(op.getarg(0))
+        opinfo = self.getptrinfo(op.getarg(0))
         expectedclassbox = op.getarg(1)
         assert isinstance(expectedclassbox, Const)
-        realclassbox = value.get_constant_class(self.optimizer.cpu)
-        if realclassbox is not None:
-            assert realclassbox.same_constant(expectedclassbox)
-            return
-        value.make_constant_class(None, expectedclassbox)
+        if opinfo is not None:
+            realclassbox = opinfo.get_known_class(self.optimizer.cpu)
+            if realclassbox is not None:
+                assert realclassbox.same_constant(expectedclassbox)
+                return
+        self.make_constant_class(op.getarg(0), expectedclassbox)
 
     def optimize_GUARD_CLASS(self, op):
         expectedclassbox = op.getarg(1)

From noreply at buildbot.pypy.org  Tue Jun  2 10:49:11 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Tue,  2 Jun 2015 10:49:11 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: minor fixes
Message-ID: <20150602084911.8CF761C024E@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77753:1ea860301814
Date: 2015-06-02 10:49 +0200
http://bitbucket.org/pypy/pypy/changeset/1ea860301814/

Log:	minor fixes

diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py
--- a/rpython/jit/backend/llsupport/rewrite.py
+++ b/rpython/jit/backend/llsupport/rewrite.py
@@ -85,6 +85,7 @@
 
     def emit_op(self, op):
         op = self.get_box_replacement(op)
+        orig_op = op
         # XXX specialize on number of args
         replaced = False
         for i in range(op.numargs()):
@@ -93,11 +94,13 @@
             if orig_arg is not arg:
                 if not replaced:
                     op = op.copy_and_change(op.getopnum())
+                    orig_op.set_forwarded(op)
                     replaced = True
                 op.setarg(i, arg)
         if op.is_guard():
             if not replaced:
                 op = op.copy_and_change(op.getopnum())
+                orig_op.set_forwarded(op)
             op.setfailargs([self.get_box_replacement(a)
                             for a in op.getfailargs()])
         self._newops.append(op)
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -2985,9 +2985,8 @@
 
     def do_not_in_trace_call(self, allboxes, descr):
         self.clear_exception()
-        resbox = executor.execute_varargs(self.cpu, self, rop.CALL,
+        executor.execute_varargs(self.cpu, self, rop.CALL_N,
                                           allboxes, descr)
-        assert resbox is None
         if self.last_exc_value_box is not None:
             # cannot trace this!  it raises, so we have to follow the
             # exception-catching path, but the trace doesn't contain

From noreply at buildbot.pypy.org  Tue Jun  2 10:50:22 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Tue,  2 Jun 2015 10:50:22 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: allow None in failargs;
Message-ID: <20150602085022.5C8861C024E@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77754:d86ae392be1c
Date: 2015-06-02 10:50 +0200
http://bitbucket.org/pypy/pypy/changeset/d86ae392be1c/

Log:	allow None in failargs;

diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py
--- a/rpython/jit/backend/llsupport/rewrite.py
+++ b/rpython/jit/backend/llsupport/rewrite.py
@@ -78,7 +78,9 @@
             self._delayed_zero_setfields[op] = d
         return d
 
-    def get_box_replacement(self, op):
+    def get_box_replacement(self, op, allow_none=False):
+        if allow_none and op is None:
+            return None # for failargs
         while op.get_forwarded():
             op = op.get_forwarded()
         return op
@@ -101,7 +103,7 @@
             if not replaced:
                 op = op.copy_and_change(op.getopnum())
                 orig_op.set_forwarded(op)
-            op.setfailargs([self.get_box_replacement(a)
+            op.setfailargs([self.get_box_replacement(a, True)
                             for a in op.getfailargs()])
         self._newops.append(op)
 

From noreply at buildbot.pypy.org  Tue Jun  2 10:59:58 2015
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Tue,  2 Jun 2015 10:59:58 +0200 (CEST)
Subject: [pypy-commit] pypy.org extradoc: update the python version we
	implement
Message-ID: <20150602085958.5C4BC1C1F3F@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: extradoc
Changeset: r616:590cf35c94dd
Date: 2015-06-02 05:00 -0400
http://bitbucket.org/pypy/pypy.org/changeset/590cf35c94dd/

Log:	update the python version we implement

diff --git a/compat.html b/compat.html
--- a/compat.html
+++ b/compat.html
@@ -69,7 +69,7 @@
 <div>
 <div id="main">
 <h1 class="title">Python compatibility</h1>
-<p>PyPy implements the Python language version 2.7.8. It supports all of the core
+<p>PyPy implements the Python language version 2.7.9. It supports all of the core
 language, passing Python test suite (with minor modifications that were
 already accepted in the main python in newer versions). It supports most
 of the commonly used Python <a class="reference external" href="http://docs.python.org/library/">standard library modules</a>; details below.</p>
diff --git a/features.html b/features.html
--- a/features.html
+++ b/features.html
@@ -72,7 +72,7 @@
 <p>PyPy is a replacement for CPython.  It is built using the RPython
 language that was co-developed with it.  The main reason to use it
 instead of CPython is speed: it runs generally faster (see next section).</p>
-<p><strong>PyPy 2.5</strong> implements <strong>Python 2.7.8</strong> and runs on Intel
+<p><strong>PyPy 2.5</strong> implements <strong>Python 2.7.9</strong> and runs on Intel
 <a class="reference external" href="http://en.wikipedia.org/wiki/IA-32">x86 (IA-32)</a> , <a class="reference external" href="http://en.wikipedia.org/wiki/X86_64">x86_64</a> and <a class="reference external" href="http://en.wikipedia.org/wiki/ARM">ARM</a> platforms, with PPC being
 stalled. It supports all of the core language, passing the Python test suite
 (with minor modifications that were already accepted in the main python
diff --git a/index.html b/index.html
--- a/index.html
+++ b/index.html
@@ -70,7 +70,7 @@
 <div id="main">
 <h1 class="title">Welcome to PyPy</h1>
 <p>PyPy is a <a class="reference external" href="http://speed.pypy.org/">fast</a>, <a class="reference external" href="compat.html">compliant</a> alternative implementation of the <a class="reference external" href="http://python.org/">Python</a>
-language (2.7.8 and 3.2.5). It has several advantages and distinct features:</p>
+language (2.7.9 and 3.2.5). It has several advantages and distinct features:</p>
 <blockquote>
 <ul>
 <li><p class="first"><strong>Speed:</strong> thanks to its Just-in-Time compiler, Python programs
diff --git a/source/compat.txt b/source/compat.txt
--- a/source/compat.txt
+++ b/source/compat.txt
@@ -3,7 +3,7 @@
 title: Python compatibility
 ---
 
-PyPy implements the Python language version 2.7.8. It supports all of the core
+PyPy implements the Python language version 2.7.9. It supports all of the core
 language, passing Python test suite (with minor modifications that were
 already accepted in the main python in newer versions). It supports most
 of the commonly used Python `standard library modules`_; details below.
diff --git a/source/features.txt b/source/features.txt
--- a/source/features.txt
+++ b/source/features.txt
@@ -10,7 +10,7 @@
 language that was co-developed with it.  The main reason to use it
 instead of CPython is speed: it runs generally faster (see next section).
 
-**PyPy 2.5** implements **Python 2.7.8** and runs on Intel
+**PyPy 2.5** implements **Python 2.7.9** and runs on Intel
 `x86 (IA-32)`_ , `x86_64`_ and `ARM`_ platforms, with PPC being
 stalled. It supports all of the core language, passing the Python test suite
 (with minor modifications that were already accepted in the main python
diff --git a/source/index.txt b/source/index.txt
--- a/source/index.txt
+++ b/source/index.txt
@@ -4,7 +4,7 @@
 ---
 
 PyPy is a `fast`_, `compliant`_ alternative implementation of the `Python`_
-language (2.7.8 and 3.2.5). It has several advantages and distinct features:
+language (2.7.9 and 3.2.5). It has several advantages and distinct features:
 
   * **Speed:** thanks to its Just-in-Time compiler, Python programs
     often run `faster`_ on PyPy.  `(What is a JIT compiler?)`_

From noreply at buildbot.pypy.org  Tue Jun  2 11:21:48 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Tue,  2 Jun 2015 11:21:48 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: removed the unpacking of the values in
	the trace of call2
Message-ID: <20150602092148.DA0471C0262@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r77755:dd7d048468f6
Date: 2015-06-02 11:21 +0200
http://bitbucket.org/pypy/pypy/changeset/dd7d048468f6/

Log:	removed the unpacking of the values in the trace of call2

diff --git a/pypy/module/micronumpy/iterators.py b/pypy/module/micronumpy/iterators.py
--- a/pypy/module/micronumpy/iterators.py
+++ b/pypy/module/micronumpy/iterators.py
@@ -145,7 +145,9 @@
             jit.promote(elsize)
             offset += elsize
         elif self.ndim_m1 == 0:
-            offset += self.strides[0]
+            stride = self.strides[0]
+            jit.promote(stride)
+            offset += stride
         else:
             for i in xrange(self.ndim_m1, -1, -1):
                 idx = indices[i]
diff --git a/pypy/module/micronumpy/loop.py b/pypy/module/micronumpy/loop.py
--- a/pypy/module/micronumpy/loop.py
+++ b/pypy/module/micronumpy/loop.py
@@ -73,9 +73,16 @@
         if right_iter:
             w_right = right_iter.getitem(right_state).convert_to(space, calc_dtype)
             right_state = right_iter.next(right_state)
-        out_iter.setitem(out_state, func(calc_dtype, w_left, w_right).convert_to(
-            space, res_dtype))
+        w_out = func(calc_dtype, w_left, w_right)
+        out_iter.setitem(out_state, w_out.convert_to(space, res_dtype))
         out_state = out_iter.next(out_state)
+        # if not set to None, the values will be loop carried, forcing
+        # the vectorization to unpack the vector registers at the end
+        # of the loop
+        if left_iter:
+            w_left = None
+        if right_iter:
+            w_right = None
     return out
 
 call1_driver = jit.JitDriver(
diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -488,7 +488,8 @@
     def test_setslice(self):
         result = self.run("setslice")
         assert result == 5.5
-        self.check_vectorized(1, 0) # TODO?
+        self.check_trace_count(1)
+        self.check_vectorized(1, 1)
 
     def define_virtual_slice():
         return """

From noreply at buildbot.pypy.org  Tue Jun  2 11:57:48 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Tue,  2 Jun 2015 11:57:48 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: excluded <OP>_GC from vectorization.
 Those ops can load complex objects which are not subject of this
 optimization
Message-ID: <20150602095748.9A0B41C03A8@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r77756:f80a25298cf3
Date: 2015-06-02 11:57 +0200
http://bitbucket.org/pypy/pypy/changeset/f80a25298cf3/

Log:	excluded <OP>_GC from vectorization. Those ops can load complex
	objects which are not subject of this optimization

diff --git a/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py b/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
@@ -172,6 +172,21 @@
         """
         self.assert_vectorize(self.parse_loop(ops), self.parse_loop(ops))
 
+    def test_vectorize_skip_impossible_2(self):
+        ops = """
+        [p0,i0]
+        i1 = int_add(i0,1)
+        i2 = int_le(i1, 10)
+        guard_true(i2) []
+        i3 = getarrayitem_gc(p0,i0,descr=intarraydescr)
+        jump(p0,i1)
+        """
+        try:
+            self.vectorize(self.parse_loop(ops))
+            py.test.fail("should not happend")
+        except NotAVectorizeableLoop:
+            pass
+
     def test_unroll_empty_stays_empty(self):
         """ has no operations in this trace, thus it stays empty
         after unrolling it 2 times """
@@ -1300,19 +1315,6 @@
 
     def test_abc(self):
         trace ="""
-        [p0, p9, i10, p2, i11, p12, i13, p7, i14, f15, p5, p6, i16, f17, i18, i19]
-        guard_early_exit() [p7, p6, p5, p2, p0, i10, i14, i11, p12, i13, f15, p9]
-        i20 = raw_load(i16, i11, descr=floatarraydescr)
-        guard_not_invalidated() [p7, p6, p5, p2, p0, i20, i10, i14, i11, p12, i13, None, p9]
-        f22 = cast_int_to_float(i20)
-        i24 = int_add(i11, 8)
-        f25 = float_add(f22, f17)
-        raw_store(i18, i14, f25, descr=floatarraydescr)
-        i27 = int_add(i13, 1)
-        i29 = int_add(i14, 8)
-        i30 = int_ge(i27, i19)
-        guard_false(i30) [p7, p6, p5, p2, p0, i24, i27, i29, f22, i10, None, None, p12, None, None, p9]
-        jump(p0, p9, i10, p2, i24, p12, i27, p7, i29, f22, p5, p6, i16, f17, i18, i19)
         """
         opt = self.vectorize(self.parse_loop(trace))
         self.debug_print_operations(opt.loop)
@@ -1335,52 +1337,6 @@
         opt = self.vectorize(self.parse_loop(trace))
         self.debug_print_operations(opt.loop)
         return
-        pass # TODO
-        trace = """
-        # Loop unroll (pre vectorize) : -2 with 23 ops
-[i0, i1, p2, p3, p4, p5, p6, p7, p8, p9]
-label(i1, p2, p3, p10, i11, p7, i12, p6, p8, p13, i14, i15, i16, i17, i18, i19, i20, i21, i22, i23, descr=TargetToken(140567134602960))
-debug_merge_point(0, 0, '(numpy_axis_reduce: no get_printable_location)')
-guard_early_exit(descr=<rpython.jit.metainterp.compile.ResumeAtLoopHeaderDescr object at 0x7fd857537510>) [i1, p8, p7, p6, p3, p2, p10, p13, i12, i14, i15, i11]
-f24 = raw_load(i16, i15, descr=<ArrayF 8>)
-guard_not_invalidated(descr=<rpython.jit.metainterp.compile.ResumeGuardNotInvalidated object at 0x7fd857563a90>) [i1, p8, p7, p6, p3, p2, f24, p10, p13, i12, i14, i15, i11]
-i26 = int_add(i15, 8)
-i27 = getarrayitem_gc(p10, i1, descr=<ArrayS 8>)
-i28 = int_is_zero(i27)
-guard_false(i28, descr=<rpython.jit.metainterp.compile.ResumeGuardFalseDescr object at 0x7fd85753f550>) [i1, p8, p7, p6, p3, p2, f24, i26, p10, p13, i12, i14, None, i11]
-f30 = raw_load(i17, i12, descr=<ArrayF 8>)
-f31 = float_add(f30, f24)
-raw_store(i18, i12, f31, descr=<ArrayF 8>)
-i33 = int_add(i11, 1)
-i34 = getarrayitem_gc(p10, i19, descr=<ArrayS 8>)
-i35 = int_lt(i34, i20)
-guard_true(i35, descr=<rpython.jit.metainterp.compile.ResumeGuardTrueDescr object at 0x7fd857537290>) [i1, p8, p7, p6, p3, p2, i21, i34, i12, i33, i19, p10, f31, None, i26, None, p13, None, i14, None, i11]
-i37 = int_add(i34, 1)
-setarrayitem_gc(p10, i19, i37, descr=<ArrayS 8>)
-i38 = int_add(i12, i22)
-i39 = int_ge(i33, i23)
-guard_false(i39, descr=<rpython.jit.metainterp.compile.ResumeGuardFalseDescr object at 0x7fd8575487d0>) [i1, p8, p7, p6, p3, p2, i38, i33, None, None, i26, p10, p13, None, i14, None, None]
-debug_merge_point(0, 0, '(numpy_axis_reduce: no get_printable_location)')
-jump(i1, p2, p3, p10, i33, p7, i38, p6, p8, p13, i14, i26, i16, i17, i18, i19, i20, i21, i22, i23, descr=TargetToken(140567134602960))
-        """
-        trace = """ # fail fail RuntimeError('guard_true/false has no operation that returns the bool for the arg 0',)
-        # Loop unroll (pre vectorize) : -2 with 14 ops
-        [p0, p1, p2]
-        label(p3, i4, p2, i5, i6, i7, descr=TargetToken(140567130056592))
-        debug_merge_point(0, 0, '(numpy_reduce: no get_printable_location)')
-        guard_early_exit(descr=<rpython.jit.metainterp.compile.ResumeAtLoopHeaderDescr object at 0x7fd855dc6bd0>) [p2, p3, i4, i5]
-        f8 = raw_load(i6, i5, descr=<ArrayF 8>)
-        guard_not_invalidated(descr=<rpython.jit.metainterp.compile.ResumeGuardNotInvalidated object at 0x7fd855dbcad0>) [p2, f8, p3, i4, i5]
-        i9 = cast_float_to_int(f8)
-        i11 = int_and(i9, 255)
-        guard_false(i11, descr=<rpython.jit.metainterp.compile.ResumeGuardFalseDescr object at 0x7fd855dca390>) [p2, p3, i4, i5]
-        i13 = int_add(i4, 1)
-        i15 = int_add(i5, 8)
-        i16 = int_ge(i13, i7)
-        guard_false(i16, descr=<rpython.jit.metainterp.compile.ResumeGuardFalseDescr object at 0x7fd8560c6150>) [p2, i13, i15, p3, None, None]
-        debug_merge_point(0, 0, '(numpy_reduce: no get_printable_location)')
-        jump(p3, i13, p2, i15, i6, i7, descr=TargetToken(140567130056592))
-        """
 
 class TestLLtype(BaseTestVectorize, LLtypeMixin):
     pass
diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/vectorize.py
@@ -245,7 +245,7 @@
         for i,op in enumerate(loop.operations):
             if op.getopnum() == rop.GUARD_EARLY_EXIT:
                 self.early_exit_idx = i
-            if op.is_array_op():
+            if op.is_raw_array_access():
                 descr = op.getdescr()
                 if not descr.is_array_of_pointers():
                     byte_count = descr.get_item_size_in_bytes()
@@ -1296,7 +1296,7 @@
         """ Blocks the packing of some operations """
         if inquestion.vector == -1:
             return True
-        if packed.is_array_op():
+        if packed.is_raw_array_access():
             if packed.getarg(1) == inquestion.result:
                 return True
         return False
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -171,10 +171,14 @@
     def is_vector_arithmetic(self):
         return rop._VEC_ARITHMETIC_FIRST <= self.getopnum() <= rop._VEC_ARITHMETIC_LAST
 
-    def is_array_op(self):
-        on = self.getopnum()
-        return rop.GETARRAYITEM_GC <= on <= rop.VEC_RAW_LOAD or \
-               rop.SETARRAYITEM_GC <= on <= rop.VEC_RAW_STORE
+    def is_raw_array_access(self):
+        return self.is_raw_load() or self.is_raw_store()
+
+    def is_raw_load(self):
+        return rop._RAW_LOAD_FIRST < self.getopnum() < rop._RAW_LOAD_LAST
+
+    def is_raw_store(self):
+        return rop._RAW_LOAD_FIRST < self.getopnum() < rop._RAW_LOAD_LAST
 
     def is_comparison(self):
         return self.is_always_pure() and self.returns_bool_result()
@@ -530,10 +534,14 @@
     '_ALWAYS_PURE_LAST',  # ----- end of always_pure operations -----
 
     'GETARRAYITEM_GC/2d',
+
+    '_RAW_LOAD_FIRST',
     'GETARRAYITEM_RAW/2d',
     'VEC_GETARRAYITEM_RAW/3d',
     'RAW_LOAD/2d',
     'VEC_RAW_LOAD/3d',
+    '_RAW_LOAD_LAST',
+
     'GETINTERIORFIELD_GC/2d',
     'GETFIELD_GC/1d',
     'GETFIELD_RAW/1d',
@@ -554,10 +562,14 @@
 
     'INCREMENT_DEBUG_COUNTER/1',
     'SETARRAYITEM_GC/3d',
+
+    '_RAW_STORE_FIRST',
     'SETARRAYITEM_RAW/3d',
     'VEC_SETARRAYITEM_RAW/3d',
     'RAW_STORE/3d',
     'VEC_RAW_STORE/3d',
+    '_RAW_STORE_LAST',
+
     'SETINTERIORFIELD_GC/3d',
     'SETINTERIORFIELD_RAW/3d',    # right now, only used by tests
     'SETFIELD_GC/2d',

From noreply at buildbot.pypy.org  Tue Jun  2 13:47:06 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Tue,  2 Jun 2015 13:47:06 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: fix the backend
Message-ID: <20150602114706.873D11C1C3E@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77757:4180412b78a9
Date: 2015-06-02 13:47 +0200
http://bitbucket.org/pypy/pypy/changeset/4180412b78a9/

Log:	fix the backend

diff --git a/rpython/jit/backend/llsupport/descr.py b/rpython/jit/backend/llsupport/descr.py
--- a/rpython/jit/backend/llsupport/descr.py
+++ b/rpython/jit/backend/llsupport/descr.py
@@ -66,7 +66,7 @@
 
 BaseSizeDescr = SizeDescr
 
-def get_size_descr(cpu, gccache, STRUCT, is_object):
+def get_size_descr(gccache, STRUCT, is_object):
     cache = gccache._cache_size
     try:
         return cache[STRUCT]
@@ -74,13 +74,13 @@
         size = symbolic.get_size(STRUCT, gccache.translate_support_code)
         count_fields_if_immut = heaptracker.count_fields_if_immutable(STRUCT)
         gc_fielddescrs = heaptracker.gc_fielddescrs(gccache, STRUCT)
-        if is_object: #heaptracker.has_gcstruct_a_vtable(STRUCT):
-            #assert is_object
+        if is_object:
+            assert heaptracker.has_gcstruct_a_vtable(STRUCT)
             sizedescr = SizeDescrWithVTable(size, count_fields_if_immut,
                                             gc_fielddescrs, None,
-                heaptracker.get_vtable_for_gcstruct(cpu, STRUCT))
+                heaptracker.get_vtable_for_gcstruct(gccache, STRUCT))
         else:
-            #assert not is_object
+            assert not heaptracker.has_gcstruct_a_vtable(STRUCT)
             sizedescr = SizeDescr(size, count_fields_if_immut,
                                   gc_fielddescrs, None)
         gccache.init_size_descr(STRUCT, sizedescr)
@@ -172,7 +172,8 @@
         fielddescr = FieldDescr(name, offset, size, flag, index_in_parent)
         cachedict = cache.setdefault(STRUCT, {})
         cachedict[fieldname] = fielddescr
-        fielddescr.parent_descr = get_size_descr(None, gccache, STRUCT, False)
+        fielddescr.parent_descr = get_size_descr(gccache, STRUCT,
+                                  heaptracker.has_gcstruct_a_vtable(STRUCT))
         return fielddescr
 
 def get_type_flag(TYPE):
diff --git a/rpython/jit/backend/llsupport/llmodel.py b/rpython/jit/backend/llsupport/llmodel.py
--- a/rpython/jit/backend/llsupport/llmodel.py
+++ b/rpython/jit/backend/llsupport/llmodel.py
@@ -299,7 +299,7 @@
         return rffi.cast(TYPE, x)
 
     def sizeof(self, S, is_object):
-        return get_size_descr(self, self.gc_ll_descr, S, is_object)
+        return get_size_descr(self.gc_ll_descr, S, is_object)
 
     def fielddescrof(self, STRUCT, fieldname):
         return get_field_descr(self.gc_ll_descr, STRUCT, fieldname)
diff --git a/rpython/jit/codewriter/heaptracker.py b/rpython/jit/codewriter/heaptracker.py
--- a/rpython/jit/codewriter/heaptracker.py
+++ b/rpython/jit/codewriter/heaptracker.py
@@ -56,24 +56,24 @@
             return False
     return True
 
-def get_vtable_for_gcstruct(cpu, GCSTRUCT):
+def get_vtable_for_gcstruct(gccache, GCSTRUCT):
     # xxx hack: from a GcStruct representing an instance's
     # lowleveltype, return the corresponding vtable pointer.
     # Returns None if the GcStruct does not belong to an instance.
     assert isinstance(GCSTRUCT, lltype.GcStruct)
     if not has_gcstruct_a_vtable(GCSTRUCT):
         return None
-    setup_cache_gcstruct2vtable(cpu)
-    return cpu._cache_gcstruct2vtable[GCSTRUCT]
+    setup_cache_gcstruct2vtable(gccache)
+    return gccache._cache_gcstruct2vtable[GCSTRUCT]
 
-def setup_cache_gcstruct2vtable(cpu):
-    if not hasattr(cpu, '_cache_gcstruct2vtable'):
+def setup_cache_gcstruct2vtable(gccache):
+    if not hasattr(gccache, '_cache_gcstruct2vtable'):
         cache = {}
         cache.update(testing_gcstruct2vtable)
-        if cpu.rtyper:
-            for rinstance in cpu.rtyper.instance_reprs.values():
+        if gccache.rtyper:
+            for rinstance in gccache.rtyper.instance_reprs.values():
                 cache[rinstance.lowleveltype.TO] = rinstance.rclass.getvtable()
-        cpu._cache_gcstruct2vtable = cache
+            gccache._cache_gcstruct2vtable = cache
 
 def set_testing_vtable_for_gcstruct(GCSTRUCT, vtable, name):
     # only for tests that need to register the vtable of their malloc'ed
diff --git a/rpython/jit/metainterp/virtualref.py b/rpython/jit/metainterp/virtualref.py
--- a/rpython/jit/metainterp/virtualref.py
+++ b/rpython/jit/metainterp/virtualref.py
@@ -27,6 +27,7 @@
         adr = heaptracker.adr2int(adr)
         self.jit_virtual_ref_const_class = history.ConstInt(adr)
         fielddescrof = self.cpu.fielddescrof
+        self.cpu.gc_ll_descr._cache_gcstruct2vtable[self.JIT_VIRTUAL_REF] = self.jit_virtual_ref_vtable
         self.descr_virtual_token = fielddescrof(self.JIT_VIRTUAL_REF,
                                                 'virtual_token')
         self.descr_forced = fielddescrof(self.JIT_VIRTUAL_REF, 'forced')

From noreply at buildbot.pypy.org  Tue Jun  2 14:00:11 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Tue,  2 Jun 2015 14:00:11 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: wrong assumption: it can happen that a
 call that is exception guarded is not immediatley after the call
Message-ID: <20150602120011.2A7C51C1C3E@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r77758:cd53c1c662a5
Date: 2015-06-02 14:00 +0200
http://bitbucket.org/pypy/pypy/changeset/cd53c1c662a5/

Log:	wrong assumption: it can happen that a call that is exception
	guarded is not immediatley after the call

diff --git a/rpython/jit/metainterp/optimizeopt/dependency.py b/rpython/jit/metainterp/optimizeopt/dependency.py
--- a/rpython/jit/metainterp/optimizeopt/dependency.py
+++ b/rpython/jit/metainterp/optimizeopt/dependency.py
@@ -598,22 +598,32 @@
         elif guard_opnum in (rop.GUARD_OVERFLOW, rop.GUARD_NO_OVERFLOW):
             # previous operation must be an ovf_operation
             guard_node.setpriority(100)
-            prev_node = self.nodes[guard_node.getindex()-1]
-            assert prev_node.getoperation().is_ovf()
-            prev_node.edge_to(guard_node, None, label='overflow')
-        elif guard_opnum == rop.GUARD_NOT_FORCED:
-            # previous op must be one that can raise
-            guard_node.setpriority(100)
-            prev_node = self.nodes[guard_node.getindex()-1]
-            assert prev_node.getoperation().can_raise()
-            prev_node.edge_to(guard_node, None, label='forced')
-        elif guard_opnum in (rop.GUARD_NO_EXCEPTION, rop.GUARD_EXCEPTION):
+            i = guard_node.getindex()-1
+            while i >= 0:
+                node = self.nodes[i]
+                op = node.getoperation()
+                if node.is_ovf():
+                    break
+                i -= 1
+            else:
+                raise AssertionError("(no)overflow: no overflowing op present")
+            node.edge_to(guard_node, None, label='overflow')
+        elif guard_opnum in (rop.GUARD_NO_EXCEPTION, rop.GUARD_EXCEPTION, rop.GUARD_NOT_FORCED):
             # previous op must be one that can raise or a not forced guard
             guard_node.setpriority(100)
-            prev_node = self.nodes[guard_node.getindex()-1]
-            prev_node.edge_to(guard_node, None, label='exception')
-            if not prev_node.getoperation().getopnum() == rop.GUARD_NOT_FORCED:
-                assert prev_node.getoperation().can_raise()
+            i = guard_node.getindex() - 1
+            while i >= 0:
+                node = self.nodes[i]
+                op = node.getoperation()
+                if op.can_raise():
+                    node.edge_to(guard_node, None, label='exception/notforced')
+                    break
+                if op.is_guard():
+                    node.edge_to(guard_node, None, label='exception/notforced')
+                    break
+                i -= 1
+            else:
+                raise AssertionError("(no)exception/not_forced: not op raises for them")
         else:
             pass # not invalidated, early exit, future condition!
 
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py b/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
@@ -57,6 +57,8 @@
             raise NotAVectorizeableLoop()
         if unroll_factor == -1:
             unroll_factor = opt.get_unroll_count(ARCH_VEC_REG_SIZE)
+            print ""
+            print "unroll factor: ", unroll_factor, opt.smallest_type_bytes
         opt.analyse_index_calculations()
         if opt.dependency_graph is not None:
             self._write_dot_and_convert_to_svg(opt.dependency_graph, "ee" + self.test_name)
@@ -831,6 +833,7 @@
     def test_packset_vector_operation(self, op, descr, stride):
         ops = """
         [p0,p1,p2,i0]
+        guard_early_exit() []
         i1 = int_add(i0, {stride})
         i10 = int_le(i1, 128)
         guard_true(i10) []
@@ -845,20 +848,14 @@
         assert len(vopt.dependency_graph.memory_refs) == 12
         assert len(vopt.packset.packs) == 4
 
-        for opindices in [(4,11,18,25),(5,12,19,26),
-                          (6,13,20,27),(7,14,21,28)]:
+        for opindices in [(5,12,19,26),(6,13,20,27),
+                          (7,14,21,28),(8,15,22,29)]:
             self.assert_has_pack_with(vopt.packset, opindices)
 
     @pytest.mark.parametrize('op,descr,stride',
-            [('int_add','char',1),
-             ('int_sub','char',1),
-             ('int_mul','char',1),
-             ('float_add','float',8),
+            [('float_add','float',8),
              ('float_sub','float',8),
              ('float_mul','float',8),
-             ('float_add','singlefloat',4),
-             ('float_sub','singlefloat',4),
-             ('float_mul','singlefloat',4),
              ('int_add','int',8),
              ('int_sub','int',8),
              ('int_mul','int',8),
@@ -1314,6 +1311,7 @@
 
 
     def test_abc(self):
+        py.test.skip()
         trace ="""
         """
         opt = self.vectorize(self.parse_loop(trace))
@@ -1336,7 +1334,6 @@
         """
         opt = self.vectorize(self.parse_loop(trace))
         self.debug_print_operations(opt.loop)
-        return
 
 class TestLLtype(BaseTestVectorize, LLtypeMixin):
     pass
diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/vectorize.py
@@ -214,7 +214,8 @@
                     if not isinstance(target_guard.getdescr(), ResumeAtLoopHeaderDescr):
                         descr = invent_fail_descr_for_op(copied_op.getopnum(), self)
                         olddescr = copied_op.getdescr()
-                        descr.copy_all_attributes_from(olddescr)
+                        if olddescr:
+                            descr.copy_all_attributes_from(olddescr)
                         copied_op.setdescr(descr)
 
                     if oi < ee_pos:
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -178,7 +178,7 @@
         return rop._RAW_LOAD_FIRST < self.getopnum() < rop._RAW_LOAD_LAST
 
     def is_raw_store(self):
-        return rop._RAW_LOAD_FIRST < self.getopnum() < rop._RAW_LOAD_LAST
+        return rop._RAW_STORE_FIRST < self.getopnum() < rop._RAW_STORE_LAST
 
     def is_comparison(self):
         return self.is_always_pure() and self.returns_bool_result()

From noreply at buildbot.pypy.org  Tue Jun  2 14:45:47 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Tue,  2 Jun 2015 14:45:47 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: removed primitive type mixin
Message-ID: <20150602124547.9A8851C024E@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r77759:20d16b623806
Date: 2015-06-02 14:43 +0200
http://bitbucket.org/pypy/pypy/changeset/20d16b623806/

Log:	removed primitive type mixin added two schedule tests for
	constant/variable expansion, need to move the instructions before
	the label

diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py
--- a/rpython/jit/metainterp/history.py
+++ b/rpython/jit/metainterp/history.py
@@ -519,38 +519,7 @@
 
 # ____________________________________________________________
 
-class PrimitiveTypeMixin(object):
-    _mixin_ = True
-
-    def gettype(self):
-        raise NotImplementedError
-    def getsize(self):
-        raise NotImplementedError
-    def getsigned(self):
-        raise NotImplementedError
-
-    def matches_type(self, other):
-        assert isinstance(other, PrimitiveTypeMixin)
-        return self.gettype() == other.gettype()
-
-    def matches_size(self, other):
-        assert isinstance(other, PrimitiveTypeMixin)
-        return self.getsize() == other.getsize()
-
-    def matches_sign(self, other):
-        assert isinstance(other, PrimitiveTypeMixin)
-        return self.getsigend() == other.signed()
-
-    def matches(self, other):
-        if isinstance(other, PrimitiveTypeMixin):
-            return self.matches_type(other) and \
-                   self.matches_size(other) and \
-                   self.matches_sign(other)
-        return False
-
-
-
-class BoxVector(Box, PrimitiveTypeMixin):
+class BoxVector(Box):
     type = VECTOR
     _attrs_ = ('item_type','item_count','item_size','item_signed')
     _extended_display = False
diff --git a/rpython/jit/metainterp/optimizeopt/dependency.py b/rpython/jit/metainterp/optimizeopt/dependency.py
--- a/rpython/jit/metainterp/optimizeopt/dependency.py
+++ b/rpython/jit/metainterp/optimizeopt/dependency.py
@@ -602,7 +602,7 @@
             while i >= 0:
                 node = self.nodes[i]
                 op = node.getoperation()
-                if node.is_ovf():
+                if op.is_ovf():
                     break
                 i -= 1
             else:
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_schedule.py b/rpython/jit/metainterp/optimizeopt/test/test_schedule.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_schedule.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_schedule.py
@@ -93,3 +93,34 @@
         v3[f64#2] = vec_cast_int_to_float(v2[i32#2])
         """, False)
         self.assert_equal(loop2, loop3)
+
+    def test_scalar_pack(self):
+        loop1 = self.parse("""
+        i10 = int_add(i0, 73)
+        i11 = int_add(i1, 73)
+        """)
+        pack1 = self.pack(loop1, 0, 2)
+        loop2 = self.schedule(loop1, [pack1])
+        loop3 = self.parse("""
+        v1[i64#2] = vec_box(2)
+        v2[i64#2] = vec_int_pack(v1[i64#2], i0, 0, 1)
+        v3[i64#2] = vec_int_pack(v2[i64#2], i1, 1, 1)
+        v4[i64#2] = vec_int_expand(73)
+        v5[i64#2] = vec_int_add(v3[i64#2], v4[i64#2])
+        """, False)
+        self.assert_equal(loop2, loop3)
+
+        loop1 = self.parse("""
+        f10 = float_add(f0, 73.0)
+        f11 = float_add(f1, 73.0)
+        """)
+        pack1 = self.pack(loop1, 0, 2)
+        loop2 = self.schedule(loop1, [pack1])
+        loop3 = self.parse("""
+        v1[f64#2] = vec_box(2)
+        v2[f64#2] = vec_float_pack(v1[f64#2], f0, 0, 1)
+        v3[f64#2] = vec_float_pack(v2[f64#2], f1, 1, 1)
+        v4[f64#2] = vec_float_expand(73.0)
+        v5[f64#2] = vec_float_add(v3[f64#2], v4[f64#2])
+        """, False)
+        self.assert_equal(loop2, loop3)
diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/vectorize.py
@@ -5,7 +5,7 @@
 from rpython.jit.metainterp.optimizeopt.unroll import optimize_unroll
 from rpython.jit.metainterp.compile import ResumeAtLoopHeaderDescr, invent_fail_descr_for_op
 from rpython.jit.metainterp.history import (ConstInt, VECTOR, FLOAT, INT,
-        BoxVector, TargetToken, JitCellToken, Box, PrimitiveTypeMixin)
+        BoxVector, BoxFloat, BoxInt, ConstFloat, TargetToken, JitCellToken, Box)
 from rpython.jit.metainterp.optimizeopt.optimizer import Optimizer, Optimization
 from rpython.jit.metainterp.optimizeopt.util import make_dispatcher_method
 from rpython.jit.metainterp.optimizeopt.dependency import (DependencyGraph, 
@@ -593,7 +593,7 @@
             assert isinstance(key2, IndexVar)
             return key1.compare(key2)
         #
-        raise RuntimeError("cannot compare: " + str(key1) + " <=> " + str(key2))
+        raise AssertionError("cannot compare: " + str(key1) + " <=> " + str(key2))
 
     def emit_varops(self, opt, var, old_arg):
         if isinstance(var, IndexVar):
@@ -630,7 +630,7 @@
                 return op
             i -= 1
 
-        raise RuntimeError("guard_true/false first arg not defined")
+        raise AssertionError("guard_true/false first arg not defined")
 
     def _get_key(self, cmp_op):
         if cmp_op and rop.INT_LT <= cmp_op.getopnum() <= rop.INT_GE:
@@ -772,7 +772,7 @@
         return 1
 
 
-class PackType(PrimitiveTypeMixin):
+class PackType(object):
     UNKNOWN_TYPE = '-'
 
     def __init__(self, type, size, signed, count=-1, scalar_cost=1, vector_cost=1):
@@ -845,11 +845,18 @@
         return self.input_type.getsize()
 
     def determine_input_type(self, op):
+        arg = op.getarg(0)
         _, vbox = self.sched_data.getvector_of_box(op.getarg(0))
         if vbox:
             return PackType.of(vbox)
         else:
-            raise RuntimeError("fatal: box %s is not in a vector box" % (op.getarg(0),))
+            vec_reg_size = self.sched_data.vec_reg_size
+            if isinstance(arg, ConstInt) or isinstance(arg, BoxInt):
+                return PackType(INT, 8, True, 2)
+            elif isinstance(arg, ConstFloat) or isinstance(arg, BoxFloat):
+                return PackType(FLOAT, 8, True, 2)
+            else:
+                raise NotImplementedError("arg %s not supported" % (arg,))
 
     def determine_output_type(self, op):
         return self.determine_input_type(op)
@@ -1010,7 +1017,8 @@
             op = ResOperation(opnum, [tgt_box, src_box, ConstInt(i),
                                       ConstInt(src_box.item_count)], new_box)
             self.preamble_ops.append(op)
-            self._check_vec_pack(op)
+            if not we_are_translated():
+                self._check_vec_pack(op)
             i += src_box.item_count
 
             # overwrite the new positions, arguments now live in new_box
@@ -1041,10 +1049,11 @@
         assert index.value + count.value <= result.item_count
         assert result.item_count > arg0.item_count
 
-    def expand_box_to_vector_box(self, vbox, ops, arg, argidx):
+    def expand_box_to_vector_box(self, vbox, nodes, arg, argidx):
         all_same_box = True
-        for i, op in enumerate(ops):
-            if arg is not op.getoperation().getarg(argidx):
+        for i, node in enumerate(nodes):
+            op = node.getoperation()
+            if not arg.same_box(op.getarg(argidx)):
                 all_same_box = False
                 break
             i += 1
@@ -1060,16 +1069,17 @@
             expand_op = ResOperation(expand_opnum, [arg], vbox)
             self.preamble_ops.append(expand_op)
         else:
-            resop = ResOperation(rop.VEC_BOX, [ConstInt(len(ops))], vbox)
+            resop = ResOperation(rop.VEC_BOX, [ConstInt(len(nodes))], vbox)
             self.preamble_ops.append(resop)
             opnum = rop.VEC_FLOAT_PACK
             if arg.type == INT:
                 opnum = rop.VEC_INT_PACK
-            for i,op in enumerate(ops):
-                arg = op.getoperation().getarg(argidx)
+            for i,node in enumerate(nodes):
+                op = node.getoperation()
+                arg = op.getarg(argidx)
                 new_box = vbox.clonebox()
                 resop = ResOperation(opnum,
-                                     [vbox,arg,ConstInt(i),ConstInt(0)], new_box)
+                                     [vbox,arg,ConstInt(i),ConstInt(1)], new_box)
                 vbox = new_box
                 self.preamble_ops.append(resop)
         return vbox

From noreply at buildbot.pypy.org  Tue Jun  2 15:19:01 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Tue,  2 Jun 2015 15:19:01 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: expanding variables/constants are not
 inlined before the instruction but gathered in another list to be prepended
 to the label operation
Message-ID: <20150602131901.DA5A41C0262@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r77760:692b8f945d3a
Date: 2015-06-02 15:19 +0200
http://bitbucket.org/pypy/pypy/changeset/692b8f945d3a/

Log:	expanding variables/constants are not inlined before the instruction
	but gathered in another list to be prepended to the label operation

diff --git a/rpython/jit/metainterp/optimizeopt/test/test_schedule.py b/rpython/jit/metainterp/optimizeopt/test/test_schedule.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_schedule.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_schedule.py
@@ -39,7 +39,7 @@
     def pack(self, loop, l, r):
         return [Node(op,1+l+i) for i,op in enumerate(loop.operations[1+l:1+r])]
 
-    def schedule(self, loop_orig, packs, vec_reg_size=16):
+    def schedule(self, loop_orig, packs, vec_reg_size=16, prepend_invariant=False):
         loop = get_model(False).ExtendedTreeLoop("loop")
         loop.original_jitcell_token = loop_orig.original_jitcell_token
         loop.inputargs = loop_orig.inputargs
@@ -53,6 +53,8 @@
                 for op in vsd.as_vector_operation(Pack(pack)):
                     ops.append(op)
         loop.operations = ops
+        if prepend_invariant:
+            loop.operations = vsd.invariant_oplist + ops
         return loop
 
     def assert_operations_match(self, loop_a, loop_b):
@@ -100,7 +102,7 @@
         i11 = int_add(i1, 73)
         """)
         pack1 = self.pack(loop1, 0, 2)
-        loop2 = self.schedule(loop1, [pack1])
+        loop2 = self.schedule(loop1, [pack1], prepend_invariant=True)
         loop3 = self.parse("""
         v1[i64#2] = vec_box(2)
         v2[i64#2] = vec_int_pack(v1[i64#2], i0, 0, 1)
@@ -115,7 +117,7 @@
         f11 = float_add(f1, 73.0)
         """)
         pack1 = self.pack(loop1, 0, 2)
-        loop2 = self.schedule(loop1, [pack1])
+        loop2 = self.schedule(loop1, [pack1], prepend_invariant=True)
         loop3 = self.parse("""
         v1[f64#2] = vec_box(2)
         v2[f64#2] = vec_float_pack(v1[f64#2], f0, 0, 1)
diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/vectorize.py
@@ -941,8 +941,7 @@
         box_pos, vbox = self.sched_data.getvector_of_box(arg)
         if not vbox:
             # constant/variable expand this box
-            vbox = self.input_type.new_vector_box(len(ops))
-            vbox = self.expand_box_to_vector_box(vbox, ops, arg, argidx)
+            vbox = self.expand(ops, arg, argidx)
             box_pos = 0
 
         # use the input as an indicator for the pack type
@@ -1049,39 +1048,43 @@
         assert index.value + count.value <= result.item_count
         assert result.item_count > arg0.item_count
 
-    def expand_box_to_vector_box(self, vbox, nodes, arg, argidx):
-        all_same_box = True
+    def expand(self, nodes, arg, argidx):
+        vbox = self.input_type.new_vector_box(len(nodes))
+        box_type = arg.type
+        invariant_ops = self.sched_data.invariant_oplist
+        invariant_vars = self.sched_data.invariant_vector_vars
+        if isinstance(arg, BoxVector):
+            box_type = arg.item_type
+
         for i, node in enumerate(nodes):
             op = node.getoperation()
             if not arg.same_box(op.getarg(argidx)):
-                all_same_box = False
                 break
             i += 1
+        else:
+            expand_opnum = rop.VEC_FLOAT_EXPAND
+            if box_type == INT:
+                expand_opnum = rop.VEC_INT_EXPAND
+            op = ResOperation(expand_opnum, [arg], vbox)
+            invariant_ops.append(op)
+            invariant_vars.append(vbox)
+            return vbox
 
-        box_type = arg.type
-        if isinstance(arg, BoxVector):
-            box_type = arg.item_type
-        expand_opnum = rop.VEC_FLOAT_EXPAND
-        if box_type == INT:
-            expand_opnum = rop.VEC_INT_EXPAND
-
-        if all_same_box:
-            expand_op = ResOperation(expand_opnum, [arg], vbox)
-            self.preamble_ops.append(expand_op)
-        else:
-            resop = ResOperation(rop.VEC_BOX, [ConstInt(len(nodes))], vbox)
-            self.preamble_ops.append(resop)
-            opnum = rop.VEC_FLOAT_PACK
-            if arg.type == INT:
-                opnum = rop.VEC_INT_PACK
-            for i,node in enumerate(nodes):
-                op = node.getoperation()
-                arg = op.getarg(argidx)
-                new_box = vbox.clonebox()
-                resop = ResOperation(opnum,
-                                     [vbox,arg,ConstInt(i),ConstInt(1)], new_box)
-                vbox = new_box
-                self.preamble_ops.append(resop)
+        op = ResOperation(rop.VEC_BOX, [ConstInt(len(nodes))], vbox)
+        invariant_ops.append(op)
+        opnum = rop.VEC_FLOAT_PACK
+        if arg.type == INT:
+            opnum = rop.VEC_INT_PACK
+        for i,node in enumerate(nodes):
+            op = node.getoperation()
+            arg = op.getarg(argidx)
+            new_box = vbox.clonebox()
+            ci = ConstInt(i)
+            c1 = ConstInt(1)
+            op = ResOperation(opnum, [vbox,arg,ci,c1], new_box)
+            vbox = new_box
+            invariant_ops.append(op)
+        invariant_vars.append(vbox)
         return vbox
 
 class OpToVectorOpConv(OpToVectorOp):
@@ -1232,9 +1235,9 @@
 class VecScheduleData(SchedulerData):
     def __init__(self, vec_reg_size):
         self.box_to_vbox = {}
-        self.preamble_ops = None
-        self.expansion_byte_count = -1
         self.vec_reg_size = vec_reg_size
+        self.invariant_oplist = []
+        self.invariant_vector_vars = []
 
     def as_vector_operation(self, pack):
         op_count = len(pack.operations)
@@ -1247,7 +1250,7 @@
         op0 = pack.operations[0].getoperation()
         tovector = ROP_ARG_RES_VECTOR.get(op0.vector, None)
         if tovector is None:
-            raise NotImplementedError("missing vecop for '" + op0.getopname() + "'")
+            raise NotImplementedError("missing vecop for '%s'" % (op0.getopname(),))
         oplist = []
         tovector.as_vector_operation(pack, self, oplist)
         return oplist

From noreply at buildbot.pypy.org  Tue Jun  2 15:19:27 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Tue,  2 Jun 2015 15:19:27 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: work on rpython and recursive stuff
Message-ID: <20150602131927.CF83A1C0262@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77761:c3aeb7fe8605
Date: 2015-06-02 14:56 +0200
http://bitbucket.org/pypy/pypy/changeset/c3aeb7fe8605/

Log:	work on rpython and recursive stuff

diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -995,63 +995,64 @@
     execute_call_release_gil_r = _execute_call_release_gil
     execute_call_release_gil_f = _execute_call_release_gil
 
-    def _execute_call_assembler(self, descr, *args):
-        # XXX simplify the following a bit
-        #
-        # pframe = CALL_ASSEMBLER(args..., descr=looptoken)
-        # ==>
-        #     pframe = CALL looptoken.loopaddr(*args)
-        #     JUMP_IF_FAST_PATH @fastpath
-        #     res = CALL assembler_call_helper(pframe)
-        #     jmp @done
-        #   @fastpath:
-        #     res = GETFIELD(pframe, 'result')
-        #   @done:
-        #
-        call_op = self.lltrace.operations[self.current_index]
-        guard_op = self.lltrace.operations[self.current_index + 1]
-        assert guard_op.getopnum() == rop.GUARD_NOT_FORCED
-        self.force_guard_op = guard_op
-        pframe = self.cpu._execute_token(descr, *args)
-        del self.force_guard_op
-        #
-        jd = descr.outermost_jitdriver_sd
-        assert jd is not None, ("call_assembler(): the loop_token needs "
-                                "to have 'outermost_jitdriver_sd'")
-        if jd.index_of_virtualizable != -1:
-            vable = args[jd.index_of_virtualizable]
-        else:
-            vable = lltype.nullptr(llmemory.GCREF.TO)
-        #
-        # Emulate the fast path
-        #
-        faildescr = self.cpu.get_latest_descr(pframe)
-        if faildescr == self.cpu.done_with_this_frame_descr_int:
-            return self.cpu.get_int_value(pframe, 0)
-        elif faildescr == self.cpu.done_with_this_frame_descr_ref:
-            return self.cpu.get_ref_value(pframe, 0)
-        elif faildescr == self.cpu.done_with_this_frame_descr_float:
-            return self.cpu.get_float_value(pframe, 0)
-        elif faildescr == self.cpu.done_with_this_frame_descr_void:
-            return None
+    def _new_execute_call_assembler(def_val):
+        def _execute_call_assembler(self, descr, *args):
+            # XXX simplify the following a bit
+            #
+            # pframe = CALL_ASSEMBLER(args..., descr=looptoken)
+            # ==>
+            #     pframe = CALL looptoken.loopaddr(*args)
+            #     JUMP_IF_FAST_PATH @fastpath
+            #     res = CALL assembler_call_helper(pframe)
+            #     jmp @done
+            #   @fastpath:
+            #     res = GETFIELD(pframe, 'result')
+            #   @done:
+            #
+            call_op = self.lltrace.operations[self.current_index]
+            guard_op = self.lltrace.operations[self.current_index + 1]
+            assert guard_op.getopnum() == rop.GUARD_NOT_FORCED
+            self.force_guard_op = guard_op
+            pframe = self.cpu._execute_token(descr, *args)
+            del self.force_guard_op
+            #
+            jd = descr.outermost_jitdriver_sd
+            assert jd is not None, ("call_assembler(): the loop_token needs "
+                                    "to have 'outermost_jitdriver_sd'")
+            if jd.index_of_virtualizable != -1:
+                vable = args[jd.index_of_virtualizable]
+            else:
+                vable = lltype.nullptr(llmemory.GCREF.TO)
+            #
+            # Emulate the fast path
+            #
+            faildescr = self.cpu.get_latest_descr(pframe)
+            if faildescr == self.cpu.done_with_this_frame_descr_int:
+                return self.cpu.get_int_value(pframe, 0)
+            elif faildescr == self.cpu.done_with_this_frame_descr_ref:
+                return self.cpu.get_ref_value(pframe, 0)
+            elif faildescr == self.cpu.done_with_this_frame_descr_float:
+                return self.cpu.get_float_value(pframe, 0)
+            elif faildescr == self.cpu.done_with_this_frame_descr_void:
+                return None
 
-        assembler_helper_ptr = jd.assembler_helper_adr.ptr  # fish
-        try:
-            result = assembler_helper_ptr(pframe, vable)
-        except LLException, lle:
-            assert self.last_exception is None, "exception left behind"
-            self.last_exception = lle
-            # fish op
-            op = self.current_op
-            return op.result and op.result.value
-        if isinstance(result, float):
-            result = support.cast_to_floatstorage(result)
-        return result
+            assembler_helper_ptr = jd.assembler_helper_adr.ptr  # fish
+            try:
+                result = assembler_helper_ptr(pframe, vable)
+            except LLException, lle:
+                assert self.last_exception is None, "exception left behind"
+                self.last_exception = lle
+                # fish op
+                result = def_val
+            if isinstance(result, float):
+                result = support.cast_to_floatstorage(result)
+            return result
+        return _execute_call_assembler
 
-    execute_call_assembler_i = _execute_call_assembler
-    execute_call_assembler_r = _execute_call_assembler
-    execute_call_assembler_f = _execute_call_assembler
-    execute_call_assembler_n = _execute_call_assembler
+    execute_call_assembler_i = _new_execute_call_assembler(0)
+    execute_call_assembler_r = _new_execute_call_assembler(lltype.nullptr(llmemory.GCREF.TO))
+    execute_call_assembler_f = _new_execute_call_assembler(0.0)
+    execute_call_assembler_n = _new_execute_call_assembler(None)
 
     def execute_same_as_i(self, _, x):
         return x
diff --git a/rpython/jit/codewriter/heaptracker.py b/rpython/jit/codewriter/heaptracker.py
--- a/rpython/jit/codewriter/heaptracker.py
+++ b/rpython/jit/codewriter/heaptracker.py
@@ -105,10 +105,9 @@
 
 def finish_registering(cpu):
     # annotation hack for small examples which have no vtable at all
-    pass
-    #if not hasattr(cpu.tracker, '_all_size_descrs_with_vtable'):
-    #    vtable = lltype.malloc(rclass.OBJECT_VTABLE, immortal=True)
-    #    register_known_gctype(cpu, vtable, rclass.OBJECT)
+    if not hasattr(cpu.tracker, '_all_size_descrs_with_vtable'):
+        vtable = lltype.malloc(rclass.OBJECT_VTABLE, immortal=True)
+        register_known_gctype(cpu, vtable, rclass.OBJECT)
 
 def vtable2descr(cpu, vtable):
     assert lltype.typeOf(vtable) is lltype.Signed
diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -8,14 +8,14 @@
 from rpython.rlib.jit import JitDebugInfo, Counters, dont_look_inside
 from rpython.conftest import option
 
-from rpython.jit.metainterp.resoperation import ResOperation, rop, get_deep_immutable_oplist
+from rpython.jit.metainterp.resoperation import ResOperation, rop,\
+     get_deep_immutable_oplist, OpHelpers
 from rpython.jit.metainterp.history import (TreeLoop, Const, JitCellToken,
     TargetToken, AbstractFailDescr, ConstInt)
 from rpython.jit.metainterp import history, jitexc
 from rpython.jit.metainterp.optimize import InvalidLoop
 from rpython.jit.metainterp.resume import NUMBERING, PENDINGFIELDSP, ResumeDataDirectReader
 from rpython.jit.codewriter import heaptracker, longlong
-from rpython.jit.metainterp.inliner import Inliner
 
 
 def giveup():
@@ -161,7 +161,7 @@
     if part.quasi_immutable_deps:
         loop.quasi_immutable_deps.update(part.quasi_immutable_deps)
     if part.operations[-1].getopnum() == rop.LABEL:
-        xxx
+        raise Exception("unrolling unsupported")
         d = part.operations[0].getdescr()
         assert isinstance(d, TargetToken)
         part.operations[-1] = part.operations[-1].copy_and_change(rop.JUMP,
@@ -280,8 +280,37 @@
     record_loop_or_bridge(metainterp_sd, loop)
     return target_token
 
+def get_box_replacement(op, allow_none=False):
+    if allow_none and op is None:
+        return None # for failargs
+    while op.get_forwarded():
+        op = op.get_forwarded()
+    return op
+
+def emit_op(lst, op):
+    op = get_box_replacement(op)
+    orig_op = op
+    # XXX specialize on number of args
+    replaced = False
+    for i in range(op.numargs()):
+        orig_arg = op.getarg(i)
+        arg = get_box_replacement(orig_arg)
+        if orig_arg is not arg:
+            if not replaced:
+                op = op.copy_and_change(op.getopnum())
+                orig_op.set_forwarded(op)
+                replaced = True
+            op.setarg(i, arg)
+    if op.is_guard():
+        if not replaced:
+            op = op.copy_and_change(op.getopnum())
+            orig_op.set_forwarded(op)
+        op.setfailargs([get_box_replacement(a, True)
+                        for a in op.getfailargs()])
+    lst.append(op)
+
 def patch_new_loop_to_load_virtualizable_fields(loop, jitdriver_sd):
-    xxx
+    # XXX merge with rewriting
     vinfo = jitdriver_sd.virtualizable_info
     extra_ops = []
     inputargs = loop.inputargs
@@ -291,28 +320,33 @@
     for descr in vinfo.static_field_descrs:
         assert i < len(inputargs)
         box = inputargs[i]
-        extra_ops.append(
-            ResOperation(rop.GETFIELD_GC, [vable_box], box, descr))
+        opnum = OpHelpers.getfield_for_descr(descr)
+        emit_op(extra_ops,
+                ResOperation(opnum, [vable_box], descr))
+        box.set_forwarded(extra_ops[-1])
         i += 1
     arrayindex = 0
     for descr in vinfo.array_field_descrs:
         vable = vable_box.getref_base()
         arraylen = vinfo.get_array_length(vable, arrayindex)
-        arraybox = BoxPtr()
-        extra_ops.append(
-            ResOperation(rop.GETFIELD_GC, [vable_box], arraybox, descr))
+        arrayop = ResOperation(rop.GETFIELD_GC, [vable_box], descr)
+        emit_op(extra_ops, arrayop)
         arraydescr = vinfo.array_descrs[arrayindex]
         assert i + arraylen <= len(inputargs)
         for index in range(arraylen):
+            opnum = OpHelpers.getarrayitem_for_descr(arraydescr)
             box = inputargs[i]
-            extra_ops.append(
+            emit_op(extra_ops,
                 ResOperation(rop.GETARRAYITEM_GC,
-                             [arraybox, ConstInt(index)],
-                             box, descr=arraydescr))
+                             [arrayop, ConstInt(index)],
+                             descr=arraydescr))
             i += 1
+            box.set_forwarded(extra_ops[-1])
         arrayindex += 1
     assert i == len(inputargs)
-    loop.operations = extra_ops + loop.operations
+    for op in loop.operations:
+        emit_op(extra_ops, op)
+    loop.operations = extra_ops
 
 def propagate_original_jitcell_token(trace):
     for op in trace.operations:
@@ -915,7 +949,8 @@
     calls back the interpreter.  Used temporarily: a fully compiled
     version of the code may end up replacing it.
     """
-    xxx
+    import pdb
+    pdb.set_trace()
     jitcell_token = make_jitcell_token(jitdriver_sd)
     nb_red_args = jitdriver_sd.num_red_args
     assert len(redargtypes) == nb_red_args
diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py
--- a/rpython/jit/metainterp/history.py
+++ b/rpython/jit/metainterp/history.py
@@ -162,7 +162,9 @@
 
 @specialize.argtype(0)
 def newconst(value):
-    if lltype.typeOf(value) == lltype.Signed:
+    if value is None:
+        return ConstPtr(lltype.nullptr(llmemory.GCREF.TO))
+    elif lltype.typeOf(value) == lltype.Signed:
         return ConstInt(value)
     elif type(value) is bool:
         return ConstInt(int(value))
@@ -403,7 +405,7 @@
     _attrs_ = ('value',)
 
     def __init__(self, value=0):
-        xxx
+        raise Exception("boxes no longer supported")
         if not we_are_translated():
             if is_valid_int(value):
                 value = int(value)    # bool -> int
@@ -470,7 +472,7 @@
     _attrs_ = ('value',)
 
     def __init__(self, value=lltype.nullptr(llmemory.GCREF.TO)):
-        xxx
+        raise Exception("boxes no longer supported")
         assert lltype.typeOf(value) == llmemory.GCREF
         self.value = value
 
@@ -756,7 +758,21 @@
     @specialize.argtype(3)
     def record(self, opnum, argboxes, value, descr=None):
         op = ResOperation(opnum, argboxes, descr)
-        op.setvalue(value)
+        if value is None:
+            assert op.type == 'v'
+        elif type(value) is bool:
+            assert op.type == 'i'
+            op.setint(int(value))
+        elif isinstance(value, float):
+            assert op.type == 'f'
+            op.setfloatstorage(value)
+        elif lltype.typeOf(value) == lltype.Signed:
+            assert op.type == 'i'
+            op.setint(value)
+        else:
+            assert lltype.typeOf(value) == llmemory.GCREF
+            assert op.type == 'r'
+            op.setref_base(value)
         self.operations.append(op)
         return op
 
diff --git a/rpython/jit/metainterp/jitprof.py b/rpython/jit/metainterp/jitprof.py
--- a/rpython/jit/metainterp/jitprof.py
+++ b/rpython/jit/metainterp/jitprof.py
@@ -110,9 +110,9 @@
         return self.counters[num]
 
     def count_ops(self, opnum, kind=Counters.OPS):
-        from rpython.jit.metainterp.resoperation import rop
+        from rpython.jit.metainterp.resoperation import OpHelpers
         self.counters[kind] += 1
-        if opnum == rop.CALL and kind == Counters.RECORDED_OPS:
+        if OpHelpers.is_call(opnum) and kind == Counters.RECORDED_OPS:
             self.calls += 1
 
     def print_stats(self):
diff --git a/rpython/jit/metainterp/optimizeopt/__init__.py b/rpython/jit/metainterp/optimizeopt/__init__.py
--- a/rpython/jit/metainterp/optimizeopt/__init__.py
+++ b/rpython/jit/metainterp/optimizeopt/__init__.py
@@ -59,7 +59,7 @@
                                                           loop.operations)
         optimizations, unroll = build_opt_chain(metainterp_sd, enable_opts)
         if unroll:
-            xxx
+            raise Exception("unrolling disabled")
             return optimize_unroll(metainterp_sd, jitdriver_sd, loop,
                                    optimizations,
                                    inline_short_preamble, start_state,
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -19,7 +19,7 @@
 from rpython.rlib.jit import Counters
 from rpython.rlib.objectmodel import we_are_translated, specialize
 from rpython.rlib.unroll import unrolling_iterable
-from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.rtyper.lltypesystem import lltype, rffi, llmemory
 from rpython.rtyper import rclass
 
 
@@ -682,9 +682,9 @@
     def opimpl_getfield_gc_f_pure(self, box, fielddescr):
         if isinstance(box, ConstPtr):
             # if 'box' is directly a ConstPtr, bypass the heapcache completely
-            resbox = executor.execute(self.metainterp.cpu, self.metainterp,
+            resvalue = executor.execute(self.metainterp.cpu, self.metainterp,
                                       rop.GETFIELD_GC_PURE_F, fielddescr, box)
-            return resbox.constbox()
+            return ConstPtr(resvalue)
         return self._opimpl_getfield_gc_any_pureornot(
                 rop.GETFIELD_GC_PURE_F, box, fielddescr, 'f')
 
@@ -692,9 +692,9 @@
     def opimpl_getfield_gc_r_pure(self, box, fielddescr):
         if isinstance(box, ConstPtr):
             # if 'box' is directly a ConstPtr, bypass the heapcache completely
-            resbox = executor.execute(self.metainterp.cpu, self.metainterp,
+            val = executor.execute(self.metainterp.cpu, self.metainterp,
                                       rop.GETFIELD_GC_PURE_R, fielddescr, box)
-            return resbox.constbox()
+            return ConstFloat(val)
         return self._opimpl_getfield_gc_any_pureornot(
                 rop.GETFIELD_GC_PURE_R, box, fielddescr, 'r')
 
@@ -1555,9 +1555,21 @@
                     if resbox is not None:
                         return resbox
                 self.metainterp.vable_and_vrefs_before_residual_call()
-                opnum = OpHelpers.call_may_force_for_descr(descr)
-                resbox = self.metainterp.execute_and_record_varargs(
-                    opnum, allboxes, descr=descr)
+                tp = descr.get_result_type()
+                if tp == 'i':
+                    resbox = self.metainterp.execute_and_record_varargs(
+                        rop.CALL_MAY_FORCE_I, allboxes, descr=descr)
+                elif tp == 'r':
+                    resbox = self.metainterp.execute_and_record_varargs(
+                        rop.CALL_MAY_FORCE_R, allboxes, descr=descr)
+                elif tp == 'f':
+                    resbox = self.metainterp.execute_and_record_varargs(
+                        rop.CALL_MAY_FORCE_F, allboxes, descr=descr)
+                elif tp == 'v':
+                    resbox = self.metainterp.execute_and_record_varargs(
+                        rop.CALL_MAY_FORCE_N, allboxes, descr=descr)
+                else:
+                    assert False
                 if effectinfo.is_call_release_gil():
                     self.metainterp.direct_call_release_gil()
                 self.metainterp.vrefs_after_residual_call()
@@ -1565,7 +1577,7 @@
                 if assembler_call:
                     vablebox, resbox = self.metainterp.direct_assembler_call(
                         assembler_call_jd)
-                if resbox is not None:
+                if resbox and resbox.type != 'v':
                     self.make_result_of_lastop(resbox)
                 self.metainterp.vable_after_residual_call(funcbox)
                 self.metainterp.generate_guard(rop.GUARD_NOT_FORCED, None)
@@ -1578,15 +1590,42 @@
                 return resbox
             else:
                 effect = effectinfo.extraeffect
+                tp = descr.get_result_type()
                 if effect == effectinfo.EF_LOOPINVARIANT:
-                    opnum = OpHelpers.call_loopinvariant_for_descr(descr)
-                    return self.execute_varargs(opnum,
-                                                allboxes,
-                                                descr, False, False)
+                    if tp == 'i':
+                        return self.execute_varargs(rop.CALL_LOOPINVARIANT_I,
+                                                    allboxes,
+                                                    descr, False, False)
+                    elif tp == 'r':
+                        return self.execute_varargs(rop.CALL_LOOPINVARIANT_R,
+                                                    allboxes,
+                                                    descr, False, False)
+                    elif tp == 'f':
+                        return self.execute_varargs(rop.CALL_LOOPINVARIANT_F,
+                                                    allboxes,
+                                                    descr, False, False)
+                    elif tp == 'v':
+                        return self.execute_varargs(rop.CALL_LOOPINVARIANT_N,
+                                                    allboxes,
+                                                    descr, False, False)
+                    else:
+                        assert False
                 exc = effectinfo.check_can_raise()
                 pure = effectinfo.check_is_elidable()
-                opnum = OpHelpers.call_for_descr(descr)
-                return self.execute_varargs(opnum, allboxes, descr, exc, pure)
+                if tp == 'i':
+                    return self.execute_varargs(rop.CALL_I, allboxes, descr,
+                                                exc, pure)
+                elif tp == 'r':
+                    return self.execute_varargs(rop.CALL_R, allboxes, descr,
+                                                exc, pure)
+                elif tp == 'f':
+                    return self.execute_varargs(rop.CALL_F, allboxes, descr,
+                                                exc, pure)
+                elif tp == 'v':
+                    return self.execute_varargs(rop.CALL_N, allboxes, descr,
+                                                exc, pure)
+                else:
+                    assert False
         finally:
             debug_stop("jit-residual-call")
 
@@ -1957,7 +1996,11 @@
             moreargs = [box] + extraargs
         else:
             moreargs = list(extraargs)
-        guard_op = self.history.record(opnum, moreargs, None)
+        if opnum == rop.GUARD_EXCEPTION:
+            guard_op = self.history.record(opnum, moreargs,
+                                           lltype.nullptr(llmemory.GCREF.TO))
+        else:
+            guard_op = self.history.record(opnum, moreargs, None)            
         assert isinstance(guard_op, GuardResOp)
         self.capture_resumedata(guard_op, resumepc)
         self.staticdata.profiler.count_ops(opnum, Counters.GUARDS)
@@ -2681,7 +2724,7 @@
         vrefbox = self.virtualref_boxes[i+1]
         # record VIRTUAL_REF_FINISH just before the current CALL_MAY_FORCE
         call_may_force_op = self.history.operations.pop()
-        assert call_may_force_op.getopnum() == rop.CALL_MAY_FORCE
+        assert call_may_force_op.is_call_may_force()
         self.history.record(rop.VIRTUAL_REF_FINISH,
                             [vrefbox, virtualbox], None)
         self.history.operations.append(call_may_force_op)
@@ -2971,12 +3014,13 @@
 
     def direct_call_release_gil(self):
         op = self.history.operations.pop()
-        assert op.opnum == rop.CALL_MAY_FORCE
+        assert op.is_call_may_force()
         descr = op.getdescr()
         effectinfo = descr.get_extra_info()
         realfuncaddr, saveerr = effectinfo.call_release_gil_target
         funcbox = ConstInt(heaptracker.adr2int(realfuncaddr))
         savebox = ConstInt(saveerr)
+        assert False, "not yet"
         self.history.record(rop.CALL_RELEASE_GIL,
                             [savebox, funcbox] + op.getarglist()[1:],
                             op.result, descr)
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -6,6 +6,7 @@
 class AbstractValue(object):
     _repr_memo = {}
     is_info_class = False
+    _attrs_ = ()
 
     def _get_hash_(self):
         return compute_identity_hash(self)
@@ -129,21 +130,6 @@
         if self.type != 'v':
             newop.copy_value_from(self)
         return newop
-
-    @specialize.argtype(1)
-    def setvalue(self, value):
-        if lltype.typeOf(value) == lltype.Signed:
-            self._resint = value
-        elif type(value) == bool:
-            self._resint = int(value)
-        elif isinstance(value, float):
-            self._resfloat = value
-        elif value is None:
-            pass
-        else:
-            assert lltype.typeOf(value) == llmemory.GCREF
-            self._resref = value
-
             
     def clone(self, memo):
         args = [memo.get(arg, arg) for arg in self.getarglist()]
@@ -1124,6 +1110,10 @@
         return rop.CALL_N
 
     @staticmethod
+    def is_call(opnum):
+        return rop._CALL_FIRST <= opnum <= rop._CALL_LAST
+
+    @staticmethod
     def is_call_assembler(opnum):
         return (opnum == rop.CALL_ASSEMBLER_I or
                 opnum == rop.CALL_ASSEMBLER_R or
diff --git a/rpython/jit/metainterp/virtualref.py b/rpython/jit/metainterp/virtualref.py
--- a/rpython/jit/metainterp/virtualref.py
+++ b/rpython/jit/metainterp/virtualref.py
@@ -27,7 +27,9 @@
         adr = heaptracker.adr2int(adr)
         self.jit_virtual_ref_const_class = history.ConstInt(adr)
         fielddescrof = self.cpu.fielddescrof
-        self.cpu.gc_ll_descr._cache_gcstruct2vtable[self.JIT_VIRTUAL_REF] = self.jit_virtual_ref_vtable
+        if hasattr(self.cpu, 'gc_ll_descr'):
+            heaptracker.setup_cache_gcstruct2vtable(self.cpu.gc_ll_descr)
+            self.cpu.gc_ll_descr._cache_gcstruct2vtable[self.JIT_VIRTUAL_REF] = self.jit_virtual_ref_vtable
         self.descr_virtual_token = fielddescrof(self.JIT_VIRTUAL_REF,
                                                 'virtual_token')
         self.descr_forced = fielddescrof(self.JIT_VIRTUAL_REF, 'forced')
diff --git a/rpython/memory/gctypelayout.py b/rpython/memory/gctypelayout.py
--- a/rpython/memory/gctypelayout.py
+++ b/rpython/memory/gctypelayout.py
@@ -320,7 +320,8 @@
         else:
             # no vtable from lltype2vtable -- double-check to be sure
             # that it's not a subclass of OBJECT.
-            assert not is_subclass_of_object(TYPE)
+            pass
+            #assert not is_subclass_of_object(TYPE)
 
     def get_info(self, type_id):
         res = llop.get_group_member(GCData.TYPE_INFO_PTR,

From noreply at buildbot.pypy.org  Tue Jun  2 15:19:29 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Tue,  2 Jun 2015 15:19:29 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: work on rpython and recursive stuff
Message-ID: <20150602131929.2C2CB1C0262@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77762:6e5fd3dc775a
Date: 2015-06-02 14:57 +0200
http://bitbucket.org/pypy/pypy/changeset/6e5fd3dc775a/

Log:	work on rpython and recursive stuff

diff --git a/rpython/rtyper/llinterp.py b/rpython/rtyper/llinterp.py
--- a/rpython/rtyper/llinterp.py
+++ b/rpython/rtyper/llinterp.py
@@ -83,6 +83,7 @@
                     self.tracer.dump('LLException: %s\n' % (e,))
                 raise
             except Exception, e:
+                tb = sys.exc_info()[2]
                 if getattr(e, '_go_through_llinterp_uncaught_', False):
                     raise
                 log.error("AN ERROR OCCURED: %s" % (e, ))

From noreply at buildbot.pypy.org  Tue Jun  2 15:19:30 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Tue,  2 Jun 2015 15:19:30 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: start working on the virtualizable
Message-ID: <20150602131930.5BE6E1C0262@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77763:717e62ece9d4
Date: 2015-06-02 15:04 +0200
http://bitbucket.org/pypy/pypy/changeset/717e62ece9d4/

Log:	start working on the virtualizable

diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py
--- a/rpython/jit/metainterp/optimizeopt/rewrite.py
+++ b/rpython/jit/metainterp/optimizeopt/rewrite.py
@@ -457,9 +457,8 @@
         info0 = self.getptrinfo(arg0)
         info1 = self.getptrinfo(arg1)
         if info0 and info0.is_virtual():
-            xxx
-            if value1.is_virtual():
-                intres = (value0 is value1) ^ expect_isnot
+            if info1 and info1.is_virtual():
+                intres = (info0 is info1) ^ expect_isnot
                 self.make_constant_int(op, intres)
             else:
                 self.make_constant_int(op, expect_isnot)
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -891,16 +891,26 @@
         return vinfo.static_field_by_descrs[fielddescr]
 
     @arguments("box", "descr", "orgpc")
-    def _opimpl_getfield_vable(self, box, fielddescr, pc):
+    def opimpl_getfield_vable_i(self, box, fielddescr, pc):
         if self._nonstandard_virtualizable(pc, box, fielddescr):
-            return self._opimpl_getfield_gc_any(box, fielddescr)
+            return self.opimpl_getfield_gc_i(box, fielddescr)
         self.metainterp.check_synchronized_virtualizable()
         index = self._get_virtualizable_field_index(fielddescr)
         return self.metainterp.virtualizable_boxes[index]
-
-    opimpl_getfield_vable_i = _opimpl_getfield_vable
-    opimpl_getfield_vable_r = _opimpl_getfield_vable
-    opimpl_getfield_vable_f = _opimpl_getfield_vable
+    @arguments("box", "descr", "orgpc")
+    def opimpl_getfield_vable_r(self, box, fielddescr, pc):
+        if self._nonstandard_virtualizable(pc, box, fielddescr):
+            return self.opimpl_getfield_gc_r(box, fielddescr)
+        self.metainterp.check_synchronized_virtualizable()
+        index = self._get_virtualizable_field_index(fielddescr)
+        return self.metainterp.virtualizable_boxes[index]
+    @arguments("box", "descr", "orgpc")
+    def opimpl_getfield_vable_f(self, box, fielddescr, pc):
+        if self._nonstandard_virtualizable(pc, box, fielddescr):
+            return self.opimpl_getfield_gc_f(box, fielddescr)
+        self.metainterp.check_synchronized_virtualizable()
+        index = self._get_virtualizable_field_index(fielddescr)
+        return self.metainterp.virtualizable_boxes[index]
 
     @arguments("box", "box", "descr", "orgpc")
     def _opimpl_setfield_vable(self, box, valuebox, fielddescr, pc):
@@ -2579,12 +2589,12 @@
         vbox = self.virtualizable_boxes[-1]
         if vbox is self.forced_virtualizable:
             return # we already forced it by hand
-        force_token_box = history.BoxPtr()
         # in case the force_token has not been recorded, record it here
         # to make sure we know the virtualizable can be broken. However, the
         # contents of the virtualizable should be generally correct
-        self.history.record(rop.FORCE_TOKEN, [], force_token_box)
-        self.history.record(rop.SETFIELD_GC, [vbox, force_token_box],
+        force_token = self.history.record(rop.FORCE_TOKEN, [],
+                                          lltype.nullptr(llmemory.GCREF.TO))
+        self.history.record(rop.SETFIELD_GC, [vbox, force_token],
                             None, descr=vinfo.vable_token_descr)
         self.generate_guard(rop.GUARD_NOT_FORCED_2, None)
 
@@ -2679,10 +2689,10 @@
             virtualizable = vinfo.unwrap_virtualizable_box(virtualizable_box)
             vinfo.tracing_before_residual_call(virtualizable)
             #
-            force_token_box = history.BoxPtr()
-            self.history.record(rop.FORCE_TOKEN, [], force_token_box)
+            force_token = self.history.record(rop.FORCE_TOKEN, [],
+                                lltype.nullptr(llmemory.GCREF.TO))
             self.history.record(rop.SETFIELD_GC, [virtualizable_box,
-                                                  force_token_box],
+                                                  force_token],
                                 None, descr=vinfo.vable_token_descr)
 
     def vrefs_after_residual_call(self):
diff --git a/rpython/rtyper/llinterp.py b/rpython/rtyper/llinterp.py
--- a/rpython/rtyper/llinterp.py
+++ b/rpython/rtyper/llinterp.py
@@ -83,7 +83,6 @@
                     self.tracer.dump('LLException: %s\n' % (e,))
                 raise
             except Exception, e:
-                tb = sys.exc_info()[2]
                 if getattr(e, '_go_through_llinterp_uncaught_', False):
                     raise
                 log.error("AN ERROR OCCURED: %s" % (e, ))

From noreply at buildbot.pypy.org  Tue Jun  2 15:19:31 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Tue,  2 Jun 2015 15:19:31 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: basic fixes to vstring
Message-ID: <20150602131931.82BFB1C0262@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77764:e93151054fb5
Date: 2015-06-02 15:19 +0200
http://bitbucket.org/pypy/pypy/changeset/e93151054fb5/

Log:	basic fixes to vstring

diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py
--- a/rpython/jit/metainterp/optimizeopt/virtualize.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualize.py
@@ -543,12 +543,14 @@
         return opinfo
 
     def make_virtual_raw_memory(self, size, source_op):
+        raise Exception("unsupported")
         logops = self.optimizer.loop.logops
         vvalue = VRawBufferValue(self.optimizer.cpu, logops, size, source_op)
         self.make_equal_to(source_op, vvalue)
         return vvalue
 
     def make_virtual_raw_slice(self, rawbuffer_value, offset, source_op):
+        raise Exception("unsupported")
         vvalue = VRawSliceValue(rawbuffer_value, offset, source_op)
         self.make_equal_to(source_op, vvalue)
         return vvalue
diff --git a/rpython/jit/metainterp/optimizeopt/vstring.py b/rpython/jit/metainterp/optimizeopt/vstring.py
--- a/rpython/jit/metainterp/optimizeopt/vstring.py
+++ b/rpython/jit/metainterp/optimizeopt/vstring.py
@@ -49,10 +49,11 @@
 
 
 class StrPtrInfo(info.NonNullPtrInfo):
-    _attrs_ = ('length', 'lenbound', 'lgtop', 'mode')
+    _attrs_ = ('length', 'lenbound', 'lgtop', 'mode', '_cached_vinfo')
 
     lenbound = None
     lgtop = None
+    _cached_vinfo = None
 
     def __init__(self, mode, is_virtual=False, length=-1):
         self.length = length
@@ -191,6 +192,13 @@
             offsetbox = _int_add(string_optimizer, offsetbox, CONST_1)
         return offsetbox
 
+    def visitor_walk_recursive(self, instbox, visitor, optimizer):
+        visitor.register_virtual_fields(instbox, self._chars)
+
+    @specialize.argtype(1)
+    def visitor_dispatch_virtual_type(self, visitor):
+        return visitor.visit_vstrplain(self.mode is mode_unicode)
+
 class VStringSliceInfo(StrPtrInfo):
     def __init__(self, s, start, length, mode):
         self.s = s
@@ -276,6 +284,23 @@
                                              targetbox, offsetbox, mode)
         return offsetbox
 
+    def visitor_walk_recursive(self, instbox, visitor, optimizer):
+        # we don't store the lengthvalue in guards, because the
+        # guard-failed code starts with a regular STR_CONCAT again
+        leftbox = self.vleft
+        rightbox = self.vright
+        visitor.register_virtual_fields(instbox, [leftbox, rightbox])
+        leftinfo = optimizer.getptrinfo(leftbox)
+        rightinfo = optimizer.getptrinfo(rightbox)
+        if leftinfo and leftinfo.is_virtual():
+            leftinfo.visitor_walk_recursive(leftbox, visitor, optimizer)
+        if rightinfo and rightinfo.is_virtual():
+            rightinfo.visitor_walk_recursive(rightbox, visitor, optimizer)
+
+    @specialize.argtype(1)
+    def visitor_dispatch_virtual_type(self, visitor):
+        return visitor.visit_vstrconcat(self.mode is mode_unicode)
+
 # class __extend__(optimizer.OptValue):
 #     """New methods added to the base class OptValue for this file."""
 
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -684,7 +684,7 @@
             # if 'box' is directly a ConstPtr, bypass the heapcache completely
             resvalue = executor.execute(self.metainterp.cpu, self.metainterp,
                                       rop.GETFIELD_GC_PURE_F, fielddescr, box)
-            return ConstPtr(resvalue)
+            return ConstFloat(resvalue)
         return self._opimpl_getfield_gc_any_pureornot(
                 rop.GETFIELD_GC_PURE_F, box, fielddescr, 'f')
 
@@ -694,7 +694,7 @@
             # if 'box' is directly a ConstPtr, bypass the heapcache completely
             val = executor.execute(self.metainterp.cpu, self.metainterp,
                                       rop.GETFIELD_GC_PURE_R, fielddescr, box)
-            return ConstFloat(val)
+            return ConstPtr(val)
         return self._opimpl_getfield_gc_any_pureornot(
                 rop.GETFIELD_GC_PURE_R, box, fielddescr, 'r')
 
diff --git a/rpython/jit/metainterp/resume.py b/rpython/jit/metainterp/resume.py
--- a/rpython/jit/metainterp/resume.py
+++ b/rpython/jit/metainterp/resume.py
@@ -289,7 +289,7 @@
         self.snapshot_storage = snapshot_storage
         self.memo = memo
 
-    def make_virtual_info(self, descr, info, fieldnums):
+    def make_virtual_info(self, info, fieldnums):
         assert fieldnums is not None
         vinfo = info._cached_vinfo
         if vinfo is not None and vinfo.equals(fieldnums):
@@ -455,8 +455,7 @@
                 assert info.is_virtual()
                 fieldnums = [self._gettagged(box)
                              for box in fieldboxes]
-                descr = info.vdescr
-                vinfo = self.make_virtual_info(descr, info, fieldnums)
+                vinfo = self.make_virtual_info(info, fieldnums)
                 # if a new vinfo instance is made, we get the fieldnums list we
                 # pass in as an attribute. hackish.
                 if vinfo.fieldnums is not fieldnums:

From noreply at buildbot.pypy.org  Tue Jun  2 15:31:22 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Tue,  2 Jun 2015 15:31:22 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: fixes to virtualizable
Message-ID: <20150602133122.7A8CC1C0262@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77765:24891008629e
Date: 2015-06-02 15:24 +0200
http://bitbucket.org/pypy/pypy/changeset/24891008629e/

Log:	fixes to virtualizable

diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -329,7 +329,7 @@
     for descr in vinfo.array_field_descrs:
         vable = vable_box.getref_base()
         arraylen = vinfo.get_array_length(vable, arrayindex)
-        arrayop = ResOperation(rop.GETFIELD_GC, [vable_box], descr)
+        arrayop = ResOperation(rop.GETFIELD_GC_R, [vable_box], descr)
         emit_op(extra_ops, arrayop)
         arraydescr = vinfo.array_descrs[arrayindex]
         assert i + arraylen <= len(inputargs)
@@ -337,7 +337,7 @@
             opnum = OpHelpers.getarrayitem_for_descr(arraydescr)
             box = inputargs[i]
             emit_op(extra_ops,
-                ResOperation(rop.GETARRAYITEM_GC,
+                ResOperation(opnum,
                              [arrayop, ConstInt(index)],
                              descr=arraydescr))
             i += 1
diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py
--- a/rpython/jit/metainterp/optimizeopt/virtualize.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualize.py
@@ -594,8 +594,8 @@
         effectinfo = op.getdescr().get_extra_info()
         oopspecindex = effectinfo.oopspecindex
         if oopspecindex == EffectInfo.OS_JIT_FORCE_VIRTUALIZABLE:
-            value = self.getvalue(op.getarg(2))
-            if value.is_virtual():
+            opinfo = self.getptrinfo(op.getarg(2))
+            if opinfo and opinfo.is_virtual():
                 return
         self.emit_operation(op)
 
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -876,7 +876,7 @@
         assert vinfo is not None
         token_descr = vinfo.vable_token_descr
         mi = self.metainterp
-        tokenbox = mi.execute_and_record(rop.GETFIELD_GC, token_descr, box)
+        tokenbox = mi.execute_and_record(rop.GETFIELD_GC_R, token_descr, box)
         condbox = mi.execute_and_record(rop.PTR_NE, None, tokenbox,
                                        history.CONST_NULL)
         funcbox = ConstInt(rffi.cast(lltype.Signed, vinfo.clear_vable_ptr))
@@ -945,8 +945,13 @@
     @arguments("box", "box", "descr", "descr", "orgpc")
     def _opimpl_getarrayitem_vable(self, box, indexbox, fdescr, adescr, pc):
         if self._nonstandard_virtualizable(pc, box, fdescr):
-            arraybox = self._opimpl_getfield_gc_any(box, fdescr)
-            return self._opimpl_getarrayitem_gc_any(arraybox, indexbox, adescr)
+            arraybox = self.opimpl_getfield_gc_r(box, fdescr)
+            if adescr.is_array_of_pointers():
+                return self.opimpl_getarrayitem_gc_r(arraybox, indexbox, adescr)
+            elif adescr.is_array_of_floats():
+                return self.opimpl_getarrayitem_gc_f(arraybox, indexbox, adescr)
+            else:
+                return self.opimpl_getarrayitem_gc_i(arraybox, indexbox, adescr)
         self.metainterp.check_synchronized_virtualizable()
         index = self._get_arrayitem_vable_index(pc, fdescr, indexbox)
         return self.metainterp.virtualizable_boxes[index]
@@ -959,7 +964,7 @@
     def _opimpl_setarrayitem_vable(self, box, indexbox, valuebox,
                                    fdescr, adescr, pc):
         if self._nonstandard_virtualizable(pc, box, fdescr):
-            arraybox = self._opimpl_getfield_gc_any(box, fdescr)
+            arraybox = self.opimpl_getfield_gc_r(box, fdescr)
             self._opimpl_setarrayitem_gc_any(arraybox, indexbox, valuebox,
                                              adescr)
             return
@@ -975,7 +980,7 @@
     @arguments("box", "descr", "descr", "orgpc")
     def opimpl_arraylen_vable(self, box, fdescr, adescr, pc):
         if self._nonstandard_virtualizable(pc, box, fdescr):
-            arraybox = self._opimpl_getfield_gc_any(box, fdescr)
+            arraybox = self.opimpl_getfield_gc_r(box, fdescr)
             return self.opimpl_arraylen_gc(arraybox, adescr)
         vinfo = self.metainterp.jitdriver_sd.virtualizable_info
         virtualizable_box = self.metainterp.virtualizable_boxes[-1]

From noreply at buildbot.pypy.org  Tue Jun  2 15:31:23 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Tue,  2 Jun 2015 15:31:23 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: fix tmp_callback
Message-ID: <20150602133123.994E21C0262@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77766:d4d7b1dff942
Date: 2015-06-02 15:31 +0200
http://bitbucket.org/pypy/pypy/changeset/d4d7b1dff942/

Log:	fix tmp_callback

diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -9,7 +9,8 @@
 from rpython.conftest import option
 
 from rpython.jit.metainterp.resoperation import ResOperation, rop,\
-     get_deep_immutable_oplist, OpHelpers
+     get_deep_immutable_oplist, OpHelpers, InputArgInt, InputArgRef,\
+     InputArgFloat
 from rpython.jit.metainterp.history import (TreeLoop, Const, JitCellToken,
     TargetToken, AbstractFailDescr, ConstInt)
 from rpython.jit.metainterp import history, jitexc
@@ -949,19 +950,17 @@
     calls back the interpreter.  Used temporarily: a fully compiled
     version of the code may end up replacing it.
     """
-    import pdb
-    pdb.set_trace()
     jitcell_token = make_jitcell_token(jitdriver_sd)
     nb_red_args = jitdriver_sd.num_red_args
     assert len(redargtypes) == nb_red_args
     inputargs = []
     for kind in redargtypes:
         if kind == history.INT:
-            box = BoxInt()
+            box = InputArgInt()
         elif kind == history.REF:
-            box = BoxPtr()
+            box = InputArgRef()
         elif kind == history.FLOAT:
-            box = BoxFloat()
+            box = InputArgFloat()
         else:
             raise AssertionError
         inputargs.append(box)
@@ -969,28 +968,20 @@
     funcbox = history.ConstInt(heaptracker.adr2int(k))
     callargs = [funcbox] + greenboxes + inputargs
     #
-    result_type = jitdriver_sd.result_type
-    if result_type == history.INT:
-        result = BoxInt()
-    elif result_type == history.REF:
-        result = BoxPtr()
-    elif result_type == history.FLOAT:
-        result = BoxFloat()
-    elif result_type == history.VOID:
-        result = None
-    else:
-        assert 0, "bad result_type"
-    if result is not None:
-        finishargs = [result]
+
+    jd = jitdriver_sd
+    opnum = OpHelpers.call_for_descr(jd.portal_calldescr)
+    call_op = ResOperation(opnum, callargs, descr=jd.portal_calldescr)
+    if call_op.type != 'v' is not None:
+        finishargs = [call_op]
     else:
         finishargs = []
     #
-    jd = jitdriver_sd
     faildescr = jitdriver_sd.propagate_exc_descr
     operations = [
-        ResOperation(rop.CALL, callargs, result, descr=jd.portal_calldescr),
-        ResOperation(rop.GUARD_NO_EXCEPTION, [], None, descr=faildescr),
-        ResOperation(rop.FINISH, finishargs, None, descr=jd.portal_finishtoken)
+        call_op,
+        ResOperation(rop.GUARD_NO_EXCEPTION, [], descr=faildescr),
+        ResOperation(rop.FINISH, finishargs, descr=jd.portal_finishtoken)
     ]
     operations[1].setfailargs([])
     operations = get_deep_immutable_oplist(operations)

From noreply at buildbot.pypy.org  Tue Jun  2 15:48:26 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Tue,  2 Jun 2015 15:48:26 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: prepending invariant operations before
	the label
Message-ID: <20150602134826.AF77E1C0262@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r77767:9a42391307c6
Date: 2015-06-02 15:48 +0200
http://bitbucket.org/pypy/pypy/changeset/9a42391307c6/

Log:	prepending invariant operations before the label moved inlined ops
	before header in the tests (vectorize)

diff --git a/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py b/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
@@ -30,12 +30,18 @@
 
     jitdriver_sd = FakeJitDriverStaticData()
 
-    def parse_loop(self, ops):
+    def parse_loop(self, ops, add_label=True):
         loop = self.parse(ops, postprocess=self.postprocess)
         token = JitCellToken()
-        loop.operations = \
-            [ResOperation(rop.LABEL, loop.inputargs, None, descr=TargetToken(token))] + \
-            loop.operations
+        pre = []
+        tt = TargetToken(token)
+        if add_label:
+            pre = [ResOperation(rop.LABEL, loop.inputargs, None, descr=tt)]
+        else:
+            for i,op in enumerate(loop.operations):
+                if op.getopnum() == rop.LABEL:
+                    op.setdescr(tt)
+        loop.operations = pre + loop.operations
         if loop.operations[-1].getopnum() == rop.JUMP:
             loop.operations[-1].setdescr(token)
         return loop
@@ -988,6 +994,8 @@
         """
         opt="""
         [p0,i0]
+        v3 = vec_int_expand(42)
+        label(p0,i0,v3)
         guard_early_exit() [p0,i0]
         i20 = int_add(i0, 1)
         i30 = int_lt(i20, 10)
@@ -997,12 +1005,11 @@
         i4 = int_add(i0, 2)
         i5 = int_lt(i2, 10)
         v1 = vec_getarrayitem_raw(p0, i0, 2, descr=floatarraydescr)
-        v3 = vec_int_expand(42)
         v2 = vec_int_mul(v1, v3)
-        jump(p0,i2)
+        jump(p0,i2,v3)
         """
         vopt = self.vectorize(self.parse_loop(ops),1)
-        self.assert_equal(vopt.loop, self.parse_loop(opt))
+        self.assert_equal(vopt.loop, self.parse_loop(opt,add_label=False))
 
     def test_variable_expansion(self):
         ops = """
@@ -1017,6 +1024,8 @@
         """
         opt="""
         [p0,i0,f3]
+        v3 = vec_float_expand(f3)
+        label(p0,i0,f3,v3)
         guard_early_exit() [p0,i0]
         i20 = int_add(i0, 1)
         i30 = int_lt(i20, 10)
@@ -1026,12 +1035,11 @@
         i4 = int_add(i0, 2)
         i5 = int_lt(i2, 10)
         v1 = vec_getarrayitem_raw(p0, i0, 2, descr=floatarraydescr)
-        v3 = vec_float_expand(f3)
         v2 = vec_int_mul(v1, v3)
-        jump(p0,i2,f3)
+        jump(p0,i2,f3,v3)
         """
         vopt = self.vectorize(self.parse_loop(ops),1)
-        self.assert_equal(vopt.loop, self.parse_loop(opt))
+        self.assert_equal(vopt.loop, self.parse_loop(opt, add_label=False))
 
     def test_element_f45_in_guard_failargs(self):
         ops = """
diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/vectorize.py
@@ -372,7 +372,8 @@
         if not we_are_translated():
             for node in self.dependency_graph.nodes:
                 assert node.emitted
-        self.loop.operations = self._newoperations[:]
+        self.loop.operations = \
+            sched_data.prepend_invariant_operations(self._newoperations)
         self.clear_newoperations()
 
     def unpack_from_vector(self, op, sched_data, renamer):
@@ -1261,6 +1262,26 @@
     def setvector_of_box(self, box, off, vector):
         self.box_to_vbox[box] = (off, vector)
 
+    def prepend_invariant_operations(self, oplist):
+        if len(self.invariant_oplist) > 0:
+            label = oplist[0]
+            assert label.getopnum() == rop.LABEL
+            jump = oplist[-1]
+            assert jump.getopnum() == rop.JUMP
+
+            label_args = label.getarglist()
+            jump_args = jump.getarglist()
+            for var in self.invariant_vector_vars:
+                label_args.append(var)
+                jump_args.append(var)
+
+            oplist[0] = label.copy_and_change(label.getopnum(), label_args, None, label.getdescr())
+            oplist[-1] = jump.copy_and_change(jump.getopnum(), jump_args, None, jump.getdescr())
+
+            return self.invariant_oplist + oplist
+
+        return oplist
+
 def isomorphic(l_op, r_op):
     """ Subject of definition """
     if l_op.getopnum() == r_op.getopnum():

From noreply at buildbot.pypy.org  Tue Jun  2 15:55:47 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue,  2 Jun 2015 15:55:47 +0200 (CEST)
Subject: [pypy-commit] cffi default: Test and fix for ffi.verify() only on
 PyPy (thanks Alex Gaynor for
Message-ID: <20150602135547.E56351C0262@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2157:8e0288e21778
Date: 2015-06-02 15:56 +0200
http://bitbucket.org/cffi/cffi/changeset/8e0288e21778/

Log:	Test and fix for ffi.verify() only on PyPy (thanks Alex Gaynor for
	pointing it out)

diff --git a/cffi/vengine_gen.py b/cffi/vengine_gen.py
--- a/cffi/vengine_gen.py
+++ b/cffi/vengine_gen.py
@@ -402,12 +402,16 @@
         else:
             assert tp is not None
             assert check_value is None
-            prnt(tp.get_c_name(' %s(void)' % funcname, name),)
-            prnt('{')
             if category == 'var':
                 ampersand = '&'
             else:
                 ampersand = ''
+            extra = ''
+            if category == 'const' and isinstance(tp, model.StructOrUnion):
+                extra = 'const *'
+                ampersand = '&'
+            prnt(tp.get_c_name(' %s%s(void)' % (extra, funcname), name))
+            prnt('{')
             prnt('  return (%s%s);' % (ampersand, name))
             prnt('}')
         prnt()
@@ -436,9 +440,14 @@
                 value += (1 << (8*self.ffi.sizeof(BLongLong)))
         else:
             assert check_value is None
-            BFunc = self.ffi._typeof_locked(tp.get_c_name('(*)(void)', name))[0]
+            fntypeextra = '(*)(void)'
+            if isinstance(tp, model.StructOrUnion):
+                fntypeextra = '*' + fntypeextra
+            BFunc = self.ffi._typeof_locked(tp.get_c_name(fntypeextra, name))[0]
             function = module.load_function(BFunc, funcname)
             value = function()
+            if isinstance(tp, model.StructOrUnion):
+                value = value[0]
         return value
 
     def _loaded_gen_constant(self, tp, name, module, library):
diff --git a/testing/cffi0/test_verify.py b/testing/cffi0/test_verify.py
--- a/testing/cffi0/test_verify.py
+++ b/testing/cffi0/test_verify.py
@@ -2227,3 +2227,11 @@
     ffi.cdef("static const int FOO = 123;")
     e = py.test.raises(VerificationError, ffi.verify, "#define FOO 124")
     assert str(e.value).endswith("FOO has the real value 124, not 123")
+
+def test_const_struct_global():
+    ffi = FFI()
+    ffi.cdef("typedef struct { int x; ...; } T; const T myglob;")
+    lib = ffi.verify("typedef struct { double y; int x; } T;"
+                     "const T myglob = { 0.1, 42 };")
+    assert ffi.typeof(lib.myglob) == ffi.typeof("T")
+    assert lib.myglob.x == 42

From noreply at buildbot.pypy.org  Tue Jun  2 16:08:46 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Tue,  2 Jun 2015 16:08:46 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: vector boxes in label and jump args are
 not correctly dispatched as xmm registers
Message-ID: <20150602140846.B231F1C03A8@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r77768:8ef2e618034c
Date: 2015-06-02 16:07 +0200
http://bitbucket.org/pypy/pypy/changeset/8ef2e618034c/

Log:	vector boxes in label and jump args are not correctly dispatched as
	xmm registers

diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -1329,7 +1329,7 @@
             box = op.getarg(i)
             src_loc = self.loc(box)
             dst_loc = arglocs[i]
-            if box.type != FLOAT:
+            if box.type != FLOAT and box.type != VECTOR:
                 src_locations1.append(src_loc)
                 dst_locations1.append(dst_loc)
             else:

From noreply at buildbot.pypy.org  Tue Jun  2 16:14:25 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Tue,  2 Jun 2015 16:14:25 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: an attempt to fix exception handling
	in the new model
Message-ID: <20150602141425.60F1E1C024E@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77769:22d1b419809f
Date: 2015-06-02 16:14 +0200
http://bitbucket.org/pypy/pypy/changeset/22d1b419809f/

Log:	an attempt to fix exception handling in the new model

diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py
--- a/rpython/jit/metainterp/history.py
+++ b/rpython/jit/metainterp/history.py
@@ -684,7 +684,7 @@
     @staticmethod
     def check_consistency_of_branch(operations, seen):
         "NOT_RPYTHON"
-        for op in operations:
+        for num, op in enumerate(operations):
             for i in range(op.numargs()):
                 box = op.getarg(i)
                 if not isinstance(box, Const):
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -326,11 +326,11 @@
     @arguments("label")
     def opimpl_catch_exception(self, target):
         """This is a no-op when run normally.  We can check that
-        last_exc_value_box is None; it should have been set to None
+        last_exc_value is a null ptr; it should have been set to None
         by the previous instruction.  If the previous instruction
         raised instead, finishframe_exception() should have been
         called and we would not be there."""
-        assert self.metainterp.last_exc_value_box is None
+        assert not self.metainterp.last_exc_value
 
     @arguments("label")
     def opimpl_goto(self, target):
@@ -1270,10 +1270,10 @@
     @arguments("box", "label")
     def opimpl_goto_if_exception_mismatch(self, vtablebox, next_exc_target):
         metainterp = self.metainterp
-        last_exc_value_box = metainterp.last_exc_value_box
-        assert last_exc_value_box is not None
+        last_exc_value = metainterp.last_exc_value
+        assert last_exc_value
         assert metainterp.class_of_last_exc_is_const
-        if not metainterp.cpu.ts.instanceOf(last_exc_value_box, vtablebox):
+        if not metainterp.cpu.ts.instanceOf(ConstPtr(lltype.cast_opaque_ptr(llmemory.GCREF, last_exc_value)), vtablebox):
             self.pc = next_exc_target
 
     @arguments("box", "orgpc")
@@ -1284,29 +1284,30 @@
             self.metainterp.generate_guard(rop.GUARD_CLASS, exc_value_box,
                                            [clsbox], resumepc=orgpc)
         self.metainterp.class_of_last_exc_is_const = True
-        self.metainterp.last_exc_value_box = exc_value_box
+        self.metainterp.last_exc_value = exc_value_box.getref(rclass.OBJECTPTR)
+        self.metainterp.last_exc_box = exc_value_box
         self.metainterp.popframe()
         self.metainterp.finishframe_exception()
 
     @arguments()
     def opimpl_reraise(self):
-        assert self.metainterp.last_exc_value_box is not None
+        assert self.metainterp.last_exc_value
         self.metainterp.popframe()
         self.metainterp.finishframe_exception()
 
     @arguments()
     def opimpl_last_exception(self):
         # Same comment as in opimpl_goto_if_exception_mismatch().
-        exc_value_box = self.metainterp.last_exc_value_box
-        assert exc_value_box is not None
+        exc_value = self.metainterp.last_exc_value
+        assert exc_value
         assert self.metainterp.class_of_last_exc_is_const
-        return self.metainterp.cpu.ts.cls_of_box(exc_value_box)
+        return self.metainterp.cpu.ts.cls_of_box(ConstPtr(exc_value))
 
     @arguments()
     def opimpl_last_exc_value(self):
-        exc_value_box = self.metainterp.last_exc_value_box
-        assert exc_value_box is not None
-        return exc_value_box
+        exc_value = self.metainterp.last_exc_value
+        assert exc_value
+        return ConstPtr(lltype.cast_opaque_ptr(llmemory.GCREF, exc_value))
 
     @arguments("box")
     def opimpl_debug_fatalerror(self, box):
@@ -1503,7 +1504,7 @@
         self.metainterp.clear_exception()
         op = self.metainterp.execute_and_record_varargs(opnum, argboxes,
                                                             descr=descr)
-        if pure and self.metainterp.last_exc_value_box is None and op:
+        if pure and not self.metainterp.last_exc_value and op:
             op = self.metainterp.record_result_of_call_pure(op)
             exc = exc and not isinstance(op, Const)
         if exc:
@@ -1868,6 +1869,7 @@
     portal_call_depth = 0
     cancel_count = 0
     exported_state = None
+    last_exc_box = None
 
     def __init__(self, staticdata, jitdriver_sd):
         self.staticdata = staticdata
@@ -1878,7 +1880,7 @@
         # during recursion we can also see other jitdrivers.
         self.portal_trace_positions = []
         self.free_frames_list = []
-        self.last_exc_value_box = None
+        self.last_exc_value = lltype.nullptr(rclass.OBJECT)
         self.forced_virtualizable = None
         self.partial_trace = None
         self.retracing_from = -1
@@ -1937,7 +1939,7 @@
 
     def finishframe(self, resultbox):
         # handle a non-exceptional return from the current frame
-        self.last_exc_value_box = None
+        self.last_exc_value = lltype.nullptr(rclass.OBJECT)
         self.popframe()
         if self.framestack:
             if resultbox is not None:
@@ -1963,7 +1965,7 @@
                 assert False
 
     def finishframe_exception(self):
-        excvaluebox = self.last_exc_value_box
+        excvalue = self.last_exc_value
         while self.framestack:
             frame = self.framestack[-1]
             code = frame.bytecode
@@ -1978,10 +1980,10 @@
                     raise ChangeFrame
             self.popframe()
         try:
-            self.compile_exit_frame_with_exception(excvaluebox)
+            self.compile_exit_frame_with_exception(self.last_exc_box)
         except SwitchToBlackhole, stb:
             self.aborted_tracing(stb.reason)
-        raise jitexc.ExitFrameWithExceptionRef(self.cpu, excvaluebox.getref_base())
+        raise jitexc.ExitFrameWithExceptionRef(self.cpu, lltype.cast_opaque_ptr(llmemory.GCREF, excvalue))
 
     def check_recursion_invariant(self):
         portal_call_depth = -1
@@ -2105,7 +2107,7 @@
     @specialize.argtype(2)
     def _record_helper_nonpure_varargs(self, opnum, resvalue, descr, argboxes):
         if (rop._OVF_FIRST <= opnum <= rop._OVF_LAST and
-            self.last_exc_value_box is None and
+            not self.last_exc_value and
             self._all_constants_varargs(argboxes)):
             return history.newconst(resvalue)
         # record the operation
@@ -2173,22 +2175,17 @@
     def execute_ll_raised(self, llexception, constant=False):
         # Exception handling: when execute.do_call() gets an exception it
         # calls metainterp.execute_raised(), which puts it into
-        # 'self.last_exc_value_box'.  This is used shortly afterwards
+        # 'self.last_exc_value'.  This is used shortly afterwards
         # to generate either GUARD_EXCEPTION or GUARD_NO_EXCEPTION, and also
         # to handle the following opcodes 'goto_if_exception_mismatch'.
-        llexception = self.cpu.ts.cast_to_ref(llexception)
-        if constant:
-            exc_value_box = self.cpu.ts.get_exc_value_const(llexception)
-        else:
-            exc_value_box = self.cpu.ts.get_exc_value_box(llexception)
-        self.last_exc_value_box = exc_value_box
+        self.last_exc_value = llexception
         self.class_of_last_exc_is_const = constant
         # 'class_of_last_exc_is_const' means that the class of the value
         # stored in the exc_value Box can be assumed to be a Const.  This
         # is only True after a GUARD_EXCEPTION or GUARD_CLASS.
 
     def clear_exception(self):
-        self.last_exc_value_box = None
+        self.last_exc_value = lltype.nullptr(rclass.OBJECT)
 
     def aborted_tracing(self, reason):
         self.staticdata.profiler.count(reason)
@@ -2747,28 +2744,30 @@
         self.virtualref_boxes[i+1] = self.cpu.ts.CONST_NULL
 
     def handle_possible_exception(self):
-        if self.last_exc_value_box is not None:
-            exception_box = self.cpu.ts.cls_of_box(self.last_exc_value_box)
+        if self.last_exc_value:
+            exception_box = ConstInt(heaptracker.adr2int(
+                llmemory.cast_ptr_to_adr(self.last_exc_value.typeptr)))
             op = self.generate_guard(rop.GUARD_EXCEPTION,
                                      None, [exception_box])
+            self.last_exc_box = op
+            op.setref_base(lltype.cast_opaque_ptr(llmemory.GCREF,
+                                                  self.last_exc_value))
             assert op is not None
-            op.result = self.last_exc_value_box
             self.class_of_last_exc_is_const = True
             self.finishframe_exception()
         else:
             self.generate_guard(rop.GUARD_NO_EXCEPTION, None, [])
 
     def handle_possible_overflow_error(self):
-        if self.last_exc_value_box is not None:
+        if self.last_exc_value:
             self.generate_guard(rop.GUARD_OVERFLOW, None)
-            assert isinstance(self.last_exc_value_box, Const)
             assert self.class_of_last_exc_is_const
             self.finishframe_exception()
         else:
             self.generate_guard(rop.GUARD_NO_OVERFLOW, None)
 
     def assert_no_exception(self):
-        assert self.last_exc_value_box is None
+        assert self.last_exc_value
 
     def rebuild_state_after_failure(self, resumedescr, deadframe):
         vinfo = self.jitdriver_sd.virtualizable_info
@@ -3046,7 +3045,7 @@
         self.clear_exception()
         executor.execute_varargs(self.cpu, self, rop.CALL_N,
                                           allboxes, descr)
-        if self.last_exc_value_box is not None:
+        if self.last_exc_value:
             # cannot trace this!  it raises, so we have to follow the
             # exception-catching path, but the trace doesn't contain
             # the call at all
@@ -3065,7 +3064,7 @@
         self.reason = reason
         self.raising_exception = raising_exception
         # ^^^ must be set to True if the SwitchToBlackhole is raised at a
-        #     point where the exception on metainterp.last_exc_value_box
+        #     point where the exception on metainterp.last_exc_value
         #     is supposed to be raised.  The default False means that it
         #     should just be copied into the blackhole interp, but not raised.
 
diff --git a/rpython/jit/metainterp/typesystem.py b/rpython/jit/metainterp/typesystem.py
--- a/rpython/jit/metainterp/typesystem.py
+++ b/rpython/jit/metainterp/typesystem.py
@@ -79,14 +79,6 @@
     def get_exception_box(self, etype):
         return history.ConstInt(etype)
 
-    def get_exc_value_box(self, evalue):
-        from rpython.jit.metainterp.resoperation import InputArgRef
-        
-        return InputArgRef(evalue)
-
-    def get_exc_value_const(self, evalue):
-        return history.ConstPtr(evalue)
-
     def get_exception_obj(self, evaluebox):
         # only works when translated
         obj = evaluebox.getref(lltype.Ptr(rclass.OBJECT))

From noreply at buildbot.pypy.org  Tue Jun  2 16:32:45 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Tue,  2 Jun 2015 16:32:45 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: one more thing
Message-ID: <20150602143245.42F591C03A8@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77770:7ada4b15297f
Date: 2015-06-02 16:15 +0200
http://bitbucket.org/pypy/pypy/changeset/7ada4b15297f/

Log:	one more thing

diff --git a/rpython/jit/metainterp/blackhole.py b/rpython/jit/metainterp/blackhole.py
--- a/rpython/jit/metainterp/blackhole.py
+++ b/rpython/jit/metainterp/blackhole.py
@@ -1670,8 +1670,8 @@
         nextbh = curbh
     firstbh = nextbh
     #
-    if metainterp.last_exc_value_box is not None:
-        current_exc = metainterp.last_exc_value_box.getref(rclass.OBJECTPTR)
+    if metainterp.last_exc_value:
+        current_exc = metainterp.last_exc_value
     else:
         current_exc = lltype.nullptr(rclass.OBJECTPTR.TO)
     if not raising_exception:

From noreply at buildbot.pypy.org  Tue Jun  2 16:32:46 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Tue,  2 Jun 2015 16:32:46 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: fix here
Message-ID: <20150602143246.5E5551C03A8@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77771:7c631275cd0d
Date: 2015-06-02 16:23 +0200
http://bitbucket.org/pypy/pypy/changeset/7c631275cd0d/

Log:	fix here

diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -1307,7 +1307,7 @@
     def opimpl_last_exc_value(self):
         exc_value = self.metainterp.last_exc_value
         assert exc_value
-        return ConstPtr(lltype.cast_opaque_ptr(llmemory.GCREF, exc_value))
+        return self.metainterp.last_exc_box
 
     @arguments("box")
     def opimpl_debug_fatalerror(self, box):

From noreply at buildbot.pypy.org  Tue Jun  2 16:32:47 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Tue,  2 Jun 2015 16:32:47 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: finish fixing exceptions
Message-ID: <20150602143247.7FD8A1C03A8@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77772:979818fc7953
Date: 2015-06-02 16:32 +0200
http://bitbucket.org/pypy/pypy/changeset/979818fc7953/

Log:	finish fixing exceptions

diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -915,6 +915,7 @@
     def execute_guard_overflow(self, descr):
         if not self.overflow_flag:
             self.fail_guard(descr)
+        return lltype.nullptr(llmemory.GCREF.TO) # I think it's fine....
 
     def execute_jump(self, descr, *args):
         raise Jump(descr._llgraph_target, args)
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -1301,7 +1301,7 @@
         exc_value = self.metainterp.last_exc_value
         assert exc_value
         assert self.metainterp.class_of_last_exc_is_const
-        return self.metainterp.cpu.ts.cls_of_box(ConstPtr(exc_value))
+        return self.metainterp.cpu.ts.cls_of_box(ConstPtr(lltype.cast_opaque_ptr(llmemory.GCREF, exc_value)))
 
     @arguments()
     def opimpl_last_exc_value(self):
@@ -2013,7 +2013,7 @@
             moreargs = [box] + extraargs
         else:
             moreargs = list(extraargs)
-        if opnum == rop.GUARD_EXCEPTION:
+        if opnum == rop.GUARD_EXCEPTION or opnum == rop.GUARD_OVERFLOW:
             guard_op = self.history.record(opnum, moreargs,
                                            lltype.nullptr(llmemory.GCREF.TO))
         else:
@@ -2749,9 +2749,12 @@
                 llmemory.cast_ptr_to_adr(self.last_exc_value.typeptr)))
             op = self.generate_guard(rop.GUARD_EXCEPTION,
                                      None, [exception_box])
-            self.last_exc_box = op
-            op.setref_base(lltype.cast_opaque_ptr(llmemory.GCREF,
-                                                  self.last_exc_value))
+            val = lltype.cast_opaque_ptr(llmemory.GCREF, self.last_exc_value)
+            if self.class_of_last_exc_is_const:
+                self.last_exc_box = ConstPtr(val)
+            else:
+                self.last_exc_box = op
+                op.setref_base(val)
             assert op is not None
             self.class_of_last_exc_is_const = True
             self.finishframe_exception()
@@ -2760,8 +2763,12 @@
 
     def handle_possible_overflow_error(self):
         if self.last_exc_value:
-            self.generate_guard(rop.GUARD_OVERFLOW, None)
+            op = self.generate_guard(rop.GUARD_OVERFLOW, None)
+            op.setref_base(lltype.cast_opaque_ptr(llmemory.GCREF,
+                                                  self.last_exc_value))
             assert self.class_of_last_exc_is_const
+            self.last_exc_box = ConstPtr(
+                lltype.cast_opaque_ptr(llmemory.GCREF, self.last_exc_value))
             self.finishframe_exception()
         else:
             self.generate_guard(rop.GUARD_NO_OVERFLOW, None)
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -653,7 +653,7 @@
     'GUARD_NO_EXCEPTION/0d/n',   # may be called with an exception currently set
     'GUARD_EXCEPTION/1d/r',     # may be called with an exception currently set
     'GUARD_NO_OVERFLOW/0d/n',
-    'GUARD_OVERFLOW/0d/n',
+    'GUARD_OVERFLOW/0d/r',
     'GUARD_NOT_FORCED/0d/n',      # may be called with an exception currently set
     'GUARD_NOT_FORCED_2/0d/n',    # same as GUARD_NOT_FORCED, but for finish()
     'GUARD_NOT_INVALIDATED/0d/n',

From noreply at buildbot.pypy.org  Tue Jun  2 16:35:30 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Tue,  2 Jun 2015 16:35:30 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: two more small fixes
Message-ID: <20150602143530.F05DA1C03A8@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77773:3b5889478fc5
Date: 2015-06-02 16:35 +0200
http://bitbucket.org/pypy/pypy/changeset/3b5889478fc5/

Log:	two more small fixes

diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -2774,7 +2774,7 @@
             self.generate_guard(rop.GUARD_NO_OVERFLOW, None)
 
     def assert_no_exception(self):
-        assert self.last_exc_value
+        assert not self.last_exc_value
 
     def rebuild_state_after_failure(self, resumedescr, deadframe):
         vinfo = self.jitdriver_sd.virtualizable_info
diff --git a/rpython/jit/metainterp/resume.py b/rpython/jit/metainterp/resume.py
--- a/rpython/jit/metainterp/resume.py
+++ b/rpython/jit/metainterp/resume.py
@@ -1055,7 +1055,7 @@
         str1box = self.decode_box(str1num, REF)
         str2box = self.decode_box(str2num, REF)
         return self.metainterp.execute_and_record_varargs(
-            rop.CALL, [ConstInt(func), str1box, str2box], calldescr)
+            rop.CALL_R, [ConstInt(func), str1box, str2box], calldescr)
 
     def slice_string(self, strnum, startnum, lengthnum):
         cic = self.metainterp.staticdata.callinfocollection

From noreply at buildbot.pypy.org  Tue Jun  2 16:52:37 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Tue,  2 Jun 2015 16:52:37 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: fix fix fix fix
Message-ID: <20150602145237.6F46B1C0F16@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77774:8010cbfca5a0
Date: 2015-06-02 16:52 +0200
http://bitbucket.org/pypy/pypy/changeset/8010cbfca5a0/

Log:	fix fix fix fix

diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py
--- a/rpython/jit/metainterp/history.py
+++ b/rpython/jit/metainterp/history.py
@@ -760,7 +760,7 @@
         op = ResOperation(opnum, argboxes, descr)
         if value is None:
             assert op.type == 'v'
-        elif type(value) is bool:
+        elif not we_are_translated() and type(value) is bool:
             assert op.type == 'i'
             op.setint(int(value))
         elif isinstance(value, float):
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -836,6 +836,7 @@
     def optimize_STRGETITEM(self, op):
         indexb = self.getintbound(op.getarg(1))
         if indexb.is_constant():
+            raise Exception("implement me")
             arrayvalue = self.getvalue(op.getarg(0))
             arrayvalue.make_len_gt(MODE_STR, op.getdescr(), indexvalue.box.getint())
         self.optimize_default(op)
@@ -843,6 +844,7 @@
     def optimize_UNICODEGETITEM(self, op):
         indexb = self.getintbound(op.getarg(1))
         if indexb.is_constant():
+            raise Exception("implement me")
             arrayvalue = self.getvalue(op.getarg(0))
             arrayvalue.make_len_gt(MODE_UNICODE, op.getdescr(), indexvalue.box.getint())
         self.optimize_default(op)
diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py
--- a/rpython/jit/metainterp/optimizeopt/rewrite.py
+++ b/rpython/jit/metainterp/optimizeopt/rewrite.py
@@ -262,6 +262,8 @@
             box = self.get_box_replacement(box)
             if box.is_constant():
                 if not box.same_constant(constbox):
+                    r = self.optimizer.metainterp_sd.logger_ops.repr_of_resop(
+                        op)
                     raise InvalidLoop('A GUARD_VALUE (%s) was proven '
                                       'to always fail' % r)
                 return
diff --git a/rpython/jit/metainterp/optimizeopt/simplify.py b/rpython/jit/metainterp/optimizeopt/simplify.py
--- a/rpython/jit/metainterp/optimizeopt/simplify.py
+++ b/rpython/jit/metainterp/optimizeopt/simplify.py
@@ -33,8 +33,9 @@
         pass
 
     def optimize_VIRTUAL_REF(self, op):
-        op = ResOperation(rop.SAME_AS, [op.getarg(0)], op.result)
-        self.emit_operation(op)
+        newop = ResOperation(rop.SAME_AS_R, [op.getarg(0)], op.result)
+        self.replace_op_with(op, newop)
+        self.emit_operation(newop)
 
     def optimize_QUASIIMMUT_FIELD(self, op):
         # xxx ideally we could also kill the following GUARD_NOT_INVALIDATED
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -1382,10 +1382,9 @@
         vrefinfo = metainterp.staticdata.virtualref_info
         obj = box.getref_base()
         vref = vrefinfo.virtual_ref_during_tracing(obj)
-        resbox = history.BoxPtr(vref)
+        cindex = history.ConstInt(len(metainterp.virtualref_boxes) // 2)
+        resbox = metainterp.history.record(rop.VIRTUAL_REF, [box, cindex], vref)
         self.metainterp.heapcache.new(resbox)
-        cindex = history.ConstInt(len(metainterp.virtualref_boxes) // 2)
-        metainterp.history.record(rop.VIRTUAL_REF, [box, cindex], resbox)
         # Note: we allocate a JIT_VIRTUAL_REF here
         # (in virtual_ref_during_tracing()), in order to detect when
         # the virtual escapes during tracing already.  We record it as a
@@ -2975,7 +2974,7 @@
         num_extra_guards = 0
         while True:
             op = self.history.operations[-1-num_extra_guards]
-            if op.getopnum() == rop.CALL_MAY_FORCE:
+            if op.is_call_may_force():
                 break
             assert op.is_guard()
             num_extra_guards += 1
@@ -3004,29 +3003,31 @@
         for i in range(cif_description.nargs):
             kind, descr, itemsize = get_arg_descr(self.cpu,
                                                   cif_description.atypes[i])
+            ofs = cif_description.exchange_args[i]
+            assert ofs % itemsize == 0     # alignment check
             if kind == 'i':
-                box_arg = history.BoxInt()
+                box_arg = self.history.record(rop.GETARRAYITEM_RAW_I,
+                                    [box_exchange_buffer,
+                                     ConstInt(ofs // itemsize)],
+                                     0, descr)
             elif kind == 'f':
-                box_arg = history.BoxFloat()
+                box_arg = self.history.record(rop.GETARRAYITEM_RAW_F,
+                                    [box_exchange_buffer,
+                                     ConstInt(ofs // itemsize)],
+                                     0.0, descr)
             else:
                 assert kind == 'v'
                 continue
-            ofs = cif_description.exchange_args[i]
-            assert ofs % itemsize == 0     # alignment check
-            self.history.record(rop.GETARRAYITEM_RAW,
-                                [box_exchange_buffer,
-                                 ConstInt(ofs // itemsize)],
-                                box_arg, descr)
             arg_boxes.append(box_arg)
         #
-        box_result = op.result
         # for now, any call via libffi saves and restores everything
         # (that is, errno and SetLastError/GetLastError on Windows)
         # Note these flags match the ones in clibffi.ll_callback
         c_saveall = ConstInt(rffi.RFFI_ERR_ALL | rffi.RFFI_ALT_ERRNO)
-        self.history.record(rop.CALL_RELEASE_GIL,
-                            [c_saveall, op.getarg(2)] + arg_boxes,
-                            box_result, calldescr)
+        if op.type == 'i':
+            self.history.record(rop.CALL_RELEASE_GIL,
+                                [c_saveall, op.getarg(2)] + arg_boxes,
+                                box_result, calldescr)
         #
         self.history.operations.extend(extra_guards)
         #

From noreply at buildbot.pypy.org  Tue Jun  2 17:01:19 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Tue,  2 Jun 2015 17:01:19 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: hg merge default
Message-ID: <20150602150119.A379E1C0F16@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob <me at manueljacob.de>
Branch: py3k
Changeset: r77775:0b31f311ef2f
Date: 2015-06-02 17:02 +0200
http://bitbucket.org/pypy/pypy/changeset/0b31f311ef2f/

Log:	hg merge default

diff too long, truncating to 2000 out of 34800 lines

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -10,3 +10,8 @@
 32f35069a16d819b58c1b6efb17c44e3e53397b2 release-2.3.1
 10f1b29a2bd21f837090286174a9ca030b8680b2 release-2.5.0
 9c4588d731b7fe0b08669bd732c2b676cb0a8233 release-2.5.1
+fcdb941565156385cbac04cfb891f8f4c7a92ef6 release-2.6.0
+fcdb941565156385cbac04cfb891f8f4c7a92ef6 release-2.6.0
+e03971291f3a0729ecd3ee7fae7ddb0bb82d476c release-2.6.0
+e03971291f3a0729ecd3ee7fae7ddb0bb82d476c release-2.6.0
+295ee98b69288471b0fcf2e0ede82ce5209eb90b release-2.6.0
diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -38,8 +38,8 @@
   Armin Rigo
   Maciej Fijalkowski
   Carl Friedrich Bolz
+  Amaury Forgeot d'Arc
   Antonio Cuni
-  Amaury Forgeot d'Arc
   Samuele Pedroni
   Alex Gaynor
   Brian Kearns
@@ -50,9 +50,9 @@
   Holger Krekel
   Christian Tismer
   Hakan Ardo
-  Benjamin Peterson
   Manuel Jacob
   Ronan Lamy
+  Benjamin Peterson
   Anders Chrigstrom
   Eric van Riet Paap
   Wim Lavrijsen
@@ -63,8 +63,8 @@
   Sven Hager
   Anders Lehmann
   Aurelien Campeas
+  Remi Meier
   Niklaus Haldimann
-  Remi Meier
   Camillo Bruni
   Laura Creighton
   Toon Verwaest
@@ -76,10 +76,10 @@
   David Edelsohn
   Anders Hammarquist
   Jakub Gustak
+  Gregor Wegberg
   Guido Wesdorp
   Lawrence Oluyede
   Bartosz Skowron
-  Gregor Wegberg
   Daniel Roberts
   Niko Matsakis
   Adrien Di Mascio
@@ -87,10 +87,11 @@
   Ludovic Aubry
   Jacob Hallen
   Jason Creighton
+  Richard Plangger
   Alex Martelli
   Michal Bendowski
+  stian
   Jan de Mooij
-  stian
   Tyler Wade
   Michael Foord
   Stephan Diehl
@@ -133,15 +134,15 @@
   Georg Brandl
   Bert Freudenberg
   Stian Andreassen
+  Edd Barrett
   Wanja Saatkamp
   Gerald Klix
   Mike Blume
+  Tobias Pape
   Oscar Nierstrasz
   Stefan H. Muller
-  Edd Barrett
   Jeremy Thurgood
   Rami Chowdhury
-  Tobias Pape
   Eugene Oden
   Henry Mason
   Vasily Kuznetsov
@@ -167,11 +168,13 @@
   Michael Twomey
   Lucian Branescu Mihaila
   Yichao Yu
+  Anton Gulenko
   Gabriel Lavoie
   Olivier Dormond
   Jared Grubb
   Karl Bartel
   Wouter van Heyst
+  Sebastian Pawlu&#347;
   Brian Dorsey
   Victor Stinner
   Andrews Medina
@@ -188,6 +191,7 @@
   Neil Shepperd
   Stanislaw Halik
   Mikael Sch&#246;nenberg
+  Berkin Ilbeyi
   Elmo M?ntynen
   Jonathan David Riehl
   Anders Qvist
@@ -211,11 +215,11 @@
   Carl Meyer
   Karl Ramm
   Pieter Zieschang
-  Sebastian Pawlu&#347;
   Gabriel
   Lukas Vacek
   Andrew Dalke
   Sylvain Thenault
+  Jakub Stasiak
   Nathan Taylor
   Vladimir Kryachko
   Jacek Generowicz
@@ -242,6 +246,7 @@
   Tomo Cocoa
   Toni Mattis
   Lucas Stadler
+  Julian Berman
   roberto at goyle
   Yury V. Zaytsev
   Anna Katrina Dominguez
@@ -253,6 +258,8 @@
   Ben Darnell
   Roberto De Ioris
   Juan Francisco Cantero Hurtado
+  Ruochen Huang
+  Jeong YunWon
   Godefroid Chappelle
   Joshua Gilbert
   Dan Colish
@@ -271,6 +278,7 @@
   Christian Muirhead
   Berker Peksag
   James Lan
+  Volodymyr Vladymyrov
   shoma hosaka
   Daniel Neuh&#228;user
   Ben Mather
@@ -316,6 +324,7 @@
   yasirs
   Michael Chermside
   Anna Ravencroft
+  Andrey Churin
   Dan Crosta
   Julien Phalip
   Roman Podoliaka
diff --git a/lib_pypy/_audioop_build.py b/lib_pypy/_audioop_build.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_audioop_build.py
@@ -0,0 +1,621 @@
+from cffi import FFI
+
+ffi = FFI()
+ffi.cdef("""
+typedef short PyInt16;
+
+int ratecv(char* rv, char* cp, size_t len, int size,
+           int nchannels, int inrate, int outrate,
+           int* state_d, int* prev_i, int* cur_i,
+           int weightA, int weightB);
+
+void tostereo(char* rv, char* cp, size_t len, int size,
+              double fac1, double fac2);
+void add(char* rv, char* cp1, char* cp2, size_t len1, int size);
+
+/* 2's complement (14-bit range) */
+unsigned char
+st_14linear2ulaw(PyInt16 pcm_val);
+PyInt16 st_ulaw2linear16(unsigned char);
+
+/* 2's complement (13-bit range) */
+unsigned char
+st_linear2alaw(PyInt16 pcm_val);
+PyInt16 st_alaw2linear16(unsigned char);
+
+
+void lin2adcpm(unsigned char* rv, unsigned char* cp, size_t len,
+               size_t size, int* state);
+void adcpm2lin(unsigned char* rv, unsigned char* cp, size_t len,
+               size_t size, int* state);
+""")
+
+# This code is directly copied from CPython file: Modules/audioop.c
+_AUDIOOP_C_MODULE = r"""
+typedef short PyInt16;
+typedef int Py_Int32;
+
+/* Code shamelessly stolen from sox, 12.17.7, g711.c
+** (c) Craig Reese, Joe Campbell and Jeff Poskanzer 1989 */
+
+/* From g711.c:
+ *
+ * December 30, 1994:
+ * Functions linear2alaw, linear2ulaw have been updated to correctly
+ * convert unquantized 16 bit values.
+ * Tables for direct u- to A-law and A- to u-law conversions have been
+ * corrected.
+ * Borge Lindberg, Center for PersonKommunikation, Aalborg University.
+ * bli at cpk.auc.dk
+ *
+ */
+#define BIAS 0x84   /* define the add-in bias for 16 bit samples */
+#define CLIP 32635
+#define SIGN_BIT        (0x80)          /* Sign bit for a A-law byte. */
+#define QUANT_MASK      (0xf)           /* Quantization field mask. */
+#define SEG_SHIFT       (4)             /* Left shift for segment number. */
+#define SEG_MASK        (0x70)          /* Segment field mask. */
+
+static PyInt16 seg_aend[8] = {0x1F, 0x3F, 0x7F, 0xFF,
+                              0x1FF, 0x3FF, 0x7FF, 0xFFF};
+static PyInt16 seg_uend[8] = {0x3F, 0x7F, 0xFF, 0x1FF,
+                              0x3FF, 0x7FF, 0xFFF, 0x1FFF};
+
+static PyInt16
+search(PyInt16 val, PyInt16 *table, int size)
+{
+    int i;
+
+    for (i = 0; i < size; i++) {
+        if (val <= *table++)
+            return (i);
+    }
+    return (size);
+}
+#define st_ulaw2linear16(uc) (_st_ulaw2linear16[uc])
+#define st_alaw2linear16(uc) (_st_alaw2linear16[uc])
+
+static PyInt16 _st_ulaw2linear16[256] = {
+    -32124,  -31100,  -30076,  -29052,  -28028,  -27004,  -25980,
+    -24956,  -23932,  -22908,  -21884,  -20860,  -19836,  -18812,
+    -17788,  -16764,  -15996,  -15484,  -14972,  -14460,  -13948,
+    -13436,  -12924,  -12412,  -11900,  -11388,  -10876,  -10364,
+     -9852,   -9340,   -8828,   -8316,   -7932,   -7676,   -7420,
+     -7164,   -6908,   -6652,   -6396,   -6140,   -5884,   -5628,
+     -5372,   -5116,   -4860,   -4604,   -4348,   -4092,   -3900,
+     -3772,   -3644,   -3516,   -3388,   -3260,   -3132,   -3004,
+     -2876,   -2748,   -2620,   -2492,   -2364,   -2236,   -2108,
+     -1980,   -1884,   -1820,   -1756,   -1692,   -1628,   -1564,
+     -1500,   -1436,   -1372,   -1308,   -1244,   -1180,   -1116,
+     -1052,    -988,    -924,    -876,    -844,    -812,    -780,
+      -748,    -716,    -684,    -652,    -620,    -588,    -556,
+      -524,    -492,    -460,    -428,    -396,    -372,    -356,
+      -340,    -324,    -308,    -292,    -276,    -260,    -244,
+      -228,    -212,    -196,    -180,    -164,    -148,    -132,
+      -120,    -112,    -104,     -96,     -88,     -80,     -72,
+       -64,     -56,     -48,     -40,     -32,     -24,     -16,
+    -8,       0,   32124,   31100,   30076,   29052,   28028,
+     27004,   25980,   24956,   23932,   22908,   21884,   20860,
+     19836,   18812,   17788,   16764,   15996,   15484,   14972,
+     14460,   13948,   13436,   12924,   12412,   11900,   11388,
+     10876,   10364,    9852,    9340,    8828,    8316,    7932,
+      7676,    7420,    7164,    6908,    6652,    6396,    6140,
+      5884,    5628,    5372,    5116,    4860,    4604,    4348,
+      4092,    3900,    3772,    3644,    3516,    3388,    3260,
+      3132,    3004,    2876,    2748,    2620,    2492,    2364,
+      2236,    2108,    1980,    1884,    1820,    1756,    1692,
+      1628,    1564,    1500,    1436,    1372,    1308,    1244,
+      1180,    1116,    1052,     988,     924,     876,     844,
+       812,     780,     748,     716,     684,     652,     620,
+       588,     556,     524,     492,     460,     428,     396,
+       372,     356,     340,     324,     308,     292,     276,
+       260,     244,     228,     212,     196,     180,     164,
+       148,     132,     120,     112,     104,      96,      88,
+    80,      72,      64,      56,      48,      40,      32,
+    24,      16,       8,       0
+};
+
+/*
+ * linear2ulaw() accepts a 14-bit signed integer and encodes it as u-law data
+ * stored in a unsigned char.  This function should only be called with
+ * the data shifted such that it only contains information in the lower
+ * 14-bits.
+ *
+ * In order to simplify the encoding process, the original linear magnitude
+ * is biased by adding 33 which shifts the encoding range from (0 - 8158) to
+ * (33 - 8191). The result can be seen in the following encoding table:
+ *
+ *      Biased Linear Input Code        Compressed Code
+ *      ------------------------        ---------------
+ *      00000001wxyza                   000wxyz
+ *      0000001wxyzab                   001wxyz
+ *      000001wxyzabc                   010wxyz
+ *      00001wxyzabcd                   011wxyz
+ *      0001wxyzabcde                   100wxyz
+ *      001wxyzabcdef                   101wxyz
+ *      01wxyzabcdefg                   110wxyz
+ *      1wxyzabcdefgh                   111wxyz
+ *
+ * Each biased linear code has a leading 1 which identifies the segment
+ * number. The value of the segment number is equal to 7 minus the number
+ * of leading 0's. The quantization interval is directly available as the
+ * four bits wxyz.  * The trailing bits (a - h) are ignored.
+ *
+ * Ordinarily the complement of the resulting code word is used for
+ * transmission, and so the code word is complemented before it is returned.
+ *
+ * For further information see John C. Bellamy's Digital Telephony, 1982,
+ * John Wiley & Sons, pps 98-111 and 472-476.
+ */
+static unsigned char
+st_14linear2ulaw(PyInt16 pcm_val)       /* 2's complement (14-bit range) */
+{
+    PyInt16         mask;
+    PyInt16         seg;
+    unsigned char   uval;
+
+    /* The original sox code does this in the calling function, not here */
+    pcm_val = pcm_val >> 2;
+
+    /* u-law inverts all bits */
+    /* Get the sign and the magnitude of the value. */
+    if (pcm_val < 0) {
+        pcm_val = -pcm_val;
+        mask = 0x7F;
+    } else {
+        mask = 0xFF;
+    }
+    if ( pcm_val > CLIP ) pcm_val = CLIP;           /* clip the magnitude */
+    pcm_val += (BIAS >> 2);
+
+    /* Convert the scaled magnitude to segment number. */
+    seg = search(pcm_val, seg_uend, 8);
+
+    /*
+     * Combine the sign, segment, quantization bits;
+     * and complement the code word.
+     */
+    if (seg >= 8)           /* out of range, return maximum value. */
+        return (unsigned char) (0x7F ^ mask);
+    else {
+        uval = (unsigned char) (seg << 4) | ((pcm_val >> (seg + 1)) & 0xF);
+        return (uval ^ mask);
+    }
+
+}
+
+static PyInt16 _st_alaw2linear16[256] = {
+     -5504,   -5248,   -6016,   -5760,   -4480,   -4224,   -4992,
+     -4736,   -7552,   -7296,   -8064,   -7808,   -6528,   -6272,
+     -7040,   -6784,   -2752,   -2624,   -3008,   -2880,   -2240,
+     -2112,   -2496,   -2368,   -3776,   -3648,   -4032,   -3904,
+     -3264,   -3136,   -3520,   -3392,  -22016,  -20992,  -24064,
+    -23040,  -17920,  -16896,  -19968,  -18944,  -30208,  -29184,
+    -32256,  -31232,  -26112,  -25088,  -28160,  -27136,  -11008,
+    -10496,  -12032,  -11520,   -8960,   -8448,   -9984,   -9472,
+    -15104,  -14592,  -16128,  -15616,  -13056,  -12544,  -14080,
+    -13568,    -344,    -328,    -376,    -360,    -280,    -264,
+      -312,    -296,    -472,    -456,    -504,    -488,    -408,
+      -392,    -440,    -424,     -88,     -72,    -120,    -104,
+       -24,      -8,     -56,     -40,    -216,    -200,    -248,
+      -232,    -152,    -136,    -184,    -168,   -1376,   -1312,
+     -1504,   -1440,   -1120,   -1056,   -1248,   -1184,   -1888,
+     -1824,   -2016,   -1952,   -1632,   -1568,   -1760,   -1696,
+      -688,    -656,    -752,    -720,    -560,    -528,    -624,
+      -592,    -944,    -912,   -1008,    -976,    -816,    -784,
+      -880,    -848,    5504,    5248,    6016,    5760,    4480,
+      4224,    4992,    4736,    7552,    7296,    8064,    7808,
+      6528,    6272,    7040,    6784,    2752,    2624,    3008,
+      2880,    2240,    2112,    2496,    2368,    3776,    3648,
+      4032,    3904,    3264,    3136,    3520,    3392,   22016,
+     20992,   24064,   23040,   17920,   16896,   19968,   18944,
+     30208,   29184,   32256,   31232,   26112,   25088,   28160,
+     27136,   11008,   10496,   12032,   11520,    8960,    8448,
+      9984,    9472,   15104,   14592,   16128,   15616,   13056,
+     12544,   14080,   13568,     344,     328,     376,     360,
+       280,     264,     312,     296,     472,     456,     504,
+       488,     408,     392,     440,     424,      88,      72,
+       120,     104,      24,       8,      56,      40,     216,
+       200,     248,     232,     152,     136,     184,     168,
+      1376,    1312,    1504,    1440,    1120,    1056,    1248,
+      1184,    1888,    1824,    2016,    1952,    1632,    1568,
+      1760,    1696,     688,     656,     752,     720,     560,
+       528,     624,     592,     944,     912,    1008,     976,
+       816,     784,     880,     848
+};
+
+/*
+ * linear2alaw() accepts an 13-bit signed integer and encodes it as A-law data
+ * stored in a unsigned char.  This function should only be called with
+ * the data shifted such that it only contains information in the lower
+ * 13-bits.
+ *
+ *              Linear Input Code       Compressed Code
+ *      ------------------------        ---------------
+ *      0000000wxyza                    000wxyz
+ *      0000001wxyza                    001wxyz
+ *      000001wxyzab                    010wxyz
+ *      00001wxyzabc                    011wxyz
+ *      0001wxyzabcd                    100wxyz
+ *      001wxyzabcde                    101wxyz
+ *      01wxyzabcdef                    110wxyz
+ *      1wxyzabcdefg                    111wxyz
+ *
+ * For further information see John C. Bellamy's Digital Telephony, 1982,
+ * John Wiley & Sons, pps 98-111 and 472-476.
+ */
+static unsigned char
+st_linear2alaw(PyInt16 pcm_val) /* 2's complement (13-bit range) */
+{
+    PyInt16         mask;
+    short           seg;
+    unsigned char   aval;
+
+    /* The original sox code does this in the calling function, not here */
+    pcm_val = pcm_val >> 3;
+
+    /* A-law using even bit inversion */
+    if (pcm_val >= 0) {
+        mask = 0xD5;            /* sign (7th) bit = 1 */
+    } else {
+        mask = 0x55;            /* sign bit = 0 */
+        pcm_val = -pcm_val - 1;
+    }
+
+    /* Convert the scaled magnitude to segment number. */
+    seg = search(pcm_val, seg_aend, 8);
+
+    /* Combine the sign, segment, and quantization bits. */
+
+    if (seg >= 8)           /* out of range, return maximum value. */
+        return (unsigned char) (0x7F ^ mask);
+    else {
+        aval = (unsigned char) seg << SEG_SHIFT;
+        if (seg < 2)
+            aval |= (pcm_val >> 1) & QUANT_MASK;
+        else
+            aval |= (pcm_val >> seg) & QUANT_MASK;
+        return (aval ^ mask);
+    }
+}
+/* End of code taken from sox */
+
+/* Intel ADPCM step variation table */
+static int indexTable[16] = {
+    -1, -1, -1, -1, 2, 4, 6, 8,
+    -1, -1, -1, -1, 2, 4, 6, 8,
+};
+
+static int stepsizeTable[89] = {
+    7, 8, 9, 10, 11, 12, 13, 14, 16, 17,
+    19, 21, 23, 25, 28, 31, 34, 37, 41, 45,
+    50, 55, 60, 66, 73, 80, 88, 97, 107, 118,
+    130, 143, 157, 173, 190, 209, 230, 253, 279, 307,
+    337, 371, 408, 449, 494, 544, 598, 658, 724, 796,
+    876, 963, 1060, 1166, 1282, 1411, 1552, 1707, 1878, 2066,
+    2272, 2499, 2749, 3024, 3327, 3660, 4026, 4428, 4871, 5358,
+    5894, 6484, 7132, 7845, 8630, 9493, 10442, 11487, 12635, 13899,
+    15289, 16818, 18500, 20350, 22385, 24623, 27086, 29794, 32767
+};
+
+#define CHARP(cp, i) ((signed char *)(cp+i))
+#define SHORTP(cp, i) ((short *)(cp+i))
+#define LONGP(cp, i) ((Py_Int32 *)(cp+i))
+"""
+
+C_SOURCE = _AUDIOOP_C_MODULE + r"""
+#include <math.h>
+
+static const int maxvals[] = {0, 0x7F, 0x7FFF, 0x7FFFFF, 0x7FFFFFFF};
+/* -1 trick is needed on Windows to support -0x80000000 without a warning */
+static const int minvals[] = {0, -0x80, -0x8000, -0x800000, -0x7FFFFFFF-1};
+
+static int
+fbound(double val, double minval, double maxval)
+{
+    if (val > maxval)
+        val = maxval;
+    else if (val < minval + 1)
+        val = minval;
+    return val;
+}
+
+static int
+gcd(int a, int b)
+{
+    while (b > 0) {
+        int tmp = a % b;
+        a = b;
+        b = tmp;
+    }
+    return a;
+}
+
+int ratecv(char* rv, char* cp, size_t len, int size,
+           int nchannels, int inrate, int outrate,
+           int* state_d, int* prev_i, int* cur_i,
+           int weightA, int weightB)
+{
+    char *ncp = rv;
+    int d, chan;
+
+    /* divide inrate and outrate by their greatest common divisor */
+    d = gcd(inrate, outrate);
+    inrate /= d;
+    outrate /= d;
+    /* divide weightA and weightB by their greatest common divisor */
+    d = gcd(weightA, weightB);
+    weightA /= d;
+    weightA /= d;
+
+    d = *state_d;
+
+    for (;;) {
+        while (d < 0) {
+            if (len == 0) {
+                *state_d = d;
+                return ncp - rv;
+            }
+            for (chan = 0; chan < nchannels; chan++) {
+                prev_i[chan] = cur_i[chan];
+                if (size == 1)
+                    cur_i[chan] = ((int)*CHARP(cp, 0)) << 24;
+                else if (size == 2)
+                    cur_i[chan] = ((int)*SHORTP(cp, 0)) << 16;
+                else if (size == 4)
+                    cur_i[chan] = (int)*LONGP(cp, 0);
+                cp += size;
+                /* implements a simple digital filter */
+                cur_i[chan] = (int)(
+                    ((double)weightA * (double)cur_i[chan] +
+                     (double)weightB * (double)prev_i[chan]) /
+                    ((double)weightA + (double)weightB));
+            }
+            len--;
+            d += outrate;
+        }
+        while (d >= 0) {
+            for (chan = 0; chan < nchannels; chan++) {
+                int cur_o;
+                cur_o = (int)(((double)prev_i[chan] * (double)d +
+                         (double)cur_i[chan] * (double)(outrate - d)) /
+                    (double)outrate);
+                if (size == 1)
+                    *CHARP(ncp, 0) = (signed char)(cur_o >> 24);
+                else if (size == 2)
+                    *SHORTP(ncp, 0) = (short)(cur_o >> 16);
+                else if (size == 4)
+                    *LONGP(ncp, 0) = (Py_Int32)(cur_o);
+                ncp += size;
+            }
+            d -= inrate;
+        }
+    }
+}
+
+void tostereo(char* rv, char* cp, size_t len, int size,
+              double fac1, double fac2)
+{
+    int val1, val2, val = 0;
+    double fval, maxval, minval;
+    char *ncp = rv;
+    int i;
+
+    maxval = (double) maxvals[size];
+    minval = (double) minvals[size];
+
+    for ( i=0; i < len; i += size ) {
+        if ( size == 1 )      val = (int)*CHARP(cp, i);
+        else if ( size == 2 ) val = (int)*SHORTP(cp, i);
+        else if ( size == 4 ) val = (int)*LONGP(cp, i);
+
+        fval = (double)val*fac1;
+        val1 = (int)floor(fbound(fval, minval, maxval));
+
+        fval = (double)val*fac2;
+        val2 = (int)floor(fbound(fval, minval, maxval));
+
+        if ( size == 1 )      *CHARP(ncp, i*2) = (signed char)val1;
+        else if ( size == 2 ) *SHORTP(ncp, i*2) = (short)val1;
+        else if ( size == 4 ) *LONGP(ncp, i*2) = (Py_Int32)val1;
+
+        if ( size == 1 )      *CHARP(ncp, i*2+1) = (signed char)val2;
+        else if ( size == 2 ) *SHORTP(ncp, i*2+2) = (short)val2;
+        else if ( size == 4 ) *LONGP(ncp, i*2+4) = (Py_Int32)val2;
+    }
+}
+
+void add(char* rv, char* cp1, char* cp2, size_t len1, int size)
+{
+    int i;
+    int val1 = 0, val2 = 0, minval, maxval, newval;
+    char* ncp = rv;
+
+    maxval = maxvals[size];
+    minval = minvals[size];
+
+    for ( i=0; i < len1; i += size ) {
+        if ( size == 1 )      val1 = (int)*CHARP(cp1, i);
+        else if ( size == 2 ) val1 = (int)*SHORTP(cp1, i);
+        else if ( size == 4 ) val1 = (int)*LONGP(cp1, i);
+
+        if ( size == 1 )      val2 = (int)*CHARP(cp2, i);
+        else if ( size == 2 ) val2 = (int)*SHORTP(cp2, i);
+        else if ( size == 4 ) val2 = (int)*LONGP(cp2, i);
+
+        if (size < 4) {
+            newval = val1 + val2;
+            /* truncate in case of overflow */
+            if (newval > maxval)
+                newval = maxval;
+            else if (newval < minval)
+                newval = minval;
+        }
+        else {
+            double fval = (double)val1 + (double)val2;
+            /* truncate in case of overflow */
+            newval = (int)floor(fbound(fval, minval, maxval));
+        }
+
+        if ( size == 1 )      *CHARP(ncp, i) = (signed char)newval;
+        else if ( size == 2 ) *SHORTP(ncp, i) = (short)newval;
+        else if ( size == 4 ) *LONGP(ncp, i) = (Py_Int32)newval;
+    }
+}
+
+void lin2adcpm(unsigned char* ncp, unsigned char* cp, size_t len,
+               size_t size, int* state)
+{
+    int step, outputbuffer = 0, bufferstep;
+    int val = 0;
+    int diff, vpdiff, sign, delta;
+    size_t i;
+    int valpred = state[0];
+    int index = state[1];
+
+    step = stepsizeTable[index];
+    bufferstep = 1;
+
+    for ( i=0; i < len; i += size ) {
+        if ( size == 1 )      val = ((int)*CHARP(cp, i)) << 8;
+        else if ( size == 2 ) val = (int)*SHORTP(cp, i);
+        else if ( size == 4 ) val = ((int)*LONGP(cp, i)) >> 16;
+
+        /* Step 1 - compute difference with previous value */
+        diff = val - valpred;
+        sign = (diff < 0) ? 8 : 0;
+        if ( sign ) diff = (-diff);
+
+        /* Step 2 - Divide and clamp */
+        /* Note:
+        ** This code *approximately* computes:
+        **    delta = diff*4/step;
+        **    vpdiff = (delta+0.5)*step/4;
+        ** but in shift step bits are dropped. The net result of this
+        ** is that even if you have fast mul/div hardware you cannot
+        ** put it to good use since the fixup would be too expensive.
+        */
+        delta = 0;
+        vpdiff = (step >> 3);
+
+        if ( diff >= step ) {
+            delta = 4;
+            diff -= step;
+            vpdiff += step;
+        }
+        step >>= 1;
+        if ( diff >= step  ) {
+            delta |= 2;
+            diff -= step;
+            vpdiff += step;
+        }
+        step >>= 1;
+        if ( diff >= step ) {
+            delta |= 1;
+            vpdiff += step;
+        }
+
+        /* Step 3 - Update previous value */
+        if ( sign )
+            valpred -= vpdiff;
+        else
+            valpred += vpdiff;
+
+        /* Step 4 - Clamp previous value to 16 bits */
+        if ( valpred > 32767 )
+            valpred = 32767;
+        else if ( valpred < -32768 )
+            valpred = -32768;
+
+        /* Step 5 - Assemble value, update index and step values */
+        delta |= sign;
+
+        index += indexTable[delta];
+        if ( index < 0 ) index = 0;
+        if ( index > 88 ) index = 88;
+        step = stepsizeTable[index];
+
+        /* Step 6 - Output value */
+        if ( bufferstep ) {
+            outputbuffer = (delta << 4) & 0xf0;
+        } else {
+            *ncp++ = (delta & 0x0f) | outputbuffer;
+        }
+        bufferstep = !bufferstep;
+    }
+    state[0] = valpred;
+    state[1] = index;
+}
+
+
+void adcpm2lin(unsigned char* ncp, unsigned char* cp, size_t len,
+               size_t size, int* state)
+{
+    int step, inputbuffer = 0, bufferstep;
+    int val = 0;
+    int diff, vpdiff, sign, delta;
+    size_t i;
+    int valpred = state[0];
+    int index = state[1];
+
+    step = stepsizeTable[index];
+    bufferstep = 0;
+
+    for ( i=0; i < len*size*2; i += size ) {
+        /* Step 1 - get the delta value and compute next index */
+        if ( bufferstep ) {
+            delta = inputbuffer & 0xf;
+        } else {
+            inputbuffer = *cp++;
+            delta = (inputbuffer >> 4) & 0xf;
+        }
+
+        bufferstep = !bufferstep;
+
+        /* Step 2 - Find new index value (for later) */
+        index += indexTable[delta];
+        if ( index < 0 ) index = 0;
+        if ( index > 88 ) index = 88;
+
+        /* Step 3 - Separate sign and magnitude */
+        sign = delta & 8;
+        delta = delta & 7;
+
+        /* Step 4 - Compute difference and new predicted value */
+        /*
+        ** Computes 'vpdiff = (delta+0.5)*step/4', but see comment
+        ** in adpcm_coder.
+        */
+        vpdiff = step >> 3;
+        if ( delta & 4 ) vpdiff += step;
+        if ( delta & 2 ) vpdiff += step>>1;
+        if ( delta & 1 ) vpdiff += step>>2;
+
+        if ( sign )
+            valpred -= vpdiff;
+        else
+            valpred += vpdiff;
+
+        /* Step 5 - clamp output value */
+        if ( valpred > 32767 )
+            valpred = 32767;
+        else if ( valpred < -32768 )
+            valpred = -32768;
+
+        /* Step 6 - Update step value */
+        step = stepsizeTable[index];
+
+        /* Step 6 - Output value */
+        if ( size == 1 ) *CHARP(ncp, i) = (signed char)(valpred >> 8);
+        else if ( size == 2 ) *SHORTP(ncp, i) = (short)(valpred);
+        else if ( size == 4 ) *LONGP(ncp, i) = (Py_Int32)(valpred<<16);
+    }
+    state[0] = valpred;
+    state[1] = index;
+}
+"""
+
+ffi.set_source("_audioop_cffi", C_SOURCE)
+
+if __name__ == "__main__":
+    ffi.compile()
diff --git a/lib_pypy/_curses.py b/lib_pypy/_curses.py
--- a/lib_pypy/_curses.py
+++ b/lib_pypy/_curses.py
@@ -6,326 +6,7 @@
     raise ImportError('No module named _curses')
 from functools import wraps
 
-from cffi import FFI
-
-ffi = FFI()
-
-ffi.cdef("""
-typedef ... WINDOW;
-typedef ... SCREEN;
-typedef unsigned long mmask_t;
-typedef unsigned char bool;
-typedef unsigned long chtype;
-typedef chtype attr_t;
-
-typedef struct
-{
-    short id;           /* ID to distinguish multiple devices */
-    int x, y, z;        /* event coordinates (character-cell) */
-    mmask_t bstate;     /* button state bits */
-}
-MEVENT;
-
-static const int ERR, OK;
-static const int TRUE, FALSE;
-static const int KEY_MIN, KEY_MAX;
-
-static const int COLOR_BLACK;
-static const int COLOR_RED;
-static const int COLOR_GREEN;
-static const int COLOR_YELLOW;
-static const int COLOR_BLUE;
-static const int COLOR_MAGENTA;
-static const int COLOR_CYAN;
-static const int COLOR_WHITE;
-
-static const chtype A_ATTRIBUTES;
-static const chtype A_NORMAL;
-static const chtype A_STANDOUT;
-static const chtype A_UNDERLINE;
-static const chtype A_REVERSE;
-static const chtype A_BLINK;
-static const chtype A_DIM;
-static const chtype A_BOLD;
-static const chtype A_ALTCHARSET;
-static const chtype A_INVIS;
-static const chtype A_PROTECT;
-static const chtype A_CHARTEXT;
-static const chtype A_COLOR;
-
-static const int BUTTON1_RELEASED;
-static const int BUTTON1_PRESSED;
-static const int BUTTON1_CLICKED;
-static const int BUTTON1_DOUBLE_CLICKED;
-static const int BUTTON1_TRIPLE_CLICKED;
-static const int BUTTON2_RELEASED;
-static const int BUTTON2_PRESSED;
-static const int BUTTON2_CLICKED;
-static const int BUTTON2_DOUBLE_CLICKED;
-static const int BUTTON2_TRIPLE_CLICKED;
-static const int BUTTON3_RELEASED;
-static const int BUTTON3_PRESSED;
-static const int BUTTON3_CLICKED;
-static const int BUTTON3_DOUBLE_CLICKED;
-static const int BUTTON3_TRIPLE_CLICKED;
-static const int BUTTON4_RELEASED;
-static const int BUTTON4_PRESSED;
-static const int BUTTON4_CLICKED;
-static const int BUTTON4_DOUBLE_CLICKED;
-static const int BUTTON4_TRIPLE_CLICKED;
-static const int BUTTON_SHIFT;
-static const int BUTTON_CTRL;
-static const int BUTTON_ALT;
-static const int ALL_MOUSE_EVENTS;
-static const int REPORT_MOUSE_POSITION;
-
-int setupterm(char *, int, int *);
-
-WINDOW *stdscr;
-int COLORS;
-int COLOR_PAIRS;
-int COLS;
-int LINES;
-
-int baudrate(void);
-int beep(void);
-int box(WINDOW *, chtype, chtype);
-bool can_change_color(void);
-int cbreak(void);
-int clearok(WINDOW *, bool);
-int color_content(short, short*, short*, short*);
-int copywin(const WINDOW*, WINDOW*, int, int, int, int, int, int, int);
-int curs_set(int);
-int def_prog_mode(void);
-int def_shell_mode(void);
-int delay_output(int);
-int delwin(WINDOW *);
-WINDOW * derwin(WINDOW *, int, int, int, int);
-int doupdate(void);
-int echo(void);
-int endwin(void);
-char erasechar(void);
-void filter(void);
-int flash(void);
-int flushinp(void);
-chtype getbkgd(WINDOW *);
-WINDOW * getwin(FILE *);
-int halfdelay(int);
-bool has_colors(void);
-bool has_ic(void);
-bool has_il(void);
-void idcok(WINDOW *, bool);
-int idlok(WINDOW *, bool);
-void immedok(WINDOW *, bool);
-WINDOW * initscr(void);
-int init_color(short, short, short, short);
-int init_pair(short, short, short);
-int intrflush(WINDOW *, bool);
-bool isendwin(void);
-bool is_linetouched(WINDOW *, int);
-bool is_wintouched(WINDOW *);
-const char * keyname(int);
-int keypad(WINDOW *, bool);
-char killchar(void);
-int leaveok(WINDOW *, bool);
-char * longname(void);
-int meta(WINDOW *, bool);
-int mvderwin(WINDOW *, int, int);
-int mvwaddch(WINDOW *, int, int, const chtype);
-int mvwaddnstr(WINDOW *, int, int, const char *, int);
-int mvwaddstr(WINDOW *, int, int, const char *);
-int mvwchgat(WINDOW *, int, int, int, attr_t, short, const void *);
-int mvwdelch(WINDOW *, int, int);
-int mvwgetch(WINDOW *, int, int);
-int mvwgetnstr(WINDOW *, int, int, char *, int);
-int mvwin(WINDOW *, int, int);
-chtype mvwinch(WINDOW *, int, int);
-int mvwinnstr(WINDOW *, int, int, char *, int);
-int mvwinsch(WINDOW *, int, int, chtype);
-int mvwinsnstr(WINDOW *, int, int, const char *, int);
-int mvwinsstr(WINDOW *, int, int, const char *);
-int napms(int);
-WINDOW * newpad(int, int);
-WINDOW * newwin(int, int, int, int);
-int nl(void);
-int nocbreak(void);
-int nodelay(WINDOW *, bool);
-int noecho(void);
-int nonl(void);
-void noqiflush(void);
-int noraw(void);
-int notimeout(WINDOW *, bool);
-int overlay(const WINDOW*, WINDOW *);
-int overwrite(const WINDOW*, WINDOW *);
-int pair_content(short, short*, short*);
-int pechochar(WINDOW *, const chtype);
-int pnoutrefresh(WINDOW*, int, int, int, int, int, int);
-int prefresh(WINDOW *, int, int, int, int, int, int);
-int putwin(WINDOW *, FILE *);
-void qiflush(void);
-int raw(void);
-int redrawwin(WINDOW *);
-int resetty(void);
-int reset_prog_mode(void);
-int reset_shell_mode(void);
-int savetty(void);
-int scroll(WINDOW *);
-int scrollok(WINDOW *, bool);
-int start_color(void);
-WINDOW * subpad(WINDOW *, int, int, int, int);
-WINDOW * subwin(WINDOW *, int, int, int, int);
-int syncok(WINDOW *, bool);
-chtype termattrs(void);
-char * termname(void);
-int touchline(WINDOW *, int, int);
-int touchwin(WINDOW *);
-int typeahead(int);
-int ungetch(int);
-int untouchwin(WINDOW *);
-void use_env(bool);
-int waddch(WINDOW *, const chtype);
-int waddnstr(WINDOW *, const char *, int);
-int waddstr(WINDOW *, const char *);
-int wattron(WINDOW *, int);
-int wattroff(WINDOW *, int);
-int wattrset(WINDOW *, int);
-int wbkgd(WINDOW *, chtype);
-void wbkgdset(WINDOW *, chtype);
-int wborder(WINDOW *, chtype, chtype, chtype, chtype,
-            chtype, chtype, chtype, chtype);
-int wchgat(WINDOW *, int, attr_t, short, const void *);
-int wclear(WINDOW *);
-int wclrtobot(WINDOW *);
-int wclrtoeol(WINDOW *);
-void wcursyncup(WINDOW *);
-int wdelch(WINDOW *);
-int wdeleteln(WINDOW *);
-int wechochar(WINDOW *, const chtype);
-int werase(WINDOW *);
-int wgetch(WINDOW *);
-int wgetnstr(WINDOW *, char *, int);
-int whline(WINDOW *, chtype, int);
-chtype winch(WINDOW *);
-int winnstr(WINDOW *, char *, int);
-int winsch(WINDOW *, chtype);
-int winsdelln(WINDOW *, int);
-int winsertln(WINDOW *);
-int winsnstr(WINDOW *, const char *, int);
-int winsstr(WINDOW *, const char *);
-int wmove(WINDOW *, int, int);
-int wresize(WINDOW *, int, int);
-int wnoutrefresh(WINDOW *);
-int wredrawln(WINDOW *, int, int);
-int wrefresh(WINDOW *);
-int wscrl(WINDOW *, int);
-int wsetscrreg(WINDOW *, int, int);
-int wstandout(WINDOW *);
-int wstandend(WINDOW *);
-void wsyncdown(WINDOW *);
-void wsyncup(WINDOW *);
-void wtimeout(WINDOW *, int);
-int wtouchln(WINDOW *, int, int, int);
-int wvline(WINDOW *, chtype, int);
-int tigetflag(char *);
-int tigetnum(char *);
-char * tigetstr(char *);
-int putp(const char *);
-char * tparm(const char *, ...);
-int getattrs(const WINDOW *);
-int getcurx(const WINDOW *);
-int getcury(const WINDOW *);
-int getbegx(const WINDOW *);
-int getbegy(const WINDOW *);
-int getmaxx(const WINDOW *);
-int getmaxy(const WINDOW *);
-int getparx(const WINDOW *);
-int getpary(const WINDOW *);
-
-int getmouse(MEVENT *);
-int ungetmouse(MEVENT *);
-mmask_t mousemask(mmask_t, mmask_t *);
-bool wenclose(const WINDOW *, int, int);
-int mouseinterval(int);
-
-void setsyx(int y, int x);
-const char *unctrl(chtype);
-int use_default_colors(void);
-
-int has_key(int);
-bool is_term_resized(int, int);
-
-#define _m_STRICT_SYSV_CURSES ...
-#define _m_NCURSES_MOUSE_VERSION ...
-#define _m_NetBSD ...
-int _m_ispad(WINDOW *);
-
-chtype acs_map[];
-
-// For _curses_panel:
-
-typedef ... PANEL;
-
-WINDOW *panel_window(const PANEL *);
-void update_panels(void);
-int hide_panel(PANEL *);
-int show_panel(PANEL *);
-int del_panel(PANEL *);
-int top_panel(PANEL *);
-int bottom_panel(PANEL *);
-PANEL *new_panel(WINDOW *);
-PANEL *panel_above(const PANEL *);
-PANEL *panel_below(const PANEL *);
-int set_panel_userptr(PANEL *, void *);
-const void *panel_userptr(const PANEL *);
-int move_panel(PANEL *, int, int);
-int replace_panel(PANEL *,WINDOW *);
-int panel_hidden(const PANEL *);
-
-void _m_getsyx(int *yx);
-""")
-
-
-lib = ffi.verify("""
-#ifdef __APPLE__
-/* the following define is necessary for OS X 10.6+; without it, the
-   Apple-supplied ncurses.h sets NCURSES_OPAQUE to 1, and then Python
-   can't get at the WINDOW flags field. */
-#define NCURSES_OPAQUE 0
-#endif
-
-#include <ncurses.h>
-#include <panel.h>
-#include <term.h>
-
-#if defined STRICT_SYSV_CURSES
-#define _m_STRICT_SYSV_CURSES TRUE
-#else
-#define _m_STRICT_SYSV_CURSES FALSE
-#endif
-
-#if defined NCURSES_MOUSE_VERSION
-#define _m_NCURSES_MOUSE_VERSION TRUE
-#else
-#define _m_NCURSES_MOUSE_VERSION FALSE
-#endif
-
-#if defined __NetBSD__
-#define _m_NetBSD TRUE
-#else
-#define _m_NetBSD FALSE
-#endif
-
-int _m_ispad(WINDOW *win) {
-    // <curses.h> may not have _flags (and possibly _ISPAD),
-    // but for now let's assume that <ncurses.h> always has it
-    return (win->_flags & _ISPAD);
-}
-
-void _m_getsyx(int *yx) {
-    getsyx(yx[0], yx[1]);
-}
-""", libraries=['ncurses', 'panel'])
-
+from _curses_cffi import ffi, lib
 
 def _copy_to_globals(name):
     globals()[name] = getattr(lib, name)
diff --git a/lib_pypy/_curses_build.py b/lib_pypy/_curses_build.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_curses_build.py
@@ -0,0 +1,323 @@
+from cffi import FFI
+
+ffi = FFI()
+
+ffi.set_source("_curses_cffi", """
+#ifdef __APPLE__
+/* the following define is necessary for OS X 10.6+; without it, the
+   Apple-supplied ncurses.h sets NCURSES_OPAQUE to 1, and then Python
+   can't get at the WINDOW flags field. */
+#define NCURSES_OPAQUE 0
+#endif
+
+#include <ncurses.h>
+#include <panel.h>
+#include <term.h>
+
+#if defined STRICT_SYSV_CURSES
+#define _m_STRICT_SYSV_CURSES TRUE
+#else
+#define _m_STRICT_SYSV_CURSES FALSE
+#endif
+
+#if defined NCURSES_MOUSE_VERSION
+#define _m_NCURSES_MOUSE_VERSION TRUE
+#else
+#define _m_NCURSES_MOUSE_VERSION FALSE
+#endif
+
+#if defined __NetBSD__
+#define _m_NetBSD TRUE
+#else
+#define _m_NetBSD FALSE
+#endif
+
+int _m_ispad(WINDOW *win) {
+    // <curses.h> may not have _flags (and possibly _ISPAD),
+    // but for now let's assume that <ncurses.h> always has it
+    return (win->_flags & _ISPAD);
+}
+
+void _m_getsyx(int *yx) {
+    getsyx(yx[0], yx[1]);
+}
+""", libraries=['ncurses', 'panel'])
+
+
+ffi.cdef("""
+typedef ... WINDOW;
+typedef ... SCREEN;
+typedef unsigned long... mmask_t;
+typedef unsigned char bool;
+typedef unsigned long... chtype;
+typedef chtype attr_t;
+
+typedef struct
+{
+    short id;           /* ID to distinguish multiple devices */
+    int x, y, z;        /* event coordinates (character-cell) */
+    mmask_t bstate;     /* button state bits */
+}
+MEVENT;
+
+static const int ERR, OK;
+static const int TRUE, FALSE;
+static const int KEY_MIN, KEY_MAX;
+
+static const int COLOR_BLACK;
+static const int COLOR_RED;
+static const int COLOR_GREEN;
+static const int COLOR_YELLOW;
+static const int COLOR_BLUE;
+static const int COLOR_MAGENTA;
+static const int COLOR_CYAN;
+static const int COLOR_WHITE;
+
+static const chtype A_ATTRIBUTES;
+static const chtype A_NORMAL;
+static const chtype A_STANDOUT;
+static const chtype A_UNDERLINE;
+static const chtype A_REVERSE;
+static const chtype A_BLINK;
+static const chtype A_DIM;
+static const chtype A_BOLD;
+static const chtype A_ALTCHARSET;
+static const chtype A_INVIS;
+static const chtype A_PROTECT;
+static const chtype A_CHARTEXT;
+static const chtype A_COLOR;
+
+static const int BUTTON1_RELEASED;
+static const int BUTTON1_PRESSED;
+static const int BUTTON1_CLICKED;
+static const int BUTTON1_DOUBLE_CLICKED;
+static const int BUTTON1_TRIPLE_CLICKED;
+static const int BUTTON2_RELEASED;
+static const int BUTTON2_PRESSED;
+static const int BUTTON2_CLICKED;
+static const int BUTTON2_DOUBLE_CLICKED;
+static const int BUTTON2_TRIPLE_CLICKED;
+static const int BUTTON3_RELEASED;
+static const int BUTTON3_PRESSED;
+static const int BUTTON3_CLICKED;
+static const int BUTTON3_DOUBLE_CLICKED;
+static const int BUTTON3_TRIPLE_CLICKED;
+static const int BUTTON4_RELEASED;
+static const int BUTTON4_PRESSED;
+static const int BUTTON4_CLICKED;
+static const int BUTTON4_DOUBLE_CLICKED;
+static const int BUTTON4_TRIPLE_CLICKED;
+static const int BUTTON_SHIFT;
+static const int BUTTON_CTRL;
+static const int BUTTON_ALT;
+static const int ALL_MOUSE_EVENTS;
+static const int REPORT_MOUSE_POSITION;
+
+int setupterm(char *, int, int *);
+
+WINDOW *stdscr;
+int COLORS;
+int COLOR_PAIRS;
+int COLS;
+int LINES;
+
+int baudrate(void);
+int beep(void);
+int box(WINDOW *, chtype, chtype);
+bool can_change_color(void);
+int cbreak(void);
+int clearok(WINDOW *, bool);
+int color_content(short, short*, short*, short*);
+int copywin(const WINDOW*, WINDOW*, int, int, int, int, int, int, int);
+int curs_set(int);
+int def_prog_mode(void);
+int def_shell_mode(void);
+int delay_output(int);
+int delwin(WINDOW *);
+WINDOW * derwin(WINDOW *, int, int, int, int);
+int doupdate(void);
+int echo(void);
+int endwin(void);
+char erasechar(void);
+void filter(void);
+int flash(void);
+int flushinp(void);
+chtype getbkgd(WINDOW *);
+WINDOW * getwin(FILE *);
+int halfdelay(int);
+bool has_colors(void);
+bool has_ic(void);
+bool has_il(void);
+void idcok(WINDOW *, bool);
+int idlok(WINDOW *, bool);
+void immedok(WINDOW *, bool);
+WINDOW * initscr(void);
+int init_color(short, short, short, short);
+int init_pair(short, short, short);
+int intrflush(WINDOW *, bool);
+bool isendwin(void);
+bool is_linetouched(WINDOW *, int);
+bool is_wintouched(WINDOW *);
+const char * keyname(int);
+int keypad(WINDOW *, bool);
+char killchar(void);
+int leaveok(WINDOW *, bool);
+char * longname(void);
+int meta(WINDOW *, bool);
+int mvderwin(WINDOW *, int, int);
+int mvwaddch(WINDOW *, int, int, const chtype);
+int mvwaddnstr(WINDOW *, int, int, const char *, int);
+int mvwaddstr(WINDOW *, int, int, const char *);
+int mvwchgat(WINDOW *, int, int, int, attr_t, short, const void *);
+int mvwdelch(WINDOW *, int, int);
+int mvwgetch(WINDOW *, int, int);
+int mvwgetnstr(WINDOW *, int, int, char *, int);
+int mvwin(WINDOW *, int, int);
+chtype mvwinch(WINDOW *, int, int);
+int mvwinnstr(WINDOW *, int, int, char *, int);
+int mvwinsch(WINDOW *, int, int, chtype);
+int mvwinsnstr(WINDOW *, int, int, const char *, int);
+int mvwinsstr(WINDOW *, int, int, const char *);
+int napms(int);
+WINDOW * newpad(int, int);
+WINDOW * newwin(int, int, int, int);
+int nl(void);
+int nocbreak(void);
+int nodelay(WINDOW *, bool);
+int noecho(void);
+int nonl(void);
+void noqiflush(void);
+int noraw(void);
+int notimeout(WINDOW *, bool);
+int overlay(const WINDOW*, WINDOW *);
+int overwrite(const WINDOW*, WINDOW *);
+int pair_content(short, short*, short*);
+int pechochar(WINDOW *, const chtype);
+int pnoutrefresh(WINDOW*, int, int, int, int, int, int);
+int prefresh(WINDOW *, int, int, int, int, int, int);
+int putwin(WINDOW *, FILE *);
+void qiflush(void);
+int raw(void);
+int redrawwin(WINDOW *);
+int resetty(void);
+int reset_prog_mode(void);
+int reset_shell_mode(void);
+int savetty(void);
+int scroll(WINDOW *);
+int scrollok(WINDOW *, bool);
+int start_color(void);
+WINDOW * subpad(WINDOW *, int, int, int, int);
+WINDOW * subwin(WINDOW *, int, int, int, int);
+int syncok(WINDOW *, bool);
+chtype termattrs(void);
+char * termname(void);
+int touchline(WINDOW *, int, int);
+int touchwin(WINDOW *);
+int typeahead(int);
+int ungetch(int);
+int untouchwin(WINDOW *);
+void use_env(bool);
+int waddch(WINDOW *, const chtype);
+int waddnstr(WINDOW *, const char *, int);
+int waddstr(WINDOW *, const char *);
+int wattron(WINDOW *, int);
+int wattroff(WINDOW *, int);
+int wattrset(WINDOW *, int);
+int wbkgd(WINDOW *, chtype);
+void wbkgdset(WINDOW *, chtype);
+int wborder(WINDOW *, chtype, chtype, chtype, chtype,
+            chtype, chtype, chtype, chtype);
+int wchgat(WINDOW *, int, attr_t, short, const void *);
+int wclear(WINDOW *);
+int wclrtobot(WINDOW *);
+int wclrtoeol(WINDOW *);
+void wcursyncup(WINDOW *);
+int wdelch(WINDOW *);
+int wdeleteln(WINDOW *);
+int wechochar(WINDOW *, const chtype);
+int werase(WINDOW *);
+int wgetch(WINDOW *);
+int wgetnstr(WINDOW *, char *, int);
+int whline(WINDOW *, chtype, int);
+chtype winch(WINDOW *);
+int winnstr(WINDOW *, char *, int);
+int winsch(WINDOW *, chtype);
+int winsdelln(WINDOW *, int);
+int winsertln(WINDOW *);
+int winsnstr(WINDOW *, const char *, int);
+int winsstr(WINDOW *, const char *);
+int wmove(WINDOW *, int, int);
+int wresize(WINDOW *, int, int);
+int wnoutrefresh(WINDOW *);
+int wredrawln(WINDOW *, int, int);
+int wrefresh(WINDOW *);
+int wscrl(WINDOW *, int);
+int wsetscrreg(WINDOW *, int, int);
+int wstandout(WINDOW *);
+int wstandend(WINDOW *);
+void wsyncdown(WINDOW *);
+void wsyncup(WINDOW *);
+void wtimeout(WINDOW *, int);
+int wtouchln(WINDOW *, int, int, int);
+int wvline(WINDOW *, chtype, int);
+int tigetflag(char *);
+int tigetnum(char *);
+char * tigetstr(char *);
+int putp(const char *);
+char * tparm(const char *, ...);
+int getattrs(const WINDOW *);
+int getcurx(const WINDOW *);
+int getcury(const WINDOW *);
+int getbegx(const WINDOW *);
+int getbegy(const WINDOW *);
+int getmaxx(const WINDOW *);
+int getmaxy(const WINDOW *);
+int getparx(const WINDOW *);
+int getpary(const WINDOW *);
+
+int getmouse(MEVENT *);
+int ungetmouse(MEVENT *);
+mmask_t mousemask(mmask_t, mmask_t *);
+bool wenclose(const WINDOW *, int, int);
+int mouseinterval(int);
+
+void setsyx(int y, int x);
+const char *unctrl(chtype);
+int use_default_colors(void);
+
+int has_key(int);
+bool is_term_resized(int, int);
+
+#define _m_STRICT_SYSV_CURSES ...
+#define _m_NCURSES_MOUSE_VERSION ...
+#define _m_NetBSD ...
+int _m_ispad(WINDOW *);
+
+chtype acs_map[];
+
+// For _curses_panel:
+
+typedef ... PANEL;
+
+WINDOW *panel_window(const PANEL *);
+void update_panels(void);
+int hide_panel(PANEL *);
+int show_panel(PANEL *);
+int del_panel(PANEL *);
+int top_panel(PANEL *);
+int bottom_panel(PANEL *);
+PANEL *new_panel(WINDOW *);
+PANEL *panel_above(const PANEL *);
+PANEL *panel_below(const PANEL *);
+int set_panel_userptr(PANEL *, void *);
+const void *panel_userptr(const PANEL *);
+int move_panel(PANEL *, int, int);
+int replace_panel(PANEL *,WINDOW *);
+int panel_hidden(const PANEL *);
+
+void _m_getsyx(int *yx);
+""")
+
+
+if __name__ == "__main__":
+    ffi.compile()
diff --git a/lib_pypy/_gdbm.py b/lib_pypy/_gdbm.py
--- a/lib_pypy/_gdbm.py
+++ b/lib_pypy/_gdbm.py
@@ -1,76 +1,7 @@
-import cffi, os, sys
-import thread
+from _gdbm_cffi import ffi, lib    # generated by _gdbm_build.py
+import os, thread
 _lock = thread.allocate_lock()
 
-ffi = cffi.FFI()
-ffi.cdef('''
-#define GDBM_READER ...
-#define GDBM_WRITER ...
-#define GDBM_WRCREAT ...
-#define GDBM_NEWDB ...
-#define GDBM_FAST ...
-#define GDBM_SYNC ...
-#define GDBM_NOLOCK ...
-#define GDBM_REPLACE ...
-
-void* gdbm_open(char *, int, int, int, void (*)());
-void gdbm_close(void*);
-
-typedef struct {
-    char *dptr;
-    int   dsize;
-} datum;
-
-datum gdbm_fetch(void*, datum);
-datum pygdbm_fetch(void*, char*, int);
-int gdbm_delete(void*, datum);
-int gdbm_store(void*, datum, datum, int);
-int gdbm_exists(void*, datum);
-int pygdbm_exists(void*, char*, int);
-
-int gdbm_reorganize(void*);
-
-datum gdbm_firstkey(void*);
-datum gdbm_nextkey(void*, datum);
-void gdbm_sync(void*);
-
-char* gdbm_strerror(int);
-int gdbm_errno;
-
-void free(void*);
-''')
-
-try:
-    verify_code = '''
-    #include <stdlib.h>
-    #include "gdbm.h"
-
-    static datum pygdbm_fetch(GDBM_FILE gdbm_file, char *dptr, int dsize) {
-        datum key = {dptr, dsize};
-        return gdbm_fetch(gdbm_file, key);
-    }
-
-    static int pygdbm_exists(GDBM_FILE gdbm_file, char *dptr, int dsize) {
-        datum key = {dptr, dsize};
-        return gdbm_exists(gdbm_file, key);
-    }
-    
-    '''
-    if sys.platform.startswith('freebsd'):
-        import os.path
-        _localbase = os.environ.get('LOCALBASE', '/usr/local')
-        lib = ffi.verify(verify_code, libraries=['gdbm'],
-             include_dirs=[os.path.join(_localbase, 'include')],
-             library_dirs=[os.path.join(_localbase, 'lib')]
-        )
-    else:
-        lib = ffi.verify(verify_code, libraries=['gdbm'])
-except cffi.VerificationError as e:
-    # distutils does not preserve the actual message,
-    # but the verification is simple enough that the
-    # failure must be due to missing gdbm dev libs
-    raise ImportError('%s: %s' %(e.__class__.__name__, e))
-
 class error(IOError):
     pass
 
diff --git a/lib_pypy/_gdbm_build.py b/lib_pypy/_gdbm_build.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_gdbm_build.py
@@ -0,0 +1,65 @@
+import cffi, os, sys
+
+ffi = cffi.FFI()
+ffi.cdef('''
+#define GDBM_READER ...
+#define GDBM_WRITER ...
+#define GDBM_WRCREAT ...
+#define GDBM_NEWDB ...
+#define GDBM_FAST ...
+#define GDBM_SYNC ...
+#define GDBM_NOLOCK ...
+#define GDBM_REPLACE ...
+
+void* gdbm_open(char *, int, int, int, void (*)());
+void gdbm_close(void*);
+
+typedef struct {
+    char *dptr;
+    int   dsize;
+} datum;
+
+datum gdbm_fetch(void*, datum);
+datum pygdbm_fetch(void*, char*, int);
+int gdbm_delete(void*, datum);
+int gdbm_store(void*, datum, datum, int);
+int gdbm_exists(void*, datum);
+int pygdbm_exists(void*, char*, int);
+
+int gdbm_reorganize(void*);
+
+datum gdbm_firstkey(void*);
+datum gdbm_nextkey(void*, datum);
+void gdbm_sync(void*);
+
+char* gdbm_strerror(int);
+int gdbm_errno;
+
+void free(void*);
+''')
+
+
+kwds = {}
+if sys.platform.startswith('freebsd'):
+    _localbase = os.environ.get('LOCALBASE', '/usr/local')
+    kwds['include_dirs'] = [os.path.join(_localbase, 'include')]
+    kwds['library_dirs'] = [os.path.join(_localbase, 'lib')]
+
+ffi.set_source("_gdbm_cffi", '''
+#include <stdlib.h>
+#include "gdbm.h"
+
+static datum pygdbm_fetch(GDBM_FILE gdbm_file, char *dptr, int dsize) {
+    datum key = {dptr, dsize};
+    return gdbm_fetch(gdbm_file, key);
+}
+
+static int pygdbm_exists(GDBM_FILE gdbm_file, char *dptr, int dsize) {
+    datum key = {dptr, dsize};
+    return gdbm_exists(gdbm_file, key);
+}
+''', libraries=['gdbm'], **kwds)
+
+
+if __name__ == '__main__':
+    ffi.compile()
diff --git a/lib_pypy/_pwdgrp_build.py b/lib_pypy/_pwdgrp_build.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_pwdgrp_build.py
@@ -0,0 +1,53 @@
+from cffi import FFI
+
+ffi = FFI()
+
+ffi.set_source("_pwdgrp_cffi", """
+#include <sys/types.h>
+#include <pwd.h>
+#include <grp.h>
+""")
+
+
+ffi.cdef("""
+
+typedef int... uid_t;
+typedef int... gid_t;
+
+struct passwd {
+    char *pw_name;
+    char *pw_passwd;
+    uid_t pw_uid;
+    gid_t pw_gid;
+    char *pw_gecos;
+    char *pw_dir;
+    char *pw_shell;
+    ...;
+};
+
+struct group {
+    char *gr_name;       /* group name */
+    char *gr_passwd;     /* group password */
+    gid_t gr_gid;        /* group ID */
+    char **gr_mem;        /* group members */
+};
+
+struct passwd *getpwuid(uid_t uid);
+struct passwd *getpwnam(const char *name);
+
+struct passwd *getpwent(void);
+void setpwent(void);
+void endpwent(void);
+
+struct group *getgrgid(gid_t gid);
+struct group *getgrnam(const char *name);
+
+struct group *getgrent(void);
+void setgrent(void);
+void endgrent(void);
+
+""")
+
+
+if __name__ == "__main__":
+    ffi.compile()
diff --git a/lib_pypy/_sqlite3.py b/lib_pypy/_sqlite3.py
--- a/lib_pypy/_sqlite3.py
+++ b/lib_pypy/_sqlite3.py
@@ -47,243 +47,7 @@
 else:
     _BLOB_TYPE = buffer
 
-from cffi import FFI as _FFI
-
-_ffi = _FFI()
-
-_ffi.cdef("""
-#define SQLITE_OK ...
-#define SQLITE_ERROR ...
-#define SQLITE_INTERNAL ...
-#define SQLITE_PERM ...
-#define SQLITE_ABORT ...
-#define SQLITE_BUSY ...
-#define SQLITE_LOCKED ...
-#define SQLITE_NOMEM ...
-#define SQLITE_READONLY ...
-#define SQLITE_INTERRUPT ...
-#define SQLITE_IOERR ...
-#define SQLITE_CORRUPT ...
-#define SQLITE_NOTFOUND ...
-#define SQLITE_FULL ...
-#define SQLITE_CANTOPEN ...
-#define SQLITE_PROTOCOL ...
-#define SQLITE_EMPTY ...
-#define SQLITE_SCHEMA ...
-#define SQLITE_TOOBIG ...
-#define SQLITE_CONSTRAINT ...
-#define SQLITE_MISMATCH ...
-#define SQLITE_MISUSE ...
-#define SQLITE_NOLFS ...
-#define SQLITE_AUTH ...
-#define SQLITE_FORMAT ...
-#define SQLITE_RANGE ...
-#define SQLITE_NOTADB ...
-#define SQLITE_ROW ...
-#define SQLITE_DONE ...
-#define SQLITE_INTEGER ...
-#define SQLITE_FLOAT ...
-#define SQLITE_BLOB ...
-#define SQLITE_NULL ...
-#define SQLITE_TEXT ...
-#define SQLITE3_TEXT ...
-
-#define SQLITE_TRANSIENT ...
-#define SQLITE_UTF8 ...
-
-#define SQLITE_DENY ...
-#define SQLITE_IGNORE ...
-
-#define SQLITE_CREATE_INDEX ...
-#define SQLITE_CREATE_TABLE ...
-#define SQLITE_CREATE_TEMP_INDEX ...
-#define SQLITE_CREATE_TEMP_TABLE ...
-#define SQLITE_CREATE_TEMP_TRIGGER ...
-#define SQLITE_CREATE_TEMP_VIEW ...
-#define SQLITE_CREATE_TRIGGER ...
-#define SQLITE_CREATE_VIEW ...
-#define SQLITE_DELETE ...
-#define SQLITE_DROP_INDEX ...
-#define SQLITE_DROP_TABLE ...
-#define SQLITE_DROP_TEMP_INDEX ...
-#define SQLITE_DROP_TEMP_TABLE ...
-#define SQLITE_DROP_TEMP_TRIGGER ...
-#define SQLITE_DROP_TEMP_VIEW ...
-#define SQLITE_DROP_TRIGGER ...
-#define SQLITE_DROP_VIEW ...
-#define SQLITE_INSERT ...
-#define SQLITE_PRAGMA ...
-#define SQLITE_READ ...
-#define SQLITE_SELECT ...
-#define SQLITE_TRANSACTION ...
-#define SQLITE_UPDATE ...
-#define SQLITE_ATTACH ...
-#define SQLITE_DETACH ...
-#define SQLITE_ALTER_TABLE ...
-#define SQLITE_REINDEX ...
-#define SQLITE_ANALYZE ...
-#define SQLITE_CREATE_VTABLE ...
-#define SQLITE_DROP_VTABLE ...
-#define SQLITE_FUNCTION ...
-
-const char *sqlite3_libversion(void);
-
-typedef ... sqlite3;
-typedef ... sqlite3_stmt;
-typedef ... sqlite3_context;
-typedef ... sqlite3_value;
-typedef int64_t sqlite3_int64;
-typedef uint64_t sqlite3_uint64;
-
-int sqlite3_open(
-    const char *filename,   /* Database filename (UTF-8) */
-    sqlite3 **ppDb          /* OUT: SQLite db handle */
-);
-
-int sqlite3_close(sqlite3 *);
-
-int sqlite3_busy_timeout(sqlite3*, int ms);
-int sqlite3_prepare_v2(
-    sqlite3 *db,            /* Database handle */
-    const char *zSql,       /* SQL statement, UTF-8 encoded */
-    int nByte,              /* Maximum length of zSql in bytes. */
-    sqlite3_stmt **ppStmt,  /* OUT: Statement handle */
-    const char **pzTail     /* OUT: Pointer to unused portion of zSql */
-);
-int sqlite3_finalize(sqlite3_stmt *pStmt);
-int sqlite3_data_count(sqlite3_stmt *pStmt);
-int sqlite3_column_count(sqlite3_stmt *pStmt);
-const char *sqlite3_column_name(sqlite3_stmt*, int N);
-int sqlite3_get_autocommit(sqlite3*);
-int sqlite3_reset(sqlite3_stmt *pStmt);
-int sqlite3_step(sqlite3_stmt*);
-int sqlite3_errcode(sqlite3 *db);
-const char *sqlite3_errmsg(sqlite3*);
-int sqlite3_changes(sqlite3*);
-
-int sqlite3_bind_blob(sqlite3_stmt*, int, const void*, int n, void(*)(void*));
-int sqlite3_bind_double(sqlite3_stmt*, int, double);
-int sqlite3_bind_int(sqlite3_stmt*, int, int);
-int sqlite3_bind_int64(sqlite3_stmt*, int, sqlite3_int64);
-int sqlite3_bind_null(sqlite3_stmt*, int);
-int sqlite3_bind_text(sqlite3_stmt*, int, const char*, int n, void(*)(void*));
-int sqlite3_bind_text16(sqlite3_stmt*, int, const void*, int, void(*)(void*));
-int sqlite3_bind_value(sqlite3_stmt*, int, const sqlite3_value*);
-int sqlite3_bind_zeroblob(sqlite3_stmt*, int, int n);
-
-const void *sqlite3_column_blob(sqlite3_stmt*, int iCol);
-int sqlite3_column_bytes(sqlite3_stmt*, int iCol);
-double sqlite3_column_double(sqlite3_stmt*, int iCol);
-int sqlite3_column_int(sqlite3_stmt*, int iCol);
-sqlite3_int64 sqlite3_column_int64(sqlite3_stmt*, int iCol);
-const unsigned char *sqlite3_column_text(sqlite3_stmt*, int iCol);
-const void *sqlite3_column_text16(sqlite3_stmt*, int iCol);
-int sqlite3_column_type(sqlite3_stmt*, int iCol);
-const char *sqlite3_column_decltype(sqlite3_stmt*,int);
-
-void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*);
-int sqlite3_create_collation(
-    sqlite3*,
-    const char *zName,
-    int eTextRep,
-    void*,
-    int(*xCompare)(void*,int,const void*,int,const void*)
-);
-int sqlite3_set_authorizer(
-    sqlite3*,
-    int (*xAuth)(void*,int,const char*,const char*,const char*,const char*),
-    void *pUserData
-);
-int sqlite3_create_function(
-    sqlite3 *db,
-    const char *zFunctionName,
-    int nArg,
-    int eTextRep,
-    void *pApp,
-    void (*xFunc)(sqlite3_context*,int,sqlite3_value**),
-    void (*xStep)(sqlite3_context*,int,sqlite3_value**),
-    void (*xFinal)(sqlite3_context*)
-);
-void *sqlite3_aggregate_context(sqlite3_context*, int nBytes);
-
-sqlite3_int64 sqlite3_last_insert_rowid(sqlite3*);
-int sqlite3_bind_parameter_count(sqlite3_stmt*);
-const char *sqlite3_bind_parameter_name(sqlite3_stmt*, int);
-int sqlite3_total_changes(sqlite3*);
-
-int sqlite3_prepare(
-    sqlite3 *db,            /* Database handle */
-    const char *zSql,       /* SQL statement, UTF-8 encoded */
-    int nByte,              /* Maximum length of zSql in bytes. */
-    sqlite3_stmt **ppStmt,  /* OUT: Statement handle */
-    const char **pzTail     /* OUT: Pointer to unused portion of zSql */
-);
-
-void sqlite3_result_blob(sqlite3_context*, const void*, int, void(*)(void*));
-void sqlite3_result_double(sqlite3_context*, double);
-void sqlite3_result_error(sqlite3_context*, const char*, int);
-void sqlite3_result_error16(sqlite3_context*, const void*, int);
-void sqlite3_result_error_toobig(sqlite3_context*);
-void sqlite3_result_error_nomem(sqlite3_context*);
-void sqlite3_result_error_code(sqlite3_context*, int);
-void sqlite3_result_int(sqlite3_context*, int);
-void sqlite3_result_int64(sqlite3_context*, sqlite3_int64);
-void sqlite3_result_null(sqlite3_context*);
-void sqlite3_result_text(sqlite3_context*, const char*, int, void(*)(void*));
-void sqlite3_result_text16(sqlite3_context*, const void*, int, void(*)(void*));
-void sqlite3_result_text16le(sqlite3_context*,const void*, int,void(*)(void*));
-void sqlite3_result_text16be(sqlite3_context*,const void*, int,void(*)(void*));
-void sqlite3_result_value(sqlite3_context*, sqlite3_value*);
-void sqlite3_result_zeroblob(sqlite3_context*, int n);
-
-const void *sqlite3_value_blob(sqlite3_value*);
-int sqlite3_value_bytes(sqlite3_value*);
-int sqlite3_value_bytes16(sqlite3_value*);
-double sqlite3_value_double(sqlite3_value*);
-int sqlite3_value_int(sqlite3_value*);
-sqlite3_int64 sqlite3_value_int64(sqlite3_value*);
-const unsigned char *sqlite3_value_text(sqlite3_value*);
-const void *sqlite3_value_text16(sqlite3_value*);
-const void *sqlite3_value_text16le(sqlite3_value*);
-const void *sqlite3_value_text16be(sqlite3_value*);
-int sqlite3_value_type(sqlite3_value*);
-int sqlite3_value_numeric_type(sqlite3_value*);
-""")
-
-def _has_load_extension():
-    """Only available since 3.3.6"""
-    unverified_ffi = _FFI()
-    unverified_ffi.cdef("""
-    typedef ... sqlite3;
-    int sqlite3_enable_load_extension(sqlite3 *db, int onoff);
-    """)
-    libname = 'sqlite3'
-    if sys.platform == 'win32':
-        import os
-        _libname = os.path.join(os.path.dirname(sys.executable), libname)
-        if os.path.exists(_libname + '.dll'):
-            libname = _libname
-    unverified_lib = unverified_ffi.dlopen(libname)
-    return hasattr(unverified_lib, 'sqlite3_enable_load_extension')
-
-if _has_load_extension():
-    _ffi.cdef("int sqlite3_enable_load_extension(sqlite3 *db, int onoff);")
-
-if sys.platform.startswith('freebsd'):
-    import os
-    import os.path
-    _localbase = os.environ.get('LOCALBASE', '/usr/local')
-    _lib = _ffi.verify("""
-    #include <sqlite3.h>
-    """, libraries=['sqlite3'],
-         include_dirs=[os.path.join(_localbase, 'include')],
-         library_dirs=[os.path.join(_localbase, 'lib')]
-    )
-else:
-    _lib = _ffi.verify("""
-    #include <sqlite3.h>
-    """, libraries=['sqlite3']
-    )
+from _sqlite3_cffi import ffi as _ffi, lib as _lib
 
 exported_sqlite_symbols = [
     'SQLITE_ALTER_TABLE',
@@ -322,7 +86,7 @@
 for symbol in exported_sqlite_symbols:
     globals()[symbol] = getattr(_lib, symbol)
 
-_SQLITE_TRANSIENT = _ffi.cast('void *', _lib.SQLITE_TRANSIENT)
+_SQLITE_TRANSIENT = _lib.SQLITE_TRANSIENT
 
 # pysqlite version information
 version = "2.6.0"
@@ -521,7 +285,7 @@
             raise ProgrammingError(
                 "SQLite objects created in a thread can only be used in that "
                 "same thread. The object was created in thread id %d and this "
-                "is thread id %d", self.__thread_ident, _thread_get_ident())
+                "is thread id %d" % (self.__thread_ident, _thread_get_ident()))
 
     def _check_thread_wrap(func):
         @wraps(func)
diff --git a/lib_pypy/_sqlite3_build.py b/lib_pypy/_sqlite3_build.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_sqlite3_build.py
@@ -0,0 +1,265 @@
+#-*- coding: utf-8 -*-
+# pysqlite2/dbapi.py: pysqlite DB-API module
+#
+# Copyright (C) 2007-2008 Gerhard H&#228;ring <gh at ghaering.de>
+#
+# This file is part of pysqlite.
+#
+# This software is provided 'as-is', without any express or implied
+# warranty.  In no event will the authors be held liable for any damages
+# arising from the use of this software.
+#
+# Permission is granted to anyone to use this software for any purpose,
+# including commercial applications, and to alter it and redistribute it
+# freely, subject to the following restrictions:
+#
+# 1. The origin of this software must not be misrepresented; you must not
+#    claim that you wrote the original software. If you use this software
+#    in a product, an acknowledgment in the product documentation would be
+#    appreciated but is not required.
+# 2. Altered source versions must be plainly marked as such, and must not be
+#    misrepresented as being the original software.
+# 3. This notice may not be removed or altered from any source distribution.
+#
+# Note: This software has been modified for use in PyPy.
+
+import sys, os
+from cffi import FFI as _FFI
+
+_ffi = _FFI()
+
+_ffi.cdef("""
+#define SQLITE_OK ...
+#define SQLITE_ERROR ...
+#define SQLITE_INTERNAL ...
+#define SQLITE_PERM ...
+#define SQLITE_ABORT ...
+#define SQLITE_BUSY ...
+#define SQLITE_LOCKED ...
+#define SQLITE_NOMEM ...
+#define SQLITE_READONLY ...
+#define SQLITE_INTERRUPT ...
+#define SQLITE_IOERR ...
+#define SQLITE_CORRUPT ...
+#define SQLITE_NOTFOUND ...
+#define SQLITE_FULL ...
+#define SQLITE_CANTOPEN ...
+#define SQLITE_PROTOCOL ...
+#define SQLITE_EMPTY ...
+#define SQLITE_SCHEMA ...
+#define SQLITE_TOOBIG ...
+#define SQLITE_CONSTRAINT ...
+#define SQLITE_MISMATCH ...
+#define SQLITE_MISUSE ...
+#define SQLITE_NOLFS ...
+#define SQLITE_AUTH ...
+#define SQLITE_FORMAT ...
+#define SQLITE_RANGE ...
+#define SQLITE_NOTADB ...
+#define SQLITE_ROW ...
+#define SQLITE_DONE ...
+#define SQLITE_INTEGER ...
+#define SQLITE_FLOAT ...
+#define SQLITE_BLOB ...
+#define SQLITE_NULL ...
+#define SQLITE_TEXT ...
+#define SQLITE3_TEXT ...
+
+static void *const SQLITE_TRANSIENT;
+#define SQLITE_UTF8 ...
+
+#define SQLITE_DENY ...
+#define SQLITE_IGNORE ...
+
+#define SQLITE_CREATE_INDEX ...
+#define SQLITE_CREATE_TABLE ...
+#define SQLITE_CREATE_TEMP_INDEX ...
+#define SQLITE_CREATE_TEMP_TABLE ...

From noreply at buildbot.pypy.org  Tue Jun  2 17:02:21 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Tue,  2 Jun 2015 17:02:21 +0200 (CEST)
Subject: [pypy-commit] pypy use_min_scalar: correct handling of scalars for
 non-simple binary ufuncs
Message-ID: <20150602150221.7AD3D1C0FD4@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: use_min_scalar
Changeset: r77776:02c9c753b06c
Date: 2015-06-02 05:31 +0100
http://bitbucket.org/pypy/pypy/changeset/02c9c753b06c/

Log:	correct handling of scalars for non-simple binary ufuncs

diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -1351,3 +1351,4 @@
         assert np.add(np.float16(0), np.complex128(0)).dtype == np.complex128
         assert np.add(np.zeros(5, dtype=np.int8), 257).dtype == np.int16
         assert np.subtract(np.zeros(5, dtype=np.int8), 257).dtype == np.int16
+        assert np.divide(np.zeros(5, dtype=np.int8), 257).dtype == np.int16
diff --git a/pypy/module/micronumpy/ufuncs.py b/pypy/module/micronumpy/ufuncs.py
--- a/pypy/module/micronumpy/ufuncs.py
+++ b/pypy/module/micronumpy/ufuncs.py
@@ -21,7 +21,8 @@
 from pypy.module.micronumpy.support import (_parse_signature, product,
         get_storage_as_int, is_rhs_priority_higher)
 from .casting import (
-    can_cast_type, can_cast_to, find_result_type, promote_types)
+    can_cast_type, can_cast_array, can_cast_to,
+    find_result_type, promote_types)
 from .boxes import W_GenericBox, W_ObjectBox
 
 def done_if_true(dtype, val):
@@ -642,12 +643,6 @@
 
     def _find_specialization(self, space, l_dtype, r_dtype, out, casting,
                              w_arg1, w_arg2):
-        if (self.are_common_types(l_dtype, r_dtype) and
-                w_arg1 is not None and w_arg2 is not None):
-            if not w_arg1.is_scalar() and w_arg2.is_scalar():
-                r_dtype = l_dtype
-            elif w_arg1.is_scalar() and not w_arg2.is_scalar():
-                l_dtype = r_dtype
         if (not self.allow_bool and (l_dtype.is_bool() or
                                          r_dtype.is_bool()) or
                 not self.allow_complex and (l_dtype.is_complex() or
@@ -659,7 +654,8 @@
             dtype = find_result_type(space, [], [l_dtype, r_dtype])
             bool_dtype = get_dtype_cache(space).w_booldtype
             return dtype, bool_dtype, self.func
-        dt_in, dt_out = self._calc_dtype(space, l_dtype, r_dtype, out, casting)
+        dt_in, dt_out = self._calc_dtype(
+            space, l_dtype, r_dtype, out, casting, w_arg1, w_arg2)
         return dt_in, dt_out, self.func
 
     def find_specialization(self, space, l_dtype, r_dtype, out, casting,
@@ -695,15 +691,21 @@
             "requested type has type code '%s'" % (self.name, dtype.char))
 
 
-    def _calc_dtype(self, space, l_dtype, r_dtype, out=None, casting='unsafe'):
-        use_min_scalar = False
+    def _calc_dtype(self, space, l_dtype, r_dtype, out, casting,
+                    w_arg1, w_arg2):
         if l_dtype.is_object() or r_dtype.is_object():
             dtype = get_dtype_cache(space).w_objectdtype
             return dtype, dtype
+        use_min_scalar = (w_arg1 is not None and w_arg2 is not None and
+                          ((w_arg1.is_scalar() and not w_arg2.is_scalar()) or
+                           (not w_arg1.is_scalar() and w_arg2.is_scalar())))
         in_casting = safe_casting_mode(casting)
         for dt_in, dt_out in self.dtypes:
             if use_min_scalar:
-                if not can_cast_array(space, w_arg, dt_in, in_casting):
+                w_arg1 = convert_to_array(space, w_arg1)
+                w_arg2 = convert_to_array(space, w_arg2)
+                if not (can_cast_array(space, w_arg1, dt_in, in_casting) and
+                        can_cast_array(space, w_arg2, dt_in, in_casting)):
                     continue
             else:
                 if not (can_cast_type(space, l_dtype, dt_in, in_casting) and

From noreply at buildbot.pypy.org  Tue Jun  2 17:36:24 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Tue,  2 Jun 2015 17:36:24 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Use threading.Lock() instead of
	thread.allocate_lock().
Message-ID: <20150602153624.A6DC51C1473@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob <me at manueljacob.de>
Branch: py3k
Changeset: r77777:dbda865fd16e
Date: 2015-06-02 17:36 +0200
http://bitbucket.org/pypy/pypy/changeset/dbda865fd16e/

Log:	Use threading.Lock() instead of thread.allocate_lock().

diff --git a/lib_pypy/_gdbm.py b/lib_pypy/_gdbm.py
--- a/lib_pypy/_gdbm.py
+++ b/lib_pypy/_gdbm.py
@@ -1,6 +1,6 @@
 from _gdbm_cffi import ffi, lib    # generated by _gdbm_build.py
-import os, thread
-_lock = thread.allocate_lock()
+import os, threading
+_lock = threading.Lock()
 
 class error(IOError):
     pass

From noreply at buildbot.pypy.org  Tue Jun  2 18:30:25 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Tue,  2 Jun 2015 18:30:25 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: fix quasiimmut + rpython fixes
Message-ID: <20150602163025.AD7E51C1473@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77778:63a86d7eb28c
Date: 2015-06-02 18:26 +0200
http://bitbucket.org/pypy/pypy/changeset/63a86d7eb28c/

Log:	fix quasiimmut + rpython fixes

diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -95,6 +95,9 @@
         self.all_fielddescrs = heaptracker.all_fielddescrs(runner, S,
                                     get_field_descr=LLGraphCPU.fielddescrof)
 
+    def get_all_fielddescrs(self):
+        return self.all_fielddescrs
+
     def is_object(self):
         return self._is_object
 
@@ -114,9 +117,15 @@
         self.FIELD = getattr(S, fieldname)
         self.index = heaptracker.get_fielddescr_index_in(S, fieldname)
 
+    def get_parent_descr(self):
+        return self.parent_descr
+
     def get_vinfo(self):
         return self.vinfo
 
+    def get_index(self):
+        return self.index
+
     def __repr__(self):
         return 'FieldDescr(%r, %r)' % (self.S, self.fieldname)
 
diff --git a/rpython/jit/backend/llsupport/descr.py b/rpython/jit/backend/llsupport/descr.py
--- a/rpython/jit/backend/llsupport/descr.py
+++ b/rpython/jit/backend/llsupport/descr.py
@@ -45,6 +45,9 @@
         self.all_fielddescrs = all_fielddescrs
         self.vtable = vtable
 
+    def get_all_fielddescrs(self):
+        return self.all_fielddescrs
+
     def count_fields_if_immutable(self):
         return self.count_fields_if_immut
 
@@ -157,6 +160,12 @@
     def repr_of_descr(self):
         return '<Field%s %s %s>' % (self.flag, self.name, self.offset)
 
+    def get_parent_descr(self):
+        return self.parent_descr
+
+    def get_index(self):
+        return self.index
+
 
 def get_field_descr(gccache, STRUCT, fieldname):
     cache = gccache._cache_field
@@ -294,6 +303,9 @@
         self.arraydescr = arraydescr
         self.fielddescr = fielddescr
 
+    def get_arraydescr(self):
+        return self.arraydescr
+
     def sort_key(self):
         return self.fielddescr.sort_key()
 
diff --git a/rpython/jit/backend/llsupport/gc.py b/rpython/jit/backend/llsupport/gc.py
--- a/rpython/jit/backend/llsupport/gc.py
+++ b/rpython/jit/backend/llsupport/gc.py
@@ -152,6 +152,7 @@
     def _rewrite_changeable_constptrs(self, op, ops_with_movable_const_ptr, moving_obj_tracker):
         newops = []
         for arg_i in ops_with_movable_const_ptr[op]:
+            raise Exception("implement me")
             v = op.getarg(arg_i)
             # assert to make sure we got what we expected
             assert isinstance(v, ConstPtr)
diff --git a/rpython/jit/metainterp/executor.py b/rpython/jit/metainterp/executor.py
--- a/rpython/jit/metainterp/executor.py
+++ b/rpython/jit/metainterp/executor.py
@@ -76,6 +76,7 @@
                 metainterp.execute_raised(e)
             return None
         raise AssertionError("bad rettype")
+    do_call.func_name = "do_call_" + rettype
     return do_call
 
 do_call_r = new_do_call("r")
@@ -144,6 +145,7 @@
         cpu.bh_setarrayitem_raw_i(array, index, itembox.getint(), arraydescr)
 
 def do_getinteriorfield_gc(cpu, _, arraybox, indexbox, descr):
+    raise Exception("implement me")
     xxxx
     array = arraybox.getref_base()
     index = indexbox.getint()
@@ -179,6 +181,7 @@
     return cpu.bh_getfield_gc_f(struct, fielddescr)
 
 def do_getfield_raw(cpu, _, structbox, fielddescr):
+    raise Exception("implement me")    
     xxxx
     check_descr(fielddescr)
     struct = structbox.getint()
@@ -217,6 +220,7 @@
         cpu.bh_raw_store_i(addr, offset, valuebox.getint(), arraydescr)
 
 def do_raw_load(cpu, _, addrbox, offsetbox, arraydescr):
+    raise Exception("implement me")    
     xxx
     addr = addrbox.getint()
     offset = offsetbox.getint()
@@ -422,8 +426,8 @@
     # constant-folded away.  Only works if opnum and num_args are
     # constants, of course.
     func = EXECUTE_BY_NUM_ARGS[num_args, withdescr][opnum]
-    assert func is not None, "EXECUTE_BY_NUM_ARGS[%s, %s][%s]" % (
-        num_args, withdescr, resoperation.opname[opnum])
+    #assert func is not None, "EXECUTE_BY_NUM_ARGS[%s, %s][%s]" % (
+    #    num_args, withdescr, resoperation.opname[opnum])
     return func
 get_execute_function._annspecialcase_ = 'specialize:memo'
 
@@ -472,47 +476,52 @@
     else:
         assert op.type == 'f'
         return ConstFloat(op.getfloatstorage())
+
+unrolled_range = unrolling_iterable(range(rop._LAST))
     
 def execute_nonspec_const(cpu, metainterp, opnum, argboxes, descr=None,
                           type='i'):
-    return wrap_constant(_execute_nonspec(cpu, metainterp, opnum, argboxes,
-                                          descr))
+    for num in unrolled_range:
+        if num == opnum:
+            return wrap_constant(_execute_arglist(cpu, metainterp, num,
+                                                  argboxes, descr))
+    assert False
 
 @specialize.arg(2)
-def _execute_nonspec(cpu, metainterp, opnum, argboxes, descr=None):
-    arity = resoperation.oparity[opnum]
+def _execute_arglist(cpu, metainterp, opnum, argboxes, descr=None):
+    arity = resoperation.oparity[opnum]    
     assert arity == -1 or len(argboxes) == arity
     if resoperation.opwithdescr[opnum]:
         check_descr(descr)
         if arity == -1:
-            func = get_execute_funclist(-1, True)[opnum]
+            func = get_execute_function(opnum, -1, True)
             return func(cpu, metainterp, argboxes, descr)
         if arity == 0:
-            func = get_execute_funclist(0, True)[opnum]
+            func = get_execute_function(opnum, 0, True)
             return func(cpu, metainterp, descr)
         if arity == 1:
-            func = get_execute_funclist(1, True)[opnum]
+            func = get_execute_function(opnum, 1, True)
             return func(cpu, metainterp, argboxes[0], descr)
         if arity == 2:
-            func = get_execute_funclist(2, True)[opnum]
+            func = get_execute_function(opnum, 2, True)
             return func(cpu, metainterp, argboxes[0], argboxes[1], descr)
         if arity == 3:
-            func = get_execute_funclist(3, True)[opnum]
+            func = get_execute_function(opnum, 3, True)
             return func(cpu, metainterp, argboxes[0], argboxes[1], argboxes[2],
                         descr)
     else:
         assert descr is None
         if arity == 1:
-            func = get_execute_funclist(1, False)[opnum]
+            func = get_execute_function(opnum, 1, False)
             return func(cpu, metainterp, argboxes[0])
         if arity == 2:
-            func = get_execute_funclist(2, False)[opnum]
+            func = get_execute_function(opnum, 2, False)
             return func(cpu, metainterp, argboxes[0], argboxes[1])
         if arity == 3:
-            func = get_execute_funclist(3, False)[opnum]
+            func = get_execute_function(opnum, 3, False)
             return func(cpu, metainterp, argboxes[0], argboxes[1], argboxes[2])
         if arity == 5:    # copystrcontent, copyunicodecontent
-            func = get_execute_funclist(5, False)[opnum]
+            func = get_execute_function(opnum, 5, False)
             return func(cpu, metainterp, argboxes[0], argboxes[1],
                         argboxes[2], argboxes[3], argboxes[4])
     raise NotImplementedError
diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py
--- a/rpython/jit/metainterp/history.py
+++ b/rpython/jit/metainterp/history.py
@@ -137,6 +137,8 @@
     def get_vinfo(self):
         raise NotImplementedError
 
+DONT_CHANGE = AbstractDescr()
+
 class AbstractFailDescr(AbstractDescr):
     index = -1
     final_descr = False
diff --git a/rpython/jit/metainterp/optimizeopt/heap.py b/rpython/jit/metainterp/optimizeopt/heap.py
--- a/rpython/jit/metainterp/optimizeopt/heap.py
+++ b/rpython/jit/metainterp/optimizeopt/heap.py
@@ -11,6 +11,7 @@
 from rpython.jit.metainterp.optimize import InvalidLoop
 from rpython.jit.metainterp.resoperation import rop, ResOperation, OpHelpers
 from rpython.rlib.objectmodel import we_are_translated
+from rpython.jit.metainterp.optimizeopt import info
 
 
 class BogusPureField(JitException):
@@ -40,8 +41,9 @@
         self.cached_infos.append(info)
 
     def invalidate(self, descr):
-        for info in self.cached_infos:
-            info._fields[descr.index] = None
+        for opinfo in self.cached_infos:
+            assert isinstance(opinfo, info.AbstractStructPtrInfo)
+            opinfo._fields[descr.get_index()] = None
         self.cached_infos = []
 
 
@@ -151,8 +153,9 @@
         opinfo.setitem(self.index, arg, self, optheap)
 
     def invalidate(self, descr):
-        for info in self.cached_infos:
-            info._items = None
+        for opinfo in self.cached_infos:
+            assert isinstance(opinfo, info.ArrayPtrInfo)
+            opinfo._items = None
         self.cached_infos = []
 
 class OptHeap(Optimization):
@@ -402,7 +405,7 @@
             return
         for idx, cf in submap.iteritems():
             if indexb is None or indexb.contains(idx):
-                cf.force_lazy_setfield(self, idx, can_cache)
+                cf.force_lazy_setfield(self, None, can_cache)
 
     def force_all_lazy_setfields_and_arrayitems(self):
         # XXX fix the complexity here
@@ -410,7 +413,7 @@
             cf.force_lazy_setfield(self, descr)
         for submap in self.cached_arrayitems.itervalues():
             for index, cf in submap.iteritems():
-                cf.force_lazy_setfield(self, index)
+                cf.force_lazy_setfield(self, None)
 
     def force_lazy_setfields_and_arrayitems_for_guard(self):
         pendingfields = []
diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py
--- a/rpython/jit/metainterp/optimizeopt/info.py
+++ b/rpython/jit/metainterp/optimizeopt/info.py
@@ -11,6 +11,8 @@
 INFO_UNKNOWN = 2
 
 class AbstractInfo(AbstractValue):
+    _attrs_ = ()
+    
     is_info_class = True
 
     def force_box(self, op, optforce):
@@ -58,6 +60,9 @@
             return None
         return optimizer._newoperations[self.last_guard_pos]
 
+    def get_last_guard_pos(self):
+        return self.last_guard_pos
+
     def reset_last_guard_pos(self):
         self.last_guard_pos = -1
 
@@ -92,7 +97,7 @@
     _attrs_ = ('_fields',)
 
     def init_fields(self, descr):
-        self._fields = [None] * len(descr.all_fielddescrs)
+        self._fields = [None] * len(descr.get_all_fielddescrs())
 
     def clear_cache(self):
         assert not self.is_virtual()
@@ -100,22 +105,22 @@
 
     def setfield(self, descr, op, optheap=None, cf=None):
         if self._fields is None:
-            self.init_fields(descr.parent_descr)
-        self._fields[descr.index] = op
+            self.init_fields(descr.get_parent_descr())
+        self._fields[descr.get_index()] = op
         if cf is not None:
             assert not self.is_virtual()
             cf.register_dirty_field(self)
 
     def getfield(self, descr, optheap=None):
         if self._fields is None:
-            self.init_fields(descr.parent_descr)
-        return self._fields[descr.index]
+            self.init_fields(descr.get_parent_descr())
+        return self._fields[descr.get_index()]
 
     def _force_elements(self, op, optforce, descr):
         if self._fields is None:
             return 0
         count = 0
-        for i, flddescr in enumerate(descr.all_fielddescrs):
+        for i, flddescr in enumerate(descr.get_all_fielddescrs()):
             fld = self._fields[i]
             if fld is not None:
                 subbox = optforce.force_box(fld)
@@ -129,7 +134,7 @@
     def visitor_walk_recursive(self, instbox, visitor, optimizer):
         if visitor.already_seen_virtual(instbox):
             return
-        lst = self.vdescr.all_fielddescrs
+        lst = self.vdescr.get_all_fielddescrs()
         assert self.is_virtual()
         visitor.register_virtual_fields(instbox,
                                         [optimizer.get_box_replacement(box)
@@ -155,7 +160,7 @@
 
     @specialize.argtype(1)
     def visitor_dispatch_virtual_type(self, visitor):
-        fielddescrs = self.vdescr.all_fielddescrs
+        fielddescrs = self.vdescr.get_all_fielddescrs()
         assert self.is_virtual()
         return visitor.visit_virtual(self.vdescr, fielddescrs)
 
@@ -165,7 +170,7 @@
 
     @specialize.argtype(1)
     def visitor_dispatch_virtual_type(self, visitor):
-        fielddescrs = self.vdescr.all_fielddescrs
+        fielddescrs = self.vdescr.get_all_fielddescrs()
         assert self.is_virtual()
         return visitor.visit_vstruct(self.vdescr, fielddescrs)
 
@@ -187,6 +192,7 @@
 
     def getlenbound(self):
         if self.lenbound is None:
+            raise Exception("implement me - lenbound")
             xxx
         return self.lenbound
 
@@ -249,13 +255,14 @@
 class ArrayStructInfo(ArrayPtrInfo):
     def __init__(self, size, vdescr=None):
         self.length = size
-        lgt = len(vdescr.all_interiorfielddescrs)
+        lgt = len(vdescr.get_all_fielddescrs())
         self.vdescr = vdescr
         self._items = [None] * (size * lgt)
 
     def _compute_index(self, index, fielddescr):
-        one_size = len(fielddescr.arraydescr.all_interiorfielddescrs)
-        return index * one_size + fielddescr.fielddescr.index
+        raise Exception("implement virtual array of structs")
+        one_size = len(fielddescr.get_arraydescr().get_all_fielddescrs())
+        return index * one_size + fielddescr.fielddescr.get_index()
         
     def setinteriorfield_virtual(self, index, fielddescr, fld):
         index = self._compute_index(index, fielddescr)
@@ -267,7 +274,7 @@
 
     def _force_elements(self, op, optforce, descr):
         i = 0
-        fielddescrs = op.getdescr().all_interiorfielddescrs
+        fielddescrs = op.getdescr().get_all_fielddescrs()
         count = 0
         for index in range(self.length):
             for flddescr in fielddescrs:
@@ -294,7 +301,7 @@
         info = optheap.const_infos.get(ref, None)
         if info is None:
             info = StructPtrInfo()
-            info.init_fields(descr.parent_descr)
+            info.init_fields(descr.get_parent_descr())
             optheap.const_infos[ref] = info
         return info
 
@@ -314,7 +321,7 @@
         info = self._get_array_info(optheap)
         return info.getitem(index)
 
-    def setitem(self, index, op, cf, optheap=None):
+    def setitem(self, index, op, cf=None, optheap=None):
         info = self._get_array_info(optheap)
         info.setitem(index, op, cf)
 
@@ -349,25 +356,35 @@
 
     # --------------------- vstring -------------------
 
+    @specialize.arg(1)
     def _unpack_str(self, mode):
         return mode.hlstr(lltype.cast_opaque_ptr(
             lltype.Ptr(mode.LLTYPE), self._const.getref_base()))
 
+    @specialize.arg(2)
     def get_constant_string_spec(self, optforce, mode):
         return self._unpack_str(mode)
     
     def getstrlen(self, op, string_optimizer, mode, create_ops=True):
-        s = self._unpack_str(mode)
-        if s is None:
-            return None
-        return ConstInt(len(s))
+        from rpython.jit.metainterp.optimizeopt import vstring
+        
+        if mode is vstring.mode_string:
+            s = self._unpack_str(vstring.mode_string)
+            if s is None:
+                return None
+            return ConstInt(len(s))
+        else:
+            s = self._unpack_str(vstring.mode_unicode)            
+            if s is None:
+                return None
+            return ConstInt(len(s))
 
     def string_copy_parts(self, op, string_optimizer, targetbox, offsetbox,
                           mode):
         from rpython.jit.metainterp.optimizeopt import vstring
         from rpython.jit.metainterp.optimizeopt.optimizer import CONST_0
 
-        lgt = self.getstrlen(op, string_optimizer, mode, None)
+        lgt = self.getstrlen(op, string_optimizer, mode, False)
         return vstring.copy_str_content(string_optimizer, self._const,
                                         targetbox, CONST_0, offsetbox,
                                         lgt, mode)
diff --git a/rpython/jit/metainterp/optimizeopt/intbounds.py b/rpython/jit/metainterp/optimizeopt/intbounds.py
--- a/rpython/jit/metainterp/optimizeopt/intbounds.py
+++ b/rpython/jit/metainterp/optimizeopt/intbounds.py
@@ -569,12 +569,12 @@
             self.propagate_bounds_backward(op.getarg(1))
 
     def propagate_bounds_INT_LSHIFT(self, op):
-        v1 = self.getvalue(op.getarg(0))
-        v2 = self.getvalue(op.getarg(1))
-        r = self.getvalue(op)
-        b = r.getintbound().rshift_bound(v2.getintbound())
-        if v1.getintbound().intersect(b):
-            self.propagate_bounds_backward(op.getarg(0), v1)
+        b1 = self.getintbound(op.getarg(0))
+        b2 = self.getintbound(op.getarg(1))
+        r = self.getintbound(op)
+        b = r.rshift_bound(b2)
+        if b1.intersect(b):
+            self.propagate_bounds_backward(op.getarg(0))
 
     propagate_bounds_INT_ADD_OVF = propagate_bounds_INT_ADD
     propagate_bounds_INT_SUB_OVF = propagate_bounds_INT_SUB
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -305,7 +305,7 @@
             opinfo._known_class = class_const
         else:
             if opinfo is not None:
-                last_guard_pos = opinfo.last_guard_pos
+                last_guard_pos = opinfo.get_last_guard_pos()
             else:
                 last_guard_pos = -1
             opinfo = info.InstancePtrInfo(class_const)
@@ -633,17 +633,18 @@
         if isinstance(opinfo, info.AbstractVirtualPtrInfo):
             return opinfo
         elif opinfo is not None:
-            last_guard_pos = opinfo.last_guard_pos
+            last_guard_pos = opinfo.get_last_guard_pos()
         else:
             last_guard_pos = -1
         assert opinfo is None or opinfo.__class__ is info.NonNullPtrInfo
-        if op.is_getfield() or op.getopnum() == rop.SETFIELD_GC:
-            is_object = op.getdescr().parent_descr.is_object()
+        if (op.is_getfield() or op.getopnum() == rop.SETFIELD_GC or
+            op.getopnum() == rop.QUASIIMMUT_FIELD):
+            is_object = op.getdescr().get_parent_descr().is_object()
             if is_object:
                 opinfo = info.InstancePtrInfo()
             else:
                 opinfo = info.StructPtrInfo()
-            opinfo.init_fields(op.getdescr().parent_descr)
+            opinfo.init_fields(op.getdescr().get_parent_descr())
         elif op.is_getarrayitem() or op.getopnum() == rop.SETARRAYITEM_GC:
             opinfo = info.ArrayPtrInfo(op.getdescr())
         elif op.getopnum() == rop.GUARD_CLASS:
@@ -653,7 +654,8 @@
         elif op.getopnum() in (rop.UNICODELEN,):
             opinfo = vstring.StrPtrInfo(vstring.mode_unicode)
         else:
-            xxx
+            assert False, "operations %s unsupported" % op
+        assert isinstance(opinfo, info.NonNullPtrInfo)
         opinfo.last_guard_pos = last_guard_pos
         arg0.set_forwarded(opinfo)
         return opinfo
diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py
--- a/rpython/jit/metainterp/optimizeopt/rewrite.py
+++ b/rpython/jit/metainterp/optimizeopt/rewrite.py
@@ -305,7 +305,7 @@
             info = self.getptrinfo(arg0)
             if info:
                 if info.is_virtual():
-                    xxx
+                    raise InvalidLoop("promote of a virtual")
                 old_guard_op = info.get_last_guard(self.optimizer)
                 if old_guard_op is not None:
                     op = self.replace_guard_class_with_guard_value(op, info,
@@ -393,9 +393,8 @@
         self.make_constant_class(op.getarg(0), expectedclassbox)
 
     def optimize_GUARD_NONNULL_CLASS(self, op):
-        xxx
-        value = self.getvalue(op.getarg(0))
-        if value.is_null():
+        info = self.getptrinfo(op.getarg(0))
+        if info and info.is_null():
             r = self.optimizer.metainterp_sd.logger_ops.repr_of_resop(op)
             raise InvalidLoop('A GUARD_NONNULL_CLASS (%s) was proven to '
                               'always fail' % r)
diff --git a/rpython/jit/metainterp/optimizeopt/simplify.py b/rpython/jit/metainterp/optimizeopt/simplify.py
--- a/rpython/jit/metainterp/optimizeopt/simplify.py
+++ b/rpython/jit/metainterp/optimizeopt/simplify.py
@@ -23,7 +23,8 @@
     optimize_CALL_PURE_N = optimize_CALL_PURE_I
 
     def optimize_CALL_LOOPINVARIANT_I(self, op):
-        op = op.copy_and_change(rop.CALL)
+        opnum = OpHelpers.call_for_descr(op.getdescr())
+        op = op.copy_and_change(opnum)
         self.emit_operation(op)
     optimize_CALL_LOOPINVARIANT_R = optimize_CALL_LOOPINVARIANT_I
     optimize_CALL_LOOPINVARIANT_F = optimize_CALL_LOOPINVARIANT_I
@@ -33,8 +34,7 @@
         pass
 
     def optimize_VIRTUAL_REF(self, op):
-        newop = ResOperation(rop.SAME_AS_R, [op.getarg(0)], op.result)
-        self.replace_op_with(op, newop)
+        newop = self.replace_op_with(op, rop.SAME_AS_R, [op.getarg(0)])
         self.emit_operation(newop)
 
     def optimize_QUASIIMMUT_FIELD(self, op):
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -8,7 +8,7 @@
 from rpython.jit.metainterp.optimizeopt.optimizer import Optimizer, Optimization
 from rpython.jit.metainterp.optimizeopt.virtualstate import (VirtualStateConstructor,
         ShortBoxes, BadVirtualState, VirtualStatesCantMatch)
-from rpython.jit.metainterp.resoperation import rop, ResOperation, DONT_CHANGE,\
+from rpython.jit.metainterp.resoperation import rop, ResOperation,\
      OpHelpers, AbstractInputArg, GuardResOp
 from rpython.jit.metainterp.resume import Snapshot
 from rpython.jit.metainterp import compile
diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py
--- a/rpython/jit/metainterp/optimizeopt/virtualize.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualize.py
@@ -654,6 +654,7 @@
         # was already forced).
 
     def _optimize_JIT_FORCE_VIRTUAL(self, op):
+        raise Exception("implement me")
         vref = self.getvalue(op.getarg(1))
         vrefinfo = self.optimizer.metainterp_sd.virtualref_info
         if vref.is_virtual():
@@ -683,6 +684,7 @@
         if opinfo and opinfo.is_virtual():
             fieldop = opinfo.getfield(op.getdescr())
             if fieldop is None:
+                raise Exception("I think this is plain illegal")
                 xxx
                 fieldvalue = self.optimizer.new_const(op.getdescr())
             self.make_equal_to(op, fieldop)
@@ -736,8 +738,8 @@
             self.do_RAW_FREE(op)
         elif effectinfo.oopspecindex == EffectInfo.OS_JIT_FORCE_VIRTUALIZABLE:
             # we might end up having CALL here instead of COND_CALL
-            value = self.getvalue(op.getarg(1))
-            if value.is_virtual():
+            info = self.getptrinfo(op.getarg(1))
+            if info and info.is_virtual():
                 return
         else:
             self.emit_operation(op)
@@ -830,6 +832,7 @@
     def optimize_GETARRAYITEM_RAW_I(self, op):
         opinfo = self.getrawptrinfo(op.getarg(0))
         if opinfo and opinfo.is_virtual():
+            raise Exception("implement raw virtuals")
             xxx
             indexbox = self.get_constant_box(op.getarg(1))
             if indexbox is not None:
@@ -850,6 +853,7 @@
         if opinfo and opinfo.is_virtual():
             indexbox = self.get_constant_box(op.getarg(1))
             if indexbox is not None:
+                raise Exception("implement raw virtuals")
                 offset, itemsize, descr = self._unpack_arrayitem_raw_op(op, indexbox)
                 itemvalue = self.getvalue(op.getarg(2))
                 try:
@@ -868,6 +872,7 @@
         return offset, itemsize, descr
 
     def optimize_RAW_LOAD_I(self, op):
+        raise Exception("implement me")
         value = self.getvalue(op.getarg(0))
         if value.is_virtual():
             offsetbox = self.get_constant_box(op.getarg(1))
@@ -907,6 +912,7 @@
                 descr = op.getdescr()
                 fld = opinfo.getinteriorfield_virtual(indexbox.getint(), descr)
                 if fld is None:
+                    raise Exception("I think this is illegal")
                     xxx
                     fieldvalue = self.new_const(descr)
                 self.make_equal_to(op, fld)
diff --git a/rpython/jit/metainterp/optimizeopt/vstring.py b/rpython/jit/metainterp/optimizeopt/vstring.py
--- a/rpython/jit/metainterp/optimizeopt/vstring.py
+++ b/rpython/jit/metainterp/optimizeopt/vstring.py
@@ -1,11 +1,12 @@
 from rpython.jit.codewriter.effectinfo import EffectInfo
 from rpython.jit.metainterp.history import (BoxInt, Const, ConstInt, ConstPtr,
-    get_const_ptr_for_string, get_const_ptr_for_unicode, BoxPtr, REF, INT)
+    get_const_ptr_for_string, get_const_ptr_for_unicode, BoxPtr, REF, INT,
+    DONT_CHANGE)
 from rpython.jit.metainterp.optimizeopt import optimizer, virtualize
 from rpython.jit.metainterp.optimizeopt.optimizer import CONST_0, CONST_1
 from rpython.jit.metainterp.optimizeopt.optimizer import llhelper, REMOVED
 from rpython.jit.metainterp.optimizeopt.util import make_dispatcher_method
-from rpython.jit.metainterp.resoperation import rop, ResOperation, DONT_CHANGE,\
+from rpython.jit.metainterp.resoperation import rop, ResOperation,\
      AbstractResOp
 from rpython.jit.metainterp.optimizeopt import info
 from rpython.rlib.objectmodel import specialize, we_are_translated
@@ -48,8 +49,8 @@
 
 
 
-class StrPtrInfo(info.NonNullPtrInfo):
-    _attrs_ = ('length', 'lenbound', 'lgtop', 'mode', '_cached_vinfo')
+class StrPtrInfo(info.AbstractVirtualPtrInfo):
+    #_attrs_ = ('length', 'lenbound', 'lgtop', 'mode', '_cached_vinfo', '_is_virtual')
 
     lenbound = None
     lgtop = None
@@ -134,7 +135,7 @@
                                  CONST_0, offsetbox, lengthbox, mode)
 
 class VStringPlainInfo(StrPtrInfo):
-    _attrs_ = ('mode', '_is_virtual')
+    #_attrs_ = ('mode', '_is_virtual')
 
     _chars = None
     
@@ -143,7 +144,7 @@
             self._chars = [None] * length
         StrPtrInfo.__init__(self, mode, is_virtual, length)
 
-    def setitem(self, index, item):
+    def setitem(self, index, item, cf=None, optheap=None):
         self._chars[index] = item
 
     def setup_slice(self, longerlist, start, stop):
@@ -151,7 +152,7 @@
         self._chars = longerlist[start:stop]
         # slice the 'longerlist', which may also contain Nones
 
-    def getitem(self, index):
+    def getitem(self, index, optheap=None):
         return self._chars[index]
 
     def is_virtual(self):
@@ -162,7 +163,7 @@
             self.lgtop = ConstInt(len(self._chars))
         return self.lgtop
 
-    @specialize.arg(1)
+    @specialize.arg(2)
     def get_constant_string_spec(self, optforce, mode):
         for c in self._chars:
             if c is None or not c.is_constant():
@@ -172,7 +173,9 @@
 
     def string_copy_parts(self, op, string_optimizer, targetbox, offsetbox,
                           mode):
-        if not self.is_virtual() and not self.is_completely_initialized():
+        if not self.is_virtual():
+            # and not self.is_completely_initialized():
+            raise Exception("implement me")
             return VAbstractStringValue.string_copy_parts(
                 self, string_optimizer, targetbox, offsetbox, mode)
         else:
@@ -215,12 +218,12 @@
         return copy_str_content(string_optimizer, self.s, targetbox,
                                 self.start, offsetbox, self.lgtop, mode)
 
-    @specialize.arg(1)
+    @specialize.arg(2)
     def get_constant_string_spec(self, string_optimizer, mode):
         vstart = string_optimizer.getintbound(self.start)
         vlength = string_optimizer.getintbound(self.lgtop)
         if vstart.is_constant() and vlength.is_constant():
-            xxx
+            raise Exception("implement me")
             s1 = self.vstr.get_constant_string_spec(mode)
             if s1 is None:
                 return None
@@ -235,7 +238,7 @@
         return self.lgtop
 
 class VStringConcatInfo(StrPtrInfo):
-    _attrs_ = ('mode', 'vleft', 'vright', '_is_virtual')
+    #_attrs_ = ('mode', 'vleft', 'vright', '_is_virtual')
     
     def __init__(self, mode, vleft, vright, is_virtual):
         self.vleft = vleft
@@ -262,7 +265,7 @@
             # ^^^ may still be None, if string_optimizer is None
         return self.lgtop
 
-    @specialize.arg(1)
+    @specialize.arg(2)
     def get_constant_string_spec(self, string_optimizer, mode):
         ileft = string_optimizer.getptrinfo(self.vleft)
         s1 = ileft.get_constant_string_spec(string_optimizer, mode)
@@ -717,6 +720,7 @@
         self.make_vstring_slice(op, strbox, startbox, mode, lengthbox)
         return True
 
+    @specialize.arg(2)
     def opt_call_stroruni_STR_EQUAL(self, op, mode):
         arg1 = self.get_box_replacement(op.getarg(1))
         arg2 = self.get_box_replacement(op.getarg(2))
@@ -845,6 +849,7 @@
         return False
 
     def opt_call_stroruni_STR_CMP(self, op, mode):
+        raise Exception('implement me')
         v1 = self.getvalue(op.getarg(1))
         v2 = self.getvalue(op.getarg(2))
         l1box = v1.getstrlen(None, mode, None)
@@ -865,6 +870,7 @@
         return False
 
     def opt_call_SHRINK_ARRAY(self, op):
+        raise Exception('implement me')
         v1 = self.getvalue(op.getarg(1))
         v2 = self.getvalue(op.getarg(2))
         # If the index is constant, if the argument is virtual (we only support
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -818,7 +818,8 @@
                                        mutatefielddescr, orgpc):
         from rpython.jit.metainterp.quasiimmut import QuasiImmutDescr
         cpu = self.metainterp.cpu
-        descr = QuasiImmutDescr(cpu, box, fielddescr, mutatefielddescr)
+        descr = QuasiImmutDescr(cpu, box.getref_base(), fielddescr,
+                                mutatefielddescr)
         self.metainterp.history.record(rop.QUASIIMMUT_FIELD, [box],
                                        None, descr=descr)
         self.metainterp.generate_guard(rop.GUARD_NOT_INVALIDATED,
@@ -834,7 +835,8 @@
         # null, and the guard will be removed.  So the fact that the field is
         # quasi-immutable will have no effect, and instead it will work as a
         # regular, probably virtual, structure.
-        mutatebox = self.execute_with_descr(rop.GETFIELD_GC,
+        opnum = OpHelpers.getfield_for_descr(mutatefielddescr)
+        mutatebox = self.execute_with_descr(opnum,
                                             mutatefielddescr, box)
         if mutatebox.nonnull():
             from rpython.jit.metainterp.quasiimmut import do_force_quasi_immutable
@@ -1600,8 +1602,10 @@
                     self.metainterp.history.record(rop.KEEPALIVE, [vablebox], None)
                 self.metainterp.handle_possible_exception()
                 # XXX refactor: direct_libffi_call() is a hack
+                # does not work in the new system
                 if effectinfo.oopspecindex == effectinfo.OS_LIBFFI_CALL:
-                    self.metainterp.direct_libffi_call()
+                    raise Exception("implement OS_LIBFFI_CALL properly")
+                #    self.metainterp.direct_libffi_call()
                 return resbox
             else:
                 effect = effectinfo.extraeffect
diff --git a/rpython/jit/metainterp/quasiimmut.py b/rpython/jit/metainterp/quasiimmut.py
--- a/rpython/jit/metainterp/quasiimmut.py
+++ b/rpython/jit/metainterp/quasiimmut.py
@@ -109,6 +109,9 @@
         self.qmut = get_current_qmut_instance(cpu, struct, mutatefielddescr)
         self.constantfieldbox = self.get_current_constant_fieldvalue()
 
+    def get_parent_descr(self):
+        return self.fielddescr.get_parent_descr()
+
     def get_current_constant_fieldvalue(self):
         struct = self.struct
         fielddescr = self.fielddescr
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -35,8 +35,6 @@
             orig_op.set_forwarded(op)
         return op
 
-DONT_CHANGE = AbstractValue()
-
 def ResOperation(opnum, args, descr=None):
     cls = opclasses[opnum]
     op = cls()
@@ -120,6 +118,8 @@
 
     def copy_and_change(self, opnum, args=None, descr=None):
         "shallow copy: the returned operation is meant to be used in place of self"
+        from rpython.jit.metainterp.history import DONT_CHANGE
+        
         if args is None:
             args = self.getarglist()
         if descr is None:
diff --git a/rpython/jit/metainterp/resume.py b/rpython/jit/metainterp/resume.py
--- a/rpython/jit/metainterp/resume.py
+++ b/rpython/jit/metainterp/resume.py
@@ -12,6 +12,7 @@
 from rpython.rtyper.lltypesystem import lltype, llmemory, rffi, rstr
 from rpython.rtyper.rclass import OBJECTPTR
 from rpython.jit.metainterp.walkvirtual import VirtualVisitor
+from rpython.jit.metainterp.optimizeopt.info import AbstractVirtualPtrInfo
 
 
 # Logic to encode the chain of frames and the state of the boxes at a
@@ -453,6 +454,7 @@
                 num, _ = untag(self.liveboxes[virtualbox])
                 info = optimizer.getptrinfo(virtualbox)
                 assert info.is_virtual()
+                assert isinstance(info, AbstractVirtualPtrInfo)
                 fieldnums = [self._gettagged(box)
                              for box in fieldboxes]
                 vinfo = self.make_virtual_info(info, fieldnums)
@@ -1038,7 +1040,7 @@
         cic = self.metainterp.staticdata.callinfocollection
         calldescr, func = cic.callinfo_for_oopspec(EffectInfo.OS_RAW_MALLOC_VARSIZE_CHAR)
         return self.metainterp.execute_and_record_varargs(
-            rop.CALL, [ConstInt(func), ConstInt(size)], calldescr)
+            rop.CALL_I, [ConstInt(func), ConstInt(size)], calldescr)
 
     def allocate_string(self, length):
         return self.metainterp.execute_and_record(rop.NEWSTR,
@@ -1066,7 +1068,7 @@
         stopbox = self.metainterp.execute_and_record(rop.INT_ADD, None,
                                                      startbox, lengthbox)
         return self.metainterp.execute_and_record_varargs(
-            rop.CALL, [ConstInt(func), strbox, startbox, stopbox], calldescr)
+            rop.CALL_R, [ConstInt(func), strbox, startbox, stopbox], calldescr)
 
     def allocate_unicode(self, length):
         return self.metainterp.execute_and_record(rop.NEWUNICODE,
@@ -1083,7 +1085,7 @@
         str1box = self.decode_box(str1num, REF)
         str2box = self.decode_box(str2num, REF)
         return self.metainterp.execute_and_record_varargs(
-            rop.CALL, [ConstInt(func), str1box, str2box], calldescr)
+            rop.CALL_R, [ConstInt(func), str1box, str2box], calldescr)
 
     def slice_unicode(self, strnum, startnum, lengthnum):
         cic = self.metainterp.staticdata.callinfocollection
@@ -1094,7 +1096,7 @@
         stopbox = self.metainterp.execute_and_record(rop.INT_ADD, None,
                                                      startbox, lengthbox)
         return self.metainterp.execute_and_record_varargs(
-            rop.CALL, [ConstInt(func), strbox, startbox, stopbox], calldescr)
+            rop.CALL_R, [ConstInt(func), strbox, startbox, stopbox], calldescr)
 
     def setfield(self, structbox, fieldnum, descr):
         if descr.is_pointer_field():
diff --git a/rpython/jit/metainterp/test/test_resume.py b/rpython/jit/metainterp/test/test_resume.py
--- a/rpython/jit/metainterp/test/test_resume.py
+++ b/rpython/jit/metainterp/test/test_resume.py
@@ -135,6 +135,15 @@
     vinfo4 = modifier.make_virtual_info(v1, [1, 2, 6])
     assert vinfo3 is vinfo4
 
+def setvalue(op, val):
+    if op.type == 'i':
+        op.setint(val)
+    elif op.type == 'r':
+        op.setref_base(val)
+    elif op.type == 'f':
+        op.setfloatstorage(val)
+    else:
+        assert op.type == 'v'
 
 class MyMetaInterp:
     _already_allocated_resume_virtuals = None
@@ -155,7 +164,7 @@
     def execute_and_record(self, opnum, descr, *argboxes):
         resvalue = executor.execute(self.cpu, None, opnum, descr, *argboxes)
         op = ResOperation(opnum, list(argboxes), descr)
-        op.setvalue(resvalue)
+        setvalue(op, resvalue)
         self.trace.append((opnum, list(argboxes), resvalue, descr))
         return op
 

From noreply at buildbot.pypy.org  Tue Jun  2 18:30:26 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Tue,  2 Jun 2015 18:30:26 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: few fixes
Message-ID: <20150602163026.C9A631C1473@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77779:cf289fc49521
Date: 2015-06-02 18:30 +0200
http://bitbucket.org/pypy/pypy/changeset/cf289fc49521/

Log:	few fixes

diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py
--- a/rpython/jit/metainterp/optimizeopt/virtualize.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualize.py
@@ -890,6 +890,7 @@
     optimize_RAW_LOAD_F = optimize_RAW_LOAD_I
 
     def optimize_RAW_STORE(self, op):
+        raise Exception("implement me")
         value = self.getvalue(op.getarg(0))
         if value.is_virtual():
             offsetbox = self.get_constant_box(op.getarg(1))
diff --git a/rpython/jit/metainterp/resume.py b/rpython/jit/metainterp/resume.py
--- a/rpython/jit/metainterp/resume.py
+++ b/rpython/jit/metainterp/resume.py
@@ -12,7 +12,6 @@
 from rpython.rtyper.lltypesystem import lltype, llmemory, rffi, rstr
 from rpython.rtyper.rclass import OBJECTPTR
 from rpython.jit.metainterp.walkvirtual import VirtualVisitor
-from rpython.jit.metainterp.optimizeopt.info import AbstractVirtualPtrInfo
 
 
 # Logic to encode the chain of frames and the state of the boxes at a
@@ -413,6 +412,8 @@
         return liveboxes[:]
 
     def _number_virtuals(self, liveboxes, optimizer, num_env_virtuals):
+        from rpython.jit.metainterp.optimizeopt.info import AbstractVirtualPtrInfo
+        
         # !! 'liveboxes' is a list that is extend()ed in-place !!
         memo = self.memo
         new_liveboxes = [None] * memo.num_cached_boxes()

From noreply at buildbot.pypy.org  Tue Jun  2 18:40:54 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Tue,  2 Jun 2015 18:40:54 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: 2to3
Message-ID: <20150602164054.287AC1C024E@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob <me at manueljacob.de>
Branch: py3k
Changeset: r77780:94efbc9c28df
Date: 2015-06-02 18:06 +0200
http://bitbucket.org/pypy/pypy/changeset/94efbc9c28df/

Log:	2to3

diff --git a/pypy/module/__pypy__/test/test_magic.py b/pypy/module/__pypy__/test/test_magic.py
--- a/pypy/module/__pypy__/test/test_magic.py
+++ b/pypy/module/__pypy__/test/test_magic.py
@@ -3,12 +3,12 @@
     spaceconfig = dict(usemodules=['__pypy__'])
 
     def test_save_module_content_for_future_reload(self):
-        import sys, __pypy__
+        import sys, __pypy__, imp
         d = sys.dont_write_bytecode
         sys.dont_write_bytecode = "hello world"
         __pypy__.save_module_content_for_future_reload(sys)
         sys.dont_write_bytecode = d
-        reload(sys)
+        imp.reload(sys)
         assert sys.dont_write_bytecode == "hello world"
         #
         sys.dont_write_bytecode = d

From noreply at buildbot.pypy.org  Tue Jun  2 18:40:55 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Tue,  2 Jun 2015 18:40:55 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: 2to3
Message-ID: <20150602164055.5CF441C024E@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob <me at manueljacob.de>
Branch: py3k
Changeset: r77781:a13d93cc5971
Date: 2015-06-02 18:20 +0200
http://bitbucket.org/pypy/pypy/changeset/a13d93cc5971/

Log:	2to3

diff --git a/pypy/module/_cffi_backend/ffi_obj.py b/pypy/module/_cffi_backend/ffi_obj.py
--- a/pypy/module/_cffi_backend/ffi_obj.py
+++ b/pypy/module/_cffi_backend/ffi_obj.py
@@ -132,7 +132,7 @@
     def ffi_type(self, w_x, accept):
         space = self.space
         if (accept & ACCEPT_STRING) and (
-                space.isinstance_w(w_x, space.w_basestring)):
+                space.isinstance_w(w_x, space.w_unicode)):
             string = space.str_w(w_x)
             consider_fn_as_fnptr = (accept & CONSIDER_FN_AS_FNPTR) != 0
             if jit.isconstant(string):

From noreply at buildbot.pypy.org  Tue Jun  2 18:52:09 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Tue,  2 Jun 2015 18:52:09 +0200 (CEST)
Subject: [pypy-commit] pypy use_min_scalar: Switch to the scalar fast path
 earlier in W_Ufunc2.call()
Message-ID: <20150602165209.5F92F1C024E@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: use_min_scalar
Changeset: r77782:22a1e48d9fc0
Date: 2015-06-02 17:52 +0100
http://bitbucket.org/pypy/pypy/changeset/22a1e48d9fc0/

Log:	Switch to the scalar fast path earlier in W_Ufunc2.call()

diff --git a/pypy/module/micronumpy/ufuncs.py b/pypy/module/micronumpy/ufuncs.py
--- a/pypy/module/micronumpy/ufuncs.py
+++ b/pypy/module/micronumpy/ufuncs.py
@@ -605,18 +605,18 @@
                             w_rdtype.get_name(), w_ldtype.get_name(),
                             self.name)
 
-        calc_dtype, dt_out, func = self.find_specialization(
-            space, w_ldtype, w_rdtype, out, casting, w_lhs, w_rhs)
-
         if (isinstance(w_lhs, W_GenericBox) and
                 isinstance(w_rhs, W_GenericBox) and out is None):
-            return self.call_scalar(space, w_lhs, w_rhs, calc_dtype)
+            return self.call_scalar(space, w_lhs, w_rhs, casting)
         if isinstance(w_lhs, W_GenericBox):
             w_lhs = W_NDimArray.from_scalar(space, w_lhs)
         assert isinstance(w_lhs, W_NDimArray)
         if isinstance(w_rhs, W_GenericBox):
             w_rhs = W_NDimArray.from_scalar(space, w_rhs)
         assert isinstance(w_rhs, W_NDimArray)
+        calc_dtype, dt_out, func = self.find_specialization(
+            space, w_ldtype, w_rdtype, out, casting, w_lhs, w_rhs)
+
         new_shape = shape_agreement(space, w_lhs.get_shape(), w_rhs)
         new_shape = shape_agreement(space, new_shape, out, broadcast_down=False)
         w_highpriority, out_subtype = array_priority(space, w_lhs, w_rhs)
@@ -633,7 +633,10 @@
             w_res = space.call_method(w_highpriority, '__array_wrap__', w_res)
         return w_res
 
-    def call_scalar(self, space, w_lhs, w_rhs, in_dtype):
+    def call_scalar(self, space, w_lhs, w_rhs, casting):
+        in_dtype, out_dtype, func = self.find_specialization(
+            space, w_lhs.get_dtype(space), w_rhs.get_dtype(space),
+            out=None, casting=casting)
         w_val = self.func(in_dtype,
                           w_lhs.convert_to(space, in_dtype),
                           w_rhs.convert_to(space, in_dtype))

From noreply at buildbot.pypy.org  Tue Jun  2 19:08:48 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Tue,  2 Jun 2015 19:08:48 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: 2to3
Message-ID: <20150602170848.0930C1C024E@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob <me at manueljacob.de>
Branch: py3k
Changeset: r77783:a4114a083024
Date: 2015-06-02 19:03 +0200
http://bitbucket.org/pypy/pypy/changeset/a4114a083024/

Log:	2to3

diff --git a/pypy/module/_cffi_backend/test/test_ffi_obj.py b/pypy/module/_cffi_backend/test/test_ffi_obj.py
--- a/pypy/module/_cffi_backend/test/test_ffi_obj.py
+++ b/pypy/module/_cffi_backend/test/test_ffi_obj.py
@@ -186,7 +186,7 @@
         import _cffi_backend as _cffi1_backend
         ffi = _cffi1_backend.FFI()
         a = ffi.new("signed char[]", [5, 6, 7])
-        assert ffi.buffer(a)[:] == '\x05\x06\x07'
+        assert ffi.buffer(a)[:] == b'\x05\x06\x07'
 
     def test_ffi_from_buffer(self):
         import _cffi_backend as _cffi1_backend

From noreply at buildbot.pypy.org  Tue Jun  2 19:08:49 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Tue,  2 Jun 2015 19:08:49 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: 2to3
Message-ID: <20150602170849.507271C024E@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob <me at manueljacob.de>
Branch: py3k
Changeset: r77784:56eddd4b5785
Date: 2015-06-02 19:03 +0200
http://bitbucket.org/pypy/pypy/changeset/56eddd4b5785/

Log:	2to3

diff --git a/pypy/module/_cffi_backend/test/test_recompiler.py b/pypy/module/_cffi_backend/test/test_recompiler.py
--- a/pypy/module/_cffi_backend/test/test_recompiler.py
+++ b/pypy/module/_cffi_backend/test/test_recompiler.py
@@ -100,7 +100,7 @@
             del self.space._cleanup_ffi
         self.space.appexec([self._w_modules], """(old_modules):
             import sys
-            for key in sys.modules.keys():
+            for key in list(sys.modules.keys()):
                 if key not in old_modules:
                     del sys.modules[key]
         """)

From noreply at buildbot.pypy.org  Tue Jun  2 19:17:40 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Tue,  2 Jun 2015 19:17:40 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Remove merge conflict marker which was
	accidently left.
Message-ID: <20150602171740.E4A3E1C024E@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob <me at manueljacob.de>
Branch: py3k
Changeset: r77785:baaee72e978a
Date: 2015-06-02 19:17 +0200
http://bitbucket.org/pypy/pypy/changeset/baaee72e978a/

Log:	Remove merge conflict marker which was accidently left.

diff --git a/rpython/translator/driver.py b/rpython/translator/driver.py
--- a/rpython/translator/driver.py
+++ b/rpython/translator/driver.py
@@ -487,7 +487,6 @@
                     exename = exe.new(purebasename=exe.purebasename + 'w')
                     shutil_copy(str(exename), str(newexename))
                     ext_to_copy = ['lib', 'pdb']
->>>>>>> other
                     for ext in ext_to_copy:
                         name = soname.new(ext=ext)
                         newname = newexename.new(basename=soname.basename)

From noreply at buildbot.pypy.org  Tue Jun  2 19:28:01 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue,  2 Jun 2015 19:28:01 +0200 (CEST)
Subject: [pypy-commit] cffi default: In the type parser,
 escape error messages and don't display the input
Message-ID: <20150602172801.918A21C024E@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2158:fa3cf2db4a04
Date: 2015-06-02 19:22 +0200
http://bitbucket.org/cffi/cffi/changeset/fa3cf2db4a04/

Log:	In the type parser, escape error messages and don't display the
	input type if too huge

diff --git a/c/ffi_obj.c b/c/ffi_obj.c
--- a/c/ffi_obj.c
+++ b/c/ffi_obj.c
@@ -140,6 +140,38 @@
 #define ACCEPT_ALL      (ACCEPT_STRING | ACCEPT_CTYPE | ACCEPT_CDATA)
 #define CONSIDER_FN_AS_FNPTR  8
 
+static CTypeDescrObject *_ffi_bad_type(FFIObject *ffi, char *input_text)
+{
+    size_t length = strlen(input_text);
+    char *extra;
+
+    if (length > 500) {
+        extra = "";
+    }
+    else {
+        char *p;
+        size_t i, num_spaces = ffi->info.error_location;
+        extra = alloca(length + num_spaces + 4);
+        p = extra;
+        *p++ = '\n';
+        for (i = 0; i < length; i++) {
+            if (' ' <= input_text[i] && input_text[i] < 0x7f)
+                *p++ = input_text[i];
+            else if (input_text[i] == '\t' || input_text[i] == '\n')
+                *p++ = ' ';
+            else
+                *p++ = '?';
+        }
+        *p++ = '\n';
+        memset(p, ' ', num_spaces);
+        p += num_spaces;
+        *p++ = '^';
+        *p++ = 0;
+    }
+    PyErr_Format(FFIError, "%s%s", ffi->info.error_message, extra);
+    return NULL;
+}
+
 static CTypeDescrObject *_ffi_type(FFIObject *ffi, PyObject *arg,
                                    int accept)
 {
@@ -153,15 +185,9 @@
         if (x == NULL) {
             char *input_text = PyText_AS_UTF8(arg);
             int err, index = parse_c_type(&ffi->info, input_text);
-            if (index < 0) {
-                size_t num_spaces = ffi->info.error_location;
-                char *spaces = alloca(num_spaces + 1);
-                memset(spaces, ' ', num_spaces);
-                spaces[num_spaces] = '\0';
-                PyErr_Format(FFIError, "%s\n%s\n%s^", ffi->info.error_message,
-                             input_text, spaces);
-                return NULL;
-            }
+            if (index < 0)
+                return _ffi_bad_type(ffi, input_text);
+
             x = realize_c_type_or_func(&ffi->types_builder,
                                        ffi->info.output, index);
             if (x == NULL)
diff --git a/testing/cffi1/test_ffi_obj.py b/testing/cffi1/test_ffi_obj.py
--- a/testing/cffi1/test_ffi_obj.py
+++ b/testing/cffi1/test_ffi_obj.py
@@ -158,6 +158,12 @@
     assert str(e.value) == ("undefined struct/union name\n"
                             "struct never_heard_of_s\n"
                             "       ^")
+    e = py.test.raises(ffi.error, ffi.cast, "\t\n\x01\x1f~\x7f\x80\xff", 0)
+    assert str(e.value) == ("identifier expected\n"
+                            "  ??~???\n"
+                            "  ^")
+    e = py.test.raises(ffi.error, ffi.cast, "X" * 600, 0)
+    assert str(e.value) == ("undefined type name")
 
 def test_ffi_buffer():
     ffi = _cffi1_backend.FFI()

From noreply at buildbot.pypy.org  Tue Jun  2 19:28:48 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue,  2 Jun 2015 19:28:48 +0200 (CEST)
Subject: [pypy-commit] pypy default: In the type parser,
 escape error messages and don't display the
Message-ID: <20150602172848.F22061C024E@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r77786:f891985cf1f3
Date: 2015-06-02 19:28 +0200
http://bitbucket.org/pypy/pypy/changeset/f891985cf1f3/

Log:	In the type parser, escape error messages and don't display the
	input type if too huge

diff --git a/pypy/module/_cffi_backend/ffi_obj.py b/pypy/module/_cffi_backend/ffi_obj.py
--- a/pypy/module/_cffi_backend/ffi_obj.py
+++ b/pypy/module/_cffi_backend/ffi_obj.py
@@ -95,6 +95,23 @@
             else:
                 raise KeyError    # don't handle this error case here
 
+    def _ffi_bad_type(self, input_text):
+        info = self.ctxobj.info
+        errmsg = rffi.charp2str(info.c_error_message)
+        if len(input_text) > 500:
+            raise oefmt(self.w_FFIError, errmsg)
+        printable_text = ['?'] * len(input_text)
+        for i in range(len(input_text)):
+            if ' ' <= input_text[i] < '\x7f':
+                printable_text[i] = input_text[i]
+            elif input_text[i] == '\t' or input_text[i] == '\n':
+                printable_text[i] = ' '
+        num_spaces = rffi.getintfield(info, 'c_error_location')
+        raise oefmt(self.w_FFIError, "%s\n%s\n%s^",
+                    rffi.charp2str(info.c_error_message),
+                    ''.join(printable_text),
+                    " " * num_spaces)
+
     @jit.dont_look_inside
     def parse_string_to_type(self, string, consider_fn_as_fnptr):
         # This cannot be made @elidable because it calls general space
@@ -108,11 +125,7 @@
             info = self.ctxobj.info
             index = parse_c_type.parse_c_type(info, string)
             if index < 0:
-                num_spaces = rffi.getintfield(info, 'c_error_location')
-                raise oefmt(self.w_FFIError, "%s\n%s\n%s^",
-                            rffi.charp2str(info.c_error_message),
-                            string,
-                            " " * num_spaces)
+                raise self._ffi_bad_type(string)
             x = realize_c_type.realize_c_type_or_func(
                 self, self.ctxobj.info.c_output, index)
             assert x is not None
diff --git a/pypy/module/_cffi_backend/test/test_ffi_obj.py b/pypy/module/_cffi_backend/test/test_ffi_obj.py
--- a/pypy/module/_cffi_backend/test/test_ffi_obj.py
+++ b/pypy/module/_cffi_backend/test/test_ffi_obj.py
@@ -181,6 +181,12 @@
         assert str(e.value) == ("undefined struct/union name\n"
                                 "struct never_heard_of_s\n"
                                 "       ^")
+        e = raises(ffi.error, ffi.cast, "\t\n\x01\x1f~\x7f\x80\xff", 0)
+        assert str(e.value) == ("identifier expected\n"
+                                "  ??~???\n"
+                                "  ^")
+        e = raises(ffi.error, ffi.cast, "X" * 600, 0)
+        assert str(e.value) == ("undefined type name")
 
     def test_ffi_buffer(self):
         import _cffi_backend as _cffi1_backend

From noreply at buildbot.pypy.org  Tue Jun  2 20:12:10 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Tue,  2 Jun 2015 20:12:10 +0200 (CEST)
Subject: [pypy-commit] pypy default: fix translation ("argument not
	constant")
Message-ID: <20150602181210.6AD8F1C034D@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: 
Changeset: r77787:61595f3c9846
Date: 2015-06-02 21:12 +0300
http://bitbucket.org/pypy/pypy/changeset/61595f3c9846/

Log:	fix translation ("argument not constant")

diff --git a/pypy/module/_cffi_backend/ffi_obj.py b/pypy/module/_cffi_backend/ffi_obj.py
--- a/pypy/module/_cffi_backend/ffi_obj.py
+++ b/pypy/module/_cffi_backend/ffi_obj.py
@@ -99,7 +99,7 @@
         info = self.ctxobj.info
         errmsg = rffi.charp2str(info.c_error_message)
         if len(input_text) > 500:
-            raise oefmt(self.w_FFIError, errmsg)
+            raise oefmt(self.w_FFIError, "%s", errmsg)
         printable_text = ['?'] * len(input_text)
         for i in range(len(input_text)):
             if ' ' <= input_text[i] < '\x7f':

From noreply at buildbot.pypy.org  Tue Jun  2 20:12:11 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Tue,  2 Jun 2015 20:12:11 +0200 (CEST)
Subject: [pypy-commit] pypy default: test,
 fix to support creating a record array from a different record array
Message-ID: <20150602181211.9E0121C034D@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: 
Changeset: r77788:0c9994fe9a36
Date: 2015-06-02 21:05 +0300
http://bitbucket.org/pypy/pypy/changeset/0c9994fe9a36/

Log:	test, fix to support creating a record array from a different record
	array

diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py
--- a/pypy/module/micronumpy/test/test_dtypes.py
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -1129,6 +1129,16 @@
         exc = raises(ValueError, "dtype([('a', '<i8'), ('a', '<f8')])")
         assert exc.value[0] == 'two fields with the same name'
 
+    def test_array_from_record(self):
+        import numpy as np
+        a = np.array(('???', -999, -12345678.9), 
+                     dtype=[('c', '|S3'), ('a', '<i8'), ('b', '<f8')])
+        # Change the order of the keys
+        b = np.array(a, dtype=[('a', '<i8'), ('b', '<f8'), ('c', '|S3')])
+        assert b.base is None
+        assert b.dtype.fields['a'][1] == 0
+        assert b['a'] == -999
+
     def test_create_from_dict(self):
         import numpy as np
         import sys
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -2363,8 +2363,16 @@
     def coerce(self, space, dtype, w_item):
         from pypy.module.micronumpy.base import W_NDimArray
         if isinstance(w_item, boxes.W_VoidBox):
-            return w_item
-        if w_item is not None:
+            if dtype == w_item.dtype:
+                return w_item
+            else:
+                # match up the field names
+                items_w = [None] * len(dtype.fields)
+                for i in range(len(dtype.fields)):
+                    name = dtype.names[i]
+                    if name in w_item.dtype.names:
+                        items_w[i] = w_item.descr_getitem(space, space.wrap(name))
+        elif w_item is not None:
             if space.isinstance_w(w_item, space.w_tuple):
                 if len(dtype.fields) != space.len_w(w_item):
                     raise OperationError(space.w_ValueError, space.wrap(

From noreply at buildbot.pypy.org  Tue Jun  2 21:00:07 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Tue,  2 Jun 2015 21:00:07 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: 2to3
Message-ID: <20150602190007.BBD491C024E@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob <me at manueljacob.de>
Branch: py3k
Changeset: r77789:b0fb0320e5d9
Date: 2015-06-02 20:49 +0200
http://bitbucket.org/pypy/pypy/changeset/b0fb0320e5d9/

Log:	2to3

diff --git a/pypy/module/micronumpy/casting.py b/pypy/module/micronumpy/casting.py
--- a/pypy/module/micronumpy/casting.py
+++ b/pypy/module/micronumpy/casting.py
@@ -308,8 +308,6 @@
     if space.isinstance_w(w_obj, space.w_bool):
         return bool_dtype
     elif space.isinstance_w(w_obj, space.w_int):
-        return long_dtype
-    elif space.isinstance_w(w_obj, space.w_long):
         try:
             space.int_w(w_obj)
         except OperationError, e:

From noreply at buildbot.pypy.org  Tue Jun  2 21:12:24 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue,  2 Jun 2015 21:12:24 +0200 (CEST)
Subject: [pypy-commit] cffi default: Document that this example is also
 possible in the out-of-line mode
Message-ID: <20150602191224.8D4691C024E@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2159:25dc5648d49e
Date: 2015-06-02 21:13 +0200
http://bitbucket.org/cffi/cffi/changeset/25dc5648d49e/

Log:	Document that this example is also possible in the out-of-line mode

diff --git a/doc/source/overview.rst b/doc/source/overview.rst
--- a/doc/source/overview.rst
+++ b/doc/source/overview.rst
@@ -202,6 +202,13 @@
 .. _struct: http://docs.python.org/library/struct.html
 .. _array: http://docs.python.org/library/array.html
 
+This example also admits an out-of-line equivalent.  It is similar to
+`Out-of-line example (ABI level, out-of-line)`_ above, but without any
+call to ``ffi.dlopen()``.  In the main program, you write ``from
+_simple_example import ffi`` and then the same content as the in-line
+example above starting from the line ``image = ffi.new("pixel_t[]",
+800*600)``.
+
 
 .. _performance:
 

From noreply at buildbot.pypy.org  Tue Jun  2 21:16:03 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Tue,  2 Jun 2015 21:16:03 +0200 (CEST)
Subject: [pypy-commit] pypy use_min_scalar: use_min_scalar can never be true
	for unary ufuncs
Message-ID: <20150602191603.EEBBC1C024E@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: use_min_scalar
Changeset: r77790:f1baee239d59
Date: 2015-06-02 20:16 +0100
http://bitbucket.org/pypy/pypy/changeset/f1baee239d59/

Log:	use_min_scalar can never be true for unary ufuncs

diff --git a/pypy/module/micronumpy/ufuncs.py b/pypy/module/micronumpy/ufuncs.py
--- a/pypy/module/micronumpy/ufuncs.py
+++ b/pypy/module/micronumpy/ufuncs.py
@@ -495,17 +495,12 @@
         return dt_in, dt_out, self.func
 
     def _calc_dtype(self, space, arg_dtype, out=None, casting='unsafe'):
-        use_min_scalar = False
         if arg_dtype.is_object():
             return arg_dtype, arg_dtype
         in_casting = safe_casting_mode(casting)
         for dt_in, dt_out in self.dtypes:
-            if use_min_scalar:
-                if not can_cast_array(space, w_arg, dt_in, in_casting):
-                    continue
-            else:
-                if not can_cast_type(space, arg_dtype, dt_in, in_casting):
-                    continue
+            if not can_cast_type(space, arg_dtype, dt_in, in_casting):
+                continue
             if out is not None:
                 res_dtype = out.get_dtype()
                 if not can_cast_type(space, dt_out, res_dtype, casting):

From noreply at buildbot.pypy.org  Tue Jun  2 22:18:21 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Tue,  2 Jun 2015 22:18:21 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Fix.
Message-ID: <20150602201821.0EF071C2026@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob <me at manueljacob.de>
Branch: py3k
Changeset: r77791:fbba4b4c3b08
Date: 2015-06-02 22:18 +0200
http://bitbucket.org/pypy/pypy/changeset/fbba4b4c3b08/

Log:	Fix.

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -1169,7 +1169,7 @@
     raise oefmt(space.w_ImportError,
                 "function %s not found in library %s", look_for, path)
 
-initfunctype = lltype.Ptr(lltype.FuncType([], lltype.Void))
+initfunctype = lltype.Ptr(lltype.FuncType([], PyObject))
 
 def load_cpyext_module(space, name, path, dll, initptr):
     from rpython.rlib import rdynload

From noreply at buildbot.pypy.org  Tue Jun  2 22:48:28 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Tue,  2 Jun 2015 22:48:28 +0200 (CEST)
Subject: [pypy-commit] pypy default: add context argument to __array_wrap__
	in ufuncs
Message-ID: <20150602204828.B9A601C034D@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: 
Changeset: r77792:1137f0723a07
Date: 2015-06-02 22:15 +0300
http://bitbucket.org/pypy/pypy/changeset/1137f0723a07/

Log:	add context argument to __array_wrap__ in ufuncs

diff --git a/pypy/module/micronumpy/test/test_subtype.py b/pypy/module/micronumpy/test/test_subtype.py
--- a/pypy/module/micronumpy/test/test_subtype.py
+++ b/pypy/module/micronumpy/test/test_subtype.py
@@ -656,6 +656,7 @@
                 self.output += 'In __array_wrap__:'
                 self.output += '   self is %s' % repr(self)
                 self.output += '   arr is %r\n' % (out_arr,)
+                self.output += '   context is %r\n' % (context,)
                 # then just call the parent
                 ret = np.ndarray.__array_wrap__(self, out_arr, context)
                 print 'wrap',self.output
diff --git a/pypy/module/micronumpy/ufuncs.py b/pypy/module/micronumpy/ufuncs.py
--- a/pypy/module/micronumpy/ufuncs.py
+++ b/pypy/module/micronumpy/ufuncs.py
@@ -474,7 +474,8 @@
         if out is None:
             if w_res.is_scalar():
                 return w_res.get_scalar_value()
-            w_res = space.call_method(w_obj, '__array_wrap__', w_res)
+            ctxt = space.newtuple([self, space.newtuple([w_obj]), space.wrap(0)])
+            w_res = space.call_method(w_obj, '__array_wrap__', w_res, ctxt)
         return w_res
 
     def call_scalar(self, space, w_arg, in_dtype):
@@ -632,7 +633,8 @@
         if out is None:
             if w_res.is_scalar():
                 return w_res.get_scalar_value()
-            w_res = space.call_method(w_highpriority, '__array_wrap__', w_res)
+            ctxt = space.newtuple([self, space.newtuple([w_lhs, w_rhs]), space.wrap(0)])
+            w_res = space.call_method(w_highpriority, '__array_wrap__', w_res, ctxt)
         return w_res
 
     def call_scalar(self, space, w_lhs, w_rhs, in_dtype):

From noreply at buildbot.pypy.org  Tue Jun  2 23:43:11 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Tue,  2 Jun 2015 23:43:11 +0200 (CEST)
Subject: [pypy-commit] pypy default: test,
 fix record array creation with int value broadcast to internal array
Message-ID: <20150602214311.A1A781C024E@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: 
Changeset: r77793:d7275799e550
Date: 2015-06-03 00:43 +0300
http://bitbucket.org/pypy/pypy/changeset/d7275799e550/

Log:	test, fix record array creation with int value broadcast to internal
	array

diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py
--- a/pypy/module/micronumpy/test/test_dtypes.py
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -1138,6 +1138,10 @@
         assert b.base is None
         assert b.dtype.fields['a'][1] == 0
         assert b['a'] == -999
+        a = np.array(('N/A', 1e+20, 1e+20, 999999),
+                     dtype=[('name', '|S4'), ('x', '<f8'), 
+                            ('y', '<f8'), ('block', '<i8', (2, 3))])
+        assert (a['block'] == 999999).all()
 
     def test_create_from_dict(self):
         import numpy as np
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -20,7 +20,7 @@
      cast_gcref_to_instance
 from rpython.rtyper.lltypesystem import lltype, rffi, llmemory
 from rpython.tool.sourcetools import func_with_new_name
-from pypy.module.micronumpy import boxes
+from pypy.module.micronumpy import boxes, support
 from pypy.module.micronumpy.concrete import SliceArray, VoidBoxStorage, V_OBJECTSTORE
 from pypy.module.micronumpy.strides import calc_strides
 from . import constants as NPY
@@ -2265,10 +2265,12 @@
     def _coerce(self, space, arr, ofs, dtype, w_items, shape):
         # TODO: Make sure the shape and the array match
         from pypy.module.micronumpy.descriptor import W_Dtype
-        if w_items is not None:
+        if w_items is None:
+            items_w = [None] * shape[0]
+        elif support.issequence_w(space, w_items):
             items_w = space.fixedview(w_items)
         else:
-            items_w = [None] * shape[0]
+            items_w = [w_items] * shape[0]
         subdtype = dtype.subdtype
         assert isinstance(subdtype, W_Dtype)
         itemtype = subdtype.itemtype

From noreply at buildbot.pypy.org  Wed Jun  3 00:40:08 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Wed,  3 Jun 2015 00:40:08 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Fix.
Message-ID: <20150602224008.DFFB21C024E@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob <me at manueljacob.de>
Branch: py3k
Changeset: r77794:c759743c891a
Date: 2015-06-02 22:37 +0200
http://bitbucket.org/pypy/pypy/changeset/c759743c891a/

Log:	Fix.

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -1153,7 +1153,7 @@
             return
     #
     if space.config.objspace.usemodules.cpyext:
-        also_look_for = 'init%s' % (basename,)
+        also_look_for = 'PyInit_%s' % (basename,)
         try:
             initptr = rdynload.dlsym(dll, also_look_for)
         except KeyError:

From noreply at buildbot.pypy.org  Wed Jun  3 01:03:33 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Wed,  3 Jun 2015 01:03:33 +0200 (CEST)
Subject: [pypy-commit] pypy use_min_scalar: Close branch use_min_scalar
Message-ID: <20150602230333.279331C0262@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: use_min_scalar
Changeset: r77795:17f781c28235
Date: 2015-06-03 00:04 +0100
http://bitbucket.org/pypy/pypy/changeset/17f781c28235/

Log:	Close branch use_min_scalar


From noreply at buildbot.pypy.org  Wed Jun  3 01:03:44 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Wed,  3 Jun 2015 01:03:44 +0200 (CEST)
Subject: [pypy-commit] pypy default: Merged use_min_scalar into default
Message-ID: <20150602230344.728FD1C0262@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r77796:cad6015d5380
Date: 2015-06-03 00:04 +0100
http://bitbucket.org/pypy/pypy/changeset/cad6015d5380/

Log:	Merged use_min_scalar into default

diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -1349,3 +1349,6 @@
         assert np.add(np.float16(0), np.longdouble(0)).dtype == np.longdouble
         assert np.add(np.float16(0), np.complex64(0)).dtype == np.complex64
         assert np.add(np.float16(0), np.complex128(0)).dtype == np.complex128
+        assert np.add(np.zeros(5, dtype=np.int8), 257).dtype == np.int16
+        assert np.subtract(np.zeros(5, dtype=np.int8), 257).dtype == np.int16
+        assert np.divide(np.zeros(5, dtype=np.int8), 257).dtype == np.int16
diff --git a/pypy/module/micronumpy/ufuncs.py b/pypy/module/micronumpy/ufuncs.py
--- a/pypy/module/micronumpy/ufuncs.py
+++ b/pypy/module/micronumpy/ufuncs.py
@@ -21,7 +21,8 @@
 from pypy.module.micronumpy.support import (_parse_signature, product,
         get_storage_as_int, is_rhs_priority_higher)
 from .casting import (
-    can_cast_type, can_cast_to, find_result_type, promote_types)
+    can_cast_type, can_cast_array, can_cast_to,
+    find_result_type, promote_types)
 from .boxes import W_GenericBox, W_ObjectBox
 
 def done_if_true(dtype, val):
@@ -495,17 +496,12 @@
         return dt_in, dt_out, self.func
 
     def _calc_dtype(self, space, arg_dtype, out=None, casting='unsafe'):
-        use_min_scalar = False
         if arg_dtype.is_object():
             return arg_dtype, arg_dtype
         in_casting = safe_casting_mode(casting)
         for dt_in, dt_out in self.dtypes:
-            if use_min_scalar:
-                if not can_cast_array(space, w_arg, dt_in, in_casting):
-                    continue
-            else:
-                if not can_cast_type(space, arg_dtype, dt_in, in_casting):
-                    continue
+            if not can_cast_type(space, arg_dtype, dt_in, in_casting):
+                continue
             if out is not None:
                 res_dtype = out.get_dtype()
                 if not can_cast_type(space, dt_out, res_dtype, casting):
@@ -605,21 +601,18 @@
                             w_rdtype.get_name(), w_ldtype.get_name(),
                             self.name)
 
-        if self.are_common_types(w_ldtype, w_rdtype):
-            if not w_lhs.is_scalar() and w_rhs.is_scalar():
-                w_rdtype = w_ldtype
-            elif w_lhs.is_scalar() and not w_rhs.is_scalar():
-                w_ldtype = w_rdtype
-        calc_dtype, dt_out, func = self.find_specialization(space, w_ldtype, w_rdtype, out, casting)
         if (isinstance(w_lhs, W_GenericBox) and
                 isinstance(w_rhs, W_GenericBox) and out is None):
-            return self.call_scalar(space, w_lhs, w_rhs, calc_dtype)
+            return self.call_scalar(space, w_lhs, w_rhs, casting)
         if isinstance(w_lhs, W_GenericBox):
             w_lhs = W_NDimArray.from_scalar(space, w_lhs)
         assert isinstance(w_lhs, W_NDimArray)
         if isinstance(w_rhs, W_GenericBox):
             w_rhs = W_NDimArray.from_scalar(space, w_rhs)
         assert isinstance(w_rhs, W_NDimArray)
+        calc_dtype, dt_out, func = self.find_specialization(
+            space, w_ldtype, w_rdtype, out, casting, w_lhs, w_rhs)
+
         new_shape = shape_agreement(space, w_lhs.get_shape(), w_rhs)
         new_shape = shape_agreement(space, new_shape, out, broadcast_down=False)
         w_highpriority, out_subtype = array_priority(space, w_lhs, w_rhs)
@@ -637,7 +630,10 @@
             w_res = space.call_method(w_highpriority, '__array_wrap__', w_res, ctxt)
         return w_res
 
-    def call_scalar(self, space, w_lhs, w_rhs, in_dtype):
+    def call_scalar(self, space, w_lhs, w_rhs, casting):
+        in_dtype, out_dtype, func = self.find_specialization(
+            space, w_lhs.get_dtype(space), w_rhs.get_dtype(space),
+            out=None, casting=casting)
         w_val = self.func(in_dtype,
                           w_lhs.convert_to(space, in_dtype),
                           w_rhs.convert_to(space, in_dtype))
@@ -645,7 +641,8 @@
             return w_val.w_obj
         return w_val
 
-    def _find_specialization(self, space, l_dtype, r_dtype, out, casting):
+    def _find_specialization(self, space, l_dtype, r_dtype, out, casting,
+                             w_arg1, w_arg2):
         if (not self.allow_bool and (l_dtype.is_bool() or
                                          r_dtype.is_bool()) or
                 not self.allow_complex and (l_dtype.is_complex() or
@@ -657,15 +654,23 @@
             dtype = find_result_type(space, [], [l_dtype, r_dtype])
             bool_dtype = get_dtype_cache(space).w_booldtype
             return dtype, bool_dtype, self.func
-        dt_in, dt_out = self._calc_dtype(space, l_dtype, r_dtype, out, casting)
+        dt_in, dt_out = self._calc_dtype(
+            space, l_dtype, r_dtype, out, casting, w_arg1, w_arg2)
         return dt_in, dt_out, self.func
 
-    def find_specialization(self, space, l_dtype, r_dtype, out, casting):
+    def find_specialization(self, space, l_dtype, r_dtype, out, casting,
+                            w_arg1=None, w_arg2=None):
         if self.simple_binary:
             if out is None and not (l_dtype.is_object() or r_dtype.is_object()):
-                dtype = promote_types(space, l_dtype, r_dtype)
+                if w_arg1 is not None and w_arg2 is not None:
+                    w_arg1 = convert_to_array(space, w_arg1)
+                    w_arg2 = convert_to_array(space, w_arg2)
+                    dtype = find_result_type(space, [w_arg1, w_arg2], [])
+                else:
+                    dtype = promote_types(space, l_dtype, r_dtype)
                 return dtype, dtype, self.func
-        return self._find_specialization(space, l_dtype, r_dtype, out, casting)
+        return self._find_specialization(
+            space, l_dtype, r_dtype, out, casting, w_arg1, w_arg2)
 
     def find_binop_type(self, space, dtype):
         """Find a valid dtype signature of the form xx->x"""
@@ -686,15 +691,21 @@
             "requested type has type code '%s'" % (self.name, dtype.char))
 
 
-    def _calc_dtype(self, space, l_dtype, r_dtype, out=None, casting='unsafe'):
-        use_min_scalar = False
+    def _calc_dtype(self, space, l_dtype, r_dtype, out, casting,
+                    w_arg1, w_arg2):
         if l_dtype.is_object() or r_dtype.is_object():
             dtype = get_dtype_cache(space).w_objectdtype
             return dtype, dtype
+        use_min_scalar = (w_arg1 is not None and w_arg2 is not None and
+                          ((w_arg1.is_scalar() and not w_arg2.is_scalar()) or
+                           (not w_arg1.is_scalar() and w_arg2.is_scalar())))
         in_casting = safe_casting_mode(casting)
         for dt_in, dt_out in self.dtypes:
             if use_min_scalar:
-                if not can_cast_array(space, w_arg, dt_in, in_casting):
+                w_arg1 = convert_to_array(space, w_arg1)
+                w_arg2 = convert_to_array(space, w_arg2)
+                if not (can_cast_array(space, w_arg1, dt_in, in_casting) and
+                        can_cast_array(space, w_arg2, dt_in, in_casting)):
                     continue
             else:
                 if not (can_cast_type(space, l_dtype, dt_in, in_casting) and

From noreply at buildbot.pypy.org  Wed Jun  3 01:05:43 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Wed,  3 Jun 2015 01:05:43 +0200 (CEST)
Subject: [pypy-commit] pypy default: update whatsnew
Message-ID: <20150602230543.5BB651C0262@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r77797:41393326513b
Date: 2015-06-03 00:05 +0100
http://bitbucket.org/pypy/pypy/changeset/41393326513b/

Log:	update whatsnew

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -5,4 +5,5 @@
 .. this is a revision shortly after release-2.6.0
 .. startrev: 91904d5c5188
 
-
+.. branch: use_min_scalar
+Correctly resolve the output dtype of ufunc(array, scalar) calls.

From noreply at buildbot.pypy.org  Wed Jun  3 01:32:54 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Wed,  3 Jun 2015 01:32:54 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: 2to3
Message-ID: <20150602233254.782BE1C034D@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob <me at manueljacob.de>
Branch: py3k
Changeset: r77798:9cbfdd1c8b6b
Date: 2015-06-03 01:32 +0200
http://bitbucket.org/pypy/pypy/changeset/9cbfdd1c8b6b/

Log:	2to3

diff --git a/pypy/module/_cffi_backend/test/test_re_python.py b/pypy/module/_cffi_backend/test/test_re_python.py
--- a/pypy/module/_cffi_backend/test/test_re_python.py
+++ b/pypy/module/_cffi_backend/test/test_re_python.py
@@ -171,7 +171,7 @@
     def test_global_const_nonint(self):
         from re_python_pysrc import ffi
         lib = ffi.dlopen(self.extmod)
-        assert ffi.string(lib.globalconsthello, 8) == "hello"
+        assert ffi.string(lib.globalconsthello, 8) == b"hello"
         raises(AttributeError, ffi.addressof, lib, 'globalconsthello')
 
     def test_rtld_constants(self):
diff --git a/pypy/module/_cffi_backend/test/test_recompiler.py b/pypy/module/_cffi_backend/test/test_recompiler.py
--- a/pypy/module/_cffi_backend/test/test_recompiler.py
+++ b/pypy/module/_cffi_backend/test/test_recompiler.py
@@ -855,7 +855,7 @@
         # but we can get its address
         p = ffi.addressof(lib, 'globvar')
         assert ffi.typeof(p) == ffi.typeof('opaque_t *')
-        assert ffi.string(ffi.cast("char *", p), 8) == "hello"
+        assert ffi.string(ffi.cast("char *", p), 8) == b"hello"
 
     def test_constant_of_value_unknown_to_the_compiler(self):
         extra_c_source = self.udir + self.os_sep + (

From noreply at buildbot.pypy.org  Wed Jun  3 07:58:25 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Wed,  3 Jun 2015 07:58:25 +0200 (CEST)
Subject: [pypy-commit] stmgc c8-gil-like: A branch to have "GIL-like"
 behavior for inevitable transactions: one
Message-ID: <20150603055825.49BFC1C1291@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: c8-gil-like
Changeset: r1788:7bd179c15062
Date: 2015-06-02 16:57 +0200
http://bitbucket.org/pypy/stmgc/changeset/7bd179c15062/

Log:	A branch to have "GIL-like" behavior for inevitable transactions:
	one not-too-short inevitable transaction that is passed around
	multiple threads


From noreply at buildbot.pypy.org  Wed Jun  3 07:58:26 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Wed,  3 Jun 2015 07:58:26 +0200 (CEST)
Subject: [pypy-commit] stmgc c8-gil-like: Plan
Message-ID: <20150603055826.5FC111C1291@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: c8-gil-like
Changeset: r1789:2ee1030752df
Date: 2015-06-03 07:59 +0200
http://bitbucket.org/pypy/stmgc/changeset/2ee1030752df/

Log:	Plan

diff --git a/c8/CALL_RELEASE_GIL b/c8/CALL_RELEASE_GIL
new file mode 100644
--- /dev/null
+++ b/c8/CALL_RELEASE_GIL
@@ -0,0 +1,88 @@
+
+c8-gil-like
+===========
+
+A branch to have "GIL-like" behavior for inevitable transactions: one
+not-too-short inevitable transaction that is passed around multiple
+threads.
+
+The goal is to have good fast-case behavior with the PyPy JIT around
+CALL_RELEASE_GIL.  This is how it works in default (with shadowstack):
+
+
+- "rpy_fastgil" is a global variable.  The value 0 means the GIL is
+  definitely unlocked; the value 1 means it is probably locked (it is
+  actually locked only if some mutex object is acquired too).
+
+- before CALL_RELEASE_GIL, we know that we have the GIL and we need to
+  release it.  So we know that "rpy_fastgil" is 1, and we just write 0
+  there.
+
+- then we do the external call.
+
+- after CALL_RELEASE_GIL, two cases:
+
+  - if "rpy_fastgil" has been changed to 1 by some other thread *or*
+    if the (non-thread-local) shadowstack pointer changed, then we
+    call reacqgil_addr();
+
+  - otherwise, we swap rpy_fastgil back to 1 and we're done.
+
+- a different mechanism is used when we voluntarily release the GIL,
+  based on the mutex mentioned above.  The mutex is also used by the
+  the reacqgil_addr() function if it actually needs to wait.
+
+
+Plan for porting this idea to stmgc:
+
+- we add a few macros to stmgc.h which can be used by C code, around
+  external calls; and we also inline these macros manually around
+  CALL_RELEASE_GIL in PyPy's JIT.
+
+- we add the "detached" mode to inevitable transactions: it means that
+  no thread is actively running this inevitable transaction for now,
+  but it was not committed yet.  It is meant to be reattached, by the
+  same or a different thread.
+
+- we add a global variable, "stm_detached_inevitable_from_thread".  It
+  is equal to the shadowstack pointer of the thread that detached
+  inevitable transaction (like rpy_fastgil == 0), or NULL if there is
+  no detached inevitable transaction (like rpy_fastgil == 1).
+
+- the macro stm_detach_inevitable_transaction() simply writes the
+  current thread's shadowstack pointer into the global variable
+  stm_detached_inevitable_from_thread.  It can only be used if the
+  current transaction is inevitable (and in particular the inevitable
+  transaction was not detached already, because we're running it).
+  After the macro is called, the current thread is assumed not to be
+  running in a transaction any more (no more object or shadowstack
+  access).
+
+- the macro stm_reattach_transaction() does an atomic swap on
+  stm_detached_inevitable_from_thread to change it to NULL.  If the
+  old value was equal to our own shadowstack pointer, we are done.  If
+  not, we call a helper, _stm_reattach_transaction().
+
+- we also add the macro stm_detach_transation().  If the current
+  thread is inevitable it calls stm_detach_inevitable_transaction().
+  Otherwise it calls a helper, _stm_detach_noninevitable_transaction().
+
+- _stm_reattach_transaction(old): called with the old value from
+  stm_detach_inevitable_transaction (which was swapped to be NULL just
+  now).  If old != NULL, this swap had the effect that we took over
+  the inevitable transaction originally detached from a different
+  thread; we need to fix a few things like the shadowstack and %gs but
+  then we can continue running this reattached inevitable transaction.
+  If old == NULL, we need to fall back to the current
+  stm_start_transaction().
+
+- _stm_detach_noninevitable_transaction(): we try to make the
+  transaction inevitable.  If it works we can then use
+  stm_detach_inevitable_transaction().  On the other hand, if we can't
+  make it inevitable without waiting, then instead we just commit it
+  and continue.  In the latter case,
+  stm_detached_inevitable_from_thread is still NULL.
+
+- other place to fix: major collections.  Maybe simply look inside
+  stm_detached_inevitable_from_thread, and if not NULL, grab the
+  inevitable transaction and commit it now.

From noreply at buildbot.pypy.org  Wed Jun  3 08:30:27 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Wed,  3 Jun 2015 08:30:27 +0200 (CEST)
Subject: [pypy-commit] cffi default: ffi.dlopen(None) does not work on
 Windows, and ffi.dlopen("foo") no
Message-ID: <20150603063027.5E3AD1C0F16@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2160:1a2efa6a7f27
Date: 2015-06-03 08:31 +0200
http://bitbucket.org/cffi/cffi/changeset/1a2efa6a7f27/

Log:	ffi.dlopen(None) does not work on Windows, and ffi.dlopen("foo") no
	longer works generally.

diff --git a/doc/source/overview.rst b/doc/source/overview.rst
--- a/doc/source/overview.rst
+++ b/doc/source/overview.rst
@@ -82,9 +82,19 @@
 
     from _simple_example import ffi
 
-    lib = ffi.dlopen(None)         # or path to a library
+    lib = ffi.dlopen(None)        # or path to a library (see below for Windows)
     lib.printf(b"hi there, number %d\n", ffi.cast("int", 2))
 
+Note that this ``ffi.dlopen()``, unlike the one from in-line mode,
+does not invoke any additional magic to locate the library: it must be
+a path name (with or without a directory), as required by the C
+``dlopen()`` or ``LoadLibrary()`` functions.  This means that
+``ffi.dlopen("libfoo.so")`` is ok, but ``ffi.dlopen("foo")`` is not.
+In the latter case, you could replace it with
+``ffi.dlopen(ctypes.util.find_library("foo"))``.  Also, on
+Windows, passing None to open the standard C library does not work;
+try instead ``ffi.dlopen(ctypes.util.find_library("c"))``.
+
 For distribution purposes, remember that there is a new
 ``_simple_example.py`` file generated.  You can either include it
 statically within your project's source files, or, with Setuptools,

From noreply at buildbot.pypy.org  Wed Jun  3 08:43:22 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Wed,  3 Jun 2015 08:43:22 +0200 (CEST)
Subject: [pypy-commit] cffi default: Rewords
Message-ID: <20150603064322.5E6C91C047D@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2161:21154b1d1666
Date: 2015-06-03 08:44 +0200
http://bitbucket.org/cffi/cffi/changeset/21154b1d1666/

Log:	Rewords

diff --git a/doc/source/overview.rst b/doc/source/overview.rst
--- a/doc/source/overview.rst
+++ b/doc/source/overview.rst
@@ -82,7 +82,10 @@
 
     from _simple_example import ffi
 
-    lib = ffi.dlopen(None)        # or path to a library (see below for Windows)
+    lib = ffi.dlopen(None)      # Unix: open the standard C library
+    #import ctypes.util         # or, try this on Windows:
+    #lib = ffi.dlopen(ctypes.util.find_library("c"))
+
     lib.printf(b"hi there, number %d\n", ffi.cast("int", 2))
 
 Note that this ``ffi.dlopen()``, unlike the one from in-line mode,
@@ -91,9 +94,8 @@
 ``dlopen()`` or ``LoadLibrary()`` functions.  This means that
 ``ffi.dlopen("libfoo.so")`` is ok, but ``ffi.dlopen("foo")`` is not.
 In the latter case, you could replace it with
-``ffi.dlopen(ctypes.util.find_library("foo"))``.  Also, on
-Windows, passing None to open the standard C library does not work;
-try instead ``ffi.dlopen(ctypes.util.find_library("c"))``.
+``ffi.dlopen(ctypes.util.find_library("foo"))``.  Also, None is only
+recognized on Unix to open the standard C library.
 
 For distribution purposes, remember that there is a new
 ``_simple_example.py`` file generated.  You can either include it

From noreply at buildbot.pypy.org  Wed Jun  3 08:53:07 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Wed,  3 Jun 2015 08:53:07 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: remember expansion (reduces register
 pressure if constant/variable is use more often). but not for heterogeneous
 expanded vectors, I don't think this happens frequently
Message-ID: <20150603065307.AB6F51C0FB8@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r77799:95db7332c363
Date: 2015-06-03 08:53 +0200
http://bitbucket.org/pypy/pypy/changeset/95db7332c363/

Log:	remember expansion (reduces register pressure if constant/variable
	is use more often). but not for heterogeneous expanded vectors, I
	don't think this happens frequently

diff --git a/rpython/jit/metainterp/optimizeopt/test/test_schedule.py b/rpython/jit/metainterp/optimizeopt/test/test_schedule.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_schedule.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_schedule.py
@@ -108,6 +108,7 @@
         v2[i64#2] = vec_int_pack(v1[i64#2], i0, 0, 1)
         v3[i64#2] = vec_int_pack(v2[i64#2], i1, 1, 1)
         v4[i64#2] = vec_int_expand(73)
+        #
         v5[i64#2] = vec_int_add(v3[i64#2], v4[i64#2])
         """, False)
         self.assert_equal(loop2, loop3)
@@ -123,6 +124,28 @@
         v2[f64#2] = vec_float_pack(v1[f64#2], f0, 0, 1)
         v3[f64#2] = vec_float_pack(v2[f64#2], f1, 1, 1)
         v4[f64#2] = vec_float_expand(73.0)
+        #
         v5[f64#2] = vec_float_add(v3[f64#2], v4[f64#2])
         """, False)
         self.assert_equal(loop2, loop3)
+
+    def test_scalar_remember_expansion(self):
+        loop1 = self.parse("""
+        f10 = float_add(f0, f5)
+        f11 = float_add(f1, f5)
+        f12 = float_add(f10, f5)
+        f13 = float_add(f11, f5)
+        """)
+        pack1 = self.pack(loop1, 0, 2)
+        pack2 = self.pack(loop1, 2, 4)
+        loop2 = self.schedule(loop1, [pack1, pack2], prepend_invariant=True)
+        loop3 = self.parse("""
+        v1[f64#2] = vec_box(2)
+        v2[f64#2] = vec_float_pack(v1[f64#2], f0, 0, 1)
+        v3[f64#2] = vec_float_pack(v2[f64#2], f1, 1, 1)
+        v4[f64#2] = vec_float_expand(f5) # only expaned once
+        #
+        v5[f64#2] = vec_float_add(v3[f64#2], v4[f64#2])
+        v6[f64#2] = vec_float_add(v5[f64#2], v4[f64#2])
+        """, False)
+        self.assert_equal(loop2, loop3)
diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/vectorize.py
@@ -1052,11 +1052,17 @@
     def expand(self, nodes, arg, argidx):
         vbox = self.input_type.new_vector_box(len(nodes))
         box_type = arg.type
+        expanded_map = self.sched_data.expanded_map
         invariant_ops = self.sched_data.invariant_oplist
         invariant_vars = self.sched_data.invariant_vector_vars
         if isinstance(arg, BoxVector):
             box_type = arg.item_type
 
+        # note that heterogenous nodes are not yet tracked
+        already_expanded = expanded_map.get(arg, None)
+        if already_expanded:
+            return already_expanded
+
         for i, node in enumerate(nodes):
             op = node.getoperation()
             if not arg.same_box(op.getarg(argidx)):
@@ -1069,6 +1075,7 @@
             op = ResOperation(expand_opnum, [arg], vbox)
             invariant_ops.append(op)
             invariant_vars.append(vbox)
+            expanded_map[arg] = vbox
             return vbox
 
         op = ResOperation(rop.VEC_BOX, [ConstInt(len(nodes))], vbox)
@@ -1085,6 +1092,7 @@
             op = ResOperation(opnum, [vbox,arg,ci,c1], new_box)
             vbox = new_box
             invariant_ops.append(op)
+
         invariant_vars.append(vbox)
         return vbox
 
@@ -1239,6 +1247,7 @@
         self.vec_reg_size = vec_reg_size
         self.invariant_oplist = []
         self.invariant_vector_vars = []
+        self.expanded_map = {}
 
     def as_vector_operation(self, pack):
         op_count = len(pack.operations)

From noreply at buildbot.pypy.org  Wed Jun  3 09:17:19 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Wed,  3 Jun 2015 09:17:19 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: various random rpython fixes
Message-ID: <20150603071719.304FA1C024E@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77800:402518a1edbf
Date: 2015-06-02 19:17 +0200
http://bitbucket.org/pypy/pypy/changeset/402518a1edbf/

Log:	various random rpython fixes

diff --git a/rpython/jit/metainterp/executor.py b/rpython/jit/metainterp/executor.py
--- a/rpython/jit/metainterp/executor.py
+++ b/rpython/jit/metainterp/executor.py
@@ -495,34 +495,44 @@
         check_descr(descr)
         if arity == -1:
             func = get_execute_function(opnum, -1, True)
-            return func(cpu, metainterp, argboxes, descr)
+            if func:
+                return func(cpu, metainterp, argboxes, descr)
         if arity == 0:
             func = get_execute_function(opnum, 0, True)
-            return func(cpu, metainterp, descr)
+            if func:
+                return func(cpu, metainterp, descr)
         if arity == 1:
             func = get_execute_function(opnum, 1, True)
-            return func(cpu, metainterp, argboxes[0], descr)
+            if func:
+                return func(cpu, metainterp, argboxes[0], descr)
         if arity == 2:
             func = get_execute_function(opnum, 2, True)
-            return func(cpu, metainterp, argboxes[0], argboxes[1], descr)
+            if func:
+                return func(cpu, metainterp, argboxes[0], argboxes[1], descr)
         if arity == 3:
             func = get_execute_function(opnum, 3, True)
-            return func(cpu, metainterp, argboxes[0], argboxes[1], argboxes[2],
-                        descr)
+            if func:
+                return func(cpu, metainterp, argboxes[0], argboxes[1],
+                            argboxes[2], descr)
     else:
         assert descr is None
         if arity == 1:
             func = get_execute_function(opnum, 1, False)
-            return func(cpu, metainterp, argboxes[0])
+            if func:
+                return func(cpu, metainterp, argboxes[0])
         if arity == 2:
             func = get_execute_function(opnum, 2, False)
-            return func(cpu, metainterp, argboxes[0], argboxes[1])
+            if func:
+                return func(cpu, metainterp, argboxes[0], argboxes[1])
         if arity == 3:
             func = get_execute_function(opnum, 3, False)
-            return func(cpu, metainterp, argboxes[0], argboxes[1], argboxes[2])
+            if func:
+                return func(cpu, metainterp, argboxes[0], argboxes[1],
+                            argboxes[2])
         if arity == 5:    # copystrcontent, copyunicodecontent
             func = get_execute_function(opnum, 5, False)
-            return func(cpu, metainterp, argboxes[0], argboxes[1],
+            if func:
+                return func(cpu, metainterp, argboxes[0], argboxes[1],
                         argboxes[2], argboxes[3], argboxes[4])
     raise NotImplementedError
 
diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py
--- a/rpython/jit/metainterp/history.py
+++ b/rpython/jit/metainterp/history.py
@@ -168,7 +168,7 @@
         return ConstPtr(lltype.nullptr(llmemory.GCREF.TO))
     elif lltype.typeOf(value) == lltype.Signed:
         return ConstInt(value)
-    elif type(value) is bool:
+    elif not we_are_translated() and type(value) is bool:
         return ConstInt(int(value))
     elif isinstance(value, float):
         return ConstFloat(value)
diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py
--- a/rpython/jit/metainterp/optimizeopt/info.py
+++ b/rpython/jit/metainterp/optimizeopt/info.py
@@ -41,7 +41,7 @@
     def same_info(self, other):
         return self is other
 
-    def getstrlen(self, arg, opt, mode, create_ops=True):
+    def getstrlen(self, op, string_optimizer, mode, create_ops=True):
         return None
 
     
@@ -218,12 +218,12 @@
                 count += 1
         return count
 
-    def setitem(self, index, item, cf=None, optheap=None):
+    def setitem(self, index, op, cf=None, optheap=None):
         if self._items is None:
             self._items = [None] * (index + 1)
         if index >= len(self._items):
             self._items = self._items + [None] * (index - len(self._items) + 1)
-        self._items[index] = item
+        self._items[index] = op
         if cf is not None:
             assert not self.is_virtual()
             cf.register_dirty_field(self)
diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py
--- a/rpython/jit/metainterp/optimizeopt/rewrite.py
+++ b/rpython/jit/metainterp/optimizeopt/rewrite.py
@@ -430,7 +430,7 @@
             if b.getint() == 0:
                 self.last_emitted_operation = REMOVED
                 return
-            opnum = OpHelpers.call_for_type(op)
+            opnum = OpHelpers.call_for_type(op.type)
             op = op.copy_and_change(opnum, args=op.getarglist()[1:])
         self.emit_operation(op)
 
diff --git a/rpython/jit/metainterp/optimizeopt/vstring.py b/rpython/jit/metainterp/optimizeopt/vstring.py
--- a/rpython/jit/metainterp/optimizeopt/vstring.py
+++ b/rpython/jit/metainterp/optimizeopt/vstring.py
@@ -71,6 +71,7 @@
                 self.lenbound = intutils.ConstIntBound(self.length)
         return self.lenbound
 
+    @specialize.arg(2)
     def get_constant_string_spec(self, string_optimizer, mode):
         return None # can't be constant
 
@@ -144,8 +145,8 @@
             self._chars = [None] * length
         StrPtrInfo.__init__(self, mode, is_virtual, length)
 
-    def setitem(self, index, item, cf=None, optheap=None):
-        self._chars[index] = item
+    def setitem(self, index, op, cf=None, optheap=None):
+        self._chars[index] = op
 
     def setup_slice(self, longerlist, start, stop):
         assert 0 <= start <= stop <= len(longerlist)
diff --git a/rpython/jit/metainterp/quasiimmut.py b/rpython/jit/metainterp/quasiimmut.py
--- a/rpython/jit/metainterp/quasiimmut.py
+++ b/rpython/jit/metainterp/quasiimmut.py
@@ -101,6 +101,11 @@
 
 
 class QuasiImmutDescr(AbstractDescr):
+    # those fields are necessary for translation without quasi immutable
+    # fields
+    struct = None
+    fielddescr = None
+    
     def __init__(self, cpu, struct, fielddescr, mutatefielddescr):
         self.cpu = cpu
         self.struct = struct
@@ -110,7 +115,8 @@
         self.constantfieldbox = self.get_current_constant_fieldvalue()
 
     def get_parent_descr(self):
-        return self.fielddescr.get_parent_descr()
+        if self.fielddescr is not None:
+            return self.fielddescr.get_parent_descr()
 
     def get_current_constant_fieldvalue(self):
         struct = self.struct

From noreply at buildbot.pypy.org  Wed Jun  3 09:17:20 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Wed,  3 Jun 2015 09:17:20 +0200 (CEST)
Subject: [pypy-commit] pypy default: seems we fixed ll2ctypes
Message-ID: <20150603071720.608441C024E@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r77801:4a7ddd10e8f8
Date: 2015-06-03 09:15 +0200
http://bitbucket.org/pypy/pypy/changeset/4a7ddd10e8f8/

Log:	seems we fixed ll2ctypes

diff --git a/rpython/jit/backend/x86/test/test_basic.py b/rpython/jit/backend/x86/test/test_basic.py
--- a/rpython/jit/backend/x86/test/test_basic.py
+++ b/rpython/jit/backend/x86/test/test_basic.py
@@ -32,10 +32,5 @@
         res = self.meta_interp(f, [31], enable_opts='')
         assert res == -4
 
-    def test_r_dict(self):
-        # a Struct that belongs to the hash table is not seen as being
-        # included in the larger Array
-        py.test.skip("issue with ll2ctypes")
-
     def test_free_object(self):
         py.test.skip("issue of freeing, probably with ll2ctypes")

From noreply at buildbot.pypy.org  Wed Jun  3 09:17:22 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Wed,  3 Jun 2015 09:17:22 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge
Message-ID: <20150603071722.20E541C024E@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r77802:05a0a598cdd6
Date: 2015-06-03 09:17 +0200
http://bitbucket.org/pypy/pypy/changeset/05a0a598cdd6/

Log:	merge

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -5,4 +5,5 @@
 .. this is a revision shortly after release-2.6.0
 .. startrev: 91904d5c5188
 
-
+.. branch: use_min_scalar
+Correctly resolve the output dtype of ufunc(array, scalar) calls.
diff --git a/pypy/module/_cffi_backend/ffi_obj.py b/pypy/module/_cffi_backend/ffi_obj.py
--- a/pypy/module/_cffi_backend/ffi_obj.py
+++ b/pypy/module/_cffi_backend/ffi_obj.py
@@ -95,6 +95,23 @@
             else:
                 raise KeyError    # don't handle this error case here
 
+    def _ffi_bad_type(self, input_text):
+        info = self.ctxobj.info
+        errmsg = rffi.charp2str(info.c_error_message)
+        if len(input_text) > 500:
+            raise oefmt(self.w_FFIError, "%s", errmsg)
+        printable_text = ['?'] * len(input_text)
+        for i in range(len(input_text)):
+            if ' ' <= input_text[i] < '\x7f':
+                printable_text[i] = input_text[i]
+            elif input_text[i] == '\t' or input_text[i] == '\n':
+                printable_text[i] = ' '
+        num_spaces = rffi.getintfield(info, 'c_error_location')
+        raise oefmt(self.w_FFIError, "%s\n%s\n%s^",
+                    rffi.charp2str(info.c_error_message),
+                    ''.join(printable_text),
+                    " " * num_spaces)
+
     @jit.dont_look_inside
     def parse_string_to_type(self, string, consider_fn_as_fnptr):
         # This cannot be made @elidable because it calls general space
@@ -108,11 +125,7 @@
             info = self.ctxobj.info
             index = parse_c_type.parse_c_type(info, string)
             if index < 0:
-                num_spaces = rffi.getintfield(info, 'c_error_location')
-                raise oefmt(self.w_FFIError, "%s\n%s\n%s^",
-                            rffi.charp2str(info.c_error_message),
-                            string,
-                            " " * num_spaces)
+                raise self._ffi_bad_type(string)
             x = realize_c_type.realize_c_type_or_func(
                 self, self.ctxobj.info.c_output, index)
             assert x is not None
diff --git a/pypy/module/_cffi_backend/test/test_ffi_obj.py b/pypy/module/_cffi_backend/test/test_ffi_obj.py
--- a/pypy/module/_cffi_backend/test/test_ffi_obj.py
+++ b/pypy/module/_cffi_backend/test/test_ffi_obj.py
@@ -181,6 +181,12 @@
         assert str(e.value) == ("undefined struct/union name\n"
                                 "struct never_heard_of_s\n"
                                 "       ^")
+        e = raises(ffi.error, ffi.cast, "\t\n\x01\x1f~\x7f\x80\xff", 0)
+        assert str(e.value) == ("identifier expected\n"
+                                "  ??~???\n"
+                                "  ^")
+        e = raises(ffi.error, ffi.cast, "X" * 600, 0)
+        assert str(e.value) == ("undefined type name")
 
     def test_ffi_buffer(self):
         import _cffi_backend as _cffi1_backend
diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py
--- a/pypy/module/micronumpy/test/test_dtypes.py
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -1129,6 +1129,20 @@
         exc = raises(ValueError, "dtype([('a', '<i8'), ('a', '<f8')])")
         assert exc.value[0] == 'two fields with the same name'
 
+    def test_array_from_record(self):
+        import numpy as np
+        a = np.array(('???', -999, -12345678.9), 
+                     dtype=[('c', '|S3'), ('a', '<i8'), ('b', '<f8')])
+        # Change the order of the keys
+        b = np.array(a, dtype=[('a', '<i8'), ('b', '<f8'), ('c', '|S3')])
+        assert b.base is None
+        assert b.dtype.fields['a'][1] == 0
+        assert b['a'] == -999
+        a = np.array(('N/A', 1e+20, 1e+20, 999999),
+                     dtype=[('name', '|S4'), ('x', '<f8'), 
+                            ('y', '<f8'), ('block', '<i8', (2, 3))])
+        assert (a['block'] == 999999).all()
+
     def test_create_from_dict(self):
         import numpy as np
         import sys
diff --git a/pypy/module/micronumpy/test/test_subtype.py b/pypy/module/micronumpy/test/test_subtype.py
--- a/pypy/module/micronumpy/test/test_subtype.py
+++ b/pypy/module/micronumpy/test/test_subtype.py
@@ -656,6 +656,7 @@
                 self.output += 'In __array_wrap__:'
                 self.output += '   self is %s' % repr(self)
                 self.output += '   arr is %r\n' % (out_arr,)
+                self.output += '   context is %r\n' % (context,)
                 # then just call the parent
                 ret = np.ndarray.__array_wrap__(self, out_arr, context)
                 print 'wrap',self.output
diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -1349,3 +1349,6 @@
         assert np.add(np.float16(0), np.longdouble(0)).dtype == np.longdouble
         assert np.add(np.float16(0), np.complex64(0)).dtype == np.complex64
         assert np.add(np.float16(0), np.complex128(0)).dtype == np.complex128
+        assert np.add(np.zeros(5, dtype=np.int8), 257).dtype == np.int16
+        assert np.subtract(np.zeros(5, dtype=np.int8), 257).dtype == np.int16
+        assert np.divide(np.zeros(5, dtype=np.int8), 257).dtype == np.int16
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -20,7 +20,7 @@
      cast_gcref_to_instance
 from rpython.rtyper.lltypesystem import lltype, rffi, llmemory
 from rpython.tool.sourcetools import func_with_new_name
-from pypy.module.micronumpy import boxes
+from pypy.module.micronumpy import boxes, support
 from pypy.module.micronumpy.concrete import SliceArray, VoidBoxStorage, V_OBJECTSTORE
 from pypy.module.micronumpy.strides import calc_strides
 from . import constants as NPY
@@ -2265,10 +2265,12 @@
     def _coerce(self, space, arr, ofs, dtype, w_items, shape):
         # TODO: Make sure the shape and the array match
         from pypy.module.micronumpy.descriptor import W_Dtype
-        if w_items is not None:
+        if w_items is None:
+            items_w = [None] * shape[0]
+        elif support.issequence_w(space, w_items):
             items_w = space.fixedview(w_items)
         else:
-            items_w = [None] * shape[0]
+            items_w = [w_items] * shape[0]
         subdtype = dtype.subdtype
         assert isinstance(subdtype, W_Dtype)
         itemtype = subdtype.itemtype
@@ -2363,8 +2365,16 @@
     def coerce(self, space, dtype, w_item):
         from pypy.module.micronumpy.base import W_NDimArray
         if isinstance(w_item, boxes.W_VoidBox):
-            return w_item
-        if w_item is not None:
+            if dtype == w_item.dtype:
+                return w_item
+            else:
+                # match up the field names
+                items_w = [None] * len(dtype.fields)
+                for i in range(len(dtype.fields)):
+                    name = dtype.names[i]
+                    if name in w_item.dtype.names:
+                        items_w[i] = w_item.descr_getitem(space, space.wrap(name))
+        elif w_item is not None:
             if space.isinstance_w(w_item, space.w_tuple):
                 if len(dtype.fields) != space.len_w(w_item):
                     raise OperationError(space.w_ValueError, space.wrap(
diff --git a/pypy/module/micronumpy/ufuncs.py b/pypy/module/micronumpy/ufuncs.py
--- a/pypy/module/micronumpy/ufuncs.py
+++ b/pypy/module/micronumpy/ufuncs.py
@@ -21,7 +21,8 @@
 from pypy.module.micronumpy.support import (_parse_signature, product,
         get_storage_as_int, is_rhs_priority_higher)
 from .casting import (
-    can_cast_type, can_cast_to, find_result_type, promote_types)
+    can_cast_type, can_cast_array, can_cast_to,
+    find_result_type, promote_types)
 from .boxes import W_GenericBox, W_ObjectBox
 
 def done_if_true(dtype, val):
@@ -474,7 +475,8 @@
         if out is None:
             if w_res.is_scalar():
                 return w_res.get_scalar_value()
-            w_res = space.call_method(w_obj, '__array_wrap__', w_res)
+            ctxt = space.newtuple([self, space.newtuple([w_obj]), space.wrap(0)])
+            w_res = space.call_method(w_obj, '__array_wrap__', w_res, ctxt)
         return w_res
 
     def call_scalar(self, space, w_arg, in_dtype):
@@ -494,17 +496,12 @@
         return dt_in, dt_out, self.func
 
     def _calc_dtype(self, space, arg_dtype, out=None, casting='unsafe'):
-        use_min_scalar = False
         if arg_dtype.is_object():
             return arg_dtype, arg_dtype
         in_casting = safe_casting_mode(casting)
         for dt_in, dt_out in self.dtypes:
-            if use_min_scalar:
-                if not can_cast_array(space, w_arg, dt_in, in_casting):
-                    continue
-            else:
-                if not can_cast_type(space, arg_dtype, dt_in, in_casting):
-                    continue
+            if not can_cast_type(space, arg_dtype, dt_in, in_casting):
+                continue
             if out is not None:
                 res_dtype = out.get_dtype()
                 if not can_cast_type(space, dt_out, res_dtype, casting):
@@ -604,21 +601,18 @@
                             w_rdtype.get_name(), w_ldtype.get_name(),
                             self.name)
 
-        if self.are_common_types(w_ldtype, w_rdtype):
-            if not w_lhs.is_scalar() and w_rhs.is_scalar():
-                w_rdtype = w_ldtype
-            elif w_lhs.is_scalar() and not w_rhs.is_scalar():
-                w_ldtype = w_rdtype
-        calc_dtype, dt_out, func = self.find_specialization(space, w_ldtype, w_rdtype, out, casting)
         if (isinstance(w_lhs, W_GenericBox) and
                 isinstance(w_rhs, W_GenericBox) and out is None):
-            return self.call_scalar(space, w_lhs, w_rhs, calc_dtype)
+            return self.call_scalar(space, w_lhs, w_rhs, casting)
         if isinstance(w_lhs, W_GenericBox):
             w_lhs = W_NDimArray.from_scalar(space, w_lhs)
         assert isinstance(w_lhs, W_NDimArray)
         if isinstance(w_rhs, W_GenericBox):
             w_rhs = W_NDimArray.from_scalar(space, w_rhs)
         assert isinstance(w_rhs, W_NDimArray)
+        calc_dtype, dt_out, func = self.find_specialization(
+            space, w_ldtype, w_rdtype, out, casting, w_lhs, w_rhs)
+
         new_shape = shape_agreement(space, w_lhs.get_shape(), w_rhs)
         new_shape = shape_agreement(space, new_shape, out, broadcast_down=False)
         w_highpriority, out_subtype = array_priority(space, w_lhs, w_rhs)
@@ -632,10 +626,14 @@
         if out is None:
             if w_res.is_scalar():
                 return w_res.get_scalar_value()
-            w_res = space.call_method(w_highpriority, '__array_wrap__', w_res)
+            ctxt = space.newtuple([self, space.newtuple([w_lhs, w_rhs]), space.wrap(0)])
+            w_res = space.call_method(w_highpriority, '__array_wrap__', w_res, ctxt)
         return w_res
 
-    def call_scalar(self, space, w_lhs, w_rhs, in_dtype):
+    def call_scalar(self, space, w_lhs, w_rhs, casting):
+        in_dtype, out_dtype, func = self.find_specialization(
+            space, w_lhs.get_dtype(space), w_rhs.get_dtype(space),
+            out=None, casting=casting)
         w_val = self.func(in_dtype,
                           w_lhs.convert_to(space, in_dtype),
                           w_rhs.convert_to(space, in_dtype))
@@ -643,7 +641,8 @@
             return w_val.w_obj
         return w_val
 
-    def _find_specialization(self, space, l_dtype, r_dtype, out, casting):
+    def _find_specialization(self, space, l_dtype, r_dtype, out, casting,
+                             w_arg1, w_arg2):
         if (not self.allow_bool and (l_dtype.is_bool() or
                                          r_dtype.is_bool()) or
                 not self.allow_complex and (l_dtype.is_complex() or
@@ -655,15 +654,23 @@
             dtype = find_result_type(space, [], [l_dtype, r_dtype])
             bool_dtype = get_dtype_cache(space).w_booldtype
             return dtype, bool_dtype, self.func
-        dt_in, dt_out = self._calc_dtype(space, l_dtype, r_dtype, out, casting)
+        dt_in, dt_out = self._calc_dtype(
+            space, l_dtype, r_dtype, out, casting, w_arg1, w_arg2)
         return dt_in, dt_out, self.func
 
-    def find_specialization(self, space, l_dtype, r_dtype, out, casting):
+    def find_specialization(self, space, l_dtype, r_dtype, out, casting,
+                            w_arg1=None, w_arg2=None):
         if self.simple_binary:
             if out is None and not (l_dtype.is_object() or r_dtype.is_object()):
-                dtype = promote_types(space, l_dtype, r_dtype)
+                if w_arg1 is not None and w_arg2 is not None:
+                    w_arg1 = convert_to_array(space, w_arg1)
+                    w_arg2 = convert_to_array(space, w_arg2)
+                    dtype = find_result_type(space, [w_arg1, w_arg2], [])
+                else:
+                    dtype = promote_types(space, l_dtype, r_dtype)
                 return dtype, dtype, self.func
-        return self._find_specialization(space, l_dtype, r_dtype, out, casting)
+        return self._find_specialization(
+            space, l_dtype, r_dtype, out, casting, w_arg1, w_arg2)
 
     def find_binop_type(self, space, dtype):
         """Find a valid dtype signature of the form xx->x"""
@@ -684,15 +691,21 @@
             "requested type has type code '%s'" % (self.name, dtype.char))
 
 
-    def _calc_dtype(self, space, l_dtype, r_dtype, out=None, casting='unsafe'):
-        use_min_scalar = False
+    def _calc_dtype(self, space, l_dtype, r_dtype, out, casting,
+                    w_arg1, w_arg2):
         if l_dtype.is_object() or r_dtype.is_object():
             dtype = get_dtype_cache(space).w_objectdtype
             return dtype, dtype
+        use_min_scalar = (w_arg1 is not None and w_arg2 is not None and
+                          ((w_arg1.is_scalar() and not w_arg2.is_scalar()) or
+                           (not w_arg1.is_scalar() and w_arg2.is_scalar())))
         in_casting = safe_casting_mode(casting)
         for dt_in, dt_out in self.dtypes:
             if use_min_scalar:
-                if not can_cast_array(space, w_arg, dt_in, in_casting):
+                w_arg1 = convert_to_array(space, w_arg1)
+                w_arg2 = convert_to_array(space, w_arg2)
+                if not (can_cast_array(space, w_arg1, dt_in, in_casting) and
+                        can_cast_array(space, w_arg2, dt_in, in_casting)):
                     continue
             else:
                 if not (can_cast_type(space, l_dtype, dt_in, in_casting) and

From noreply at buildbot.pypy.org  Wed Jun  3 09:22:16 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Wed,  3 Jun 2015 09:22:16 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: start implementing virtual array of
	structs
Message-ID: <20150603072216.DDD351C0F16@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77803:6de34441a3cc
Date: 2015-06-03 09:21 +0200
http://bitbucket.org/pypy/pypy/changeset/6de34441a3cc/

Log:	start implementing virtual array of structs

diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -164,11 +164,16 @@
             rffi.cast(TYPE, -1) == -1)
 
 class ArrayDescr(AbstractDescr):
+    all_interiorfielddescrs = None
+    
     def __init__(self, A, runner):
         self.A = self.OUTERA = A
         if isinstance(A, lltype.Struct):
             self.A = A._flds[A._arrayfld]
 
+    def get_all_fielddescrs(self):
+        return self.all_interiorfielddescrs
+
     def __repr__(self):
         return 'ArrayDescr(%r)' % (self.OUTERA,)
 
@@ -211,6 +216,9 @@
         self.arraydescr = runner.arraydescrof(A)
         self.fielddescr = runner.fielddescrof(A.OF, fieldname)
 
+    def get_arraydescr(self):
+        return self.arraydescr
+
     def __repr__(self):
         return 'InteriorFieldDescr(%r, %r)' % (self.A, self.fieldname)
 
diff --git a/rpython/jit/backend/llsupport/descr.py b/rpython/jit/backend/llsupport/descr.py
--- a/rpython/jit/backend/llsupport/descr.py
+++ b/rpython/jit/backend/llsupport/descr.py
@@ -223,6 +223,7 @@
     lendescr = None
     flag = '\x00'
     vinfo = None
+    all_interiorfielddescrs = None
 
     def __init__(self, basesize, itemsize, lendescr, flag):
         self.basesize = basesize
@@ -230,6 +231,9 @@
         self.lendescr = lendescr    # or None, if no length
         self.flag = flag
 
+    def get_all_fielddescrs(self):
+        return self.all_interiorfielddescrs
+
     def is_array_of_pointers(self):
         return self.flag == FLAG_POINTER
 
@@ -285,9 +289,13 @@
             lendescr = get_field_arraylen_descr(gccache, ARRAY_OR_STRUCT)
         flag = get_type_flag(ARRAY_INSIDE.OF)
         arraydescr = ArrayDescr(basesize, itemsize, lendescr, flag)
+        cache[ARRAY_OR_STRUCT] = arraydescr
+        if isinstance(ARRAY_INSIDE.OF, lltype.Struct):
+            descrs = heaptracker.all_interiorfielddescrs(gccache,
+                ARRAY_INSIDE, get_field_descr=get_interiorfield_descr)
+            arraydescr.all_interiorfielddescrs = descrs
         if ARRAY_OR_STRUCT._gckind == 'gc':
             gccache.init_array_descr(ARRAY_OR_STRUCT, arraydescr)
-        cache[ARRAY_OR_STRUCT] = arraydescr
         return arraydescr
 
 
diff --git a/rpython/jit/backend/x86/test/test_basic.py b/rpython/jit/backend/x86/test/test_basic.py
--- a/rpython/jit/backend/x86/test/test_basic.py
+++ b/rpython/jit/backend/x86/test/test_basic.py
@@ -32,10 +32,5 @@
         res = self.meta_interp(f, [31], enable_opts='')
         assert res == -4
 
-    def test_r_dict(self):
-        # a Struct that belongs to the hash table is not seen as being
-        # included in the larger Array
-        py.test.skip("issue with ll2ctypes")
-
     def test_free_object(self):
         py.test.skip("issue of freeing, probably with ll2ctypes")
diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py
--- a/rpython/jit/metainterp/history.py
+++ b/rpython/jit/metainterp/history.py
@@ -168,7 +168,7 @@
         return ConstPtr(lltype.nullptr(llmemory.GCREF.TO))
     elif lltype.typeOf(value) == lltype.Signed:
         return ConstInt(value)
-    elif not we_are_translated() and type(value) is bool:
+    elif isinstance(value, bool):
         return ConstInt(int(value))
     elif isinstance(value, float):
         return ConstFloat(value)
@@ -762,7 +762,7 @@
         op = ResOperation(opnum, argboxes, descr)
         if value is None:
             assert op.type == 'v'
-        elif not we_are_translated() and type(value) is bool:
+        elif isinstance(value, bool):
             assert op.type == 'i'
             op.setint(int(value))
         elif isinstance(value, float):
diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py
--- a/rpython/jit/metainterp/optimizeopt/info.py
+++ b/rpython/jit/metainterp/optimizeopt/info.py
@@ -260,7 +260,6 @@
         self._items = [None] * (size * lgt)
 
     def _compute_index(self, index, fielddescr):
-        raise Exception("implement virtual array of structs")
         one_size = len(fielddescr.get_arraydescr().get_all_fielddescrs())
         return index * one_size + fielddescr.fielddescr.get_index()
         
diff --git a/rpython/jit/metainterp/test/test_executor.py b/rpython/jit/metainterp/test/test_executor.py
--- a/rpython/jit/metainterp/test/test_executor.py
+++ b/rpython/jit/metainterp/test/test_executor.py
@@ -2,7 +2,7 @@
 import sys, random
 from rpython.rlib.rarithmetic import r_uint, intmask
 from rpython.jit.metainterp.executor import execute, wrap_constant
-from rpython.jit.metainterp.executor import execute_varargs, _execute_nonspec
+from rpython.jit.metainterp.executor import execute_varargs, _execute_arglist
 from rpython.jit.metainterp.resoperation import rop, opname, opclasses,\
      InputArgInt, InputArgFloat, InputArgRef
 from rpython.jit.metainterp.history import ConstInt, ConstPtr, ConstFloat

From noreply at buildbot.pypy.org  Wed Jun  3 10:40:45 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Wed,  3 Jun 2015 10:40:45 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c8-gil-like: A branch to simplify
 stmgcintf.* and see exactly what kind of interface
Message-ID: <20150603084045.4D4AD1C024E@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c8-gil-like
Changeset: r77804:927dfa4e5b29
Date: 2015-06-02 16:48 +0200
http://bitbucket.org/pypy/pypy/changeset/927dfa4e5b29/

Log:	A branch to simplify stmgcintf.* and see exactly what kind of
	interface from stmgc.h we'd need to implement a "GIL-like" behavior
	for inevitable transactions: one not-too-short inevitable
	transaction that is passed around multiple threads


From noreply at buildbot.pypy.org  Wed Jun  3 10:40:46 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Wed,  3 Jun 2015 10:40:46 +0200 (CEST)
Subject: [pypy-commit] pypy default: Use .decode("latin1") instead of
 .decode("ascii") here, which is safer
Message-ID: <20150603084046.691311C024E@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r77805:5cf9f578ca18
Date: 2015-06-03 10:40 +0200
http://bitbucket.org/pypy/pypy/changeset/5cf9f578ca18/

Log:	Use .decode("latin1") instead of .decode("ascii") here, which is
	safer and more in line with what CPython does

diff --git a/pypy/objspace/std/newformat.py b/pypy/objspace/std/newformat.py
--- a/pypy/objspace/std/newformat.py
+++ b/pypy/objspace/std/newformat.py
@@ -528,7 +528,7 @@
 
         def _lit(self, s):
             if self.is_unicode:
-                return s.decode("ascii")
+                return s.decode("latin1")
             else:
                 return s
 
@@ -586,8 +586,8 @@
                 thousands = ""
                 grouping = "\xFF"    # special value to mean 'stop'
             if self.is_unicode:
-                self._loc_dec = dec.decode("ascii")
-                self._loc_thousands = thousands.decode("ascii")
+                self._loc_dec = dec.decode("latin1")
+                self._loc_thousands = thousands.decode("latin1")
             else:
                 self._loc_dec = dec
                 self._loc_thousands = thousands
@@ -725,7 +725,7 @@
                 out.append_multiple_char(fill_char[0], spec.n_lpadding)
             if spec.n_sign:
                 if self.is_unicode:
-                    sign = spec.sign.decode("ascii")
+                    sign = spec.sign.decode("latin1")
                 else:
                     sign = spec.sign
                 out.append(sign)
@@ -828,14 +828,14 @@
                 prefix = "0x"
             as_str = value.format(LONG_DIGITS[:base], prefix)
             if self.is_unicode:
-                return as_str.decode("ascii")
+                return as_str.decode("latin1")
             return as_str
 
         def _int_to_base(self, base, value):
             if base == 10:
                 s = str(value)
                 if self.is_unicode:
-                    return s.decode("ascii")
+                    return s.decode("latin1")
                 return s
             # This part is slow.
             negative = value < 0
@@ -954,7 +954,7 @@
             have_dec_point, to_remainder = self._parse_number(result, to_number)
             n_remainder = len(result) - to_remainder
             if self.is_unicode:
-                digits = result.decode("ascii")
+                digits = result.decode("latin1")
             else:
                 digits = result
             spec = self._calc_num_width(0, sign, to_number, n_digits,
@@ -1059,8 +1059,8 @@
                                                                to_imag_number)
 
             if self.is_unicode:
-                re_num = re_num.decode("ascii")
-                im_num = im_num.decode("ascii")
+                re_num = re_num.decode("latin1")
+                im_num = im_num.decode("latin1")
 
             #set remainder, in CPython _parse_number sets this
             #using n_re_digits causes tests to fail

From noreply at buildbot.pypy.org  Wed Jun  3 11:02:47 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Wed,  3 Jun 2015 11:02:47 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: fight resume of virtual arrays of
	structs
Message-ID: <20150603090247.B04181C034D@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77806:240e3bad4204
Date: 2015-06-03 09:38 +0200
http://bitbucket.org/pypy/pypy/changeset/240e3bad4204/

Log:	fight resume of virtual arrays of structs

diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -219,6 +219,9 @@
     def get_arraydescr(self):
         return self.arraydescr
 
+    def get_fielddescr(self):
+        return self.fielddescr
+
     def __repr__(self):
         return 'InteriorFieldDescr(%r, %r)' % (self.A, self.fieldname)
 
diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py
--- a/rpython/jit/metainterp/optimizeopt/info.py
+++ b/rpython/jit/metainterp/optimizeopt/info.py
@@ -238,7 +238,7 @@
 
     def visitor_walk_recursive(self, instbox, visitor, optimizer):
         itemops = [optimizer.get_box_replacement(item)
-                   for item in self._items if item]
+                   for item in self._items]
         visitor.register_virtual_fields(instbox, itemops)
         for i in range(self.getlength()):
             itemop = self._items[i]
@@ -289,6 +289,29 @@
                 i += 1
         return count
 
+    def visitor_walk_recursive(self, instbox, visitor, optimizer):
+        itemops = [optimizer.get_box_replacement(item)
+                   for item in self._items]
+        visitor.register_virtual_fields(instbox, itemops)
+        fielddescrs = self.vdescr.get_all_fielddescrs()
+        i = 0
+        for index in range(self.getlength()):
+            for flddescr in fielddescrs:
+                itemop = self._items[i]
+                if (itemop is not None and itemop.type == 'r' and
+                    not isinstance(itemop, Const)):
+                    ptrinfo = optimizer.getptrinfo(itemop)
+                    if ptrinfo and ptrinfo.is_virtual():
+                        ptrinfo.visitor_walk_recursive(itemop, visitor,
+                                                       optimizer)
+                i += 1
+
+    @specialize.argtype(1)
+    def visitor_dispatch_virtual_type(self, visitor):
+        flddescrs = self.vdescr.get_all_fielddescrs()
+        return visitor.visit_varraystruct(self.vdescr, self.getlength(),
+                                          flddescrs)
+
 class ConstPtrInfo(PtrInfo):
     _attrs_ = ('_const',)
     
diff --git a/rpython/jit/metainterp/resume.py b/rpython/jit/metainterp/resume.py
--- a/rpython/jit/metainterp/resume.py
+++ b/rpython/jit/metainterp/resume.py
@@ -314,8 +314,8 @@
         else:
             return VArrayInfoNotClear(arraydescr)
 
-    def visit_varraystruct(self, arraydescr, fielddescrs):
-        return VArrayStructInfo(arraydescr, fielddescrs)
+    def visit_varraystruct(self, arraydescr, size, fielddescrs):
+        return VArrayStructInfo(arraydescr, size, fielddescrs)
 
     def visit_vrawbuffer(self, size, offsets, descrs):
         return VRawBufferInfo(size, offsets, descrs)
@@ -676,7 +676,8 @@
 
 
 class VArrayStructInfo(AbstractVirtualInfo):
-    def __init__(self, arraydescr, fielddescrs):
+    def __init__(self, arraydescr, size, fielddescrs):
+        self.size = size
         self.arraydescr = arraydescr
         self.fielddescrs = fielddescrs
 
@@ -687,14 +688,16 @@
 
     @specialize.argtype(1)
     def allocate(self, decoder, index):
-        array = decoder.allocate_array(len(self.fielddescrs), self.arraydescr,
+        array = decoder.allocate_array(self.size, self.arraydescr,
                                        clear=True)
         decoder.virtuals_cache.set_ptr(index, array)
         p = 0
-        for i in range(len(self.fielddescrs)):
-            for j in range(len(self.fielddescrs[i])):
-                decoder.setinteriorfield(i, array, self.fieldnums[p],
-                                         self.fielddescrs[i][j])
+        for i in range(self.size):
+            for j in range(len(self.fielddescrs)):
+                num = self.fieldnums[p]
+                if not tagged_eq(num, UNINITIALIZED):
+                    decoder.setinteriorfield(i, array, num,
+                                             self.fielddescrs[j])
                 p += 1
         return array
 

From noreply at buildbot.pypy.org  Wed Jun  3 11:02:48 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Wed,  3 Jun 2015 11:02:48 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: fix test_blackhole
Message-ID: <20150603090248.EFD151C034D@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77807:ccfdbf0b97ce
Date: 2015-06-03 09:41 +0200
http://bitbucket.org/pypy/pypy/changeset/ccfdbf0b97ce/

Log:	fix test_blackhole

diff --git a/rpython/jit/metainterp/test/test_blackhole.py b/rpython/jit/metainterp/test/test_blackhole.py
--- a/rpython/jit/metainterp/test/test_blackhole.py
+++ b/rpython/jit/metainterp/test/test_blackhole.py
@@ -130,7 +130,7 @@
                 def start_blackhole(): pass
                 @staticmethod
                 def end_blackhole(): pass
-        last_exc_value_box = None
+        last_exc_value = None
         framestack = [MyMIFrame()]
     MyMetaInterp.staticdata.blackholeinterpbuilder = getblackholeinterp(
         {'int_add/ii>i': 0, 'int_return/i': 1}).builder

From noreply at buildbot.pypy.org  Wed Jun  3 11:02:50 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Wed,  3 Jun 2015 11:02:50 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: minor fixes
Message-ID: <20150603090250.182441C034D@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77808:de61e02e323b
Date: 2015-06-03 09:52 +0200
http://bitbucket.org/pypy/pypy/changeset/de61e02e323b/

Log:	minor fixes

diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -427,7 +427,7 @@
         except KeyError:
             descr = ArrayDescr(A, self)
             self.descrs[key] = descr
-            if isinstance(A.OF, lltype.Struct):
+            if isinstance(A, lltype.Array) and isinstance(A.OF, lltype.Struct):
                 descrs = heaptracker.all_interiorfielddescrs(self,
                         A, get_field_descr=LLGraphCPU.interiorfielddescrof)
                 descr.all_interiorfielddescrs = descrs
@@ -506,7 +506,9 @@
     bh_getfield_gc_f = bh_getfield_gc
 
     bh_getfield_raw = bh_getfield_gc
-    bh_getfield_raw_pure = bh_getfield_raw
+    bh_getfield_raw_pure_i = bh_getfield_raw
+    bh_getfield_raw_pure_r = bh_getfield_raw
+    bh_getfield_raw_pure_f = bh_getfield_raw
     bh_getfield_raw_i = bh_getfield_raw
     bh_getfield_raw_r = bh_getfield_raw
     bh_getfield_raw_f = bh_getfield_raw
diff --git a/rpython/jit/metainterp/optimizeopt/pure.py b/rpython/jit/metainterp/optimizeopt/pure.py
--- a/rpython/jit/metainterp/optimizeopt/pure.py
+++ b/rpython/jit/metainterp/optimizeopt/pure.py
@@ -101,9 +101,8 @@
         if op.returns_bool_result():
             self.getintbound(op).make_bool()
         if save:
-            realop = self.get_box_replacement(op)
-            recentops = self.getrecentops(realop.getopnum())
-            recentops.add(realop)
+            recentops = self.getrecentops(op.getopnum())
+            recentops.add(op)
         if nextop:
             self.emit_operation(nextop)
 

From noreply at buildbot.pypy.org  Wed Jun  3 11:02:51 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Wed,  3 Jun 2015 11:02:51 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: start fighting with raw buffers
Message-ID: <20150603090251.55D681C034D@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77809:fc54d02f8b1c
Date: 2015-06-03 11:02 +0200
http://bitbucket.org/pypy/pypy/changeset/fc54d02f8b1c/

Log:	start fighting with raw buffers

diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py
--- a/rpython/jit/metainterp/optimizeopt/info.py
+++ b/rpython/jit/metainterp/optimizeopt/info.py
@@ -4,6 +4,7 @@
      rop
 from rpython.jit.metainterp.history import ConstInt, Const
 from rpython.rtyper.lltypesystem import lltype
+from rpython.jit.metainterp.optimizeopt.rawbuffer import RawBuffer, InvalidRawOperation
 
 
 INFO_NULL = 0
@@ -141,11 +142,10 @@
                                          for box in self._fields])
         for i in range(len(lst)):
             op = self._fields[i]
-            if op and op.type == 'r':
-                op = op.get_box_replacement()
-                fieldinfo = optimizer.getptrinfo(op)
-                if fieldinfo and fieldinfo.is_virtual():
-                    fieldinfo.visitor_walk_recursive(op, visitor, optimizer)
+            op = op.get_box_replacement()
+            fieldinfo = optimizer.getptrinfo(op)
+            if fieldinfo and fieldinfo.is_virtual():
+                fieldinfo.visitor_walk_recursive(op, visitor, optimizer)
 
 class InstancePtrInfo(AbstractStructPtrInfo):
     _attrs_ = ('_known_class',)
@@ -174,8 +174,85 @@
         assert self.is_virtual()
         return visitor.visit_vstruct(self.vdescr, fielddescrs)
 
-class RawStructPtrInfo(StructPtrInfo):
-    pass
+class AbstractRawPtrInfo(AbstractVirtualPtrInfo):
+    def visitor_walk_recursive(self, op, visitor, optimizer):
+        xxx
+        box = self.rawbuffer_value.get_key_box()
+        visitor.register_virtual_fields(self.keybox, [box])
+        self.rawbuffer_value.visitor_walk_recursive(visitor)
+
+    @specialize.argtype(1)
+    def visitor_dispatch_virtual_type(self, visitor):
+        yyy
+        return visitor.visit_vrawslice(self.offset)    
+
+class RawBufferPtrInfo(AbstractRawPtrInfo):
+    buffer = None
+    
+    def __init__(self, cpu, size=-1):
+        self.size = size
+        if self.size != -1:
+            self.buffer = RawBuffer(cpu, None)
+
+    def getitem_raw(self, offset, itemsize, descr):
+        if not self.is_virtual():
+            raise InvalidRawOperation
+            # see 'test_virtual_raw_buffer_forced_but_slice_not_forced'
+            # for the test above: it's not enough to check is_virtual()
+            # on the original object, because it might be a VRawSliceValue
+            # instead.  If it is a virtual one, then we'll reach here anway.
+        return self.buffer.read_value(offset, itemsize, descr)
+
+    def setitem_raw(self, offset, itemsize, descr, itemop):
+        if not self.is_virtual():
+            raise InvalidRawOperation
+        self.buffer.write_value(offset, itemsize, descr, itemop)
+
+    def is_virtual(self):
+        return self.size != -1
+
+    def _force_elements(self, op, optforce, descr):
+        xxx
+
+    def visitor_walk_recursive(self, op, visitor, optimizer):
+        itemboxes = self.buffer.values
+        visitor.register_virtual_fields(op, itemboxes)
+        #for itembox in itemboxes:
+        #    xxx
+        #    itemvalue = self.get_item_value(i)
+        #    if itemvalue is not None:
+        #        itemvalue.visitor_walk_recursive(visitor)
+
+    @specialize.argtype(1)
+    def visitor_dispatch_virtual_type(self, visitor):
+        return visitor.visit_vrawbuffer(self.size,
+                                        self.buffer.offsets[:],
+                                        self.buffer.descrs[:])
+
+class RawStructPtrInfo(AbstractRawPtrInfo):
+    def __init__(self):
+        import pdb
+        pdb.set_trace()
+    
+    def _force_elements(self, op, optforce, descr):
+        xxx
+
+class RawSlicePtrInfo(AbstractRawPtrInfo):
+    def __init__(self, offset, parent):
+        self.offset = offset
+        self.parent = parent
+
+    def is_virtual(self):
+        return self.parent.is_virtual()
+
+    def getitem_raw(self, offset, itemsize, descr):
+        return self.parent.getitem_raw(self.offset+offset, itemsize, descr)        
+
+    def setitem_raw(self, offset, itemsize, descr, itemop):
+        self.parent.setitem_raw(self.offset+offset, itemsize, descr, itemop)
+    
+    def _force_elements(self, op, optforce, descr):
+        xxx
 
 class ArrayPtrInfo(AbstractVirtualPtrInfo):
     _attrs_ = ('length', '_items', 'lenbound', '_clear')
@@ -242,7 +319,7 @@
         visitor.register_virtual_fields(instbox, itemops)
         for i in range(self.getlength()):
             itemop = self._items[i]
-            if (itemop is not None and itemop.type == 'r' and
+            if (itemop is not None and
                 not isinstance(itemop, Const)):
                 ptrinfo = optimizer.getptrinfo(itemop)
                 if ptrinfo and ptrinfo.is_virtual():
@@ -298,7 +375,7 @@
         for index in range(self.getlength()):
             for flddescr in fielddescrs:
                 itemop = self._items[i]
-                if (itemop is not None and itemop.type == 'r' and
+                if (itemop is not None and
                     not isinstance(itemop, Const)):
                     ptrinfo = optimizer.getptrinfo(itemop)
                     if ptrinfo and ptrinfo.is_virtual():
diff --git a/rpython/jit/metainterp/optimizeopt/intbounds.py b/rpython/jit/metainterp/optimizeopt/intbounds.py
--- a/rpython/jit/metainterp/optimizeopt/intbounds.py
+++ b/rpython/jit/metainterp/optimizeopt/intbounds.py
@@ -117,6 +117,8 @@
 
     def optimize_INT_ADD(self, op):
         self.emit_operation(op)
+        if self.is_raw_ptr(op.getarg(0)) or self.is_raw_ptr(op.getarg(1)):
+            return
         b1 = self.getintbound(op.getarg(0))
         b2 = self.getintbound(op.getarg(1))
         r = self.getintbound(op)
@@ -536,6 +538,8 @@
         self._propagate_int_is_true_or_zero(op, 0, 1)
 
     def propagate_bounds_INT_ADD(self, op):
+        if self.is_raw_ptr(op.getarg(0)) or self.is_raw_ptr(op.getarg(1)):
+            return
         b1 = self.getintbound(op.getarg(0))
         b2 = self.getintbound(op.getarg(1))
         r = self.getintbound(op)
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -316,6 +316,10 @@
         return opinfo
 
     def getptrinfo(self, op, create=False, is_object=False):
+        if op.type == 'i':
+            return self.getrawptrinfo(op, create)
+        elif op.type == 'f':
+            return None
         assert op.type == 'r'
         op = self.get_box_replacement(op)
         assert op.type == 'r'
@@ -327,6 +331,12 @@
             return fw
         return None
 
+    def is_raw_ptr(self, op):
+        fw = self.get_box_replacement(op).get_forwarded()
+        if isinstance(fw, info.AbstractRawPtrInfo):
+            return True
+        return False
+    
     def getrawptrinfo(self, op, create=False, is_object=False):
         assert op.type == 'i'
         op = self.get_box_replacement(op)
@@ -334,11 +344,14 @@
         if isinstance(op, ConstInt):
             return info.ConstPtrInfo(op)
         fw = op.get_forwarded()
+        if isinstance(fw, IntBound) and not create:
+            return None
         if fw is not None:
-            if isinstance(fw, info.NonNullPtrInfo):
-                fw = info.RawStructPtrInfo()
-                op.set_forwarded(fw)
-            assert isinstance(fw, info.RawStructPtrInfo)
+            if isinstance(fw, info.AbstractRawPtrInfo):
+                return fw
+            fw = info.RawStructPtrInfo()
+            op.set_forwarded(fw)
+            assert isinstance(fw, info.AbstractRawPtrInfo)
             return fw
         return None
 
diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py
--- a/rpython/jit/metainterp/optimizeopt/rewrite.py
+++ b/rpython/jit/metainterp/optimizeopt/rewrite.py
@@ -121,6 +121,9 @@
             self.optimizer.pure_reverse(op)
 
     def optimize_INT_ADD(self, op):
+        if self.is_raw_ptr(op.getarg(0)) or self.is_raw_ptr(op.getarg(1)):
+            self.emit_operation(op)
+            return
         arg1 = self.get_box_replacement(op.getarg(0))
         b1 = self.getintbound(arg1)
         arg2 = self.get_box_replacement(op.getarg(1))
diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py
--- a/rpython/jit/metainterp/optimizeopt/virtualize.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualize.py
@@ -543,17 +543,14 @@
         return opinfo
 
     def make_virtual_raw_memory(self, size, source_op):
-        raise Exception("unsupported")
-        logops = self.optimizer.loop.logops
-        vvalue = VRawBufferValue(self.optimizer.cpu, logops, size, source_op)
-        self.make_equal_to(source_op, vvalue)
-        return vvalue
+        opinfo = info.RawBufferPtrInfo(self.optimizer.cpu, size)
+        source_op.set_forwarded(opinfo)
+        return opinfo
 
-    def make_virtual_raw_slice(self, rawbuffer_value, offset, source_op):
-        raise Exception("unsupported")
-        vvalue = VRawSliceValue(rawbuffer_value, offset, source_op)
-        self.make_equal_to(source_op, vvalue)
-        return vvalue
+    def make_virtual_raw_slice(self, offset, parent, source_op):
+        opinfo = info.RawSlicePtrInfo(offset, parent)
+        source_op.set_forwarded(opinfo)
+        return opinfo
 
     def optimize_GUARD_NO_EXCEPTION(self, op):
         if self.last_emitted_operation is REMOVED:
@@ -761,23 +758,20 @@
         self.emit_operation(op)
 
     def optimize_INT_ADD(self, op):
-        if 0:
-            XXX
-            value = self.getvalue(op.getarg(0))
-            offsetbox = self.get_constant_box(op.getarg(1))
-            if value.is_virtual() and offsetbox is not None:
-                offset = offsetbox.getint()
-                # the following check is constant-folded to False if the
-                # translation occurs without any VRawXxxValue instance around
-                if value.is_about_raw:
-                    if isinstance(value, VRawBufferValue):
-                        self.make_virtual_raw_slice(value, offset, op)
-                        return
-                    elif isinstance(value, VRawSliceValue):
-                        offset = offset + value.offset
-                        self.make_virtual_raw_slice(value.rawbuffer_value, offset,
-                                                    op)
-                        return
+        opinfo = self.getrawptrinfo(op.getarg(0), create=False)
+        offsetbox = self.get_constant_box(op.getarg(1))
+        if opinfo and opinfo.is_virtual() and offsetbox is not None:
+            offset = offsetbox.getint()
+            # the following check is constant-folded to False if the
+            # translation occurs without any VRawXxxValue instance around
+            if isinstance(opinfo, info.RawBufferPtrInfo):
+                self.make_virtual_raw_slice(offset, opinfo, op)
+                return
+            elif isinstance(value, VRawSliceValue):
+                offset = offset + value.offset
+                self.make_virtual_raw_slice(value.rawbuffer_value, offset,
+                                            op)
+                return
         self.emit_operation(op)
 
     def optimize_ARRAYLEN_GC(self, op):
@@ -832,13 +826,12 @@
     def optimize_GETARRAYITEM_RAW_I(self, op):
         opinfo = self.getrawptrinfo(op.getarg(0))
         if opinfo and opinfo.is_virtual():
-            raise Exception("implement raw virtuals")
-            xxx
             indexbox = self.get_constant_box(op.getarg(1))
             if indexbox is not None:
-                offset, itemsize, descr = self._unpack_arrayitem_raw_op(op, indexbox)
+                offset, itemsize, descr = self._unpack_arrayitem_raw_op(op,
+                                                                indexbox)
                 try:
-                    itemvalue = value.getitem_raw(offset, itemsize, descr)
+                    itemvalue = opinfo.getitem_raw(offset, itemsize, descr)
                 except InvalidRawOperation:
                     pass
                 else:
@@ -853,11 +846,10 @@
         if opinfo and opinfo.is_virtual():
             indexbox = self.get_constant_box(op.getarg(1))
             if indexbox is not None:
-                raise Exception("implement raw virtuals")
                 offset, itemsize, descr = self._unpack_arrayitem_raw_op(op, indexbox)
-                itemvalue = self.getvalue(op.getarg(2))
+                itemop = self.get_box_replacement(op.getarg(2))
                 try:
-                    value.setitem_raw(offset, itemsize, descr, itemvalue)
+                    opinfo.setitem_raw(offset, itemsize, descr, itemop)
                     return
                 except InvalidRawOperation:
                     pass
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -1573,7 +1573,10 @@
                         return resbox
                 self.metainterp.vable_and_vrefs_before_residual_call()
                 tp = descr.get_result_type()
-                if tp == 'i':
+                if effectinfo.oopspecindex == effectinfo.OS_LIBFFI_CALL:
+                    resbox = self.metainterp.direct_libffi_call(allboxes, descr,
+                                                                tp)
+                elif tp == 'i':
                     resbox = self.metainterp.execute_and_record_varargs(
                         rop.CALL_MAY_FORCE_I, allboxes, descr=descr)
                 elif tp == 'r':
@@ -1601,11 +1604,6 @@
                 if vablebox is not None:
                     self.metainterp.history.record(rop.KEEPALIVE, [vablebox], None)
                 self.metainterp.handle_possible_exception()
-                # XXX refactor: direct_libffi_call() is a hack
-                # does not work in the new system
-                if effectinfo.oopspecindex == effectinfo.OS_LIBFFI_CALL:
-                    raise Exception("implement OS_LIBFFI_CALL properly")
-                #    self.metainterp.direct_libffi_call()
                 return resbox
             else:
                 effect = effectinfo.extraeffect
@@ -2963,9 +2961,8 @@
         else:
             return None, op
 
-    def direct_libffi_call(self):
-        """Generate a direct call to C code, patching the CALL_MAY_FORCE
-        to jit_ffi_call() that occurred just now.
+    def direct_libffi_call(self, argboxes, descr, tp):
+        """Generate a direct call to C code using jit_ffi_call()
         """
         # an 'assert' that constant-folds away the rest of this function
         # if the codewriter didn't produce any OS_LIBFFI_CALL at all.
@@ -2975,33 +2972,19 @@
         from rpython.rlib.jit_libffi import CIF_DESCRIPTION_P
         from rpython.jit.backend.llsupport.ffisupport import get_arg_descr
         #
-        num_extra_guards = 0
-        while True:
-            op = self.history.operations[-1-num_extra_guards]
-            if op.is_call_may_force():
-                break
-            assert op.is_guard()
-            num_extra_guards += 1
-        #
-        box_cif_description = op.getarg(1)
+        box_cif_description = argboxes[1]
         if not isinstance(box_cif_description, ConstInt):
             return
         cif_description = box_cif_description.getint()
         cif_description = llmemory.cast_int_to_adr(cif_description)
         cif_description = llmemory.cast_adr_to_ptr(cif_description,
                                                    CIF_DESCRIPTION_P)
-        extrainfo = op.getdescr().get_extra_info()
+        extrainfo = descr.get_extra_info()
         calldescr = self.cpu.calldescrof_dynamic(cif_description, extrainfo)
         if calldescr is None:
             return
         #
-        extra_guards = []
-        for i in range(num_extra_guards):
-            extra_guards.append(self.history.operations.pop())
-        extra_guards.reverse()
-        #
-        box_exchange_buffer = op.getarg(3)
-        self.history.operations.pop()
+        box_exchange_buffer = argboxes[3]
         arg_boxes = []
 
         for i in range(cif_description.nargs):
@@ -3010,12 +2993,14 @@
             ofs = cif_description.exchange_args[i]
             assert ofs % itemsize == 0     # alignment check
             if kind == 'i':
-                box_arg = self.history.record(rop.GETARRAYITEM_RAW_I,
+                box_arg = self.history.record(
+                    rop.GETARRAYITEM_RAW_I,
                                     [box_exchange_buffer,
                                      ConstInt(ofs // itemsize)],
                                      0, descr)
             elif kind == 'f':
-                box_arg = self.history.record(rop.GETARRAYITEM_RAW_F,
+                box_arg = self.history.record(
+                    rop.GETARRAYITEM_RAW_F,
                                     [box_exchange_buffer,
                                      ConstInt(ofs // itemsize)],
                                      0.0, descr)
@@ -3028,15 +3013,22 @@
         # (that is, errno and SetLastError/GetLastError on Windows)
         # Note these flags match the ones in clibffi.ll_callback
         c_saveall = ConstInt(rffi.RFFI_ERR_ALL | rffi.RFFI_ALT_ERRNO)
-        if op.type == 'i':
-            self.history.record(rop.CALL_RELEASE_GIL,
-                                [c_saveall, op.getarg(2)] + arg_boxes,
-                                box_result, calldescr)
-        #
-        self.history.operations.extend(extra_guards)
+        if tp == 'i':
+            value = executor.execute_varargs(self.cpu, self,
+                                             rop.CALL_MAY_FORCE_I,
+                                             argboxes, descr)
+            box_result = self.history.record(
+                rop.CALL_RELEASE_GIL_I, [c_saveall, argboxes[2]] + arg_boxes,
+                value, descr=calldescr)
+        elif tp == 'f':
+            xxx
+            box_result = self.history.record(
+                rop.CALL_RELEASE_GIL_F, [c_saveall, argboxes[2]] + arg_boxes,
+                value, descr=calldescr)
         #
         # note that the result is written back to the exchange_buffer by the
         # special op libffi_save_result_{int,float}
+        return box_result
 
     def direct_call_release_gil(self):
         op = self.history.operations.pop()
diff --git a/rpython/jit/metainterp/resume.py b/rpython/jit/metainterp/resume.py
--- a/rpython/jit/metainterp/resume.py
+++ b/rpython/jit/metainterp/resume.py
@@ -220,11 +220,13 @@
             elif box in liveboxes:
                 tagged = liveboxes[box]
             else:
+                is_virtual = False
                 if box.type == 'r':
                     info = optimizer.getptrinfo(box)
-                    is_virtual = (info is not None and info.is_virtual())
-                else:
-                    is_virtual = False
+                    is_virtual = (info and info.is_virtual())
+                if box.type == 'i':
+                    info = optimizer.getrawptrinfo(box, create=False)
+                    is_virtual = (info and info.is_virtual()) 
                 if is_virtual:
                     tagged = tag(v, TAGVIRTUAL)
                     v += 1
@@ -390,7 +392,11 @@
                 liveboxes[i] = box
             else:
                 assert tagbits == TAGVIRTUAL
-                info = optimizer.getptrinfo(box)
+                if box.type == 'r':
+                    info = optimizer.getptrinfo(box)
+                else:
+                    assert box.type == 'i'
+                    info = optimizer.getrawptrinfo(box)
                 info.visitor_walk_recursive(box, self, optimizer)
 
         for setfield_op in pending_setfields:
diff --git a/rpython/jit/metainterp/test/test_executor.py b/rpython/jit/metainterp/test/test_executor.py
--- a/rpython/jit/metainterp/test/test_executor.py
+++ b/rpython/jit/metainterp/test/test_executor.py
@@ -97,38 +97,38 @@
     # cases with a descr
     # arity == -1
     argboxes = [InputArgInt(321), ConstInt(123)]
-    box = _execute_nonspec(cpu, FakeMetaInterp(), rop.CALL_F,
+    box = _execute_arglist(cpu, FakeMetaInterp(), rop.CALL_F,
                            argboxes, FakeCallDescr())
     assert box == 42.5
     # arity == 0
-    box = _execute_nonspec(cpu, None, rop.NEW, [], descr)
+    box = _execute_arglist(cpu, None, rop.NEW, [], descr)
     assert box.fakeargs == ('new', descr)
     # arity == 1
     box1 = InputArgRef()
-    box = _execute_nonspec(cpu, None, rop.ARRAYLEN_GC, [box1], descr)
+    box = _execute_arglist(cpu, None, rop.ARRAYLEN_GC, [box1], descr)
     assert box == 55
     # arity == 2
     box2 = boxfloat(222.2)
     fielddescr = FakeFieldDescr()
-    _execute_nonspec(cpu, None, rop.SETFIELD_GC, [box1, box2], fielddescr)
+    _execute_arglist(cpu, None, rop.SETFIELD_GC, [box1, box2], fielddescr)
     assert cpu.fakesetfield == (box1.getref_base(), box2.getfloat(),
                                 fielddescr)
     # arity == 3
     box3 = InputArgInt(33)
     arraydescr = FakeArrayDescr()
-    _execute_nonspec(cpu, None, rop.SETARRAYITEM_GC, [box1, box3, box2],
+    _execute_arglist(cpu, None, rop.SETARRAYITEM_GC, [box1, box3, box2],
                     arraydescr)
     assert cpu.fakesetarrayitem == (box1.getref_base(), box3.getint(),
                                     box2.getfloat(), arraydescr)
     # cases without descr
     # arity == 1
-    box = _execute_nonspec(cpu, None, rop.INT_INVERT, [box3])
+    box = _execute_arglist(cpu, None, rop.INT_INVERT, [box3])
     assert box == ~33
     # arity == 2
-    box = _execute_nonspec(cpu, None, rop.INT_LSHIFT, [box3, InputArgInt(3)])
+    box = _execute_arglist(cpu, None, rop.INT_LSHIFT, [box3, InputArgInt(3)])
     assert box == 33 << 3
     # arity == 3
-    _execute_nonspec(cpu, None, rop.STRSETITEM, [box1, InputArgInt(3), box3])
+    _execute_arglist(cpu, None, rop.STRSETITEM, [box1, InputArgInt(3), box3])
     assert cpu.fakestrsetitem == (box1.getref_base(), 3, box3.getint())
 
 # ints
@@ -245,7 +245,7 @@
 def test_int_ops():
     cpu = FakeCPU()
     for opnum, boxargs, retvalue in get_int_tests():
-        r = _execute_nonspec(cpu, None, opnum, boxargs)
+        r = _execute_arglist(cpu, None, opnum, boxargs)
         assert r == retvalue
 
 # floats
@@ -315,7 +315,7 @@
 def test_float_ops():
     cpu = FakeCPU()
     for opnum, boxargs, rettype, retvalue in get_float_tests(cpu):
-        res = _execute_nonspec(cpu, None, opnum, boxargs)
+        res = _execute_arglist(cpu, None, opnum, boxargs)
         assert res == retvalue
 
 def make_args_for_op(op, a, b):

From noreply at buildbot.pypy.org  Wed Jun  3 11:06:48 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Wed,  3 Jun 2015 11:06:48 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: finish test_fficall
Message-ID: <20150603090648.0E2061C034D@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77810:aa6e8a681364
Date: 2015-06-03 11:06 +0200
http://bitbucket.org/pypy/pypy/changeset/aa6e8a681364/

Log:	finish test_fficall

diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py
--- a/rpython/jit/metainterp/optimizeopt/info.py
+++ b/rpython/jit/metainterp/optimizeopt/info.py
@@ -176,15 +176,11 @@
 
 class AbstractRawPtrInfo(AbstractVirtualPtrInfo):
     def visitor_walk_recursive(self, op, visitor, optimizer):
-        xxx
-        box = self.rawbuffer_value.get_key_box()
-        visitor.register_virtual_fields(self.keybox, [box])
-        self.rawbuffer_value.visitor_walk_recursive(visitor)
+        raise NotImplementedError("abstract")
 
     @specialize.argtype(1)
     def visitor_dispatch_virtual_type(self, visitor):
-        yyy
-        return visitor.visit_vrawslice(self.offset)    
+        raise NotImplementedError("abstract")
 
 class RawBufferPtrInfo(AbstractRawPtrInfo):
     buffer = None
@@ -254,6 +250,15 @@
     def _force_elements(self, op, optforce, descr):
         xxx
 
+    def visitor_walk_recursive(self, op, visitor, optimizer):
+        source_op = optimizer.get_box_replacement(op.getarg(0))
+        visitor.register_virtual_fields(op, [source_op])
+        self.parent.visitor_walk_recursive(source_op, visitor, optimizer)
+
+    @specialize.argtype(1)
+    def visitor_dispatch_virtual_type(self, visitor):
+        return visitor.visit_vrawslice(self.offset)
+
 class ArrayPtrInfo(AbstractVirtualPtrInfo):
     _attrs_ = ('length', '_items', 'lenbound', '_clear')
 
diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py
--- a/rpython/jit/metainterp/optimizeopt/virtualize.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualize.py
@@ -767,10 +767,9 @@
             if isinstance(opinfo, info.RawBufferPtrInfo):
                 self.make_virtual_raw_slice(offset, opinfo, op)
                 return
-            elif isinstance(value, VRawSliceValue):
-                offset = offset + value.offset
-                self.make_virtual_raw_slice(value.rawbuffer_value, offset,
-                                            op)
+            elif isinstance(opinfo, info.RawSlicePtrInfo):
+                offset = offset + opinfo.offset
+                self.make_virtual_raw_slice(offset, opinfo.parent, op)
                 return
         self.emit_operation(op)
 
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -3021,10 +3021,14 @@
                 rop.CALL_RELEASE_GIL_I, [c_saveall, argboxes[2]] + arg_boxes,
                 value, descr=calldescr)
         elif tp == 'f':
-            xxx
+            value = executor.execute_varargs(self.cpu, self,
+                                             rop.CALL_MAY_FORCE_F,
+                                             argboxes, descr)
             box_result = self.history.record(
                 rop.CALL_RELEASE_GIL_F, [c_saveall, argboxes[2]] + arg_boxes,
                 value, descr=calldescr)
+        else:
+            assert False
         #
         # note that the result is written back to the exchange_buffer by the
         # special op libffi_save_result_{int,float}

From noreply at buildbot.pypy.org  Wed Jun  3 13:02:31 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Wed,  3 Jun 2015 13:02:31 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: whack until most tests pass
Message-ID: <20150603110231.6DB901C1041@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77811:1991339ececb
Date: 2015-06-03 13:02 +0200
http://bitbucket.org/pypy/pypy/changeset/1991339ececb/

Log:	whack until most tests pass

diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -544,7 +544,9 @@
     bh_getarrayitem_gc_f = bh_getarrayitem_gc
 
     bh_getarrayitem_raw = bh_getarrayitem_gc
-    bh_getarrayitem_raw_pure = bh_getarrayitem_raw
+    bh_getarrayitem_raw_pure_i = bh_getarrayitem_raw
+    bh_getarrayitem_raw_pure_r = bh_getarrayitem_raw
+    bh_getarrayitem_raw_pure_f = bh_getarrayitem_raw
     bh_getarrayitem_raw_i = bh_getarrayitem_raw
     bh_getarrayitem_raw_r = bh_getarrayitem_raw
     bh_getarrayitem_raw_f = bh_getarrayitem_raw
diff --git a/rpython/jit/codewriter/heaptracker.py b/rpython/jit/codewriter/heaptracker.py
--- a/rpython/jit/codewriter/heaptracker.py
+++ b/rpython/jit/codewriter/heaptracker.py
@@ -190,11 +190,12 @@
             continue # dealt otherwise
         elif isinstance(FIELD, lltype.Struct):
             r = get_fielddescr_index_in(FIELD, fieldname, cur_index)
-            if r != -1:
+            if r >= 0:
                 return r
+            cur_index += -r - 1
             continue
         elif name == fieldname:
             return cur_index
         cur_index += 1
-    return -1 # not found
+    return -cur_index - 1 # not found
     
diff --git a/rpython/jit/metainterp/executor.py b/rpython/jit/metainterp/executor.py
--- a/rpython/jit/metainterp/executor.py
+++ b/rpython/jit/metainterp/executor.py
@@ -460,8 +460,10 @@
 
 @specialize.argtype(0)
 def wrap_constant(value):
-    if isinstance(value, int):
+    if lltype.typeOf(value) == lltype.Signed:
         return ConstInt(value)
+    elif isinstance(value, bool):
+        return ConstInt(int(value))
     elif isinstance(value, float):
         return ConstFloat(value)
     else:
diff --git a/rpython/jit/metainterp/heapcache.py b/rpython/jit/metainterp/heapcache.py
--- a/rpython/jit/metainterp/heapcache.py
+++ b/rpython/jit/metainterp/heapcache.py
@@ -1,5 +1,5 @@
 from rpython.jit.metainterp.history import ConstInt
-from rpython.jit.metainterp.resoperation import rop
+from rpython.jit.metainterp.resoperation import rop, OpHelpers
 
 class HeapCacheValue(object):
     def __init__(self, box):
@@ -191,7 +191,9 @@
             rop._NOSIDEEFFECT_FIRST <= opnum <= rop._NOSIDEEFFECT_LAST or
             rop._GUARD_FIRST <= opnum <= rop._GUARD_LAST):
             return
-        if rop._CALL_FIRST <= opnum <= rop._CALL_LAST:
+        if (OpHelpers.is_plain_call(opnum) or
+            OpHelpers.is_call_loopinvariant(opnum) or
+            opnum == rop.COND_CALL):
             effectinfo = descr.get_extra_info()
             ef = effectinfo.extraeffect
             if (ef == effectinfo.EF_LOOPINVARIANT or
diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py
--- a/rpython/jit/metainterp/optimizeopt/info.py
+++ b/rpython/jit/metainterp/optimizeopt/info.py
@@ -96,25 +96,29 @@
 
 class AbstractStructPtrInfo(AbstractVirtualPtrInfo):
     _attrs_ = ('_fields',)
+    _fields = None
 
-    def init_fields(self, descr):
-        self._fields = [None] * len(descr.get_all_fielddescrs())
+    def init_fields(self, descr, index):
+        if self._fields is None:
+            self._fields = [None] * len(descr.get_all_fielddescrs())
+        if index >= len(self._fields):
+            # we found out a subclass with more fields
+            extra_len = len(descr.get_all_fielddescrs()) - len(self._fields)
+            self._fields = self._fields + [None] * extra_len
 
     def clear_cache(self):
         assert not self.is_virtual()
         self._fields = [None] * len(self._fields)
 
     def setfield(self, descr, op, optheap=None, cf=None):
-        if self._fields is None:
-            self.init_fields(descr.get_parent_descr())
+        self.init_fields(descr.get_parent_descr(), descr.get_index())
         self._fields[descr.get_index()] = op
         if cf is not None:
             assert not self.is_virtual()
             cf.register_dirty_field(self)
 
     def getfield(self, descr, optheap=None):
-        if self._fields is None:
-            self.init_fields(descr.get_parent_descr())
+        self.init_fields(descr.get_parent_descr(), descr.get_index())
         return self._fields[descr.get_index()]
 
     def _force_elements(self, op, optforce, descr):
@@ -142,10 +146,11 @@
                                          for box in self._fields])
         for i in range(len(lst)):
             op = self._fields[i]
-            op = op.get_box_replacement()
-            fieldinfo = optimizer.getptrinfo(op)
-            if fieldinfo and fieldinfo.is_virtual():
-                fieldinfo.visitor_walk_recursive(op, visitor, optimizer)
+            if op:
+                op = op.get_box_replacement()
+                fieldinfo = optimizer.getptrinfo(op)
+                if fieldinfo and fieldinfo.is_virtual():
+                    fieldinfo.visitor_walk_recursive(op, visitor, optimizer)
 
 class InstancePtrInfo(AbstractStructPtrInfo):
     _attrs_ = ('_known_class',)
@@ -227,8 +232,7 @@
 
 class RawStructPtrInfo(AbstractRawPtrInfo):
     def __init__(self):
-        import pdb
-        pdb.set_trace()
+        pass
     
     def _force_elements(self, op, optforce, descr):
         xxx
@@ -405,7 +409,7 @@
         info = optheap.const_infos.get(ref, None)
         if info is None:
             info = StructPtrInfo()
-            info.init_fields(descr.get_parent_descr())
+            info.init_fields(descr.get_parent_descr(), descr.get_index())
             optheap.const_infos[ref] = info
         return info
 
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -657,7 +657,8 @@
                 opinfo = info.InstancePtrInfo()
             else:
                 opinfo = info.StructPtrInfo()
-            opinfo.init_fields(op.getdescr().get_parent_descr())
+            opinfo.init_fields(op.getdescr().get_parent_descr(),
+                               op.getdescr().get_index())
         elif op.is_getarrayitem() or op.getopnum() == rop.SETARRAYITEM_GC:
             opinfo = info.ArrayPtrInfo(op.getdescr())
         elif op.getopnum() == rop.GUARD_CLASS:
@@ -851,9 +852,10 @@
     def optimize_STRGETITEM(self, op):
         indexb = self.getintbound(op.getarg(1))
         if indexb.is_constant():
-            raise Exception("implement me")
-            arrayvalue = self.getvalue(op.getarg(0))
-            arrayvalue.make_len_gt(MODE_STR, op.getdescr(), indexvalue.box.getint())
+            pass
+            #raise Exception("implement me")
+            #arrayvalue = self.getvalue(op.getarg(0))
+            #arrayvalue.make_len_gt(MODE_STR, op.getdescr(), indexvalue.box.getint())
         self.optimize_default(op)
 
     def optimize_UNICODEGETITEM(self, op):
diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py
--- a/rpython/jit/metainterp/optimizeopt/virtualize.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualize.py
@@ -522,7 +522,7 @@
 
     def make_virtual(self, known_class, source_op, descr):
         opinfo = info.InstancePtrInfo(known_class, vdescr=descr)
-        opinfo.init_fields(descr)
+        opinfo.init_fields(descr, 0)
         source_op.set_forwarded(opinfo)
         return opinfo
 
@@ -538,7 +538,7 @@
 
     def make_vstruct(self, structdescr, source_op):
         opinfo = info.StructPtrInfo(vdescr=structdescr)
-        opinfo.init_fields(structdescr)
+        opinfo.init_fields(structdescr, 0)
         source_op.set_forwarded(opinfo)
         return opinfo
 
@@ -863,37 +863,32 @@
         return offset, itemsize, descr
 
     def optimize_RAW_LOAD_I(self, op):
-        raise Exception("implement me")
-        value = self.getvalue(op.getarg(0))
-        if value.is_virtual():
+        opinfo = self.getrawptrinfo(op.getarg(0))
+        if opinfo and opinfo.is_virtual():
             offsetbox = self.get_constant_box(op.getarg(1))
             if offsetbox is not None:
                 offset, itemsize, descr = self._unpack_raw_load_store_op(op, offsetbox)
                 try:
-                    itemvalue = value.getitem_raw(offset, itemsize, descr)
+                    itemop = opinfo.getitem_raw(offset, itemsize, descr)
                 except InvalidRawOperation:
                     pass
                 else:
-                    self.make_equal_to(op, itemvalue)
+                    self.make_equal_to(op, itemop)
                     return
-        value.ensure_nonnull()
         self.emit_operation(op)
     optimize_RAW_LOAD_F = optimize_RAW_LOAD_I
 
     def optimize_RAW_STORE(self, op):
-        raise Exception("implement me")
-        value = self.getvalue(op.getarg(0))
-        if value.is_virtual():
+        opinfo = self.getrawptrinfo(op.getarg(0))
+        if opinfo and opinfo.is_virtual():
             offsetbox = self.get_constant_box(op.getarg(1))
             if offsetbox is not None:
                 offset, itemsize, descr = self._unpack_raw_load_store_op(op, offsetbox)
-                itemvalue = self.getvalue(op.getarg(2))
                 try:
-                    value.setitem_raw(offset, itemsize, descr, itemvalue)
+                    opinfo.setitem_raw(offset, itemsize, descr, op.getarg(2))
                     return
                 except InvalidRawOperation:
                     pass
-        value.ensure_nonnull()
         self.emit_operation(op)
 
     def optimize_GETINTERIORFIELD_GC_I(self, op):
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -494,12 +494,12 @@
                                        arraydescr, arraybox, indexbox)
 
     @arguments("box", "box", "descr")
-    def opimpl_getarrayitem_raw_pure_i(self, arraybox, indexbox, arraydescr):
+    def opimpl_getarrayitem_raw_i_pure(self, arraybox, indexbox, arraydescr):
         return self.execute_with_descr(rop.GETARRAYITEM_RAW_PURE_I,
                                        arraydescr, arraybox, indexbox)
 
     @arguments("box", "box", "descr")
-    def opimpl_getarrayitem_raw_pure_f(self, arraybox, indexbox, arraydescr):
+    def opimpl_getarrayitem_raw_f_pure(self, arraybox, indexbox, arraydescr):
         return self.execute_with_descr(rop.GETARRAYITEM_RAW_PURE_F,
                                        arraydescr, arraybox, indexbox)
 
@@ -734,15 +734,15 @@
     @arguments("box", "descr", "orgpc")
     def _opimpl_getfield_gc_greenfield_any(self, box, fielddescr, pc):
         ginfo = self.metainterp.jitdriver_sd.greenfield_info
+        opnum = OpHelpers.getfield_pure_for_descr(fielddescr)
         if (ginfo is not None and fielddescr in ginfo.green_field_descrs
             and not self._nonstandard_virtualizable(pc, box, fielddescr)):
             # fetch the result, but consider it as a Const box and don't
             # record any operation
-            resbox = executor.execute(self.metainterp.cpu, self.metainterp,
-                                      rop.GETFIELD_GC_PURE, fielddescr, box)
-            return resbox.constbox()
+            return executor.execute_nonspec_const(self.metainterp.cpu,
+                                    self.metainterp, opnum, [box], fielddescr)
         # fall-back
-        return self.execute_with_descr(rop.GETFIELD_GC_PURE, fielddescr, box)
+        return self.execute_with_descr(opnum, fielddescr, box)
     opimpl_getfield_gc_i_greenfield = _opimpl_getfield_gc_greenfield_any
     opimpl_getfield_gc_r_greenfield = _opimpl_getfield_gc_greenfield_any
     opimpl_getfield_gc_f_greenfield = _opimpl_getfield_gc_greenfield_any
@@ -774,10 +774,11 @@
 
 
     @arguments("box", "descr")
-    def _opimpl_getfield_raw_any(self, box, fielddescr):
-        return self.execute_with_descr(rop.GETFIELD_RAW, fielddescr, box)
-    opimpl_getfield_raw_i = _opimpl_getfield_raw_any
-    opimpl_getfield_raw_f = _opimpl_getfield_raw_any
+    def opimpl_getfield_raw_i(self, box, fielddescr):
+        return self.execute_with_descr(rop.GETFIELD_RAW_I, fielddescr, box)
+    @arguments("box", "descr")
+    def opimpl_getfield_raw_f(self, box, fielddescr):
+        return self.execute_with_descr(rop.GETFIELD_RAW_F, fielddescr, box)
 
     @arguments("box", "descr")
     def opimpl_getfield_raw_i_pure(self, box, fielddescr):
@@ -803,11 +804,13 @@
     opimpl_raw_store_f = _opimpl_raw_store
 
     @arguments("box", "box", "descr")
-    def _opimpl_raw_load(self, addrbox, offsetbox, arraydescr):
-        return self.execute_with_descr(rop.RAW_LOAD, arraydescr,
+    def opimpl_raw_load_i(self, addrbox, offsetbox, arraydescr):
+        return self.execute_with_descr(rop.RAW_LOAD_I, arraydescr,
                                        addrbox, offsetbox)
-    opimpl_raw_load_i = _opimpl_raw_load
-    opimpl_raw_load_f = _opimpl_raw_load
+    @arguments("box", "box", "descr")
+    def opimpl_raw_load_f(self, addrbox, offsetbox, arraydescr):
+        return self.execute_with_descr(rop.RAW_LOAD_F, arraydescr,
+                                       addrbox, offsetbox)
 
     @arguments("box")
     def opimpl_hint_force_virtualizable(self, box):
diff --git a/rpython/jit/metainterp/quasiimmut.py b/rpython/jit/metainterp/quasiimmut.py
--- a/rpython/jit/metainterp/quasiimmut.py
+++ b/rpython/jit/metainterp/quasiimmut.py
@@ -118,6 +118,11 @@
         if self.fielddescr is not None:
             return self.fielddescr.get_parent_descr()
 
+    def get_index(self):
+        if self.fielddescr is not None:
+            return self.fielddescr.get_index()
+        return 0 # annotation hint
+
     def get_current_constant_fieldvalue(self):
         struct = self.struct
         fielddescr = self.fielddescr
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -1114,6 +1114,13 @@
         return rop._CALL_FIRST <= opnum <= rop._CALL_LAST
 
     @staticmethod
+    def is_plain_call(opnum):
+        return (opnum == rop.CALL_I or
+                opnum == rop.CALL_R or
+                opnum == rop.CALL_F or
+                opnum == rop.CALL_N)
+
+    @staticmethod
     def is_call_assembler(opnum):
         return (opnum == rop.CALL_ASSEMBLER_I or
                 opnum == rop.CALL_ASSEMBLER_R or
@@ -1121,6 +1128,13 @@
                 opnum == rop.CALL_ASSEMBLER_N)
 
     @staticmethod
+    def is_call_loopinvariant(opnum):
+        return (opnum == rop.CALL_LOOPINVARIANT_I or
+                opnum == rop.CALL_LOOPINVARIANT_R or
+                opnum == rop.CALL_LOOPINVARIANT_F or
+                opnum == rop.CALL_LOOPINVARIANT_N)
+
+    @staticmethod
     def is_call_may_force(opnum):
         return (opnum == rop.CALL_MAY_FORCE_I or
                 opnum == rop.CALL_MAY_FORCE_R or
diff --git a/rpython/jit/metainterp/test/support.py b/rpython/jit/metainterp/test/support.py
--- a/rpython/jit/metainterp/test/support.py
+++ b/rpython/jit/metainterp/test/support.py
@@ -211,6 +211,7 @@
         assert n == count
 
     def check_enter_count(self, count):
+        return
         """Check the number of times pyjitpl ran.  (Every time, it
         should have produced either one loop or one bridge, or aborted;
         but it is not 100% clear that this is still correct in the
diff --git a/rpython/jit/metainterp/test/test_heapcache.py b/rpython/jit/metainterp/test/test_heapcache.py
--- a/rpython/jit/metainterp/test/test_heapcache.py
+++ b/rpython/jit/metainterp/test/test_heapcache.py
@@ -1,6 +1,6 @@
 from rpython.jit.metainterp.heapcache import HeapCache
-from rpython.jit.metainterp.resoperation import rop
-from rpython.jit.metainterp.history import ConstInt, BoxInt, BasicFailDescr
+from rpython.jit.metainterp.resoperation import rop, InputArgInt
+from rpython.jit.metainterp.history import ConstInt, BasicFailDescr
 
 box1 = "box1"
 box2 = "box2"
@@ -265,7 +265,7 @@
         assert h.getarrayitem(box1, index2, descr1) is box4
 
         h.invalidate_caches(
-            rop.CALL, FakeCallDescr(FakeEffectinfo.EF_ELIDABLE_CANNOT_RAISE), [])
+            rop.CALL_N, FakeCallDescr(FakeEffectinfo.EF_ELIDABLE_CANNOT_RAISE), [])
         assert h.getfield(box1, descr1) is box2
         assert h.getarrayitem(box1, index1, descr1) is box2
         assert h.getarrayitem(box1, index2, descr1) is box4
@@ -276,10 +276,10 @@
         assert h.getarrayitem(box1, index2, descr1) is box4
 
         h.invalidate_caches(
-            rop.CALL_LOOPINVARIANT, FakeCallDescr(FakeEffectinfo.EF_LOOPINVARIANT), [])
+            rop.CALL_LOOPINVARIANT_N, FakeCallDescr(FakeEffectinfo.EF_LOOPINVARIANT), [])
 
         h.invalidate_caches(
-            rop.CALL, FakeCallDescr(FakeEffectinfo.EF_RANDOM_EFFECTS), [])
+            rop.CALL_N, FakeCallDescr(FakeEffectinfo.EF_RANDOM_EFFECTS), [])
         assert h.getfield(box1, descr1) is None
         assert h.getarrayitem(box1, index1, descr1) is None
         assert h.getarrayitem(box1, index2, descr1) is None
@@ -375,13 +375,13 @@
         h.new_array(box2, lengthbox1)
         # Just need the destination box for this call
         h.invalidate_caches(
-            rop.CALL,
+            rop.CALL_N,
             arraycopydescr1,
             [None, box5, box2, index1, index1, index1]
         )
         assert h.getarrayitem(box1, index1, descr1) is box2
         h.invalidate_caches(
-            rop.CALL,
+            rop.CALL_N,
             arraycopydescr1,
             [None, box5, box3, index1, index1, index1]
         )
@@ -390,7 +390,7 @@
         h.setarrayitem(box4, index1, box2, descr1)
         assert h.getarrayitem(box4, index1, descr1) is box2
         h.invalidate_caches(
-            rop.CALL,
+            rop.CALL_N,
             arraycopydescr1,
             [None, box3, box5, index1, index1, index2]
         )
@@ -402,7 +402,7 @@
         assert h.getarrayitem(box1, index1, descr2) is box2
         h.new_array(box2, lengthbox2)
         h.invalidate_caches(
-            rop.CALL,
+            rop.CALL_N,
             arraycopydescr1,
             [None, box3, box2, index1, index1, index2]
         )
@@ -413,9 +413,9 @@
         h.setarrayitem(box1, index1, box2, descr2)
         assert h.getarrayitem(box1, index1, descr2) is box2
         h.invalidate_caches(
-            rop.CALL,
+            rop.CALL_N,
             arraycopydescr1,
-            [None, box3, box2, index1, index1, BoxInt()]
+            [None, box3, box2, index1, index1, InputArgInt()]
         )
         assert h.getarrayitem(box1, index1, descr2) is box2
 
@@ -423,7 +423,7 @@
         h = HeapCache()
         h.setarrayitem(box1, index1, box2, descr1)
         h.invalidate_caches(
-            rop.CALL,
+            rop.CALL_N,
             arraycopydescr1,
             [None, box1, box3, index1, index1, index2]
         )
@@ -434,7 +434,7 @@
         h.new_array(box1, lengthbox1)
         h.setarrayitem(box3, index1, box4, descr1)
         h.invalidate_caches(
-            rop.CALL,
+            rop.CALL_N,
             arraycopydescr1,
             [None, box2, box1, index1, index1, index2]
         )
@@ -444,16 +444,16 @@
         h.new_array(box1, lengthbox1)
         h.new_array(box2, lengthbox2)
         h.invalidate_caches(
-            rop.CALL,
+            rop.CALL_N,
             arraycopydescr1,
             [None, box2, box1, index1, index1, index2]
         )
         assert h.is_unescaped(box1)
         assert h.is_unescaped(box2)
         h.invalidate_caches(
-            rop.CALL,
+            rop.CALL_N,
             arraycopydescr1,
-            [None, box2, box1, index1, index1, BoxInt()]
+            [None, box2, box1, index1, index1, InputArgInt()]
         )
         assert not h.is_unescaped(box1)
         assert not h.is_unescaped(box2)
@@ -478,7 +478,7 @@
         h.invalidate_caches(rop.SETFIELD_GC, None, [box1, box2])
         assert h.is_unescaped(box2)
         # Reading a field from a virtual doesn't escape it.
-        h.invalidate_caches(rop.GETFIELD_GC, None, [box1])
+        h.invalidate_caches(rop.GETFIELD_GC_I, None, [box1])
         assert h.is_unescaped(box1)
         # Escaping a virtual transitively escapes anything inside of it.
         assert not h.is_unescaped(box3)
@@ -525,7 +525,7 @@
         assert h.is_unescaped(box1)
         assert h.is_unescaped(box2)
         h.invalidate_caches(
-            rop.CALL, FakeCallDescr(FakeEffectinfo.EF_RANDOM_EFFECTS), [box1]
+            rop.CALL_N, FakeCallDescr(FakeEffectinfo.EF_RANDOM_EFFECTS), [box1]
         )
         assert not h.is_unescaped(box1)
         assert not h.is_unescaped(box2)
@@ -535,7 +535,7 @@
         h.new(box1)
         assert h.is_unescaped(box1)
         h.setfield(box1, box2, descr1)
-        h.invalidate_caches(rop.CALL,
+        h.invalidate_caches(rop.CALL_N,
             FakeCallDescr(FakeEffectinfo.EF_CAN_RAISE),
             []
         )
@@ -546,7 +546,7 @@
         h.new_array(box1, lengthbox1)
         assert h.is_unescaped(box1)
         h.setarrayitem(box1, index1, box3, descr1)
-        h.invalidate_caches(rop.CALL,
+        h.invalidate_caches(rop.CALL_N,
             FakeCallDescr(FakeEffectinfo.EF_CAN_RAISE),
             []
         )
@@ -581,7 +581,7 @@
         h.setfield(box1, box2, descr1)
         h.invalidate_caches(rop.SETFIELD_GC, None, [box1, box2])
         assert h.getfield(box1, descr1) is box2
-        h.invalidate_caches(rop.CALL_MAY_FORCE, None, [])
+        h.invalidate_caches(rop.CALL_MAY_FORCE_N, FakeCallDescr(FakeEffectinfo.EF_RANDOM_EFFECTS), [])
         assert not h.is_unescaped(box1)
         assert not h.is_unescaped(box2)
         assert h.getfield(box1, descr1) is None
@@ -602,7 +602,7 @@
             EF_ELIDABLE_CANNOT_RAISE = 2
             EF_ELIDABLE_CAN_RAISE = 3
         descr.get_extra_info = XTra
-        h.invalidate_caches(rop.CALL, descr, [])
+        h.invalidate_caches(rop.CALL_N, descr, [])
         assert h.is_unescaped(box1)
         assert h.is_unescaped(box2)
         assert h.getfield(box1, descr1) is box2
diff --git a/rpython/jit/metainterp/test/test_history.py b/rpython/jit/metainterp/test/test_history.py
--- a/rpython/jit/metainterp/test/test_history.py
+++ b/rpython/jit/metainterp/test/test_history.py
@@ -13,18 +13,6 @@
     const = ConstPtr(lltype.cast_opaque_ptr(llmemory.GCREF, s))
     assert const._getrepr_() == "*T"
 
-def test_repr_ll2ctypes():
-    ptr = lltype.malloc(rffi.VOIDPP.TO, 10, flavor='raw')
-    # force it to be a ll2ctypes object
-    ptr = rffi.cast(rffi.VOIDPP, rffi.cast(rffi.LONG, ptr))
-    adr = llmemory.cast_ptr_to_adr(ptr)
-    lltype.free(ptr, flavor='raw')
-    intval = llmemory.cast_adr_to_int(adr, 'symbolic')
-    box = BoxInt(intval)
-    s = box.repr_rpython()
-    assert s.startswith('12345/') # the arbitrary hash value used by
-                                  # make_hashable_int
-
 def test_same_constant():
     c1a = ConstInt(0)
     c1b = ConstInt(0)
diff --git a/rpython/jit/metainterp/test/test_logger.py b/rpython/jit/metainterp/test/test_logger.py
--- a/rpython/jit/metainterp/test/test_logger.py
+++ b/rpython/jit/metainterp/test/test_logger.py
@@ -194,22 +194,6 @@
         assert lastline.startswith("guard_true(i0, descr=<")
         assert not lastline.startswith("guard_true(i0, descr=<Guard")
 
-    def test_class_name(self):
-        from rpython.rtyper.lltypesystem import lltype
-        AbcVTable = lltype.Struct('AbcVTable')
-        abcvtable = lltype.malloc(AbcVTable, immortal=True)
-        namespace = {'Name': abcvtable}
-        inp = '''
-        [i0]
-        p = new_with_vtable(ConstClass(Name))
-        '''
-        loop = pure_parse(inp, namespace=namespace)
-        logger = Logger(self.make_metainterp_sd())
-        output = logger.log_loop(loop)
-        assert output.splitlines()[-1].endswith(
-            " = new_with_vtable(ConstClass(Name))")
-        pure_parse(output, namespace=namespace)
-
     def test_intro_loop(self):
         bare_logger = logger.Logger(self.make_metainterp_sd())
         output = capturing(bare_logger.log_loop, [], [], 1, "foo")
diff --git a/rpython/jit/metainterp/test/test_pyjitpl.py b/rpython/jit/metainterp/test/test_pyjitpl.py
--- a/rpython/jit/metainterp/test/test_pyjitpl.py
+++ b/rpython/jit/metainterp/test/test_pyjitpl.py
@@ -3,9 +3,9 @@
 
 from rpython.jit.metainterp import pyjitpl
 from rpython.jit.metainterp import jitprof
-from rpython.jit.metainterp.history import BoxInt, ConstInt
+from rpython.jit.metainterp.history import ConstInt
 from rpython.jit.metainterp.history import History
-from rpython.jit.metainterp.resoperation import ResOperation, rop
+from rpython.jit.metainterp.resoperation import ResOperation, rop, InputArgInt
 from rpython.jit.metainterp.optimizeopt.util import equaloplists
 from rpython.jit.codewriter.jitcode import JitCode
 
@@ -58,13 +58,13 @@
         warmrunnerdesc = None
     def is_another_box_like(box, referencebox):
         assert box is not referencebox
-        assert isinstance(box, referencebox.clonebox().__class__)
-        assert box.value == referencebox.value
+        assert box.type == referencebox.type
+        assert box.getint() == referencebox.getint()
         return True
     metainterp = pyjitpl.MetaInterp(FakeStaticData(), None)
     metainterp.history = History()
-    b1 = BoxInt(1)
-    b2 = BoxInt(2)
+    b1 = InputArgInt(1)
+    b2 = InputArgInt(2)
     c3 = ConstInt(3)
     boxes = [b1, b2, b1, c3]
     dup = {}
@@ -74,19 +74,19 @@
     assert is_another_box_like(boxes[2], b1)
     assert is_another_box_like(boxes[3], c3)
     assert equaloplists(metainterp.history.operations, [
-        ResOperation(rop.SAME_AS, [b1], boxes[2]),
-        ResOperation(rop.SAME_AS, [c3], boxes[3]),
+        ResOperation(rop.SAME_AS_I, [b1]),
+        ResOperation(rop.SAME_AS_I, [c3]),
         ])
     assert dup == {b1: None, b2: None}
     #
     del metainterp.history.operations[:]
-    b4 = BoxInt(4)
+    b4 = InputArgInt(4)
     boxes = [b2, b4, "something random"]
     metainterp.remove_consts_and_duplicates(boxes, 2, dup)
     assert is_another_box_like(boxes[0], b2)
     assert boxes[1] is b4
     assert equaloplists(metainterp.history.operations, [
-        ResOperation(rop.SAME_AS, [b2], boxes[0]),
+        ResOperation(rop.SAME_AS_I, [b2]),
         ])
 
 def test_get_name_from_address():
diff --git a/rpython/jit/metainterp/test/test_resoperation.py b/rpython/jit/metainterp/test/test_resoperation.py
--- a/rpython/jit/metainterp/test/test_resoperation.py
+++ b/rpython/jit/metainterp/test/test_resoperation.py
@@ -33,10 +33,10 @@
     assert issubclass(cls, rop.BinaryOp)
     assert cls.getopnum.im_func(cls) == rop.rop.INT_ADD
 
-    cls = rop.opclasses[rop.rop.CALL]
+    cls = rop.opclasses[rop.rop.CALL_N]
     assert issubclass(cls, rop.ResOpWithDescr)
     assert issubclass(cls, rop.N_aryOp)
-    assert cls.getopnum.im_func(cls) == rop.rop.CALL
+    assert cls.getopnum.im_func(cls) == rop.rop.CALL_N
 
     cls = rop.opclasses[rop.rop.GUARD_TRUE]
     assert issubclass(cls, rop.GuardResOp)
@@ -48,36 +48,34 @@
     assert len(INT_ADD.__bases__) == 1
     BinaryPlainResOp = INT_ADD.__bases__[0]
     assert BinaryPlainResOp.__name__ == 'BinaryPlainResOp'
-    assert BinaryPlainResOp.__bases__ == (rop.BinaryOp, rop.PlainResOp)
+    assert BinaryPlainResOp.__bases__ == (rop.BinaryOp, rop.IntOp, rop.PlainResOp)
     INT_SUB = rop.opclasses[rop.rop.INT_SUB]
     assert INT_SUB.__bases__[0] is BinaryPlainResOp
 
 def test_instantiate():
-    op = rop.ResOperation(rop.rop.INT_ADD, ['a', 'b'], 'c')
+    op = rop.ResOperation(rop.rop.INT_ADD, ['a', 'b'])
     assert op.getarglist() == ['a', 'b']
-    assert op.result == 'c'
-    assert repr(op) == "c = int_add(a, b)"
+    #assert re.match(".*= int_add(a, b)", repr(op))
 
     mydescr = AbstractDescr()
-    op = rop.ResOperation(rop.rop.CALL, ['a', 'b'], 'c', descr=mydescr)
+    op = rop.ResOperation(rop.rop.CALL_I, ['a', 'b'], descr=mydescr)
     assert op.getarglist() == ['a', 'b']
-    assert op.result == 'c'
     assert op.getdescr() is mydescr
-    assert re.match("c = call\(a, b, descr=<.+>\)$", repr(op))
+    #assert re.match(".* = call\(a, b, descr=<.+>\)$", repr(op))
 
     mydescr = AbstractFailDescr()
-    op = rop.ResOperation(rop.rop.GUARD_NO_EXCEPTION, [], None, descr=mydescr)
-    assert re.match("guard_no_exception\(descr=<.+>\)$", repr(op))
+    op = rop.ResOperation(rop.rop.GUARD_NO_EXCEPTION, [], descr=mydescr)
+    #assert re.match("guard_no_exception\(descr=<.+>\)$", repr(op))
 
 def test_can_malloc():
     mydescr = AbstractDescr()
-    assert rop.ResOperation(rop.rop.NEW, [], 'b').can_malloc()
-    call = rop.ResOperation(rop.rop.CALL, ['a', 'b'], 'c', descr=mydescr)
+    assert rop.ResOperation(rop.rop.NEW, []).can_malloc()
+    call = rop.ResOperation(rop.rop.CALL_N, ['a', 'b'], descr=mydescr)
     assert call.can_malloc()
-    assert not rop.ResOperation(rop.rop.INT_ADD, ['a', 'b'], 'c').can_malloc()
+    assert not rop.ResOperation(rop.rop.INT_ADD, ['a', 'b']).can_malloc()
 
 def test_get_deep_immutable_oplist():
-    ops = [rop.ResOperation(rop.rop.INT_ADD, ['a', 'b'], 'c')]
+    ops = [rop.ResOperation(rop.rop.INT_ADD, ['a', 'b'])]
     newops = rop.get_deep_immutable_oplist(ops)
     py.test.raises(TypeError, "newops.append('foobar')")
     py.test.raises(TypeError, "newops[0] = 'foobar'")

From noreply at buildbot.pypy.org  Wed Jun  3 13:50:40 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Wed,  3 Jun 2015 13:50:40 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: implement more of vstring
Message-ID: <20150603115040.122E71C1017@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77812:64dd6e2e7f33
Date: 2015-06-03 13:20 +0200
http://bitbucket.org/pypy/pypy/changeset/64dd6e2e7f33/

Log:	implement more of vstring

diff --git a/rpython/jit/metainterp/optimizeopt/intbounds.py b/rpython/jit/metainterp/optimizeopt/intbounds.py
--- a/rpython/jit/metainterp/optimizeopt/intbounds.py
+++ b/rpython/jit/metainterp/optimizeopt/intbounds.py
@@ -384,13 +384,13 @@
         self.emit_operation(op)
         self.make_nonnull_str(op.getarg(0), vstring.mode_string)
         array = self.getptrinfo(op.getarg(0))
-        self.get_box_replacement(op).set_forwarded(array.getlenbound())
+        #self.get_box_replacement(op).set_forwarded(array.getlenbound())
 
     def optimize_UNICODELEN(self, op):
         self.emit_operation(op)
         self.make_nonnull_str(op.getarg(0), vstring.mode_unicode)
         array = self.getptrinfo(op.getarg(0))
-        self.get_box_replacement(op).set_forwarded(array.getlenbound())
+        #self.get_box_replacement(op).set_forwarded(array.getlenbound())
 
     def optimize_STRGETITEM(self, op):
         self.emit_operation(op)
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -777,7 +777,12 @@
                 raise resume.TagOverflow
         except resume.TagOverflow:
             raise compile.giveup()
-        descr.store_final_boxes(op, newboxes, self.metainterp_sd)
+        _newboxes = []
+        for box in newboxes:
+            if box is not None:
+                box = self.get_box_replacement(box)
+            _newboxes.append(box)
+        descr.store_final_boxes(op, _newboxes, self.metainterp_sd)
         #
         if op.getopnum() == rop.GUARD_VALUE:
             if op.getarg(0).type == 'i':
diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py
--- a/rpython/jit/metainterp/optimizeopt/virtualize.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualize.py
@@ -681,9 +681,7 @@
         if opinfo and opinfo.is_virtual():
             fieldop = opinfo.getfield(op.getdescr())
             if fieldop is None:
-                raise Exception("I think this is plain illegal")
-                xxx
-                fieldvalue = self.optimizer.new_const(op.getdescr())
+                fieldop = self.optimizer.new_const(op.getdescr())
             self.make_equal_to(op, fieldop)
         else:
             self.make_nonnull(op.getarg(0))
diff --git a/rpython/jit/metainterp/optimizeopt/vstring.py b/rpython/jit/metainterp/optimizeopt/vstring.py
--- a/rpython/jit/metainterp/optimizeopt/vstring.py
+++ b/rpython/jit/metainterp/optimizeopt/vstring.py
@@ -60,6 +60,7 @@
         self.length = length
         self._is_virtual = is_virtual
         self.mode = mode
+        self.length = length
 
     def getlenbound(self):
         from rpython.jit.metainterp.optimizeopt import intutils
@@ -204,6 +205,8 @@
         return visitor.visit_vstrplain(self.mode is mode_unicode)
 
 class VStringSliceInfo(StrPtrInfo):
+    length = -1
+    
     def __init__(self, s, start, length, mode):
         self.s = s
         self.start = start
@@ -224,12 +227,12 @@
         vstart = string_optimizer.getintbound(self.start)
         vlength = string_optimizer.getintbound(self.lgtop)
         if vstart.is_constant() and vlength.is_constant():
-            raise Exception("implement me")
-            s1 = self.vstr.get_constant_string_spec(mode)
+            vstr = string_optimizer.getptrinfo(self.s)
+            s1 = vstr.get_constant_string_spec(string_optimizer, mode)
             if s1 is None:
                 return None
-            start = self.vstart.box.getint()
-            length = self.vlength.box.getint()
+            start = vstart.getint()
+            length = vlength.getint()
             assert start >= 0
             assert length >= 0
             return s1[start : start + length]
@@ -238,6 +241,17 @@
     def getstrlen(self, op, string_optimizer, mode, create_ops=True):
         return self.lgtop
 
+    def visitor_walk_recursive(self, instbox, visitor, optimizer):
+        boxes = [self.s, self.start, self.lgtop]
+        visitor.register_virtual_fields(instbox, boxes)
+        opinfo = optimizer.getptrinfo(self.s)
+        if opinfo and opinfo.is_virtual():
+            opinfo.visitor_walk_recursive(visitor)
+
+    @specialize.argtype(1)
+    def visitor_dispatch_virtual_type(self, visitor):
+        return visitor.visit_vstrslice(self.mode is mode_unicode)
+
 class VStringConcatInfo(StrPtrInfo):
     #_attrs_ = ('mode', 'vleft', 'vright', '_is_virtual')
     
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -1168,9 +1168,9 @@
         if isinstance(box, Const):
             return box     # no promotion needed, already a Const
         else:
-            constbox = box.constbox()
+            constbox = ConstPtr(box.getref_base())
             resbox = self.do_residual_call(funcbox, [box, constbox], descr, orgpc)
-            promoted_box = resbox.constbox()
+            promoted_box = ConstInt(resbox.getint())
             # This is GUARD_VALUE because GUARD_TRUE assumes the existance
             # of a label when computing resumepc
             self.metainterp.generate_guard(rop.GUARD_VALUE, resbox,

From noreply at buildbot.pypy.org  Wed Jun  3 13:50:41 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Wed,  3 Jun 2015 13:50:41 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: finish vstring, I hope
Message-ID: <20150603115041.576FB1C1017@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77813:259ba1933861
Date: 2015-06-03 13:36 +0200
http://bitbucket.org/pypy/pypy/changeset/259ba1933861/

Log:	finish vstring, I hope

diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -866,9 +866,9 @@
     def optimize_UNICODEGETITEM(self, op):
         indexb = self.getintbound(op.getarg(1))
         if indexb.is_constant():
-            raise Exception("implement me")
-            arrayvalue = self.getvalue(op.getarg(0))
-            arrayvalue.make_len_gt(MODE_UNICODE, op.getdescr(), indexvalue.box.getint())
+            #arrayvalue = self.getvalue(op.getarg(0))
+            #arrayvalue.make_len_gt(MODE_UNICODE, op.getdescr(), indexvalue.box.getint())
+            pass
         self.optimize_default(op)
 
     # These are typically removed already by OptRewrite, but it can be
diff --git a/rpython/jit/metainterp/optimizeopt/vstring.py b/rpython/jit/metainterp/optimizeopt/vstring.py
--- a/rpython/jit/metainterp/optimizeopt/vstring.py
+++ b/rpython/jit/metainterp/optimizeopt/vstring.py
@@ -149,6 +149,10 @@
     def setitem(self, index, op, cf=None, optheap=None):
         self._chars[index] = op
 
+    def shrink(self, length):
+        assert length >= 0
+        del self._chars[length:]
+
     def setup_slice(self, longerlist, start, stop):
         assert 0 <= start <= stop <= len(longerlist)
         self._chars = longerlist[start:stop]
@@ -532,12 +536,14 @@
             self.pure_from_args(mode.STRLEN, [op], op.getarg(0))
 
     def optimize_STRSETITEM(self, op):
-        value = self.getptrinfo(op.getarg(0))
-        assert not value.is_constant() # strsetitem(ConstPtr) never makes sense
-        if value and value.is_virtual():
+        opinfo = self.getptrinfo(op.getarg(0))
+        if opinfo:
+            assert not opinfo.is_constant()
+            # strsetitem(ConstPtr) never makes sense
+        if opinfo and opinfo.is_virtual():
             indexbox = self.get_constant_box(op.getarg(1))
             if indexbox is not None:
-                value.setitem(indexbox.getint(),
+                opinfo.setitem(indexbox.getint(),
                               self.get_box_replacement(op.getarg(2)))
                 return
         self.make_nonnull(op.getarg(0))
@@ -864,37 +870,37 @@
         return False
 
     def opt_call_stroruni_STR_CMP(self, op, mode):
-        raise Exception('implement me')
-        v1 = self.getvalue(op.getarg(1))
-        v2 = self.getvalue(op.getarg(2))
-        l1box = v1.getstrlen(None, mode, None)
-        l2box = v2.getstrlen(None, mode, None)
+        i1 = self.getptrinfo(op.getarg(1))
+        i2 = self.getptrinfo(op.getarg(2))
+        if not i1 or not i2:
+            return False
+        l1box = i1.getstrlen(None, self, mode, False)
+        l2box = i2.getstrlen(None, self, mode, False)
         if (l1box is not None and l2box is not None and
             isinstance(l1box, ConstInt) and
             isinstance(l2box, ConstInt) and
-            l1box.value == l2box.value == 1):
+            l1box.getint() == l2box.getint() == 1):
             # comparing two single chars
-            vchar1 = self.strgetitem(v1, optimizer.CVAL_ZERO, mode)
-            vchar2 = self.strgetitem(v2, optimizer.CVAL_ZERO, mode)
+            char1 = self.strgetitem(None, op.getarg(1), optimizer.CONST_0, mode)
+            char2 = self.strgetitem(None, op.getarg(2), optimizer.CONST_0, mode)
             seo = self.optimizer.send_extra_operation
-            op = self.replace_op_with(op, rop.INT_SUB,
-                                      [vchar1.force_box(self),
-                                       vchar2.force_box(self)])
+            op = self.replace_op_with(op, rop.INT_SUB, [char1, char2],
+                                      descr=DONT_CHANGE)
             seo(op)
             return True
         return False
 
     def opt_call_SHRINK_ARRAY(self, op):
-        raise Exception('implement me')
-        v1 = self.getvalue(op.getarg(1))
-        v2 = self.getvalue(op.getarg(2))
+        i1 = self.getptrinfo(op.getarg(1))
+        i2 = self.getintbound(op.getarg(2))
         # If the index is constant, if the argument is virtual (we only support
         # VStringPlainValue for now) we can optimize away the call.
-        if v2.is_constant() and v1.is_virtual() and isinstance(v1, VStringPlainValue):
-            length = v2.box.getint()
-            v1.shrink(length)
+        if (i2 and i2.is_constant() and i1 and i1.is_virtual() and
+            isinstance(i1, VStringPlainInfo)):
+            length = i2.getint()
+            i1.shrink(length)
             self.last_emitted_operation = REMOVED
-            self.make_equal_to(op, v1)
+            self.make_equal_to(op, op.getarg(1))
             return True
         return False
 

From noreply at buildbot.pypy.org  Wed Jun  3 13:50:42 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Wed,  3 Jun 2015 13:50:42 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: small fixes
Message-ID: <20150603115042.6C5311C1017@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77814:4af0d7e2cd93
Date: 2015-06-03 13:50 +0200
http://bitbucket.org/pypy/pypy/changeset/4af0d7e2cd93/

Log:	small fixes

diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py
--- a/rpython/jit/metainterp/optimizeopt/info.py
+++ b/rpython/jit/metainterp/optimizeopt/info.py
@@ -1,5 +1,5 @@
 
-from rpython.rlib.objectmodel import specialize
+from rpython.rlib.objectmodel import specialize, we_are_translated
 from rpython.jit.metainterp.resoperation import AbstractValue, ResOperation,\
      rop
 from rpython.jit.metainterp.history import ConstInt, Const
@@ -213,7 +213,15 @@
         return self.size != -1
 
     def _force_elements(self, op, optforce, descr):
-        xxx
+        self.size = -1
+        for i in range(len(self.buffer.offsets)):
+            # write the value
+            offset = self.buffer.offsets[i]
+            descr = self.buffer.descrs[i]
+            itembox = self.buffer.values[i]
+            op = ResOperation(rop.RAW_STORE,
+                              [op, ConstInt(offset), itembox], descr=descr)
+            optforce.emit_operation(op)
 
     def visitor_walk_recursive(self, op, visitor, optimizer):
         itemboxes = self.buffer.values
diff --git a/rpython/jit/metainterp/test/test_virtualref.py b/rpython/jit/metainterp/test/test_virtualref.py
--- a/rpython/jit/metainterp/test/test_virtualref.py
+++ b/rpython/jit/metainterp/test/test_virtualref.py
@@ -103,10 +103,10 @@
         [guard_op] = [op for op in ops
                          if op.getopnum() == rop.GUARD_NOT_FORCED]
         bxs1 = [box for box in guard_op.getfailargs()
-                  if str(box._getrepr_()).endswith('.X')]
+                  if '.X' in str(box)]
         assert len(bxs1) == 1
         bxs2 = [box for box in guard_op.getfailargs()
-                  if str(box._getrepr_()).endswith('JitVirtualRef')]
+                  if 'JitVirtualRef' in str(box)]
         assert len(bxs2) == 1
         JIT_VIRTUAL_REF = self.vrefinfo.JIT_VIRTUAL_REF
         FOO = lltype.GcStruct('FOO')

From noreply at buildbot.pypy.org  Wed Jun  3 13:53:24 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Wed,  3 Jun 2015 13:53:24 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Remove lib_pypy/_struct.py which doesn't
 work on py3k and is redundant anyway.
Message-ID: <20150603115324.BEB1A1C1017@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob <me at manueljacob.de>
Branch: py3k
Changeset: r77815:a7c8ec8466c0
Date: 2015-06-03 13:53 +0200
http://bitbucket.org/pypy/pypy/changeset/a7c8ec8466c0/

Log:	Remove lib_pypy/_struct.py which doesn't work on py3k and is
	redundant anyway.

diff --git a/lib_pypy/_struct.py b/lib_pypy/_struct.py
deleted file mode 100644
--- a/lib_pypy/_struct.py
+++ /dev/null
@@ -1,421 +0,0 @@
-#
-# This module is a pure Python version of pypy.module.struct.
-# It is only imported if the vastly faster pypy.module.struct is not
-# compiled in.  For now we keep this version for reference and
-# because pypy.module.struct is not ootype-backend-friendly yet.
-#
-
-"""Functions to convert between Python values and C structs.
-Python strings are used to hold the data representing the C struct
-and also as format strings to describe the layout of data in the C struct.
-
-The optional first format char indicates byte order, size and alignment:
- @: native order, size & alignment (default)
- =: native order, std. size & alignment
- <: little-endian, std. size & alignment
- >: big-endian, std. size & alignment
- !: same as >
-
-The remaining chars indicate types of args and must match exactly;
-these can be preceded by a decimal repeat count:
-   x: pad byte (no data);
-   c:char;
-   b:signed byte;
-   B:unsigned byte;
-   h:short;
-   H:unsigned short;
-   i:int;
-   I:unsigned int;
-   l:long;
-   L:unsigned long;
-   f:float;
-   d:double.
-Special cases (preceding decimal count indicates length):
-   s:string (array of char); p: pascal string (with count byte).
-Special case (only available in native format):
-   P:an integer type that is wide enough to hold a pointer.
-Special case (not in native mode unless 'long long' in platform C):
-   q:long long;
-   Q:unsigned long long
-Whitespace between formats is ignored.
-
-The variable struct.error is an exception raised on errors."""
-
-import math, sys
-
-# TODO: XXX Find a way to get information on native sizes and alignments
-class StructError(Exception):
-    pass
-error = StructError
-def unpack_int(data,index,size,le):
-    bytes = [b for b in data[index:index+size]]
-    if le == 'little':
-        bytes.reverse()
-    number = 0
-    for b in bytes:
-        number = number << 8 | b
-    return int(number)
-
-def unpack_signed_int(data,index,size,le):
-    number = unpack_int(data,index,size,le)
-    max = 2**(size*8)
-    if number > 2**(size*8 - 1) - 1:
-        number = int(-1*(max - number))
-    return number
-
-INFINITY = 1e200 * 1e200
-NAN = INFINITY / INFINITY
-
-def unpack_char(data,index,size,le):
-    return data[index:index+size]
-
-def pack_int(number,size,le):
-    x=number
-    res=[]
-    for i in range(size):
-        res.append(x&0xff)
-        x >>= 8
-    if le == 'big':
-        res.reverse()
-    return bytes(res)
-
-def pack_signed_int(number,size,le):
-    if not isinstance(number, int):
-        raise StructError("argument for i,I,l,L,q,Q,h,H must be integer")
-    if  number > 2**(8*size-1)-1 or number < -1*2**(8*size-1):
-        raise OverflowError("Number:%i too large to convert" % number)
-    return pack_int(number,size,le)
-
-def pack_unsigned_int(number,size,le):
-    if not isinstance(number, int):
-        raise StructError("argument for i,I,l,L,q,Q,h,H must be integer")
-    if number < 0:
-        raise TypeError("can't convert negative long to unsigned")
-    if number > 2**(8*size)-1:
-        raise OverflowError("Number:%i too large to convert" % number)
-    return pack_int(number,size,le)
-
-def pack_char(char,size,le):
-    return bytes(char)
-
-def isinf(x):
-    return x != 0.0 and x / 2 == x
-def isnan(v):
-    return v != v*1.0 or (v == 1.0 and v == 2.0)
-
-def pack_float(x, size, le):
-    unsigned = float_pack(x, size)
-    result = []
-    for i in range(8):
-        result.append((unsigned >> (i * 8)) & 0xFF)
-    if le == "big":
-        result.reverse()
-    return bytes(result)
-
-def unpack_float(data, index, size, le):
-    binary = [data[i] for i in range(index, index + 8)]
-    if le == "big":
-        binary.reverse()
-    unsigned = 0
-    for i in range(8):
-        unsigned |= binary[i] << (i * 8)
-    return float_unpack(unsigned, size, le)
-
-def round_to_nearest(x):
-    """Python 3 style round:  round a float x to the nearest int, but
-    unlike the builtin Python 2.x round function:
-
-      - return an int, not a float
-      - do round-half-to-even, not round-half-away-from-zero.
-
-    We assume that x is finite and nonnegative; except wrong results
-    if you use this for negative x.
-
-    """
-    int_part = int(x)
-    frac_part = x - int_part
-    if frac_part > 0.5 or frac_part == 0.5 and int_part & 1 == 1:
-        int_part += 1
-    return int_part
-
-def float_unpack(Q, size, le):
-    """Convert a 32-bit or 64-bit integer created
-    by float_pack into a Python float."""
-
-    if size == 8:
-        MIN_EXP = -1021  # = sys.float_info.min_exp
-        MAX_EXP = 1024   # = sys.float_info.max_exp
-        MANT_DIG = 53    # = sys.float_info.mant_dig
-        BITS = 64
-    elif size == 4:
-        MIN_EXP = -125   # C's FLT_MIN_EXP
-        MAX_EXP = 128    # FLT_MAX_EXP
-        MANT_DIG = 24    # FLT_MANT_DIG
-        BITS = 32
-    else:
-        raise ValueError("invalid size value")
-
-    if Q >> BITS:
-         raise ValueError("input out of range")
-
-    # extract pieces
-    sign = Q >> BITS - 1
-    exp = (Q & ((1 << BITS - 1) - (1 << MANT_DIG - 1))) >> MANT_DIG - 1
-    mant = Q & ((1 << MANT_DIG - 1) - 1)
-
-    if exp == MAX_EXP - MIN_EXP + 2:
-        # nan or infinity
-        result = float('nan') if mant else float('inf')
-    elif exp == 0:
-        # subnormal or zero
-        result = math.ldexp(float(mant), MIN_EXP - MANT_DIG)
-    else:
-        # normal
-        mant += 1 << MANT_DIG - 1
-        result = math.ldexp(float(mant), exp + MIN_EXP - MANT_DIG - 1)
-    return -result if sign else result
-
-
-def float_pack(x, size):
-    """Convert a Python float x into a 64-bit unsigned integer
-    with the same byte representation."""
-
-    if size == 8:
-        MIN_EXP = -1021  # = sys.float_info.min_exp
-        MAX_EXP = 1024   # = sys.float_info.max_exp
-        MANT_DIG = 53    # = sys.float_info.mant_dig
-        BITS = 64
-    elif size == 4:
-        MIN_EXP = -125   # C's FLT_MIN_EXP
-        MAX_EXP = 128    # FLT_MAX_EXP
-        MANT_DIG = 24    # FLT_MANT_DIG
-        BITS = 32
-    else:
-        raise ValueError("invalid size value")
-
-    sign = math.copysign(1.0, x) < 0.0
-    if math.isinf(x):
-        mant = 0
-        exp = MAX_EXP - MIN_EXP + 2
-    elif math.isnan(x):
-        mant = 1 << (MANT_DIG-2) # other values possible
-        exp = MAX_EXP - MIN_EXP + 2
-    elif x == 0.0:
-        mant = 0
-        exp = 0
-    else:
-        m, e = math.frexp(abs(x))  # abs(x) == m * 2**e
-        exp = e - (MIN_EXP - 1)
-        if exp > 0:
-            # Normal case.
-            mant = round_to_nearest(m * (1 << MANT_DIG))
-            mant -= 1 << MANT_DIG - 1
-        else:
-            # Subnormal case.
-            if exp + MANT_DIG - 1 >= 0:
-                mant = round_to_nearest(m * (1 << exp + MANT_DIG - 1))
-            else:
-                mant = 0
-            exp = 0
-
-        # Special case: rounding produced a MANT_DIG-bit mantissa.
-        assert 0 <= mant <= 1 << MANT_DIG - 1
-        if mant == 1 << MANT_DIG - 1:
-            mant = 0
-            exp += 1
-
-        # Raise on overflow (in some circumstances, may want to return
-        # infinity instead).
-        if exp >= MAX_EXP - MIN_EXP + 2:
-             raise OverflowError("float too large to pack in this format")
-
-    # check constraints
-    assert 0 <= mant < 1 << MANT_DIG - 1
-    assert 0 <= exp <= MAX_EXP - MIN_EXP + 2
-    assert 0 <= sign <= 1
-    return ((sign << BITS - 1) | (exp << MANT_DIG - 1)) | mant
-
-
-big_endian_format = {
-    'x':{ 'size' : 1, 'alignment' : 0, 'pack' : None, 'unpack' : None},
-    'b':{ 'size' : 1, 'alignment' : 0, 'pack' : pack_signed_int, 'unpack' : unpack_signed_int},
-    'B':{ 'size' : 1, 'alignment' : 0, 'pack' : pack_unsigned_int, 'unpack' : unpack_int},
-    'c':{ 'size' : 1, 'alignment' : 0, 'pack' : pack_char, 'unpack' : unpack_char},
-    's':{ 'size' : 1, 'alignment' : 0, 'pack' : None, 'unpack' : None},
-    'p':{ 'size' : 1, 'alignment' : 0, 'pack' : None, 'unpack' : None},
-    'h':{ 'size' : 2, 'alignment' : 0, 'pack' : pack_signed_int, 'unpack' : unpack_signed_int},
-    'H':{ 'size' : 2, 'alignment' : 0, 'pack' : pack_unsigned_int, 'unpack' : unpack_int},
-    'i':{ 'size' : 4, 'alignment' : 0, 'pack' : pack_signed_int, 'unpack' : unpack_signed_int},
-    'I':{ 'size' : 4, 'alignment' : 0, 'pack' : pack_unsigned_int, 'unpack' : unpack_int},
-    'l':{ 'size' : 4, 'alignment' : 0, 'pack' : pack_signed_int, 'unpack' : unpack_signed_int},
-    'L':{ 'size' : 4, 'alignment' : 0, 'pack' : pack_unsigned_int, 'unpack' : unpack_int},
-    'q':{ 'size' : 8, 'alignment' : 0, 'pack' : pack_signed_int, 'unpack' : unpack_signed_int},
-    'Q':{ 'size' : 8, 'alignment' : 0, 'pack' : pack_unsigned_int, 'unpack' : unpack_int},
-    'f':{ 'size' : 4, 'alignment' : 0, 'pack' : pack_float, 'unpack' : unpack_float},
-    'd':{ 'size' : 8, 'alignment' : 0, 'pack' : pack_float, 'unpack' : unpack_float},
-    }
-default = big_endian_format
-formatmode={ '<' : (default, 'little'),
-             '>' : (default, 'big'),
-             '!' : (default, 'big'),
-             '=' : (default, sys.byteorder),
-             '@' : (default, sys.byteorder)
-            }
-
-def getmode(fmt):
-    try:
-        formatdef,endianness = formatmode[fmt[0]]
-        index = 1
-    except (IndexError, KeyError):
-        formatdef,endianness = formatmode['@']
-        index = 0
-    return formatdef,endianness,index
-def getNum(fmt,i):
-    num=None
-    cur = fmt[i]
-    while ('0'<= cur ) and ( cur <= '9'):
-        if num == None:
-            num = int(cur)
-        else:
-            num = 10*num + int(cur)
-        i += 1
-        cur = fmt[i]
-    return num,i
-
-def calcsize(fmt):
-    """calcsize(fmt) -> int
-    Return size of C struct described by format string fmt.
-    See struct.__doc__ for more on format strings."""
-
-    formatdef,endianness,i = getmode(fmt)
-    num = 0
-    result = 0
-    while i<len(fmt):
-        num,i = getNum(fmt,i)
-        cur = fmt[i]
-        try:
-            format = formatdef[cur]
-        except KeyError:
-            raise StructError("%s is not a valid format" % cur)
-        if num != None :
-            result += num*format['size']
-        else:
-            result += format['size']
-        num = 0
-        i += 1
-    return result
-
-def pack(fmt,*args):
-    """pack(fmt, v1, v2, ...) -> string
-       Return string containing values v1, v2, ... packed according to fmt.
-       See struct.__doc__ for more on format strings."""
-    formatdef,endianness,i = getmode(fmt)
-    args = list(args)
-    n_args = len(args)
-    result = []
-    while i<len(fmt):
-        num,i = getNum(fmt,i)
-        cur = fmt[i]
-        try:
-            format = formatdef[cur]
-        except KeyError:
-            raise StructError("%s is not a valid format" % cur)
-        if num == None :
-            num_s = 0
-            num = 1
-        else:
-            num_s = num
-
-        if cur == 'x':
-            result += [b'\0'*num]
-        elif cur == 's':
-            if isinstance(args[0], bytes):
-                padding = num - len(args[0])
-                result += [args[0][:num] + b'\0'*padding]
-                args.pop(0)
-            else:
-                raise StructError("arg for string format not a string")
-        elif cur == 'p':
-            if isinstance(args[0], bytes):
-                padding = num - len(args[0]) - 1
-
-                if padding > 0:
-                    result += [bytes([len(args[0])]) + args[0][:num-1] + b'\0'*padding]
-                else:
-                    if num<255:
-                        result += [bytes([num-1]) + args[0][:num-1]]
-                    else:
-                        result += [bytes([255]) + args[0][:num-1]]
-                args.pop(0)
-            else:
-                raise StructError("arg for string format not a string")
-
-        else:
-            if len(args) < num:
-                raise StructError("insufficient arguments to pack")
-            for var in args[:num]:
-                result += [format['pack'](var,format['size'],endianness)]
-            args=args[num:]
-        num = None
-        i += 1
-    if len(args) != 0:
-        raise StructError("too many arguments for pack format")
-    return b''.join(result)
-
-def unpack(fmt,data):
-    """unpack(fmt, string) -> (v1, v2, ...)
-       Unpack the string, containing packed C structure data, according
-       to fmt.  Requires len(string)==calcsize(fmt).
-       See struct.__doc__ for more on format strings."""
-    formatdef,endianness,i = getmode(fmt)
-    j = 0
-    num = 0
-    result = []
-    length= calcsize(fmt)
-    if length != len (data):
-        raise StructError("unpack str size does not match format")
-    while i<len(fmt):
-        num,i=getNum(fmt,i)
-        cur = fmt[i]
-        i += 1
-        try:
-            format = formatdef[cur]
-        except KeyError:
-            raise StructError("%s is not a valid format" % cur)
-
-        if not num :
-            num = 1
-
-        if cur == 'x':
-            j += num
-        elif cur == 's':
-            result.append(data[j:j+num])
-            j += num
-        elif cur == 'p':
-            n=data[j]
-            if n >= num:
-                n = num-1
-            result.append(data[j+1:j+n+1])
-            j += num
-        else:
-            for n in range(num):
-                result += [format['unpack'](data,j,format['size'],endianness)]
-                j += format['size']
-
-    return tuple(result)
-
-def pack_into(fmt, buf, offset, *args):
-    data = pack(fmt, *args)
-    buffer(buf)[offset:offset+len(data)] = data
-
-def unpack_from(fmt, buf, offset=0):
-    size = calcsize(fmt)
-    data = buffer(buf)[offset:offset+size]
-    if len(data) != size:
-        raise error("unpack_from requires a buffer of at least %d bytes"
-                    % (size,))
-    return unpack(fmt, data)
-
-def _clearcache():
-    "Clear the internal cache."
-    # No cache in this implementation

From noreply at buildbot.pypy.org  Wed Jun  3 14:03:49 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Wed,  3 Jun 2015 14:03:49 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: fix test_calling_convention
Message-ID: <20150603120349.1AEDB1C1038@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77816:e5fab419b6e2
Date: 2015-06-03 14:03 +0200
http://bitbucket.org/pypy/pypy/changeset/e5fab419b6e2/

Log:	fix test_calling_convention

diff --git a/rpython/jit/backend/test/calling_convention_test.py b/rpython/jit/backend/test/calling_convention_test.py
--- a/rpython/jit/backend/test/calling_convention_test.py
+++ b/rpython/jit/backend/test/calling_convention_test.py
@@ -1,7 +1,7 @@
 
-from rpython.jit.metainterp.history import BasicFinalDescr, BoxInt,\
-     JitCellToken, ConstInt, BoxFloat, ConstFloat
-from rpython.jit.metainterp.resoperation import rop
+from rpython.jit.metainterp.history import BasicFinalDescr,\
+     JitCellToken, ConstInt, ConstFloat
+from rpython.jit.metainterp.resoperation import rop, InputArgInt, InputArgFloat
 from rpython.jit.codewriter.effectinfo import EffectInfo
 from rpython.jit.tool.oparser import parse
 from rpython.rtyper.lltypesystem import lltype, llmemory
@@ -14,7 +14,7 @@
 import platform
 
 def boxfloat(x):
-    return BoxFloat(longlong.getfloatstorage(x))
+    return InputArgFloat(longlong.getfloatstorage(x))
 
 def constfloat(x):
     return ConstFloat(longlong.getfloatstorage(x))
@@ -98,8 +98,8 @@
 
             ops = '[%s]\n' % arguments
             ops += '%s\n' % spill_ops
-            ops += 'f99 = call(ConstClass(func_ptr), %s, descr=calldescr)\n' % arguments
-            ops += 'i99 = same_as(0)\n'
+            ops += 'f99 = call_f(ConstClass(func_ptr), %s, descr=calldescr)\n' % arguments
+            ops += 'i99 = same_as_i(0)\n'
             ops += 'guard_true(i99) [f99, %s]\n' % arguments
             ops += 'finish()\n'
 
@@ -149,10 +149,10 @@
                                         EffectInfo.MOST_GENERAL)
             funcbox = self.get_funcbox(cpu, func_ptr)
 
-            res = self.execute_operation(rop.CALL,
+            res = self.execute_operation(rop.CALL_F,
                                          [funcbox] + argslist,
                                          'float', descr=calldescr)
-            assert abs(res.getfloat() - result) < 0.0001
+            assert abs(res - result) < 0.0001
 
     def test_call_aligned_with_args_on_the_stack(self):
         cpu = self.cpu
@@ -183,7 +183,7 @@
                     args.append(I)
                     arg = local_ints.pop()
                     result += arg
-                    argslist.append(BoxInt(arg))
+                    argslist.append(InputArgInt(arg))
             FUNC = self.FuncType(args, F)
             FPTR = self.Ptr(FUNC)
             func_ptr = llhelper(FPTR, func)
@@ -191,10 +191,10 @@
                                         EffectInfo.MOST_GENERAL)
             funcbox = self.get_funcbox(cpu, func_ptr)
 
-            res = self.execute_operation(rop.CALL,
+            res = self.execute_operation(rop.CALL_F,
                                          [funcbox] + argslist,
                                          'float', descr=calldescr)
-            assert abs(res.getfloat() - result) < 0.0001
+            assert abs(res - result) < 0.0001
 
     def test_call_alignment_call_assembler(self):
         cpu = self.cpu
@@ -269,7 +269,7 @@
                 EffectInfo.MOST_GENERAL)
             ops = '''
             [%s]
-            f99 = call_assembler(%s, descr=called_looptoken)
+            f99 = call_assembler_f(%s, descr=called_looptoken)
             guard_not_forced()[]
             finish(f99, descr=fdescr4)
             ''' % (arguments, arguments)
@@ -311,10 +311,10 @@
         calldescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, EffectInfo.MOST_GENERAL)
         funcbox = self.get_funcbox(cpu, func_ptr)
 
-        res = self.execute_operation(rop.CALL,
+        res = self.execute_operation(rop.CALL_I,
                                      [funcbox] + argslist,
                                      'int', descr=calldescr)
-        assert res.value == result
+        assert res == result
 
 
     def test_call_with_singlefloats(self):
@@ -346,7 +346,7 @@
             local_ints = list(ints)
             for i in range(random.randrange(4, 20)):
                 case = random.randrange(0, 6)
-                if case & 1: boxme = BoxInt
+                if case & 1: boxme = InputArgInt
                 else:        boxme = ConstInt
                 if case < 2:
                     args.append(F)
@@ -369,11 +369,11 @@
                                         EffectInfo.MOST_GENERAL)
             funcbox = self.get_funcbox(cpu, func_ptr)
 
-            res = self.execute_operation(rop.CALL,
+            res = self.execute_operation(rop.CALL_F,
                                          [funcbox] + argslist,
                                          'float', descr=calldescr)
             expected = func(*argvalues)
-            assert abs(res.getfloat() - expected) < 0.0001
+            assert abs(res - expected) < 0.0001
 
 
     def make_function_returning_stack_pointer(self):
@@ -418,7 +418,7 @@
                                         EffectInfo.MOST_GENERAL)
 
             ops = '[%s]\n' % arguments
-            ops += 'i99 = call(%d, %s, descr=calldescr)\n' % (func_addr,
+            ops += 'i99 = call_i(%d, %s, descr=calldescr)\n' % (func_addr,
                                                               arguments)
             ops += 'finish(i99)\n'
 
diff --git a/rpython/jit/codewriter/heaptracker.py b/rpython/jit/codewriter/heaptracker.py
--- a/rpython/jit/codewriter/heaptracker.py
+++ b/rpython/jit/codewriter/heaptracker.py
@@ -64,6 +64,8 @@
     if not has_gcstruct_a_vtable(GCSTRUCT):
         return None
     setup_cache_gcstruct2vtable(gccache)
+    if not hasattr(gccache, '_cache_gcstruct2vtable'):
+        return GCSTRUCT.typeptr
     return gccache._cache_gcstruct2vtable[GCSTRUCT]
 
 def setup_cache_gcstruct2vtable(gccache):

From noreply at buildbot.pypy.org  Wed Jun  3 14:12:21 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Wed,  3 Jun 2015 14:12:21 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: fragile fragile
Message-ID: <20150603121221.3F38F1C024E@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77817:be2d3c2e4d27
Date: 2015-06-03 14:12 +0200
http://bitbucket.org/pypy/pypy/changeset/be2d3c2e4d27/

Log:	fragile fragile

diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py
--- a/rpython/jit/metainterp/optimizeopt/info.py
+++ b/rpython/jit/metainterp/optimizeopt/info.py
@@ -241,9 +241,9 @@
 class RawStructPtrInfo(AbstractRawPtrInfo):
     def __init__(self):
         pass
-    
-    def _force_elements(self, op, optforce, descr):
-        xxx
+
+    def is_virtual(self):
+        return False
 
 class RawSlicePtrInfo(AbstractRawPtrInfo):
     def __init__(self, offset, parent):
@@ -260,7 +260,7 @@
         self.parent.setitem_raw(self.offset+offset, itemsize, descr, itemop)
     
     def _force_elements(self, op, optforce, descr):
-        xxx
+        raise Exception("implement me")
 
     def visitor_walk_recursive(self, op, visitor, optimizer):
         source_op = optimizer.get_box_replacement(op.getarg(0))
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -768,7 +768,7 @@
             op.setdescr(descr)
         assert isinstance(descr, compile.ResumeGuardDescr)
         assert isinstance(op, GuardResOp)
-        modifier = resume.ResumeDataVirtualAdder(descr, op,
+        modifier = resume.ResumeDataVirtualAdder(self, descr, op,
                                                  self.resumedata_memo)
         try:
             newboxes = modifier.finish(self, pendingfields)
@@ -777,12 +777,7 @@
                 raise resume.TagOverflow
         except resume.TagOverflow:
             raise compile.giveup()
-        _newboxes = []
-        for box in newboxes:
-            if box is not None:
-                box = self.get_box_replacement(box)
-            _newboxes.append(box)
-        descr.store_final_boxes(op, _newboxes, self.metainterp_sd)
+        descr.store_final_boxes(op, newboxes, self.metainterp_sd)
         #
         if op.getopnum() == rop.GUARD_VALUE:
             if op.getarg(0).type == 'i':
diff --git a/rpython/jit/metainterp/resume.py b/rpython/jit/metainterp/resume.py
--- a/rpython/jit/metainterp/resume.py
+++ b/rpython/jit/metainterp/resume.py
@@ -223,10 +223,10 @@
                 is_virtual = False
                 if box.type == 'r':
                     info = optimizer.getptrinfo(box)
-                    is_virtual = (info and info.is_virtual())
+                    is_virtual = (info is not None and info.is_virtual())
                 if box.type == 'i':
                     info = optimizer.getrawptrinfo(box, create=False)
-                    is_virtual = (info and info.is_virtual()) 
+                    is_virtual = (info is not None and info.is_virtual()) 
                 if is_virtual:
                     tagged = tag(v, TAGVIRTUAL)
                     v += 1
@@ -286,7 +286,8 @@
 
 class ResumeDataVirtualAdder(VirtualVisitor):
 
-    def __init__(self, storage, snapshot_storage, memo):
+    def __init__(self, optimizer, storage, snapshot_storage, memo):
+        self.optimizer = optimizer
         self.storage = storage
         self.snapshot_storage = snapshot_storage
         self.memo = memo
@@ -343,9 +344,14 @@
         else:
             return VStrSliceInfo()
 
-    def register_virtual_fields(self, virtualbox, fieldboxes):
+    def register_virtual_fields(self, virtualbox, _fieldboxes):
         tagged = self.liveboxes_from_env.get(virtualbox, UNASSIGNEDVIRTUAL)
         self.liveboxes[virtualbox] = tagged
+        fieldboxes = []
+        for box in _fieldboxes:
+            if box is not None:
+                box = self.optimizer.get_box_replacement(box)
+            fieldboxes.append(box)
         self.vfieldboxes[virtualbox] = fieldboxes
         self._register_boxes(fieldboxes)
 

From noreply at buildbot.pypy.org  Wed Jun  3 15:01:41 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Wed,  3 Jun 2015 15:01:41 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: 2to3
Message-ID: <20150603130141.D516D1C0FB8@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob <me at manueljacob.de>
Branch: py3k
Changeset: r77818:b75b97b522c4
Date: 2015-06-03 15:01 +0200
http://bitbucket.org/pypy/pypy/changeset/b75b97b522c4/

Log:	2to3

diff --git a/pypy/module/test_lib_pypy/test_grp_extra.py b/pypy/module/test_lib_pypy/test_grp_extra.py
--- a/pypy/module/test_lib_pypy/test_grp_extra.py
+++ b/pypy/module/test_lib_pypy/test_grp_extra.py
@@ -20,7 +20,7 @@
             assert g.gr_gid == 0
             assert 'root' in g.gr_mem or g.gr_mem == []
             assert g.gr_name == name
-            assert isinstance(g.gr_passwd, str)    # usually just 'x', don't hope :-)
+            assert isinstance(g.gr_passwd, bytes)    # usually just 'x', don't hope :-)
             break
         else:
             raise

From noreply at buildbot.pypy.org  Wed Jun  3 15:10:10 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Wed,  3 Jun 2015 15:10:10 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Add missing import.
Message-ID: <20150603131010.0F7FF1C0FB8@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob <me at manueljacob.de>
Branch: py3k
Changeset: r77819:cd5d41f4974d
Date: 2015-06-03 15:10 +0200
http://bitbucket.org/pypy/pypy/changeset/cd5d41f4974d/

Log:	Add missing import.

diff --git a/lib_pypy/_gdbm.py b/lib_pypy/_gdbm.py
--- a/lib_pypy/_gdbm.py
+++ b/lib_pypy/_gdbm.py
@@ -1,5 +1,5 @@
 from _gdbm_cffi import ffi, lib    # generated by _gdbm_build.py
-import os, threading
+import sys, os, threading
 _lock = threading.Lock()
 
 class error(IOError):

From noreply at buildbot.pypy.org  Wed Jun  3 15:54:05 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Wed,  3 Jun 2015 15:54:05 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: use the proper descr
Message-ID: <20150603135405.DC6541C0F16@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77820:0e859127ef84
Date: 2015-06-03 14:16 +0200
http://bitbucket.org/pypy/pypy/changeset/0e859127ef84/

Log:	use the proper descr

diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -2964,7 +2964,7 @@
         else:
             return None, op
 
-    def direct_libffi_call(self, argboxes, descr, tp):
+    def direct_libffi_call(self, argboxes, orig_calldescr, tp):
         """Generate a direct call to C code using jit_ffi_call()
         """
         # an 'assert' that constant-folds away the rest of this function
@@ -2982,7 +2982,7 @@
         cif_description = llmemory.cast_int_to_adr(cif_description)
         cif_description = llmemory.cast_adr_to_ptr(cif_description,
                                                    CIF_DESCRIPTION_P)
-        extrainfo = descr.get_extra_info()
+        extrainfo = orig_calldescr.get_extra_info()
         calldescr = self.cpu.calldescrof_dynamic(cif_description, extrainfo)
         if calldescr is None:
             return
@@ -3019,14 +3019,14 @@
         if tp == 'i':
             value = executor.execute_varargs(self.cpu, self,
                                              rop.CALL_MAY_FORCE_I,
-                                             argboxes, descr)
+                                             argboxes, orig_calldescr)
             box_result = self.history.record(
                 rop.CALL_RELEASE_GIL_I, [c_saveall, argboxes[2]] + arg_boxes,
                 value, descr=calldescr)
         elif tp == 'f':
             value = executor.execute_varargs(self.cpu, self,
                                              rop.CALL_MAY_FORCE_F,
-                                             argboxes, descr)
+                                             argboxes, orig_calldescr)
             box_result = self.history.record(
                 rop.CALL_RELEASE_GIL_F, [c_saveall, argboxes[2]] + arg_boxes,
                 value, descr=calldescr)

From noreply at buildbot.pypy.org  Wed Jun  3 15:54:07 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Wed,  3 Jun 2015 15:54:07 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: unify the return value
Message-ID: <20150603135407.240E61C0F16@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77821:249cc7871bd6
Date: 2015-06-03 14:20 +0200
http://bitbucket.org/pypy/pypy/changeset/249cc7871bd6/

Log:	unify the return value

diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py
--- a/rpython/jit/metainterp/optimizeopt/info.py
+++ b/rpython/jit/metainterp/optimizeopt/info.py
@@ -123,8 +123,7 @@
 
     def _force_elements(self, op, optforce, descr):
         if self._fields is None:
-            return 0
-        count = 0
+            return
         for i, flddescr in enumerate(descr.get_all_fielddescrs()):
             fld = self._fields[i]
             if fld is not None:
@@ -133,8 +132,6 @@
                                           descr=flddescr)
                 optforce._emit_operation(setfieldop)
                 optforce.optheap.register_dirty_field(flddescr, self)
-                count += 1
-        return count
 
     def visitor_walk_recursive(self, instbox, visitor, optimizer):
         if visitor.already_seen_virtual(instbox):
@@ -299,7 +296,6 @@
 
     def _force_elements(self, op, optforce, descr):
         arraydescr = op.getdescr()
-        count = 0
         for i in range(self.length):
             item = self._items[i]
             if item is not None:
@@ -309,8 +305,6 @@
                                       descr=arraydescr)
                 optforce._emit_operation(setop)
                 # xxxx optforce.optheap
-                count += 1
-        return count
 
     def setitem(self, index, op, cf=None, optheap=None):
         if self._items is None:
@@ -368,7 +362,6 @@
     def _force_elements(self, op, optforce, descr):
         i = 0
         fielddescrs = op.getdescr().get_all_fielddescrs()
-        count = 0
         for index in range(self.length):
             for flddescr in fielddescrs:
                 fld = self._items[i]
@@ -379,9 +372,7 @@
                                               descr=flddescr)
                     optforce._emit_operation(setfieldop)
                     # XXX optforce.optheap
-                    count += 1
                 i += 1
-        return count
 
     def visitor_walk_recursive(self, instbox, visitor, optimizer):
         itemops = [optimizer.get_box_replacement(item)

From noreply at buildbot.pypy.org  Wed Jun  3 15:54:08 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Wed,  3 Jun 2015 15:54:08 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: whack at some tests
Message-ID: <20150603135408.5A94A1C0F16@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77822:7b0a6633ec26
Date: 2015-06-03 15:54 +0200
http://bitbucket.org/pypy/pypy/changeset/7b0a6633ec26/

Log:	whack at some tests

diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py
--- a/rpython/jit/backend/test/runner_test.py
+++ b/rpython/jit/backend/test/runner_test.py
@@ -1807,9 +1807,17 @@
         vtable_for_T = lltype.malloc(self.MY_VTABLE, immortal=True)
         vtable_for_T_addr = llmemory.cast_ptr_to_adr(vtable_for_T)
         cpu = self.cpu
-        if not hasattr(cpu, '_cache_gcstruct2vtable'):
-            cpu._cache_gcstruct2vtable = {}
-        cpu._cache_gcstruct2vtable.update({T: vtable_for_T})
+        class FakeGCCache(object):
+            pass
+        
+        if not hasattr(cpu.gc_ll_descr, '_cache_gcstruct2vtable'):
+            cpu.gc_ll_descr._cache_gcstruct2vtable = {}
+        cpu.gc_ll_descr._cache_gcstruct2vtable.update({T: vtable_for_T})
+        p = T
+        while hasattr(p, 'parent'):
+            vtable_for_parent = lltype.malloc(self.MY_VTABLE, immortal=True)
+            cpu.gc_ll_descr._cache_gcstruct2vtable[p.parent] = vtable_for_parent
+            p = p.parent
         t = lltype.malloc(T)
         if T == self.T:
             t.parent.parent.typeptr = vtable_for_T
@@ -1817,7 +1825,10 @@
             t.parent.parent.parent.typeptr = vtable_for_T
         t_box = InputArgRef(lltype.cast_opaque_ptr(llmemory.GCREF, t))
         T_box = ConstInt(heaptracker.adr2int(vtable_for_T_addr))
-        descr = cpu.sizeof(T, True)
+        is_object = True
+        if not hasattr(T, 'parent'):
+            is_object = False
+        descr = cpu.sizeof(T, is_object)
         return t_box, T_box, descr
 
     def null_instance(self):
@@ -3405,6 +3416,7 @@
                                   ('value', lltype.Signed),
                                   ('chr1', lltype.Char),
                                   ('chr2', lltype.Char))
+        self.alloc_instance(S)
         chr1 = self.cpu.fielddescrof(S, 'chr1').sort_key()
         value = self.cpu.fielddescrof(S, 'value').sort_key()
         chr2 = self.cpu.fielddescrof(S, 'chr2').sort_key()
diff --git a/rpython/jit/backend/x86/test/test_regalloc2.py b/rpython/jit/backend/x86/test/test_regalloc2.py
--- a/rpython/jit/backend/x86/test/test_regalloc2.py
+++ b/rpython/jit/backend/x86/test/test_regalloc2.py
@@ -1,6 +1,6 @@
-from rpython.jit.metainterp.history import ResOperation, BoxInt, ConstInt,\
+from rpython.jit.metainterp.history import ResOperation, ConstInt,\
      BasicFailDescr, JitCellToken, BasicFinalDescr, TargetToken, ConstPtr,\
-     BoxPtr, BoxFloat, ConstFloat
+     ConstFloat
 from rpython.jit.metainterp.resoperation import rop
 from rpython.jit.backend.detect_cpu import getcpuclass
 from rpython.jit.backend.x86.arch import WORD
@@ -15,11 +15,11 @@
 CPU = getcpuclass()
 
 def test_bug_rshift():
-    v1 = BoxInt()
-    v2 = BoxInt()
-    v3 = BoxInt()
-    v4 = BoxInt()
-    zero = BoxInt()
+    v1 = InputArgInt()
+    v2 = InputArgInt()
+    v3 = InputArgInt()
+    v4 = InputArgInt()
+    zero = InputArgInt()
     inputargs = [v1]
     operations = [
         ResOperation(rop.INT_ADD, [v1, v1], v2),
@@ -39,12 +39,12 @@
     assert cpu.get_int_value(deadframe, 1) == (~18)
 
 def test_bug_int_is_true_1():
-    v1 = BoxInt()
-    v2 = BoxInt()
-    v3 = BoxInt()
-    v4 = BoxInt()
-    zero = BoxInt()
-    tmp5 = BoxInt()
+    v1 = InputArgInt()
+    v2 = InputArgInt()
+    v3 = InputArgInt()
+    v4 = InputArgInt()
+    zero = InputArgInt()
+    tmp5 = InputArgInt()
     inputargs = [v1]
     operations = [
         ResOperation(rop.INT_MUL, [v1, v1], v2),
@@ -66,53 +66,53 @@
     assert cpu.get_int_value(deadframe, 2) == 1
 
 def test_bug_0():
-    v1 = BoxInt()
-    v2 = BoxInt()
-    v3 = BoxInt()
-    v4 = BoxInt()
-    v5 = BoxInt()
-    v6 = BoxInt()
-    v7 = BoxInt()
-    v8 = BoxInt()
-    v9 = BoxInt()
-    v10 = BoxInt()
-    v11 = BoxInt()
-    v12 = BoxInt()
-    v13 = BoxInt()
-    v14 = BoxInt()
-    v15 = BoxInt()
-    v16 = BoxInt()
-    v17 = BoxInt()
-    v18 = BoxInt()
-    v19 = BoxInt()
-    v20 = BoxInt()
-    v21 = BoxInt()
-    v22 = BoxInt()
-    v23 = BoxInt()
-    v24 = BoxInt()
-    v25 = BoxInt()
-    v26 = BoxInt()
-    v27 = BoxInt()
-    v28 = BoxInt()
-    v29 = BoxInt()
-    v30 = BoxInt()
-    v31 = BoxInt()
-    v32 = BoxInt()
-    v33 = BoxInt()
-    v34 = BoxInt()
-    v35 = BoxInt()
-    v36 = BoxInt()
-    v37 = BoxInt()
-    v38 = BoxInt()
-    v39 = BoxInt()
-    v40 = BoxInt()
-    zero = BoxInt()
-    tmp41 = BoxInt()
-    tmp42 = BoxInt()
-    tmp43 = BoxInt()
-    tmp44 = BoxInt()
-    tmp45 = BoxInt()
-    tmp46 = BoxInt()
+    v1 = InputArgInt()
+    v2 = InputArgInt()
+    v3 = InputArgInt()
+    v4 = InputArgInt()
+    v5 = InputArgInt()
+    v6 = InputArgInt()
+    v7 = InputArgInt()
+    v8 = InputArgInt()
+    v9 = InputArgInt()
+    v10 = InputArgInt()
+    v11 = InputArgInt()
+    v12 = InputArgInt()
+    v13 = InputArgInt()
+    v14 = InputArgInt()
+    v15 = InputArgInt()
+    v16 = InputArgInt()
+    v17 = InputArgInt()
+    v18 = InputArgInt()
+    v19 = InputArgInt()
+    v20 = InputArgInt()
+    v21 = InputArgInt()
+    v22 = InputArgInt()
+    v23 = InputArgInt()
+    v24 = InputArgInt()
+    v25 = InputArgInt()
+    v26 = InputArgInt()
+    v27 = InputArgInt()
+    v28 = InputArgInt()
+    v29 = InputArgInt()
+    v30 = InputArgInt()
+    v31 = InputArgInt()
+    v32 = InputArgInt()
+    v33 = InputArgInt()
+    v34 = InputArgInt()
+    v35 = InputArgInt()
+    v36 = InputArgInt()
+    v37 = InputArgInt()
+    v38 = InputArgInt()
+    v39 = InputArgInt()
+    v40 = InputArgInt()
+    zero = InputArgInt()
+    tmp41 = InputArgInt()
+    tmp42 = InputArgInt()
+    tmp43 = InputArgInt()
+    tmp44 = InputArgInt()
+    tmp45 = InputArgInt()
+    tmp46 = InputArgInt()
     inputargs = [v1, v2, v3, v4, v5, v6, v7, v8, v9, v10]
     operations = [
         ResOperation(rop.UINT_GT, [v3, ConstInt(-48)], v11),
@@ -179,52 +179,52 @@
     assert cpu.get_int_value(deadframe, 13) == 0
 
 def test_bug_1():
-    v1 = BoxInt()
-    v2 = BoxInt()
-    v3 = BoxInt()
-    v4 = BoxInt()
-    v5 = BoxInt()
-    v6 = BoxInt()
-    v7 = BoxInt()
-    v8 = BoxInt()
-    v9 = BoxInt()
-    v10 = BoxInt()
-    v11 = BoxInt()
-    v12 = BoxInt()
-    v13 = BoxInt()
-    v14 = BoxInt()
-    v15 = BoxInt()
-    v16 = BoxInt()
-    v17 = BoxInt()
-    v18 = BoxInt()
-    v19 = BoxInt()
-    v20 = BoxInt()
-    v21 = BoxInt()
-    v22 = BoxInt()
-    v23 = BoxInt()
-    v24 = BoxInt()
-    v25 = BoxInt()
-    v26 = BoxInt()
-    v27 = BoxInt()
-    v28 = BoxInt()
-    v29 = BoxInt()
-    v30 = BoxInt()
-    v31 = BoxInt()
-    v32 = BoxInt()
-    v33 = BoxInt()
-    v34 = BoxInt()
-    v35 = BoxInt()
-    v36 = BoxInt()
-    v37 = BoxInt()
-    v38 = BoxInt()
-    v39 = BoxInt()
-    v40 = BoxInt()
-    zero = BoxInt()
-    tmp41 = BoxInt()
-    tmp42 = BoxInt()
-    tmp43 = BoxInt()
-    tmp44 = BoxInt()
-    tmp45 = BoxInt()
+    v1 = InputArgInt()
+    v2 = InputArgInt()
+    v3 = InputArgInt()
+    v4 = InputArgInt()
+    v5 = InputArgInt()
+    v6 = InputArgInt()
+    v7 = InputArgInt()
+    v8 = InputArgInt()
+    v9 = InputArgInt()
+    v10 = InputArgInt()
+    v11 = InputArgInt()
+    v12 = InputArgInt()
+    v13 = InputArgInt()
+    v14 = InputArgInt()
+    v15 = InputArgInt()
+    v16 = InputArgInt()
+    v17 = InputArgInt()
+    v18 = InputArgInt()
+    v19 = InputArgInt()
+    v20 = InputArgInt()
+    v21 = InputArgInt()
+    v22 = InputArgInt()
+    v23 = InputArgInt()
+    v24 = InputArgInt()
+    v25 = InputArgInt()
+    v26 = InputArgInt()
+    v27 = InputArgInt()
+    v28 = InputArgInt()
+    v29 = InputArgInt()
+    v30 = InputArgInt()
+    v31 = InputArgInt()
+    v32 = InputArgInt()
+    v33 = InputArgInt()
+    v34 = InputArgInt()
+    v35 = InputArgInt()
+    v36 = InputArgInt()
+    v37 = InputArgInt()
+    v38 = InputArgInt()
+    v39 = InputArgInt()
+    v40 = InputArgInt()
+    zero = InputArgInt()
+    tmp41 = InputArgInt()
+    tmp42 = InputArgInt()
+    tmp43 = InputArgInt()
+    tmp44 = InputArgInt()
+    tmp45 = InputArgInt()
     inputargs = [v1, v2, v3, v4, v5, v6, v7, v8, v9, v10]
     operations = [
         ResOperation(rop.UINT_LT, [v6, ConstInt(0)], v11),
@@ -347,20 +347,20 @@
 
     S4 = lltype.Struct('Sx', ("f0", lltype.Char), ("f1", lltype.Signed), ("f2", lltype.Signed), ("f3", lltype.Signed))
     S5 = lltype.GcArray(S4)
-    v1 = BoxInt()
-    v2 = BoxInt()
-    v3 = BoxInt()
-    v4 = BoxInt()
-    v5 = BoxInt()
-    v6 = BoxInt()
-    v7 = BoxInt()
-    v8 = BoxInt()
-    v9 = BoxInt()
-    v10 = BoxInt()
-    tmp11 = BoxInt()
-    tmp12 = BoxPtr()
+    v1 = InputArgInt()
+    v2 = InputArgInt()
+    v3 = InputArgInt()
+    v4 = InputArgInt()
+    v5 = InputArgInt()
+    v6 = InputArgInt()
+    v7 = InputArgInt()
+    v8 = InputArgInt()
+    v9 = InputArgInt()
+    v10 = InputArgInt()
+    tmp11 = InputArgInt()
+    tmp12 = InputArgRef()
     faildescr0 = BasicFailDescr()
-    tmp13 = BoxPtr()
+    tmp13 = InputArgRef()
     faildescr1 = BasicFailDescr()
     finishdescr2 = BasicFinalDescr()
     const_ptr14 = ConstPtr(lltype.cast_opaque_ptr(llmemory.GCREF, lltype.malloc(rstr.STR, 1)))
@@ -401,39 +401,39 @@
     S7 = lltype.GcStruct('Sx', ("parent", rclass.OBJECT), ("f0", lltype.Char))
     S8 = lltype.Struct('Sx', ("f0", lltype.Char), ("f1", lltype.Signed), ("f2", lltype.Signed), ("f3", lltype.Signed))
     S9 = lltype.GcArray(S8)
-    v1 = BoxInt()
-    v2 = BoxInt()
-    v3 = BoxInt()
-    v4 = BoxInt()
-    v5 = BoxInt()
-    v6 = BoxInt()
-    v7 = BoxInt()
-    v8 = BoxInt()
-    v9 = BoxInt()
-    v10 = BoxInt()
-    v11 = BoxInt()
-    v12 = BoxInt()
-    v13 = BoxInt()
-    v14 = BoxInt()
-    v15 = BoxInt()
-    v16 = BoxInt()
-    v17 = BoxInt()
-    v18 = BoxInt()
-    v19 = BoxInt()
-    p20 = BoxPtr()
-    tmp21 = BoxPtr()
+    v1 = InputArgInt()
+    v2 = InputArgInt()
+    v3 = InputArgInt()
+    v4 = InputArgInt()
+    v5 = InputArgInt()
+    v6 = InputArgInt()
+    v7 = InputArgInt()
+    v8 = InputArgInt()
+    v9 = InputArgInt()
+    v10 = InputArgInt()
+    v11 = InputArgInt()
+    v12 = InputArgInt()
+    v13 = InputArgInt()
+    v14 = InputArgInt()
+    v15 = InputArgInt()
+    v16 = InputArgInt()
+    v17 = InputArgInt()
+    v18 = InputArgInt()
+    v19 = InputArgInt()
+    p20 = InputArgRef()
+    tmp21 = InputArgRef()
     faildescr3 = BasicFailDescr()
-    tmp22 = BoxPtr()
+    tmp22 = InputArgRef()
     faildescr4 = BasicFailDescr()
-    tmp23 = BoxInt()
-    tmp24 = BoxInt()
-    tmp25 = BoxInt()
-    tmp26 = BoxInt()
-    tmp27 = BoxInt()
-    tmp28 = BoxInt()
-    tmp29 = BoxInt()
+    tmp23 = InputArgInt()
+    tmp24 = InputArgInt()
+    tmp25 = InputArgInt()
+    tmp26 = InputArgInt()
+    tmp27 = InputArgInt()
+    tmp28 = InputArgInt()
+    tmp29 = InputArgInt()
     faildescr5 = BasicFailDescr()
-    tmp30 = BoxPtr()
+    tmp30 = InputArgRef()
     faildescr6 = BasicFailDescr()
     finishdescr7 = BasicFinalDescr()
     const_ptr31 = ConstPtr(lltype.cast_opaque_ptr(llmemory.GCREF, lltype.malloc(S4)))
@@ -508,36 +508,36 @@
     S4 = lltype.GcStruct('Sx', ("parent", rclass.OBJECT), ("f0", lltype.Signed), ("f1", lltype.Signed))
     S5 = lltype.GcStruct('Sx', ("parent", rclass.OBJECT), ("f0", lltype.Signed))
     S6 = lltype.GcStruct('Sx', ("f0", lltype.Signed), ("f1", rffi.UCHAR))
-    v1 = BoxInt()
-    v2 = BoxInt()
-    v3 = BoxInt()
-    v4 = BoxInt()
-    v5 = BoxInt()
-    v6 = BoxInt()
-    v7 = BoxInt()
-    v8 = BoxInt()
-    v9 = BoxInt()
-    v10 = BoxInt()
-    v11 = BoxInt()
-    v12 = BoxInt()
-    v13 = BoxInt()
-    v14 = BoxInt()
-    v15 = BoxInt()
-    v16 = BoxInt()
-    v17 = BoxInt()
-    v18 = BoxInt()
-    tmp19 = BoxPtr()
+    v1 = InputArgInt()
+    v2 = InputArgInt()
+    v3 = InputArgInt()
+    v4 = InputArgInt()
+    v5 = InputArgInt()
+    v6 = InputArgInt()
+    v7 = InputArgInt()
+    v8 = InputArgInt()
+    v9 = InputArgInt()
+    v10 = InputArgInt()
+    v11 = InputArgInt()
+    v12 = InputArgInt()
+    v13 = InputArgInt()
+    v14 = InputArgInt()
+    v15 = InputArgInt()
+    v16 = InputArgInt()
+    v17 = InputArgInt()
+    v18 = InputArgInt()
+    tmp19 = InputArgRef()
     faildescr8 = BasicFailDescr()
-    tmp20 = BoxInt()
-    tmp21 = BoxInt()
-    tmp22 = BoxInt()
-    tmp23 = BoxInt()
+    tmp20 = InputArgInt()
+    tmp21 = InputArgInt()
+    tmp22 = InputArgInt()
+    tmp23 = InputArgInt()
     faildescr9 = BasicFailDescr()
-    tmp24 = BoxInt()
-    tmp25 = BoxInt()
-    tmp26 = BoxInt()
-    tmp27 = BoxPtr()
-    tmp28 = BoxPtr()
+    tmp24 = InputArgInt()
+    tmp25 = InputArgInt()
+    tmp26 = InputArgInt()
+    tmp27 = InputArgRef()
+    tmp28 = InputArgRef()
     faildescr10 = BasicFailDescr()
     finishdescr11 = BasicFinalDescr()
     const_ptr29 = ConstPtr(lltype.cast_opaque_ptr(llmemory.GCREF, lltype.malloc(S4)))
@@ -587,11 +587,11 @@
     cpu.compile_bridge(faildescr6, inputargs, operations, looptoken)
     frame = cpu.execute_token(looptoken, *loop_args)
     #assert cpu.get_int_value(frame, 0) == -9223372036854775808
-    v1 = BoxInt()
-    v2 = BoxInt()
-    p3 = BoxPtr()
-    tmp4 = BoxInt()
-    tmp5 = BoxPtr()
+    v1 = InputArgInt()
+    v2 = InputArgInt()
+    p3 = InputArgRef()
+    tmp4 = InputArgInt()
+    tmp5 = InputArgRef()
     faildescr12 = BasicFailDescr()
     finishdescr13 = BasicFinalDescr()
     inputargs = [v1]
diff --git a/rpython/jit/backend/x86/test/test_runner.py b/rpython/jit/backend/x86/test/test_runner.py
--- a/rpython/jit/backend/x86/test/test_runner.py
+++ b/rpython/jit/backend/x86/test/test_runner.py
@@ -55,8 +55,8 @@
 
     def test_execute_ptr_operation(self):
         cpu = self.cpu
-        u = lltype.malloc(U)
-        u_box = InputArgRef(lltype.cast_opaque_ptr(llmemory.GCREF, u))
+        u_box, _, _ = self.alloc_instance(U)
+        u = u_box.getref(lltype.Ptr(U))
         ofs = cpu.fielddescrof(S, 'value')
         assert self.execute_operation(rop.SETFIELD_GC,
                                       [u_box, InputArgInt(3)],
diff --git a/rpython/jit/codewriter/heaptracker.py b/rpython/jit/codewriter/heaptracker.py
--- a/rpython/jit/codewriter/heaptracker.py
+++ b/rpython/jit/codewriter/heaptracker.py
@@ -65,7 +65,7 @@
         return None
     setup_cache_gcstruct2vtable(gccache)
     if not hasattr(gccache, '_cache_gcstruct2vtable'):
-        return GCSTRUCT.typeptr
+        return lltype.malloc(GCSTRUCT.typeptr.TO, flavor='raw', immortal=True)
     return gccache._cache_gcstruct2vtable[GCSTRUCT]
 
 def setup_cache_gcstruct2vtable(gccache):
diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py
--- a/rpython/jit/metainterp/optimizeopt/info.py
+++ b/rpython/jit/metainterp/optimizeopt/info.py
@@ -348,7 +348,10 @@
         self._items = [None] * (size * lgt)
 
     def _compute_index(self, index, fielddescr):
-        one_size = len(fielddescr.get_arraydescr().get_all_fielddescrs())
+        all_fdescrs = fielddescr.get_arraydescr().get_all_fielddescrs()
+        if all_fdescrs is None:
+            return 0 # annotation hack
+        one_size = len(all_fdescrs)
         return index * one_size + fielddescr.fielddescr.get_index()
         
     def setinteriorfield_virtual(self, index, fielddescr, fld):
diff --git a/rpython/jit/metainterp/optimizeopt/rawbuffer.py b/rpython/jit/metainterp/optimizeopt/rawbuffer.py
--- a/rpython/jit/metainterp/optimizeopt/rawbuffer.py
+++ b/rpython/jit/metainterp/optimizeopt/rawbuffer.py
@@ -44,10 +44,10 @@
         if not we_are_translated() and isinstance(value, str):
             return value # for tests
         if self.logops:
-            s = self.logops.repr_of_arg(value.box)
+            s = self.logops.repr_of_arg(value)
         else:
-            s = str(value.box)
-        s += " at %d" % compute_unique_id(value.box)
+            s = str(value)
+        s += " at %d" % compute_unique_id(value)
         return s
 
     def _dump_to_log(self):
diff --git a/rpython/jit/metainterp/optimizeopt/vstring.py b/rpython/jit/metainterp/optimizeopt/vstring.py
--- a/rpython/jit/metainterp/optimizeopt/vstring.py
+++ b/rpython/jit/metainterp/optimizeopt/vstring.py
@@ -250,7 +250,7 @@
         visitor.register_virtual_fields(instbox, boxes)
         opinfo = optimizer.getptrinfo(self.s)
         if opinfo and opinfo.is_virtual():
-            opinfo.visitor_walk_recursive(visitor)
+            opinfo.visitor_walk_recursive(self.s, visitor, optimizer)
 
     @specialize.argtype(1)
     def visitor_dispatch_virtual_type(self, visitor):

From noreply at buildbot.pypy.org  Wed Jun  3 16:02:48 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Wed,  3 Jun 2015 16:02:48 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: unify the interface
Message-ID: <20150603140248.26EC11C0F16@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77823:5c532c9ba4cf
Date: 2015-06-03 16:02 +0200
http://bitbucket.org/pypy/pypy/changeset/5c532c9ba4cf/

Log:	unify the interface

diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -219,7 +219,7 @@
     def get_arraydescr(self):
         return self.arraydescr
 
-    def get_fielddescr(self):
+    def get_field_descr(self):
         return self.fielddescr
 
     def __repr__(self):
diff --git a/rpython/jit/backend/llsupport/descr.py b/rpython/jit/backend/llsupport/descr.py
--- a/rpython/jit/backend/llsupport/descr.py
+++ b/rpython/jit/backend/llsupport/descr.py
@@ -314,6 +314,9 @@
     def get_arraydescr(self):
         return self.arraydescr
 
+    def get_field_descr(self):
+        return self.fielddescr
+
     def sort_key(self):
         return self.fielddescr.sort_key()
 
diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py
--- a/rpython/jit/metainterp/optimizeopt/info.py
+++ b/rpython/jit/metainterp/optimizeopt/info.py
@@ -352,7 +352,7 @@
         if all_fdescrs is None:
             return 0 # annotation hack
         one_size = len(all_fdescrs)
-        return index * one_size + fielddescr.fielddescr.get_index()
+        return index * one_size + fielddescr.get_field_descr().get_index()
         
     def setinteriorfield_virtual(self, index, fielddescr, fld):
         index = self._compute_index(index, fielddescr)

From noreply at buildbot.pypy.org  Wed Jun  3 16:07:46 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Wed,  3 Jun 2015 16:07:46 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: those asserts are no longer valid due
	to parent_descr
Message-ID: <20150603140746.92B8D1C0FB8@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77824:48f99f98e4dc
Date: 2015-06-03 16:07 +0200
http://bitbucket.org/pypy/pypy/changeset/48f99f98e4dc/

Log:	those asserts are no longer valid due to parent_descr

diff --git a/rpython/jit/backend/llsupport/gc.py b/rpython/jit/backend/llsupport/gc.py
--- a/rpython/jit/backend/llsupport/gc.py
+++ b/rpython/jit/backend/llsupport/gc.py
@@ -640,8 +640,6 @@
             return
         if self.layoutbuilder is not None:
             type_id = self.layoutbuilder.get_type_id(S)
-            assert not self.layoutbuilder.is_weakref_type(S)
-            assert not self.layoutbuilder.has_finalizer(S)
             descr.tid = llop.combine_ushort(lltype.Signed, type_id, 0)
 
     def init_array_descr(self, A, descr):

From noreply at buildbot.pypy.org  Wed Jun  3 16:10:49 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Wed,  3 Jun 2015 16:10:49 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Add a workaround for Python 2.7.6 raising
 a SyntaxError when importing a file that contains Py3k-compatible exec(code,
 d) calls in certain contexts.
Message-ID: <20150603141049.0DFDF1C1017@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob <me at manueljacob.de>
Branch: py3k
Changeset: r77825:b29711ae7758
Date: 2015-06-03 16:10 +0200
http://bitbucket.org/pypy/pypy/changeset/b29711ae7758/

Log:	Add a workaround for Python 2.7.6 raising a SyntaxError when
	importing a file that contains Py3k-compatible exec(code, d) calls
	in certain contexts.

diff --git a/pypy/module/_vmprof/test/test__vmprof.py b/pypy/module/_vmprof/test/test__vmprof.py
--- a/pypy/module/_vmprof/test/test__vmprof.py
+++ b/pypy/module/_vmprof/test/test__vmprof.py
@@ -46,13 +46,16 @@
         assert no_of_codes > 10
         d = {}
 
-        exec("""def foo():
+        def exec_(code, d):
+            exec(code, d)
+
+        exec_("""def foo():
             pass
         """, d)
 
         _vmprof.enable(self.tmpfileno2)
 
-        exec("""def foo2():
+        exec_("""def foo2():
             pass
         """, d)
 

From noreply at buildbot.pypy.org  Wed Jun  3 16:58:00 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Wed,  3 Jun 2015 16:58:00 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: disable those for now
Message-ID: <20150603145800.537931C047D@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77826:bf5cb5f4c6d4
Date: 2015-06-03 16:58 +0200
http://bitbucket.org/pypy/pypy/changeset/bf5cb5f4c6d4/

Log:	disable those for now

diff --git a/pypy/module/pypyjit/__init__.py b/pypy/module/pypyjit/__init__.py
--- a/pypy/module/pypyjit/__init__.py
+++ b/pypy/module/pypyjit/__init__.py
@@ -8,16 +8,16 @@
         'set_param':    'interp_jit.set_param',
         'residual_call': 'interp_jit.residual_call',
         'not_from_assembler': 'interp_jit.W_NotFromAssembler',
-        'set_compile_hook': 'interp_resop.set_compile_hook',
-        'set_optimize_hook': 'interp_resop.set_optimize_hook',
-        'set_abort_hook': 'interp_resop.set_abort_hook',
-        'get_stats_snapshot': 'interp_resop.get_stats_snapshot',
-        'enable_debug': 'interp_resop.enable_debug',
-        'disable_debug': 'interp_resop.disable_debug',
-        'ResOperation': 'interp_resop.WrappedOp',
-        'DebugMergePoint': 'interp_resop.DebugMergePoint',
-        'JitLoopInfo': 'interp_resop.W_JitLoopInfo',
-        'Box': 'interp_resop.WrappedBox',
+        #'set_compile_hook': 'interp_resop.set_compile_hook',
+        #'set_optimize_hook': 'interp_resop.set_optimize_hook',
+        #'set_abort_hook': 'interp_resop.set_abort_hook',
+        #'get_stats_snapshot': 'interp_resop.get_stats_snapshot',
+        #'enable_debug': 'interp_resop.enable_debug',
+        #'disable_debug': 'interp_resop.disable_debug',
+        #'ResOperation': 'interp_resop.WrappedOp',
+        #'DebugMergePoint': 'interp_resop.DebugMergePoint',
+        #'JitLoopInfo': 'interp_resop.W_JitLoopInfo',
+        #'Box': 'interp_resop.WrappedBox',
         'PARAMETER_DOCS': 'space.wrap(rpython.rlib.jit.PARAMETER_DOCS)',
     }
 

From noreply at buildbot.pypy.org  Wed Jun  3 17:09:14 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Wed,  3 Jun 2015 17:09:14 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: fixes
Message-ID: <20150603150914.117861C047D@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77827:eeecf9762142
Date: 2015-06-03 17:09 +0200
http://bitbucket.org/pypy/pypy/changeset/eeecf9762142/

Log:	fixes

diff --git a/rpython/jit/backend/llsupport/gc.py b/rpython/jit/backend/llsupport/gc.py
--- a/rpython/jit/backend/llsupport/gc.py
+++ b/rpython/jit/backend/llsupport/gc.py
@@ -643,7 +643,7 @@
             descr.tid = llop.combine_ushort(lltype.Signed, type_id, 0)
 
     def init_array_descr(self, A, descr):
-        if not isinstance(A, lltype.GcArray):
+        if not isinstance(A, (lltype.GcArray, lltype.GcStruct)):
             return
         if self.layoutbuilder is not None:
             type_id = self.layoutbuilder.get_type_id(A)
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -784,9 +784,6 @@
     def opimpl_getfield_raw_i_pure(self, box, fielddescr):
         return self.execute_with_descr(rop.GETFIELD_RAW_PURE_I, fielddescr, box)
     @arguments("box", "descr")
-    def opimpl_getfield_raw_r_pure(self, box, fielddescr):
-        return self.execute_with_descr(rop.GETFIELD_RAW_PURE_R, fielddescr, box)
-    @arguments("box", "descr")
     def opimpl_getfield_raw_f_pure(self, box, fielddescr):
         return self.execute_with_descr(rop.GETFIELD_RAW_PURE_F, fielddescr, box)
 

From noreply at buildbot.pypy.org  Wed Jun  3 17:24:24 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Wed,  3 Jun 2015 17:24:24 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: support GETFIELD_RAW_R
Message-ID: <20150603152424.C03391C047D@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77828:6bb83ce45814
Date: 2015-06-03 17:24 +0200
http://bitbucket.org/pypy/pypy/changeset/6bb83ce45814/

Log:	support GETFIELD_RAW_R

diff --git a/rpython/jit/metainterp/executor.py b/rpython/jit/metainterp/executor.py
--- a/rpython/jit/metainterp/executor.py
+++ b/rpython/jit/metainterp/executor.py
@@ -180,17 +180,20 @@
     struct = structbox.getref_base()
     return cpu.bh_getfield_gc_f(struct, fielddescr)
 
-def do_getfield_raw(cpu, _, structbox, fielddescr):
-    raise Exception("implement me")    
-    xxxx
+def do_getfield_raw_i(cpu, _, structbox, fielddescr):
     check_descr(fielddescr)
     struct = structbox.getint()
-    if fielddescr.is_pointer_field():
-        return BoxPtr(cpu.bh_getfield_raw_r(struct, fielddescr))
-    elif fielddescr.is_float_field():
-        return BoxFloat(cpu.bh_getfield_raw_f(struct, fielddescr))
-    else:
-        return BoxInt(cpu.bh_getfield_raw_i(struct, fielddescr))
+    return cpu.bh_getfield_raw_i(struct, fielddescr)
+
+def do_getfield_raw_f(cpu, _, structbox, fielddescr):
+    check_descr(fielddescr)
+    struct = structbox.getint()
+    return cpu.bh_getfield_raw_f(struct, fielddescr)
+
+def do_getfield_raw_r(cpu, _, structbox, fielddescr):
+    check_descr(fielddescr)
+    struct = structbox.getint()
+    return cpu.bh_getfield_raw_r(struct, fielddescr)
 
 def do_setfield_gc(cpu, _, structbox, itembox, fielddescr):
     struct = structbox.getref_base()
diff --git a/rpython/jit/metainterp/optimizeopt/intbounds.py b/rpython/jit/metainterp/optimizeopt/intbounds.py
--- a/rpython/jit/metainterp/optimizeopt/intbounds.py
+++ b/rpython/jit/metainterp/optimizeopt/intbounds.py
@@ -407,6 +407,7 @@
             b1.make_le(IntUpperBound(descr.get_integer_max()))
 
     optimize_GETFIELD_RAW_F = optimize_GETFIELD_RAW_I
+    optimize_GETFIELD_RAW_R = optimize_GETFIELD_RAW_I
     optimize_GETFIELD_GC_I = optimize_GETFIELD_RAW_I
     optimize_GETFIELD_GC_R = optimize_GETFIELD_RAW_I
     optimize_GETFIELD_GC_F = optimize_GETFIELD_RAW_I
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -784,6 +784,9 @@
     def opimpl_getfield_raw_i_pure(self, box, fielddescr):
         return self.execute_with_descr(rop.GETFIELD_RAW_PURE_I, fielddescr, box)
     @arguments("box", "descr")
+    def opimpl_getfield_raw_r_pure(self, box, fielddescr):
+        return self.execute_with_descr(rop.GETFIELD_RAW_PURE_R, fielddescr, box)
+    @arguments("box", "descr")
     def opimpl_getfield_raw_f_pure(self, box, fielddescr):
         return self.execute_with_descr(rop.GETFIELD_RAW_PURE_F, fielddescr, box)
 
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -725,7 +725,7 @@
     'STRLEN/1/i',
     'STRGETITEM/2/i',
     'GETFIELD_GC_PURE/1d/rfi',
-    'GETFIELD_RAW_PURE/1d/fi',
+    'GETFIELD_RAW_PURE/1d/rfi',
     'GETARRAYITEM_GC_PURE/2d/rfi',
     'GETARRAYITEM_RAW_PURE/2d/fi',
     'UNICODELEN/1/i',

From noreply at buildbot.pypy.org  Wed Jun  3 17:38:46 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Wed,  3 Jun 2015 17:38:46 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: 2to3
Message-ID: <20150603153846.690151C0F16@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob <me at manueljacob.de>
Branch: py3k
Changeset: r77829:e1465507097e
Date: 2015-06-03 17:38 +0200
http://bitbucket.org/pypy/pypy/changeset/e1465507097e/

Log:	2to3

diff --git a/pypy/module/_random/test/test_random.py b/pypy/module/_random/test/test_random.py
--- a/pypy/module/_random/test/test_random.py
+++ b/pypy/module/_random/test/test_random.py
@@ -49,8 +49,7 @@
         rnd.seed(1234)
         state = rnd.getstate()
         s = repr(state)
-        assert len(s) == 7956
-        assert s.count('L') == 625
+        assert len(s) == 7331
 
     def test_seed(self):
         import _random, sys

From noreply at buildbot.pypy.org  Wed Jun  3 17:39:39 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Wed,  3 Jun 2015 17:39:39 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: grr
Message-ID: <20150603153939.DC38B1C024E@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77830:74f6eeddfbd4
Date: 2015-06-03 17:39 +0200
http://bitbucket.org/pypy/pypy/changeset/74f6eeddfbd4/

Log:	grr

diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -738,7 +738,7 @@
     'GETINTERIORFIELD_GC/2d/rfi',
     'RAW_LOAD/2d/fi',
     'GETFIELD_GC/1d/rfi',
-    'GETFIELD_RAW/1d/fi',
+    'GETFIELD_RAW/1d/rfi',
     '_MALLOC_FIRST',
     'NEW/0d/r',           #-> GcStruct, gcptrs inside are zeroed (not the rest)
     'NEW_WITH_VTABLE/0d/r',#-> GcStruct with vtable, gcptrs inside are zeroed

From noreply at buildbot.pypy.org  Wed Jun  3 17:53:16 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Wed,  3 Jun 2015 17:53:16 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: sanity check and disable
 gen_store_back until we test it
Message-ID: <20150603155316.2CCCD1C034D@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77831:d2a9145f3990
Date: 2015-06-03 17:53 +0200
http://bitbucket.org/pypy/pypy/changeset/d2a9145f3990/

Log:	sanity check and disable gen_store_back until we test it

diff --git a/rpython/jit/backend/x86/test/test_regalloc2.py b/rpython/jit/backend/x86/test/test_regalloc2.py
--- a/rpython/jit/backend/x86/test/test_regalloc2.py
+++ b/rpython/jit/backend/x86/test/test_regalloc2.py
@@ -14,6 +14,7 @@
 
 CPU = getcpuclass()
 
+
 def test_bug_rshift():
     v1 = InputArgInt()
     v2 = InputArgInt()
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -777,6 +777,12 @@
                 raise resume.TagOverflow
         except resume.TagOverflow:
             raise compile.giveup()
+        # check no duplicates
+        seen = {}
+        for box in newboxes:
+            if box is not None:
+                assert box not in seen
+                seen[box] = None
         descr.store_final_boxes(op, newboxes, self.metainterp_sd)
         #
         if op.getopnum() == rop.GUARD_VALUE:
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -2845,6 +2845,7 @@
             self.virtualizable_boxes.append(virtualizable_box)
 
     def gen_store_back_in_vable(self, box):
+        raise Exception("untested")
         vinfo = self.jitdriver_sd.virtualizable_info
         if vinfo is not None:
             # xxx only write back the fields really modified
@@ -2866,7 +2867,7 @@
             virtualizable = vinfo.unwrap_virtualizable_box(vbox)
             for k in range(vinfo.num_arrays):
                 descr = vinfo.array_field_descrs[k]
-                abox = self.execute_and_record(rop.GETFIELD_GC, descr, vbox)
+                abox = self.execute_and_record(rop.GETFIELD_GC_R, descr, vbox)
                 descr = vinfo.array_descrs[k]
                 for j in range(vinfo.get_array_length(virtualizable, k)):
                     itembox = self.virtualizable_boxes[i]

From noreply at buildbot.pypy.org  Wed Jun  3 17:54:11 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Wed,  3 Jun 2015 17:54:11 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: arbitrary fix
Message-ID: <20150603155411.EF4A81C034D@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77832:8d7cc7c22303
Date: 2015-06-03 17:54 +0200
http://bitbucket.org/pypy/pypy/changeset/8d7cc7c22303/

Log:	arbitrary fix

diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -2845,7 +2845,6 @@
             self.virtualizable_boxes.append(virtualizable_box)
 
     def gen_store_back_in_vable(self, box):
-        raise Exception("untested")
         vinfo = self.jitdriver_sd.virtualizable_info
         if vinfo is not None:
             # xxx only write back the fields really modified
@@ -2877,7 +2876,8 @@
             assert i + 1 == len(self.virtualizable_boxes)
             # we're during tracing, so we should not execute it
             self.history.record(rop.SETFIELD_GC, [vbox, self.cpu.ts.CONST_NULL],
-                                None, descr=vinfo.vable_token_descr)
+                                self.cpu.ts.CONST_NULL.getref_base(),
+                                descr=vinfo.vable_token_descr)
 
     def replace_box(self, oldbox, newbox):
         for frame in self.framestack:

From noreply at buildbot.pypy.org  Wed Jun  3 18:10:49 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Wed,  3 Jun 2015 18:10:49 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: try to fix annotation
Message-ID: <20150603161049.D2EF91C0FB8@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77833:fa7247b692e1
Date: 2015-06-03 18:10 +0200
http://bitbucket.org/pypy/pypy/changeset/fa7247b692e1/

Log:	try to fix annotation

diff --git a/rpython/jit/metainterp/quasiimmut.py b/rpython/jit/metainterp/quasiimmut.py
--- a/rpython/jit/metainterp/quasiimmut.py
+++ b/rpython/jit/metainterp/quasiimmut.py
@@ -1,4 +1,4 @@
-from rpython.rtyper.lltypesystem import lltype
+from rpython.rtyper.lltypesystem import lltype, llmemory
 from rpython.rtyper import rclass
 from rpython.rtyper.annlowlevel import cast_base_ptr_to_instance
 from rpython.jit.metainterp.history import AbstractDescr, ConstPtr, ConstInt,\
@@ -104,7 +104,7 @@
     # those fields are necessary for translation without quasi immutable
     # fields
     struct = None
-    fielddescr = None
+    fielddescr = lltype.nullptr(llmemory.GCREF.TO)
     
     def __init__(self, cpu, struct, fielddescr, mutatefielddescr):
         self.cpu = cpu

From noreply at buildbot.pypy.org  Wed Jun  3 18:32:00 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Wed,  3 Jun 2015 18:32:00 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: a test and a fix
Message-ID: <20150603163200.B34E11C1048@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77834:9a42f91adf06
Date: 2015-06-03 18:32 +0200
http://bitbucket.org/pypy/pypy/changeset/9a42f91adf06/

Log:	a test and a fix

diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -778,11 +778,12 @@
         except resume.TagOverflow:
             raise compile.giveup()
         # check no duplicates
-        seen = {}
-        for box in newboxes:
-            if box is not None:
-                assert box not in seen
-                seen[box] = None
+        if not we_are_translated():
+            seen = {}
+            for box in newboxes:
+                if box is not None:
+                    assert box not in seen
+                    seen[box] = None
         descr.store_final_boxes(op, newboxes, self.metainterp_sd)
         #
         if op.getopnum() == rop.GUARD_VALUE:
diff --git a/rpython/jit/metainterp/resume.py b/rpython/jit/metainterp/resume.py
--- a/rpython/jit/metainterp/resume.py
+++ b/rpython/jit/metainterp/resume.py
@@ -407,10 +407,12 @@
 
         for setfield_op in pending_setfields:
             box = setfield_op.getarg(0)
+            box = optimizer.get_box_replacement(box)
             if setfield_op.getopnum() == rop.SETFIELD_GC:
                 fieldbox = setfield_op.getarg(1)
             else:
                 fieldbox = setfield_op.getarg(2)
+            fieldbox = optimizer.get_box_replacement(fieldbox)
             self.register_box(box)
             self.register_box(fieldbox)
             info = optimizer.getptrinfo(fieldbox)
@@ -418,7 +420,7 @@
             info.visitor_walk_recursive(fieldbox, self, optimizer)
 
         self._number_virtuals(liveboxes, optimizer, v)
-        self._add_pending_fields(pending_setfields)
+        self._add_pending_fields(optimizer, pending_setfields)
 
         storage.rd_consts = self.memo.consts
         return liveboxes[:]
@@ -489,14 +491,14 @@
                 return True
         return False
 
-    def _add_pending_fields(self, pending_setfields):
+    def _add_pending_fields(self, optimizer, pending_setfields):
         rd_pendingfields = lltype.nullptr(PENDINGFIELDSP.TO)
         if pending_setfields:
             n = len(pending_setfields)
             rd_pendingfields = lltype.malloc(PENDINGFIELDSP.TO, n)
             for i in range(n):
                 op = pending_setfields[i]
-                box = op.getarg(0)
+                box = optimizer.get_box_replacement(op.getarg(0))
                 descr = op.getdescr()
                 if op.getopnum() == rop.SETARRAYITEM_GC:
                     fieldbox = op.getarg(2)
@@ -504,6 +506,7 @@
                 else:
                     fieldbox = op.getarg(1)
                     itemindex = -1
+                fieldbox = optimizer.get_box_replacement(fieldbox)
                 #descr, box, fieldbox, itemindex = pending_setfields[i]
                 lldescr = annlowlevel.cast_instance_to_base_ptr(descr)
                 num = self._gettagged(box)
diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py
--- a/rpython/jit/metainterp/test/test_ajit.py
+++ b/rpython/jit/metainterp/test/test_ajit.py
@@ -4214,3 +4214,104 @@
             assert res == f(42)
             opname = "instance_ptr_%s" % cmp
             self.check_operations_history(**{opname: 0})
+
+    def test_compile_framework_9(self):
+        class X(object):
+            def __init__(self, x=0):
+                self.x = x
+
+            next = None
+
+        class CheckError(Exception):
+            pass
+
+        def check(flag):
+            if not flag:
+                raise CheckError
+
+        def before(n, x):
+            return n, x, None, None, None, None, None, None, None, None, [X(123)], None
+        def f(n, x, x0, x1, x2, x3, x4, x5, x6, x7, l, s):
+            if n < 1900:
+                check(l[0].x == 123)
+                num = 512 + (n & 7)
+                l = [None] * num
+                l[0] = X(123)
+                l[1] = X(n)
+                l[2] = X(n+10)
+                l[3] = X(n+20)
+                l[4] = X(n+30)
+                l[5] = X(n+40)
+                l[6] = X(n+50)
+                l[7] = X(n+60)
+                l[num-8] = X(n+70)
+                l[num-9] = X(n+80)
+                l[num-10] = X(n+90)
+                l[num-11] = X(n+100)
+                l[-12] = X(n+110)
+                l[-13] = X(n+120)
+                l[-14] = X(n+130)
+                l[-15] = X(n+140)
+            if n < 1800:
+                num = 512 + (n & 7)
+                check(len(l) == num)
+                check(l[0].x == 123)
+                check(l[1].x == n)
+                check(l[2].x == n+10)
+                check(l[3].x == n+20)
+                check(l[4].x == n+30)
+                check(l[5].x == n+40)
+                check(l[6].x == n+50)
+                check(l[7].x == n+60)
+                check(l[num-8].x == n+70)
+                check(l[num-9].x == n+80)
+                check(l[num-10].x == n+90)
+                check(l[num-11].x == n+100)
+                check(l[-12].x == n+110)
+                check(l[-13].x == n+120)
+                check(l[-14].x == n+130)
+                check(l[-15].x == n+140)
+            n -= x.foo
+            return n, x, x0, x1, x2, x3, x4, x5, x6, x7, l, s
+        def after(n, x, x0, x1, x2, x3, x4, x5, x6, x7, l, s):
+            check(len(l) >= 512)
+            check(l[0].x == 123)
+            check(l[1].x == 2)
+            check(l[2].x == 12)
+            check(l[3].x == 22)
+            check(l[4].x == 32)
+            check(l[5].x == 42)
+            check(l[6].x == 52)
+            check(l[7].x == 62)
+            check(l[-8].x == 72)
+            check(l[-9].x == 82)
+            check(l[-10].x == 92)
+            check(l[-11].x == 102)
+            check(l[-12].x == 112)
+            check(l[-13].x == 122)
+            check(l[-14].x == 132)
+            check(l[-15].x == 142)
+
+        def allfuncs(num, n):
+            x = X()
+            x.foo = 2
+            main_allfuncs(num, n, x)
+            x.foo = 5
+            return x
+        def main_allfuncs(num, n, x):
+            n, x, x0, x1, x2, x3, x4, x5, x6, x7, l, s = before(n, x)
+            while n > 0:
+                myjitdriver.can_enter_jit(num=num, n=n, x=x, x0=x0, x1=x1,
+                        x2=x2, x3=x3, x4=x4, x5=x5, x6=x6, x7=x7, l=l, s=s)
+                myjitdriver.jit_merge_point(num=num, n=n, x=x, x0=x0, x1=x1,
+                        x2=x2, x3=x3, x4=x4, x5=x5, x6=x6, x7=x7, l=l, s=s)
+
+                n, x, x0, x1, x2, x3, x4, x5, x6, x7, l, s = f(
+                        n, x, x0, x1, x2, x3, x4, x5, x6, x7, l, s)
+            after(n, x, x0, x1, x2, x3, x4, x5, x6, x7, l, s)
+        myjitdriver = JitDriver(greens = ['num'],
+                                reds = ['n', 'x', 'x0', 'x1', 'x2', 'x3', 'x4',
+                                        'x5', 'x6', 'x7', 'l', 's'])
+ 
+        
+        self.meta_interp(allfuncs, [9, 2000])

From noreply at buildbot.pypy.org  Wed Jun  3 18:33:03 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Wed,  3 Jun 2015 18:33:03 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: oops
Message-ID: <20150603163303.E379A1C1048@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77835:341489604c91
Date: 2015-06-03 18:32 +0200
http://bitbucket.org/pypy/pypy/changeset/341489604c91/

Log:	oops

diff --git a/rpython/jit/metainterp/quasiimmut.py b/rpython/jit/metainterp/quasiimmut.py
--- a/rpython/jit/metainterp/quasiimmut.py
+++ b/rpython/jit/metainterp/quasiimmut.py
@@ -103,8 +103,8 @@
 class QuasiImmutDescr(AbstractDescr):
     # those fields are necessary for translation without quasi immutable
     # fields
-    struct = None
-    fielddescr = lltype.nullptr(llmemory.GCREF.TO)
+    struct = lltype.nullptr(llmemory.GCREF.TO)
+    fielddescr = None
     
     def __init__(self, cpu, struct, fielddescr, mutatefielddescr):
         self.cpu = cpu

From noreply at buildbot.pypy.org  Wed Jun  3 18:49:10 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Wed,  3 Jun 2015 18:49:10 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: rpythonify
Message-ID: <20150603164910.DCD331C024E@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77836:c96f575bc6ce
Date: 2015-06-03 18:49 +0200
http://bitbucket.org/pypy/pypy/changeset/c96f575bc6ce/

Log:	rpythonify

diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -838,9 +838,15 @@
         # null, and the guard will be removed.  So the fact that the field is
         # quasi-immutable will have no effect, and instead it will work as a
         # regular, probably virtual, structure.
-        opnum = OpHelpers.getfield_for_descr(mutatefielddescr)
-        mutatebox = self.execute_with_descr(opnum,
-                                            mutatefielddescr, box)
+        if mutatefielddescr.is_pointer_field():
+            mutatebox = self.execute_with_descr(rop.GETFIELD_GC_R,
+                                                mutatefielddescr, box)
+        elif mutatefielddescr.is_float_field():
+            mutatebox = self.execute_with_descr(rop.GETFIELD_GC_R,
+                                                mutatefielddescr, box)
+        else:
+            mutatebox = self.execute_with_descr(rop.GETFIELD_GC_I,
+                                                mutatefielddescr, box)
         if mutatebox.nonnull():
             from rpython.jit.metainterp.quasiimmut import do_force_quasi_immutable
             do_force_quasi_immutable(self.metainterp.cpu, box.getref_base(),

From noreply at buildbot.pypy.org  Wed Jun  3 18:55:58 2015
From: noreply at buildbot.pypy.org (vext01)
Date: Wed,  3 Jun 2015 18:55:58 +0200 (CEST)
Subject: [pypy-commit] pypy default: Backed out changeset 5cf9f578ca18
Message-ID: <20150603165558.47B881C034D@cobra.cs.uni-duesseldorf.de>

Author: Edd Barrett <vext01 at gmail.com>
Branch: 
Changeset: r77837:131622d03fbe
Date: 2015-06-03 17:54 +0100
http://bitbucket.org/pypy/pypy/changeset/131622d03fbe/

Log:	Backed out changeset 5cf9f578ca18

	This broke translation on our debian 8 machine.

	I have tried setting various different LC_ALL/LC_CTYPE values, but
	these seem to have no effect.

	17:01 < fijal> ebarrett: well then rollback with an explanation
	17:01 < fijal> ebarrett: leaving build broken is not a good idea

	Translation error follows:

	[translation:info] File "/opt/pypy/rpython/annotator/unaryop.py",
	line 604, in method_decode [translation:info] raise
	AnnotatorError("Encoding %s not supported for strings" % (enc,))
	[translation:ERROR] AnnotatorError: [translation:ERROR]
	[translation:ERROR] Encoding latin1 not supported for strings
	[translation:ERROR] [translation:ERROR] [translation:ERROR] Occurred
	processing the following simple_call: [translation:ERROR]
	(AttributeError getting at the binding!) [translation:ERROR]
	v334 = simple_call(v333, ('latin1')) [translation:ERROR]
	[translation:ERROR] In <FunctionGraph of
	(pypy.objspace.std.newformat:529)Formatter._lit at 0x3724fa98>:
	[translation:ERROR] Happened at file
	/opt/pypy/pypy/objspace/std/newformat.py line 531
	[translation:ERROR] [translation:ERROR] ==> return
	s.decode("latin1") [translation:ERROR] [translation:ERROR] Known
	variable annotations: [translation:ERROR] v333 =
	SomeBuiltinMethod(analyser=<rpython.tool.descriptor.InstanceMethod
	object at 0x000000003802d910>, methodname='decode',
	s_self=SomeChar(const=' ', no_nul=True)) [translation:ERROR]
	[translation:ERROR] Processing block: [translation:ERROR] block at 12
	is a <class 'rpython.flowspace.flowcontext.SpamBlock'>
	[translation:ERROR] in
	(pypy.objspace.std.newformat:529)Formatter._lit [translation:ERROR]
	containing the following operations: [translation:ERROR] v333
	= getattr(s_11, ('decode')) [translation:ERROR] v334 =
	simple_call(v333, ('latin1')) [translation:ERROR] --end--

diff --git a/pypy/objspace/std/newformat.py b/pypy/objspace/std/newformat.py
--- a/pypy/objspace/std/newformat.py
+++ b/pypy/objspace/std/newformat.py
@@ -528,7 +528,7 @@
 
         def _lit(self, s):
             if self.is_unicode:
-                return s.decode("latin1")
+                return s.decode("ascii")
             else:
                 return s
 
@@ -586,8 +586,8 @@
                 thousands = ""
                 grouping = "\xFF"    # special value to mean 'stop'
             if self.is_unicode:
-                self._loc_dec = dec.decode("latin1")
-                self._loc_thousands = thousands.decode("latin1")
+                self._loc_dec = dec.decode("ascii")
+                self._loc_thousands = thousands.decode("ascii")
             else:
                 self._loc_dec = dec
                 self._loc_thousands = thousands
@@ -725,7 +725,7 @@
                 out.append_multiple_char(fill_char[0], spec.n_lpadding)
             if spec.n_sign:
                 if self.is_unicode:
-                    sign = spec.sign.decode("latin1")
+                    sign = spec.sign.decode("ascii")
                 else:
                     sign = spec.sign
                 out.append(sign)
@@ -828,14 +828,14 @@
                 prefix = "0x"
             as_str = value.format(LONG_DIGITS[:base], prefix)
             if self.is_unicode:
-                return as_str.decode("latin1")
+                return as_str.decode("ascii")
             return as_str
 
         def _int_to_base(self, base, value):
             if base == 10:
                 s = str(value)
                 if self.is_unicode:
-                    return s.decode("latin1")
+                    return s.decode("ascii")
                 return s
             # This part is slow.
             negative = value < 0
@@ -954,7 +954,7 @@
             have_dec_point, to_remainder = self._parse_number(result, to_number)
             n_remainder = len(result) - to_remainder
             if self.is_unicode:
-                digits = result.decode("latin1")
+                digits = result.decode("ascii")
             else:
                 digits = result
             spec = self._calc_num_width(0, sign, to_number, n_digits,
@@ -1059,8 +1059,8 @@
                                                                to_imag_number)
 
             if self.is_unicode:
-                re_num = re_num.decode("latin1")
-                im_num = im_num.decode("latin1")
+                re_num = re_num.decode("ascii")
+                im_num = im_num.decode("ascii")
 
             #set remainder, in CPython _parse_number sets this
             #using n_re_digits causes tests to fail

From noreply at buildbot.pypy.org  Wed Jun  3 19:03:28 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Wed,  3 Jun 2015 19:03:28 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: rpythonize this part too
Message-ID: <20150603170328.3462D1C034D@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77838:6322d9e0977e
Date: 2015-06-03 19:03 +0200
http://bitbucket.org/pypy/pypy/changeset/6322d9e0977e/

Log:	rpythonize this part too

diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -742,7 +742,12 @@
             return executor.execute_nonspec_const(self.metainterp.cpu,
                                     self.metainterp, opnum, [box], fielddescr)
         # fall-back
-        return self.execute_with_descr(opnum, fielddescr, box)
+        if fielddescr.is_pointer_field():
+            return self.execute_with_descr(rop.GETFIELD_GC_R, fielddescr, box)
+        elif fielddescr.is_float_field():
+            return self.execute_with_descr(rop.GETFIELD_GC_F, fielddescr, box)
+        else:
+            return self.execute_with_descr(rop.GETFIELD_GC_I, fielddescr, box)
     opimpl_getfield_gc_i_greenfield = _opimpl_getfield_gc_greenfield_any
     opimpl_getfield_gc_r_greenfield = _opimpl_getfield_gc_greenfield_any
     opimpl_getfield_gc_f_greenfield = _opimpl_getfield_gc_greenfield_any

From noreply at buildbot.pypy.org  Wed Jun  3 19:17:40 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Wed,  3 Jun 2015 19:17:40 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: disable it better
Message-ID: <20150603171740.CF3991C034D@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77839:a11ec25e489a
Date: 2015-06-03 19:17 +0200
http://bitbucket.org/pypy/pypy/changeset/a11ec25e489a/

Log:	disable it better

diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -295,8 +295,8 @@
 
     def jitpolicy(self, driver):
         from pypy.module.pypyjit.policy import PyPyJitPolicy
-        from pypy.module.pypyjit.hooks import pypy_hooks
-        return PyPyJitPolicy(pypy_hooks)
+        #from pypy.module.pypyjit.hooks import pypy_hooks
+        return PyPyJitPolicy()#pypy_hooks)
 
     def get_entry_point(self, config):
         from pypy.tool.lib_pypy import import_from_lib_pypy

From noreply at buildbot.pypy.org  Wed Jun  3 21:15:28 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Wed,  3 Jun 2015 21:15:28 +0200 (CEST)
Subject: [pypy-commit] pypy default: special case buffer -> uint8 dtype
Message-ID: <20150603191528.C1B381C024E@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: 
Changeset: r77840:4973a0f32579
Date: 2015-06-03 20:15 +0300
http://bitbucket.org/pypy/pypy/changeset/4973a0f32579/

Log:	special case buffer -> uint8 dtype

diff --git a/pypy/module/micronumpy/ctors.py b/pypy/module/micronumpy/ctors.py
--- a/pypy/module/micronumpy/ctors.py
+++ b/pypy/module/micronumpy/ctors.py
@@ -142,6 +142,8 @@
     else:
         # not an array
         shape, elems_w = strides.find_shape_and_elems(space, w_object, dtype)
+    if dtype is None and space.isinstance_w(w_object, space.w_buffer):
+        dtype = descriptor.get_dtype_cache(space).w_uint8dtype
     if dtype is None or (dtype.is_str_or_unicode() and dtype.elsize < 1):
         dtype = find_dtype_for_seq(space, elems_w, dtype)
         if dtype is None:
diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py
--- a/pypy/module/micronumpy/test/test_ndarray.py
+++ b/pypy/module/micronumpy/test/test_ndarray.py
@@ -3984,8 +3984,9 @@
         import sys
         builtins = getattr(__builtins__, '__dict__', __builtins__)
         _buffer = builtins.get('buffer')
-        dat = np.array(_buffer('1.0'), dtype=np.float64)
-        assert (dat == [49.0, 46.0, 48.0]).all()
+        dat = np.array(_buffer('1.0'))
+        assert (dat == [49, 46, 48]).all()
+        assert dat.dtype == np.dtype('uint8')
 
 
 class AppTestPyPy(BaseNumpyAppTest):

From noreply at buildbot.pypy.org  Wed Jun  3 21:15:30 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Wed,  3 Jun 2015 21:15:30 +0200 (CEST)
Subject: [pypy-commit] pypy default: test, fix flags.owndata
Message-ID: <20150603191530.04E991C024E@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: 
Changeset: r77841:ad5bd147db7f
Date: 2015-06-03 20:37 +0300
http://bitbucket.org/pypy/pypy/changeset/ad5bd147db7f/

Log:	test, fix flags.owndata

diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py
--- a/pypy/module/micronumpy/concrete.py
+++ b/pypy/module/micronumpy/concrete.py
@@ -438,6 +438,7 @@
     def __init__(self, shape, dtype, order, strides, backstrides,
                  storage=lltype.nullptr(RAW_STORAGE), zero=True):
         gcstruct = V_OBJECTSTORE
+        self.flags = NPY.ARRAY_ALIGNED | NPY.ARRAY_WRITEABLE
         if storage == lltype.nullptr(RAW_STORAGE):
             length = support.product(shape) 
             if dtype.num == NPY.OBJECT:
@@ -445,11 +446,11 @@
                 gcstruct = _create_objectstore(storage, length, dtype.elsize)
             else:
                 storage = dtype.itemtype.malloc(length * dtype.elsize, zero=zero)
+            self.flags |= NPY.ARRAY_OWNDATA
         start = calc_start(shape, strides)
         ConcreteArrayNotOwning.__init__(self, shape, dtype, order, strides, backstrides,
                                         storage, start=start)
         self.gcstruct = gcstruct
-        self.flags = NPY.ARRAY_ALIGNED | NPY.ARRAY_WRITEABLE
         if is_c_contiguous(self):
             self.flags |= NPY.ARRAY_C_CONTIGUOUS
         if is_f_contiguous(self):
diff --git a/pypy/module/micronumpy/test/test_flagsobj.py b/pypy/module/micronumpy/test/test_flagsobj.py
--- a/pypy/module/micronumpy/test/test_flagsobj.py
+++ b/pypy/module/micronumpy/test/test_flagsobj.py
@@ -13,6 +13,8 @@
         assert s == '%s' %('  C_CONTIGUOUS : True\n  F_CONTIGUOUS : True'
                          '\n  OWNDATA : True\n  WRITEABLE : False'
                          '\n  ALIGNED : True\n  UPDATEIFCOPY : False')
+        a = np.array(2)
+        assert a.flags.owndata
 
     def test_repr(self):
         import numpy as np

From noreply at buildbot.pypy.org  Wed Jun  3 21:15:31 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Wed,  3 Jun 2015 21:15:31 +0200 (CEST)
Subject: [pypy-commit] pypy default: test,
 implement part of dtype(align=True), missing isalignstruct in dtype.flags
Message-ID: <20150603191531.254571C024E@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: 
Changeset: r77842:5c1839b76cb0
Date: 2015-06-03 22:15 +0300
http://bitbucket.org/pypy/pypy/changeset/5c1839b76cb0/

Log:	test, implement part of dtype(align=True), missing isalignstruct in
	dtype.flags

diff --git a/pypy/module/micronumpy/descriptor.py b/pypy/module/micronumpy/descriptor.py
--- a/pypy/module/micronumpy/descriptor.py
+++ b/pypy/module/micronumpy/descriptor.py
@@ -503,11 +503,12 @@
 
 
 @specialize.arg(2)
-def dtype_from_list(space, w_lst, simple):
+def dtype_from_list(space, w_lst, simple, align=False):
     lst_w = space.listview(w_lst)
     fields = {}
     offset = 0
     names = []
+    maxalign = 0
     for i in range(len(lst_w)):
         w_elem = lst_w[i]
         if simple:
@@ -530,7 +531,11 @@
         assert isinstance(subdtype, W_Dtype)
         fields[fldname] = (offset, subdtype)
         offset += subdtype.elsize
+        maxalign = max(subdtype.elsize, maxalign)
         names.append(fldname)
+    if align:
+        # Set offset to the next power-of-two above offset
+        offset = (offset + maxalign -1) & (-maxalign)
     return W_Dtype(types.RecordType(space), space.gettypefor(boxes.W_VoidBox),
                    names=names, fields=fields, elsize=offset)
 
@@ -580,14 +585,14 @@
             sqbracket -= 1
     return False
 
-
-def descr__new__(space, w_subtype, w_dtype, w_align=None, w_copy=None, w_shape=None):
-    # w_align and w_copy are necessary for pickling
+ at unwrap_spec(align=bool)
+def descr__new__(space, w_subtype, w_dtype, align=False, w_copy=None, w_shape=None):
+    # align and w_copy are necessary for pickling
     cache = get_dtype_cache(space)
 
     if w_shape is not None and (space.isinstance_w(w_shape, space.w_int) or
                                 space.len_w(w_shape) > 0):
-        subdtype = descr__new__(space, w_subtype, w_dtype, w_align, w_copy)
+        subdtype = descr__new__(space, w_subtype, w_dtype, align, w_copy)
         assert isinstance(subdtype, W_Dtype)
         size = 1
         if space.isinstance_w(w_shape, space.w_int):
@@ -627,16 +632,16 @@
             return variable_dtype(space, name)
         raise oefmt(space.w_TypeError, 'data type "%s" not understood', name)
     elif space.isinstance_w(w_dtype, space.w_list):
-        return dtype_from_list(space, w_dtype, False)
+        return dtype_from_list(space, w_dtype, False, align=align)
     elif space.isinstance_w(w_dtype, space.w_tuple):
         w_dtype0 = space.getitem(w_dtype, space.wrap(0))
         w_dtype1 = space.getitem(w_dtype, space.wrap(1))
-        subdtype = descr__new__(space, w_subtype, w_dtype0, w_align, w_copy)
+        subdtype = descr__new__(space, w_subtype, w_dtype0, align, w_copy)
         assert isinstance(subdtype, W_Dtype)
         if subdtype.elsize == 0:
             name = "%s%d" % (subdtype.kind, space.int_w(w_dtype1))
-            return descr__new__(space, w_subtype, space.wrap(name), w_align, w_copy)
-        return descr__new__(space, w_subtype, w_dtype0, w_align, w_copy, w_shape=w_dtype1)
+            return descr__new__(space, w_subtype, space.wrap(name), align, w_copy)
+        return descr__new__(space, w_subtype, w_dtype0, align, w_copy, w_shape=w_dtype1)
     elif space.isinstance_w(w_dtype, space.w_dict):
         return dtype_from_dict(space, w_dtype)
     for dtype in cache.builtin_dtypes:
diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py
--- a/pypy/module/micronumpy/test/test_dtypes.py
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -1083,6 +1083,8 @@
     def test_create(self):
         from numpy import dtype, void
 
+        d = dtype([('x', 'i4'), ('y', 'i1')], align=True)
+        assert d.itemsize == 8
         raises(ValueError, "dtype([('x', int), ('x', float)])")
         d = dtype([("x", "<i4"), ("y", "<f4"), ("z", "<u2"), ("v", "<f8")])
         assert d.fields['x'] == (dtype('<i4'), 0)

From noreply at buildbot.pypy.org  Wed Jun  3 23:30:57 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Wed,  3 Jun 2015 23:30:57 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Correct indentation to be a multiple of
	four.
Message-ID: <20150603213057.DC1901C0FB8@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob <me at manueljacob.de>
Branch: py3k
Changeset: r77843:d4d74edf6a05
Date: 2015-06-03 19:38 +0200
http://bitbucket.org/pypy/pypy/changeset/d4d74edf6a05/

Log:	Correct indentation to be a multiple of four.

diff --git a/pypy/objspace/std/test/test_methodcache.py b/pypy/objspace/std/test/test_methodcache.py
--- a/pypy/objspace/std/test/test_methodcache.py
+++ b/pypy/objspace/std/test/test_methodcache.py
@@ -48,113 +48,113 @@
     def test_change_methods(self):
         # this test fails because of the following line in typeobject.py:427
         #             if cached_name is name:
-        
+
         # in py3k, identifiers are stored in W_UnicodeObject and unwrapped by
         # calling space.str_w, which .encode('ascii') the string, thus
         # creating new strings all the time. The problem should be solved when
         # we implement proper unicode identifiers in py3k
         @self.retry
         def run():
-          import __pypy__
-          class A(object):
-              def f(self):
-                  return 42
-          l = [A()] * 10
-          __pypy__.reset_method_cache_counter()
-          for i, a in enumerate(l):
-              assert a.f() == 42 + i
-              A.f = eval("lambda self: %s" % (42 + i + 1, ))
-          cache_counter = __pypy__.method_cache_counter("f")
-          #
-          # a bit of explanation about what's going on.  (1) is the line "a.f()"
-          # and (2) is "A.f = ...".
-          #
-          # at line (1) we do the lookup on type(a).f
-          #
-          # at line (2) we do a setattr on A. However, descr_setattr does also a
-          # lookup of type(A).f i.e. type.f, to check if by chance 'f' is a data
-          # descriptor.
-          #
-          # At the first iteration:
-          # (1) is a miss because it's the first lookup of A.f. The result is cached
-          #
-          # (2) is a miss because it is the first lookup of type.f. The
-          # (non-existant) result is cached. The version of A changes, and 'f'
-          # is changed to be a cell object, so that subsequest assignments won't
-          # change the version of A
-          #
-          # At the second iteration:
-          # (1) is a miss because the version of A changed just before
-          # (2) is a hit, because type.f is cached. The version of A no longer changes
-          #
-          # At the third and subsequent iterations:
-          # (1) is a hit, because the version of A did not change
-          # (2) is a hit, see above
-          assert cache_counter == (17, 3)
+            import __pypy__
+            class A(object):
+                def f(self):
+                    return 42
+            l = [A()] * 10
+            __pypy__.reset_method_cache_counter()
+            for i, a in enumerate(l):
+                assert a.f() == 42 + i
+                A.f = eval("lambda self: %s" % (42 + i + 1, ))
+            cache_counter = __pypy__.method_cache_counter("f")
+            #
+            # a bit of explanation about what's going on.  (1) is the line "a.f()"
+            # and (2) is "A.f = ...".
+            #
+            # at line (1) we do the lookup on type(a).f
+            #
+            # at line (2) we do a setattr on A. However, descr_setattr does also a
+            # lookup of type(A).f i.e. type.f, to check if by chance 'f' is a data
+            # descriptor.
+            #
+            # At the first iteration:
+            # (1) is a miss because it's the first lookup of A.f. The result is cached
+            #
+            # (2) is a miss because it is the first lookup of type.f. The
+            # (non-existant) result is cached. The version of A changes, and 'f'
+            # is changed to be a cell object, so that subsequest assignments won't
+            # change the version of A
+            #
+            # At the second iteration:
+            # (1) is a miss because the version of A changed just before
+            # (2) is a hit, because type.f is cached. The version of A no longer changes
+            #
+            # At the third and subsequent iterations:
+            # (1) is a hit, because the version of A did not change
+            # (2) is a hit, see above
+            assert cache_counter == (17, 3)
 
     def test_subclasses(self):
         @self.retry
         def run():
-          import __pypy__
-          class A(object):
-              def f(self):
-                  return 42
-          class B(object):
-              def f(self):
-                  return 43
-          class C(A):
-              pass
-          l = [A(), B(), C()] * 10
-          __pypy__.reset_method_cache_counter()
-          for i, a in enumerate(l):
-              assert a.f() == 42 + (i % 3 == 1)
-          cache_counter = __pypy__.method_cache_counter("f")
-          assert cache_counter[0] >= 15
-          assert cache_counter[1] >= 3 # should be (27, 3)
-          assert sum(cache_counter) == 30
-  
+            import __pypy__
+            class A(object):
+                def f(self):
+                    return 42
+            class B(object):
+                def f(self):
+                    return 43
+            class C(A):
+                pass
+            l = [A(), B(), C()] * 10
+            __pypy__.reset_method_cache_counter()
+            for i, a in enumerate(l):
+                assert a.f() == 42 + (i % 3 == 1)
+            cache_counter = __pypy__.method_cache_counter("f")
+            assert cache_counter[0] >= 15
+            assert cache_counter[1] >= 3 # should be (27, 3)
+            assert sum(cache_counter) == 30
+
     def test_many_names(self):
         @self.retry
         def run():
             import __pypy__
             laste = None
             for j in range(20):
-              class A(object):
-                  def f(self):
-                      return 42
-              class B(object):
-                  def f(self):
-                      return 43
-              class C(A):
-                  pass
-              l = [A(), B(), C()] * 10
-              __pypy__.reset_method_cache_counter()
-              for i, a in enumerate(l):
-                  assert a.f() == 42 + (i % 3 == 1)
-              cache_counter = __pypy__.method_cache_counter("f")
-              assert cache_counter[0] >= 15
-              assert cache_counter[1] >= 3 # should be (27, 3)
-              assert sum(cache_counter) == 30
+                class A(object):
+                    def f(self):
+                        return 42
+                class B(object):
+                    def f(self):
+                        return 43
+                class C(A):
+                    pass
+                l = [A(), B(), C()] * 10
+                __pypy__.reset_method_cache_counter()
+                for i, a in enumerate(l):
+                    assert a.f() == 42 + (i % 3 == 1)
+                cache_counter = __pypy__.method_cache_counter("f")
+                assert cache_counter[0] >= 15
+                assert cache_counter[1] >= 3 # should be (27, 3)
+                assert sum(cache_counter) == 30
 
-              a = A()
-              names = [name for name in A.__dict__.keys()
-                       if not name.startswith('_')]
-              names.sort()
-              names_repeated = names * 10
-              result = []
-              __pypy__.reset_method_cache_counter()
-              for name in names_repeated:
-                  result.append(getattr(a, name))
-              append_counter = __pypy__.method_cache_counter("append")
-              names_counters = [__pypy__.method_cache_counter(name)
-                                for name in names]
-              try:
-                  assert append_counter[0] >= 10 * len(names) - 1
-                  for name, count in zip(names, names_counters):
-                      assert count == (9, 1), str((name, count))
-                  break
-              except AssertionError as e:
-                  laste = e
+                a = A()
+                names = [name for name in A.__dict__.keys()
+                         if not name.startswith('_')]
+                names.sort()
+                names_repeated = names * 10
+                result = []
+                __pypy__.reset_method_cache_counter()
+                for name in names_repeated:
+                    result.append(getattr(a, name))
+                append_counter = __pypy__.method_cache_counter("append")
+                names_counters = [__pypy__.method_cache_counter(name)
+                                  for name in names]
+                try:
+                    assert append_counter[0] >= 10 * len(names) - 1
+                    for name, count in zip(names, names_counters):
+                        assert count == (9, 1), str((name, count))
+                    break
+                except AssertionError as e:
+                    laste = e
             else:
                 raise laste
 

From noreply at buildbot.pypy.org  Wed Jun  3 23:30:59 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Wed,  3 Jun 2015 23:30:59 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Call dict.copy() instead of dict.items()
 here because dict.items() does not copy the contents on Py3k.
Message-ID: <20150603213059.3064D1C0FB8@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob <me at manueljacob.de>
Branch: py3k
Changeset: r77844:6133e40a4002
Date: 2015-06-03 23:23 +0200
http://bitbucket.org/pypy/pypy/changeset/6133e40a4002/

Log:	Call dict.copy() instead of dict.items() here because dict.items()
	does not copy the contents on Py3k.

diff --git a/pypy/interpreter/mixedmodule.py b/pypy/interpreter/mixedmodule.py
--- a/pypy/interpreter/mixedmodule.py
+++ b/pypy/interpreter/mixedmodule.py
@@ -9,7 +9,7 @@
     applevel_name = None
 
     # The following attribute is None as long as the module has not been
-    # imported yet, and when it has been, it is mod.__dict__.items() just
+    # imported yet, and when it has been, it is mod.__dict__.copy() just
     # after startup().
     w_initialdict = None
     lazy = False
@@ -58,7 +58,7 @@
                 self.save_module_content_for_future_reload()
 
     def save_module_content_for_future_reload(self):
-        self.w_initialdict = self.space.call_method(self.w_dict, 'items')
+        self.w_initialdict = self.space.call_method(self.w_dict, 'copy')
 
 
     def get_applevel_name(cls):

From noreply at buildbot.pypy.org  Wed Jun  3 23:31:00 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Wed,  3 Jun 2015 23:31:00 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Add the struct module to this test's
 spaceconfig because the pure Python version of it was removed on this
 branch.
Message-ID: <20150603213100.559C41C0FB8@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob <me at manueljacob.de>
Branch: py3k
Changeset: r77845:439c34e6ed3d
Date: 2015-06-03 23:30 +0200
http://bitbucket.org/pypy/pypy/changeset/439c34e6ed3d/

Log:	Add the struct module to this test's spaceconfig because the pure
	Python version of it was removed on this branch.

diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py
--- a/pypy/module/imp/test/test_import.py
+++ b/pypy/module/imp/test/test_import.py
@@ -194,7 +194,7 @@
 
 class AppTestImport(BaseImportTest):
     spaceconfig = {
-        "usemodules": ['time'],
+        "usemodules": ['time', 'struct'],
     }
 
     def setup_class(cls):

From noreply at buildbot.pypy.org  Wed Jun  3 23:57:13 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Wed,  3 Jun 2015 23:57:13 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Make these tests,
 which rely on the GIL being released, more reliable.
Message-ID: <20150603215713.B57A81C1017@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob <me at manueljacob.de>
Branch: py3k
Changeset: r77846:f64ff1e27a9b
Date: 2015-06-03 23:57 +0200
http://bitbucket.org/pypy/pypy/changeset/f64ff1e27a9b/

Log:	Make these tests, which rely on the GIL being released, more
	reliable.

	This is necessary on this branch because the GIL semantics changed.

diff --git a/pypy/module/__pypy__/test/test_signal.py b/pypy/module/__pypy__/test/test_signal.py
--- a/pypy/module/__pypy__/test/test_signal.py
+++ b/pypy/module/__pypy__/test/test_signal.py
@@ -57,10 +57,10 @@
                 interrupted = []
                 print('--- start ---')
                 _thread.start_new_thread(subthread, ())
-                for j in range(30):
+                for j in range(100):
                     if len(done): break
                     print('.')
-                    time.sleep(0.25)
+                    time.sleep(0)
                 print('main thread loop done')
                 assert len(done) == 1
                 assert len(interrupted) == 1
diff --git a/pypy/module/select/test/test_select.py b/pypy/module/select/test/test_select.py
--- a/pypy/module/select/test/test_select.py
+++ b/pypy/module/select/test/test_select.py
@@ -287,7 +287,7 @@
             t = thread.start_new_thread(pollster.poll, ())
             try:
                 time.sleep(0.3)
-                for i in range(5): print(''),  # to release GIL untranslated
+                for i in range(100): print(''),  # to release GIL untranslated
                 # trigger ufds array reallocation
                 for fd in rfds:
                     pollster.unregister(fd)
@@ -298,7 +298,7 @@
                 # and make the call to poll() from the thread return
                 os.write(w, b'spam')
                 time.sleep(0.3)
-                for i in range(5): print(''),  # to release GIL untranslated
+                for i in range(100): print(''),  # to release GIL untranslated
         finally:
             os.close(r)
             os.close(w)

From noreply at buildbot.pypy.org  Thu Jun  4 01:58:54 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Thu,  4 Jun 2015 01:58:54 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Let the trace module ignore some modules
 which get freezed at translation-time.
Message-ID: <20150603235854.9B6911C024E@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob <me at manueljacob.de>
Branch: py3k
Changeset: r77847:f1f064b3faf1
Date: 2015-06-04 01:59 +0200
http://bitbucket.org/pypy/pypy/changeset/f1f064b3faf1/

Log:	Let the trace module ignore some modules which get freezed at
	translation-time.

	See comment for details.

diff --git a/lib-python/3/trace.py b/lib-python/3/trace.py
--- a/lib-python/3/trace.py
+++ b/lib-python/3/trace.py
@@ -245,7 +245,12 @@
         we want to have reported.
         """
         return (filename == "<string>" or
-                filename.startswith("<doctest "))
+                filename.startswith("<doctest ") or
+                # XXX PyPy freezes some (pure-Python) modules at
+                # translation-time.  These contain filenames starting with
+                # "<builtin>/" instead of their actual filenames.  Ignore them
+                # for now.
+                filename.startswith("<builtin>/"))
 
     def update(self, other):
         """Merge in the data from another CoverageResults"""

From noreply at buildbot.pypy.org  Thu Jun  4 05:37:01 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu,  4 Jun 2015 05:37:01 +0200 (CEST)
Subject: [pypy-commit] pypy release-2.6.x: Small tweaks
Message-ID: <20150604033701.0ADD01C024E@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: release-2.6.x
Changeset: r77848:b95fc4861fd5
Date: 2015-06-01 17:08 +0200
http://bitbucket.org/pypy/pypy/changeset/b95fc4861fd5/

Log:	Small tweaks

diff --git a/pypy/doc/release-2.6.0.rst b/pypy/doc/release-2.6.0.rst
--- a/pypy/doc/release-2.6.0.rst
+++ b/pypy/doc/release-2.6.0.rst
@@ -38,13 +38,13 @@
 .. _`on bitbucket`: https://www.bitbucket.org/pypy/numpy
 
 We would also like to encourage new people to join the project. PyPy has many
-layers and we need help with all of them: `PyPy`_ and `Rpython`_ documentation
+layers and we need help with all of them: `PyPy`_ and `RPython`_ documentation
 improvements, tweaking popular `modules`_ to run on pypy, or general `help`_ with making
-Rpython's JIT even better. Nine new people contributed since the last release,
+RPython's JIT even better. Nine new people contributed since the last release,
 you too could be one of them.
 
 .. _`PyPy`: http://doc.pypy.org 
-.. _`Rpython`: https://rpython.readthedocs.org
+.. _`RPython`: https://rpython.readthedocs.org
 .. _`modules`: http://doc.pypy.org/en/latest/project-ideas.html#make-more-python-modules-pypy-friendly
 .. _`help`: http://doc.pypy.org/en/latest/project-ideas.html
 
@@ -115,7 +115,7 @@
     over 7 times faster than cpython
 
 .. _`vmprof`: https://vmprof.readthedocs.org
-.. _resolved: https://doc.pypy.org/en/latest/whatsnew-2.6.0.html
+.. _resolved: http://doc.pypy.org/en/latest/whatsnew-2.6.0.html
 
 Please try it out and let us know what you think. We welcome
 success stories, `experiments`_,  or `benchmarks`_, we know you are using PyPy, please tell us about it!
diff --git a/pypy/doc/whatsnew-2.6.0.rst b/pypy/doc/whatsnew-2.6.0.rst
--- a/pypy/doc/whatsnew-2.6.0.rst
+++ b/pypy/doc/whatsnew-2.6.0.rst
@@ -1,6 +1,6 @@
-=======================
-What's new in PyPy 2.5+
-=======================
+========================
+What's new in PyPy 2.6.0
+========================
 
 .. this is a revision shortly after release-2.5.1
 .. startrev: cb01edcb59414d9d93056e54ed060673d24e67c1

From noreply at buildbot.pypy.org  Thu Jun  4 09:13:55 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu,  4 Jun 2015 09:13:55 +0200 (CEST)
Subject: [pypy-commit] stmgc c8-gil-like: More thoughts
Message-ID: <20150604071355.046F91C0F05@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: c8-gil-like
Changeset: r1790:0d8839060446
Date: 2015-06-04 09:14 +0200
http://bitbucket.org/pypy/stmgc/changeset/0d8839060446/

Log:	More thoughts

diff --git a/c8/CALL_RELEASE_GIL b/c8/CALL_RELEASE_GIL
--- a/c8/CALL_RELEASE_GIL
+++ b/c8/CALL_RELEASE_GIL
@@ -28,6 +28,11 @@
 
   - otherwise, we swap rpy_fastgil back to 1 and we're done.
 
+- if the external call is long enough, a different thread will notice
+  that rpy_fastgil == 0 by regular polling, and grab the GIL for
+  itself by swapping it back to 1.  (The changes from 0 to 1 are done
+  with atomic instructions.)
+
 - a different mechanism is used when we voluntarily release the GIL,
   based on the mutex mentioned above.  The mutex is also used by the
   the reacqgil_addr() function if it actually needs to wait.
@@ -74,7 +79,9 @@
   thread; we need to fix a few things like the shadowstack and %gs but
   then we can continue running this reattached inevitable transaction.
   If old == NULL, we need to fall back to the current
-  stm_start_transaction().
+  stm_start_transaction().  (A priori, there is no need to wait at
+  this point.  The waiting point is later, in the optional
+  stm_become_inevitable()).
 
 - _stm_detach_noninevitable_transaction(): we try to make the
   transaction inevitable.  If it works we can then use
@@ -85,4 +92,29 @@
 
 - other place to fix: major collections.  Maybe simply look inside
   stm_detached_inevitable_from_thread, and if not NULL, grab the
-  inevitable transaction and commit it now.
+  inevitable transaction and commit it now.  Or maybe not.  The point
+  is that we need to prevent a thread from asynchronously grabbing it
+  by an atomic swap of stm_detached_inevitable_from_thread; instead,
+  the parallel threads that finish their external calls should all
+  find NULL in this variable and call _stm_reattach_transaction()
+  which will wait for the major GC to end.
+
+- stm_become_inevitable(): if it finds a detached inevitable
+  transaction, it should attach and commit it as a way to get rid of
+  it.  This is why it might be better to call directly
+  stm_start_inevitable_transaction() when possible: that one is
+  allowed to attach to a detached inevitable transaction and simply
+  return, unlike stm_become_inevitable() which must continue running
+  the existing transaction.
+
+- the commit logic of a non-inevitable transaction waits if there is
+  an inevitable transaction.  Here too, if the inevitable transaction
+  is found to be detached, we could just commit it now.  Or, a better
+  approach: if we find a detached inevitable transaction we grab it
+  temporarily, and commit only the *non-inevitable* transaction if it
+  doesn't conflict.  The inevitable transaction is then detached
+  again.  (Note that the conflict detection is: we don't commit any
+  write to any of the objects in the inevitable transaction's
+  read-set.  This relies on inevitable threads maintaining their
+  read-set correctly, which should be the case in PyPy, but needs to
+  be checked.)

From noreply at buildbot.pypy.org  Thu Jun  4 09:17:06 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu,  4 Jun 2015 09:17:06 +0200 (CEST)
Subject: [pypy-commit] pypy.org extradoc: update the values
Message-ID: <20150604071706.ACE261C0F05@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r617:7b4be7c61736
Date: 2015-06-04 09:17 +0200
http://bitbucket.org/pypy/pypy.org/changeset/7b4be7c61736/

Log:	update the values

diff --git a/don1.html b/don1.html
--- a/don1.html
+++ b/don1.html
@@ -9,13 +9,13 @@
 <script>
   $(function() {
     $("#progressbar").progressbar({
-      value: 56.6
+      value: 56.7
    });
   });
 </script>
 
    <!-- Income:PyPy:Donations:Py3k / 0.95 -->
-   $59469 of $105000 (56.6%)
+   $59578 of $105000 (56.7%)
    <div id="progressbar">
    </div>
 
diff --git a/don3.html b/don3.html
--- a/don3.html
+++ b/don3.html
@@ -9,13 +9,13 @@
 <script>
   $(function() {
     $("#progressbar").progressbar({
-      value: 86.9
+      value: 87.0
    });
   });
 </script>
 
    <!-- Income:PyPy:Donations:Numpy / 0.95 -->
-   $52155 of $60000 (86.9%)
+   $52184 of $60000 (87.0%)
    <div id="progressbar">
    </div>
 
diff --git a/don4.html b/don4.html
--- a/don4.html
+++ b/don4.html
@@ -9,7 +9,7 @@
 <script>
   $(function() {
     $("#progressbar").progressbar({
-      value: 36.3
+      value: 36.4
    });
   });
 </script>
@@ -17,7 +17,7 @@
    2nd call:
    <!-- Income:PyPy:Donations:Transactional Memory from April 8, 2014,
    divided by 0.9 -->
-   $29007 of $80000 (36.3%)
+   $29112 of $80000 (36.4%)
    <div id="progressbar">
    </div>
 

From noreply at buildbot.pypy.org  Thu Jun  4 11:34:20 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Thu,  4 Jun 2015 11:34:20 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: enable this check always,
 we can always disable it later
Message-ID: <20150604093420.24ADB1C0987@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77849:9af0f17031ad
Date: 2015-06-04 11:34 +0200
http://bitbucket.org/pypy/pypy/changeset/9af0f17031ad/

Log:	enable this check always, we can always disable it later

diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -778,12 +778,12 @@
         except resume.TagOverflow:
             raise compile.giveup()
         # check no duplicates
-        if not we_are_translated():
-            seen = {}
-            for box in newboxes:
-                if box is not None:
-                    assert box not in seen
-                    seen[box] = None
+        #if not we_are_translated():
+        seen = {}
+        for box in newboxes:
+            if box is not None:
+                assert box not in seen
+                seen[box] = None
         descr.store_final_boxes(op, newboxes, self.metainterp_sd)
         #
         if op.getopnum() == rop.GUARD_VALUE:

From noreply at buildbot.pypy.org  Thu Jun  4 12:05:26 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Thu,  4 Jun 2015 12:05:26 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: skip one too direct test. fix one
	case of virtuals
Message-ID: <20150604100526.3DE641C0FD4@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77850:13cfedd23e39
Date: 2015-06-04 11:52 +0200
http://bitbucket.org/pypy/pypy/changeset/13cfedd23e39/

Log:	skip one too direct test. fix one case of virtuals

diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py
--- a/rpython/jit/metainterp/optimizeopt/info.py
+++ b/rpython/jit/metainterp/optimizeopt/info.py
@@ -80,6 +80,7 @@
 
     def force_box(self, op, optforce):
         if self.is_virtual():
+            optforce.forget_numberings()
             op.set_forwarded(None)
             optforce._emit_operation(op)
             newop = optforce.getlastop()
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -428,8 +428,8 @@
     def produce_potential_short_preamble_ops(self, potential_ops):
         pass
 
-    def forget_numberings(self, box):
-        self.optimizer.forget_numberings(box)
+    def forget_numberings(self):
+        self.optimizer.forget_numberings()
 
     def _can_optimize_call_pure(self, op):
         arg_consts = []
@@ -513,9 +513,9 @@
         for opt in self.optimizations:
             opt.produce_potential_short_preamble_ops(sb)
 
-    def forget_numberings(self, virtualbox):
+    def forget_numberings(self):
         self.metainterp_sd.profiler.count(jitprof.Counters.OPT_FORCINGS)
-        self.resumedata_memo.forget_numberings(virtualbox)
+        self.resumedata_memo.forget_numberings()
 
     def getinterned(self, box):
         constbox = self.get_constant_box(box)
diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py
--- a/rpython/jit/metainterp/optimizeopt/virtualize.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualize.py
@@ -651,13 +651,16 @@
         # was already forced).
 
     def _optimize_JIT_FORCE_VIRTUAL(self, op):
-        raise Exception("implement me")
-        vref = self.getvalue(op.getarg(1))
+        vref = self.getptrinfo(op.getarg(1))
         vrefinfo = self.optimizer.metainterp_sd.virtualref_info
-        if vref.is_virtual():
-            tokenvalue = vref.getfield(vrefinfo.descr_virtual_token, None)
-            if (tokenvalue is not None and tokenvalue.is_constant() and
-                    not tokenvalue.box.nonnull()):
+        if vref and vref.is_virtual():
+            tokenop = vref.getfield(vrefinfo.descr_virtual_token, None)
+            if tokenop is None:
+                return False
+            tokeninfo = self.getptrinfo(tokenop)
+            if (tokeninfo is not None and tokeninfo.is_constant() and
+                    not tokeninfo.is_null()):
+                xxx
                 forcedvalue = vref.getfield(vrefinfo.descr_forced, None)
                 if forcedvalue is not None and not forcedvalue.is_null():
                     self.make_equal_to(op, forcedvalue)
diff --git a/rpython/jit/metainterp/optimizeopt/vstring.py b/rpython/jit/metainterp/optimizeopt/vstring.py
--- a/rpython/jit/metainterp/optimizeopt/vstring.py
+++ b/rpython/jit/metainterp/optimizeopt/vstring.py
@@ -79,6 +79,7 @@
     def force_box(self, op, optforce):
         if not self.is_virtual():
             return op
+        optforce.forget_numberings()
         if self.mode is mode_string:
             s = self.get_constant_string_spec(optforce, mode_string)
             if s is not None:
diff --git a/rpython/jit/metainterp/resume.py b/rpython/jit/metainterp/resume.py
--- a/rpython/jit/metainterp/resume.py
+++ b/rpython/jit/metainterp/resume.py
@@ -240,7 +240,7 @@
         self.numberings[snapshot] = numb, liveboxes, v
         return numb, liveboxes.copy(), v
 
-    def forget_numberings(self, virtualbox):
+    def forget_numberings(self):
         # XXX ideally clear only the affected numberings
         self.numberings.clear()
         self.clear_box_virtual_numbers()
@@ -403,6 +403,7 @@
                 else:
                     assert box.type == 'i'
                     info = optimizer.getrawptrinfo(box)
+                assert info.is_virtual()
                 info.visitor_walk_recursive(box, self, optimizer)
 
         for setfield_op in pending_setfields:
diff --git a/rpython/jit/metainterp/test/support.py b/rpython/jit/metainterp/test/support.py
--- a/rpython/jit/metainterp/test/support.py
+++ b/rpython/jit/metainterp/test/support.py
@@ -150,13 +150,13 @@
     faildescr = cpu.get_latest_descr(deadframe)
     assert faildescr.__class__.__name__.startswith('DoneWithThisFrameDescr')
     if metainterp.jitdriver_sd.result_type == history.INT:
-        return cpu.get_int_value(deadframe, 0)
+        return deadframe, cpu.get_int_value(deadframe, 0)
     elif metainterp.jitdriver_sd.result_type == history.REF:
-        return cpu.get_ref_value(deadframe, 0)
+        return deadframe, cpu.get_ref_value(deadframe, 0)
     elif metainterp.jitdriver_sd.result_type == history.FLOAT:
-        return cpu.get_float_value(deadframe, 0)
+        return deadframe, cpu.get_float_value(deadframe, 0)
     else:
-        return None
+        return deadframe, None
 
 
 class JitMixin:
@@ -251,7 +251,8 @@
         result2 = _run_with_pyjitpl(self, args)
         assert result1 == result2 or isnan(result1) and isnan(result2)
         # try to run it by running the code compiled just before
-        result3 = _run_with_machine_code(self, args)
+        df, result3 = _run_with_machine_code(self, args)
+        self._lastframe = df
         assert result1 == result3 or result3 == NotImplemented or isnan(result1) and isnan(result3)
         #
         if (longlong.supports_longlong and
diff --git a/rpython/jit/metainterp/test/test_virtualref.py b/rpython/jit/metainterp/test/test_virtualref.py
--- a/rpython/jit/metainterp/test/test_virtualref.py
+++ b/rpython/jit/metainterp/test/test_virtualref.py
@@ -105,18 +105,20 @@
         bxs1 = [box for box in guard_op.getfailargs()
                   if '.X' in str(box)]
         assert len(bxs1) == 1
-        bxs2 = [box for box in guard_op.getfailargs()
-                  if 'JitVirtualRef' in str(box)]
+        bxs2 = [(i, box) for i, box in
+                            enumerate(guard_op.getfailargs())
+                            if 'JitVirtualRef' in str(box)]
         assert len(bxs2) == 1
         JIT_VIRTUAL_REF = self.vrefinfo.JIT_VIRTUAL_REF
         FOO = lltype.GcStruct('FOO')
         foo = lltype.malloc(FOO)
         tok = lltype.cast_opaque_ptr(llmemory.GCREF, foo)
+        cpu = self.metainterp.cpu
+        py.test.skip("rewrite this test")
         bxs2[0].getref(lltype.Ptr(JIT_VIRTUAL_REF)).virtual_token = tok
         #
         # try reloading from blackhole.py's point of view
         from rpython.jit.metainterp.resume import ResumeDataDirectReader
-        cpu = self.metainterp.cpu
         cpu.get_int_value = lambda df,i:guard_op.getfailargs()[i].getint()
         cpu.get_ref_value = lambda df,i:guard_op.getfailargs()[i].getref_base()
         class FakeMetaInterpSd:

From noreply at buildbot.pypy.org  Thu Jun  4 12:05:27 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Thu,  4 Jun 2015 12:05:27 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: fix test_w*
Message-ID: <20150604100527.586BA1C0FD4@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77851:dfd8a770d2a0
Date: 2015-06-04 12:05 +0200
http://bitbucket.org/pypy/pypy/changeset/dfd8a770d2a0/

Log:	fix test_w*

diff --git a/rpython/jit/metainterp/test/test_warmstate.py b/rpython/jit/metainterp/test/test_warmstate.py
--- a/rpython/jit/metainterp/test/test_warmstate.py
+++ b/rpython/jit/metainterp/test/test_warmstate.py
@@ -4,14 +4,15 @@
 from rpython.jit.metainterp.warmstate import wrap, unwrap, specialize_value
 from rpython.jit.metainterp.warmstate import equal_whatever, hash_whatever
 from rpython.jit.metainterp.warmstate import WarmEnterState
-from rpython.jit.metainterp.history import BoxInt, BoxFloat, BoxPtr
+from rpython.jit.metainterp.resoperation import InputArgInt, InputArgRef,\
+     InputArgFloat
 from rpython.jit.metainterp.history import ConstInt, ConstFloat, ConstPtr
 from rpython.jit.metainterp.counter import DeterministicJitCounter
 from rpython.jit.codewriter import longlong
 from rpython.rlib.rarithmetic import r_singlefloat
 
 def boxfloat(x):
-    return BoxFloat(longlong.getfloatstorage(x))
+    return InputArgFloat(longlong.getfloatstorage(x))
 
 def constfloat(x):
     return ConstFloat(longlong.getfloatstorage(x))
@@ -22,22 +23,22 @@
     RS = lltype.Struct('S')
     p = lltype.malloc(S)
     po = lltype.cast_opaque_ptr(llmemory.GCREF, p)
-    assert unwrap(lltype.Void, BoxInt(42)) is None
-    assert unwrap(lltype.Signed, BoxInt(42)) == 42
-    assert unwrap(lltype.Char, BoxInt(42)) == chr(42)
+    assert unwrap(lltype.Void, InputArgInt(42)) is None
+    assert unwrap(lltype.Signed, InputArgInt(42)) == 42
+    assert unwrap(lltype.Char, InputArgInt(42)) == chr(42)
     assert unwrap(lltype.Float, boxfloat(42.5)) == 42.5
-    assert unwrap(lltype.Ptr(S), BoxPtr(po)) == p
-    assert unwrap(lltype.Ptr(RS), BoxInt(0)) == lltype.nullptr(RS)
+    assert unwrap(lltype.Ptr(S), InputArgRef(po)) == p
+    assert unwrap(lltype.Ptr(RS), InputArgInt(0)) == lltype.nullptr(RS)
 
 def test_wrap():
     def _is(box1, box2):
         return (box1.__class__ == box2.__class__ and
-                box1.value == box2.value)
+                box1.getvalue() == box2.getvalue())
     p = lltype.malloc(lltype.GcStruct('S'))
     po = lltype.cast_opaque_ptr(llmemory.GCREF, p)
-    assert _is(wrap(None, 42), BoxInt(42))
+    assert _is(wrap(None, 42), InputArgInt(42))
     assert _is(wrap(None, 42.5), boxfloat(42.5))
-    assert _is(wrap(None, p), BoxPtr(po))
+    assert _is(wrap(None, p), InputArgRef(po))
     assert _is(wrap(None, 42, in_const_box=True), ConstInt(42))
     assert _is(wrap(None, 42.5, in_const_box=True), constfloat(42.5))
     assert _is(wrap(None, p, in_const_box=True), ConstPtr(po))
@@ -45,13 +46,13 @@
         import sys
         from rpython.rlib.rarithmetic import r_longlong, r_ulonglong
         value = r_longlong(-sys.maxint*17)
-        assert _is(wrap(None, value), BoxFloat(value))
+        assert _is(wrap(None, value), InputArgFloat(value))
         assert _is(wrap(None, value, in_const_box=True), ConstFloat(value))
         value_unsigned = r_ulonglong(-sys.maxint*17)
-        assert _is(wrap(None, value_unsigned), BoxFloat(value))
+        assert _is(wrap(None, value_unsigned), InputArgFloat(value))
     sfval = r_singlefloat(42.5)
     ival = longlong.singlefloat2int(sfval)
-    assert _is(wrap(None, sfval), BoxInt(ival))
+    assert _is(wrap(None, sfval), InputArgInt(ival))
     assert _is(wrap(None, sfval, in_const_box=True), ConstInt(ival))
 
 def test_specialize_value():
diff --git a/rpython/jit/metainterp/warmstate.py b/rpython/jit/metainterp/warmstate.py
--- a/rpython/jit/metainterp/warmstate.py
+++ b/rpython/jit/metainterp/warmstate.py
@@ -59,7 +59,8 @@
         if TYPE.TO._gckind == "gc":
             return box.getref(TYPE)
         else:
-            return llmemory.cast_adr_to_ptr(box.getaddr(), TYPE)
+            adr = heaptracker.int2adr(box.getint())
+            return llmemory.cast_adr_to_ptr(adr, TYPE)
     if TYPE == lltype.Float:
         return box.getfloat()
     else:

From noreply at buildbot.pypy.org  Thu Jun  4 13:34:13 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Thu,  4 Jun 2015 13:34:13 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: blind fix
Message-ID: <20150604113413.C4EC01C0F05@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77852:c1b80de18b21
Date: 2015-06-04 13:31 +0200
http://bitbucket.org/pypy/pypy/changeset/c1b80de18b21/

Log:	blind fix

diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py
--- a/rpython/jit/metainterp/optimizeopt/virtualize.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualize.py
@@ -660,7 +660,6 @@
             tokeninfo = self.getptrinfo(tokenop)
             if (tokeninfo is not None and tokeninfo.is_constant() and
                     not tokeninfo.is_null()):
-                xxx
                 forcedvalue = vref.getfield(vrefinfo.descr_forced, None)
                 if forcedvalue is not None and not forcedvalue.is_null():
                     self.make_equal_to(op, forcedvalue)

From noreply at buildbot.pypy.org  Thu Jun  4 13:34:15 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Thu,  4 Jun 2015 13:34:15 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: kill dead code
Message-ID: <20150604113415.191D81C0F05@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77853:20ebcf1752f1
Date: 2015-06-04 13:33 +0200
http://bitbucket.org/pypy/pypy/changeset/20ebcf1752f1/

Log:	kill dead code

diff --git a/rpython/jit/metainterp/optimizeopt/util.py b/rpython/jit/metainterp/optimizeopt/util.py
--- a/rpython/jit/metainterp/optimizeopt/util.py
+++ b/rpython/jit/metainterp/optimizeopt/util.py
@@ -69,27 +69,10 @@
         quicksort(array, pivotnewindex + 1, right)
 
 def sort_descrs(lst):
+    # unused, should I leave it or kill it?
     quicksort(lst, 0, len(lst)-1)
 
 
-def descrlist_hash(l):
-    res = 0x345678
-    for descr in l:
-        y = compute_identity_hash(descr)
-        res = intmask((1000003 * res) ^ y)
-    return res
-
-def descrlist_eq(l1, l2):
-    if len(l1) != len(l2):
-        return False
-    for i in range(len(l1)):
-        if l1[i] is not l2[i]:
-            return False
-    return True
-
-def descrlist_dict():
-    return r_dict(descrlist_eq, descrlist_hash)
-
 # ____________________________________________________________
 
 def args_eq(args1, args2):
diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py
--- a/rpython/jit/metainterp/optimizeopt/virtualize.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualize.py
@@ -1,518 +1,13 @@
 from rpython.jit.codewriter.effectinfo import EffectInfo
-from rpython.jit.metainterp.executor import execute
-from rpython.jit.codewriter.heaptracker import vtable2descr, descr2vtable
-from rpython.jit.metainterp.history import Const, ConstInt, BoxInt
-from rpython.jit.metainterp.history import CONST_NULL, BoxPtr
+from rpython.jit.codewriter.heaptracker import descr2vtable
+from rpython.jit.metainterp.history import ConstInt
+from rpython.jit.metainterp.history import CONST_NULL
 from rpython.jit.metainterp.optimizeopt import info, optimizer
 from rpython.jit.metainterp.optimizeopt.optimizer import REMOVED
-from rpython.jit.metainterp.optimizeopt.util import (make_dispatcher_method,
-                                                     descrlist_dict, sort_descrs)
+from rpython.jit.metainterp.optimizeopt.util import make_dispatcher_method
 
-from rpython.jit.metainterp.optimizeopt.rawbuffer import RawBuffer, InvalidRawOperation
+from rpython.jit.metainterp.optimizeopt.rawbuffer import InvalidRawOperation
 from rpython.jit.metainterp.resoperation import rop, ResOperation
-from rpython.rlib.objectmodel import we_are_translated, specialize
-from rpython.jit.metainterp.optimizeopt.intutils import IntUnbounded
-
-class AbstractVirtualInfo(info.PtrInfo):
-    _attrs_ = ('_cached_vinfo',)
-    is_about_raw = False
-    _cached_vinfo = None
-
-    def is_forced_virtual(self):
-        xxx
-        return self.box is not None
-    
-    #def force_box(self, optforce):
-    #    xxxx
-    #    if self.box is None:
-    #        optforce.forget_numberings(self.source_op)
-    #        self._really_force(optforce)
-    #    return self.box
-
-    def force_at_end_of_preamble(self, already_forced, optforce):
-        xxxx
-        value = already_forced.get(self, None)
-        if value:
-            return value
-        return OptValue(self.force_box(optforce))
-
-    def visitor_walk_recursive(self, visitor):
-        # checks for recursion: it is False unless
-        # we have already seen the very same keybox
-        if self.box is None and not visitor.already_seen_virtual(self.source_op):
-            self._visitor_walk_recursive(visitor)
-
-    def _visitor_walk_recursive(self, visitor):
-        raise NotImplementedError("abstract base")
-
-    @specialize.argtype(1)
-    def _visitor_dispatch_virtual_type(self, visitor):
-        raise NotImplementedError("abstract base")
-
-    def _really_force(self, optforce):
-        raise NotImplementedError("abstract base")
-
-    def import_from(self, other, optimizer):
-        raise NotImplementedError("should not be called at this level")
-
-def get_fielddescrlist_cache(cpu):
-    if not hasattr(cpu, '_optimizeopt_fielddescrlist_cache'):
-        result = descrlist_dict()
-        cpu._optimizeopt_fielddescrlist_cache = result
-        return result
-    return cpu._optimizeopt_fielddescrlist_cache
-get_fielddescrlist_cache._annspecialcase_ = "specialize:memo"
-
-class AbstractVirtualStructInfo(AbstractVirtualInfo):
-    _attrs_ = ('_fields',)
-
-    def __init__(self):
-        AbstractVirtualInfo.__init__(self)
-        #self._fields = {}
-
-    def getfield(self, ofs, default):
-        return self._fields.get(ofs, default)
-
-    def setfield(self, ofs, fieldvalue):
-        assert isinstance(fieldvalue, optimizer.OptValue)
-        self._fields[ofs] = fieldvalue
-
-    def _get_descr(self):
-        raise NotImplementedError
-
-    def _is_immutable_and_filled_with_constants(self, memo=None):
-        # check if it is possible to force the given structure into a
-        # compile-time constant: this is allowed only if it is declared
-        # immutable, if all fields are already filled, and if each field
-        # is either a compile-time constant or (recursively) a structure
-        # which also answers True to the same question.
-        #
-        # check that all fields are filled.  The following equality check
-        # also fails if count == -1, meaning "not an immutable at all".
-        count = self._get_descr().count_fields_if_immutable()
-        if count != len(self._fields):
-            return False
-        #
-        # initialize 'memo'
-        if memo is None:
-            memo = {}
-        elif self in memo:
-            return True   # recursive case: assume yes
-        memo[self] = None
-        #
-        for value in self._fields.itervalues():
-            if value.is_constant():
-                pass            # it is a constant value: ok
-            elif (isinstance(value, AbstractVirtualStructValue)
-                  and value.is_virtual()):
-                # recursive check
-                if not value._is_immutable_and_filled_with_constants(memo):
-                    return False
-            else:
-                return False    # not a constant at all
-        return True
-
-    def force_at_end_of_preamble(self, already_forced, optforce):
-        if self in already_forced:
-            return self
-        already_forced[self] = self
-        if self._fields:
-            for ofs in self._fields.keys():
-                self._fields[ofs] = self._fields[ofs].force_at_end_of_preamble(already_forced, optforce)
-        return self
-
-    def _really_force(self, optforce):
-        op = self.source_op
-        assert op is not None
-        # ^^^ This case should not occur any more (see test_bug_3).
-        #
-        if not we_are_translated():
-            op.name = 'FORCE ' + self.source_op.name
-
-        if self._is_immutable_and_filled_with_constants():
-            box = optforce.optimizer.constant_fold(op)
-            self.make_constant(box)
-            for ofs, value in self._fields.iteritems():
-                subbox = value.force_box(optforce)
-                assert isinstance(subbox, Const)
-                execute(optforce.optimizer.cpu, None, rop.SETFIELD_GC,
-                        ofs, box, subbox)
-            # keep self._fields, because it's all immutable anyway
-        else:
-            optforce.emit_operation(op)
-            op = optforce.getlastop()
-            self.box = box = op
-            #
-            iteritems = self._fields.iteritems()
-            if not we_are_translated(): #random order is fine, except for tests
-                iteritems = list(iteritems)
-                iteritems.sort(key=lambda (x, y): x.sort_key())
-            for ofs, value in iteritems:
-                subbox = value.force_box(optforce)
-                op = ResOperation(rop.SETFIELD_GC, [box, subbox], descr=ofs)
-                optforce.emit_operation(op)
-
-    def _get_field_descr_list(self):
-        _cached_sorted_fields = self._cached_sorted_fields
-        if self._fields is None:
-            nfields = 0
-        else:
-            nfields = len(self._fields)
-        if (_cached_sorted_fields is not None and
-            nfields == len(_cached_sorted_fields)):
-            lst = self._cached_sorted_fields
-        else:
-            if self._fields is None:
-                lst = []
-            else:
-                lst = self._fields.keys()
-            sort_descrs(lst)
-            cache = get_fielddescrlist_cache(self.cpu)
-            result = cache.get(lst, None)
-            if result is None:
-                cache[lst] = lst
-            else:
-                lst = result
-            # store on self, to not have to repeatedly get it from the global
-            # cache, which involves sorting
-            self._cached_sorted_fields = lst
-        return lst
-
-    def _visitor_walk_recursive(self, visitor):
-        lst = self._get_field_descr_list()
-        fieldboxes = [self._fields[ofs].get_key_box() for ofs in lst]
-        visitor.register_virtual_fields(self.source_op, fieldboxes)
-        for ofs in lst:
-            fieldvalue = self._fields[ofs]
-            fieldvalue.visitor_walk_recursive(visitor)
-
-class VirtualInfo(AbstractVirtualStructInfo):
-
-    def __init__(self, known_class, descr):
-        AbstractVirtualStructInfo.__init__(self)
-        assert isinstance(known_class, Const)
-        self.known_class = known_class
-        self.descr = descr
-
-    @specialize.argtype(1)
-    def _visitor_dispatch_virtual_type(self, visitor):
-        fielddescrs = self._get_field_descr_list()
-        return visitor.visit_virtual(self.known_class, fielddescrs)
-
-    def _get_descr(self):
-        return vtable2descr(self.cpu, self.known_class.getint())
-
-    def __repr__(self):
-        cls_name = self.known_class.value.adr.ptr._obj._TYPE._name
-        if self._fields is None:
-            return '<VirtualValue FORCED cls=%s>' % (cls_name,)
-        field_names = [field.name for field in self._fields]
-        return "<VirtualValue cls=%s fields=%s>" % (cls_name, field_names)
-
-class VStructInfo(AbstractVirtualStructInfo):
-
-    def __init__(self, cpu, structdescr, source_op):
-        xxx
-        AbstractVirtualStructValue.__init__(self, cpu, source_op)
-        self.structdescr = structdescr
-
-    @specialize.argtype(1)
-    def _visitor_dispatch_virtual_type(self, visitor):
-        fielddescrs = self._get_field_descr_list()
-        return visitor.visit_vstruct(self.structdescr, fielddescrs)
-
-    def _get_descr(self):
-        return self.structdescr
-
-class AbstractVArrayInfo(AbstractVirtualInfo):
-    """
-    Base class for VArrayValue (for normal GC arrays) and VRawBufferValue (for
-    malloc()ed memory)
-    """
-
-    def getlength(self):
-        return len(self._items)
-
-    def get_item_value(self, i):
-        raise NotImplementedError
-
-    def set_item_value(self, i, newval):
-        raise NotImplementedError
-
-    def _visitor_walk_recursive(self, visitor):
-        itemboxes = []
-        for i in range(self.getlength()):
-            itemvalue = self.get_item_value(i)
-            if itemvalue is not None:
-                box = itemvalue.get_key_box()
-            else:
-                box = None
-            itemboxes.append(box)
-        visitor.register_virtual_fields(self.source_op, itemboxes)
-        for i in range(self.getlength()):
-            itemvalue = self.get_item_value(i)
-            if itemvalue is not None:
-                itemvalue.visitor_walk_recursive(visitor)
-
-
-class VArrayInfo(AbstractVArrayInfo):
-
-    def __init__(self, arraydescr, constvalue, size, source_op,
-                 clear=False):
-        AbstractVirtualValue.__init__(self, source_op)
-        self.arraydescr = arraydescr
-        self.constvalue = constvalue
-        if clear:
-            self._items = [constvalue] * size
-        else:
-            self._items = [None] * size
-        self.clear = clear
-
-    def getlength(self):
-        return len(self._items)
-
-    def get_missing_null_value(self):
-        return self.constvalue
-
-    def get_item_value(self, i):
-        """Return the i'th item, unless it is 'constvalue' on a 'clear'
-        array.  In that case (or if the i'th item is already None),
-        return None.  The idea is that this method returns the value
-        that must be set into an array that was allocated "correctly",
-        i.e. if 'clear' is True, that means with zero=True."""
-        subvalue = self._items[i]
-        if self.clear and (subvalue is self.constvalue or
-                           subvalue.is_null()):
-            subvalue = None
-        return subvalue
-
-    def set_item_value(self, i, newval):
-        self._items[i] = newval
-
-    def getitem(self, index):
-        res = self._items[index]
-        return res
-
-    def setitem(self, index, itemvalue):
-        assert isinstance(itemvalue, optimizer.OptValue)
-        self._items[index] = itemvalue        
-
-    def force_at_end_of_preamble(self, already_forced, optforce):
-        # note that this method is on VArrayValue instead of
-        # AbstractVArrayValue because we do not want to support virtualstate
-        # for rawbuffers for now
-        if self in already_forced:
-            return self
-        already_forced[self] = self
-        for index in range(self.getlength()):
-            itemval = self._items[index]
-            # XXX should be skip alltogether, but I don't wanna know or
-            #     fight unrolling just yet
-            if itemval is None:
-                itemval = self.constvalue
-            itemval = itemval.force_at_end_of_preamble(already_forced, optforce)
-            self.set_item_value(index, itemval)
-        return self
-
-    def _really_force(self, optforce):
-        assert self.source_op is not None
-        if not we_are_translated():
-            self.source_op.name = 'FORCE ' + self.source_op.name
-        # XXX two possible optimizations:
-        # * if source_op is NEW_ARRAY_CLEAR, emit NEW_ARRAY if it's
-        #   immediately followed by SETARRAYITEM_GC into all items (hard?)
-        # * if source_op is NEW_ARRAY, emit NEW_ARRAY_CLEAR if it's
-        #   followed by setting most items to zero anyway
-        optforce.emit_operation(self.source_op)
-        op = optforce.getlastop() # potentially replaced
-        self.box = box = op
-        for index in range(len(self._items)):
-            subvalue = self._items[index]
-            if subvalue is None:
-                continue
-            if self.clear:
-                if subvalue is self.constvalue or subvalue.is_null():
-                    continue
-            subbox = subvalue.force_box(optforce)
-            op = ResOperation(rop.SETARRAYITEM_GC,
-                              [box, ConstInt(index), subbox],
-                               descr=self.arraydescr)
-            optforce.emit_operation(op)
-
-    @specialize.argtype(1)
-    def _visitor_dispatch_virtual_type(self, visitor):
-        return visitor.visit_varray(self.arraydescr, self.clear)
-
-
-class VArrayStructInfo(AbstractVirtualInfo):
-    def __init__(self, arraydescr, size, source_op):
-        AbstractVirtualValue.__init__(self, source_op)
-        self.arraydescr = arraydescr
-        self._items = [{} for _ in xrange(size)]
-
-    def getlength(self):
-        return len(self._items)
-
-    def getinteriorfield(self, index, ofs, default):
-        return self._items[index].get(ofs, default)
-
-    def setinteriorfield(self, index, ofs, itemvalue):
-        assert isinstance(itemvalue, optimizer.OptValue)
-        self._items[index][ofs] = itemvalue
-
-    def _really_force(self, optforce):
-        assert self.source_op is not None
-        if not we_are_translated():
-            self.source_op.name = 'FORCE ' + self.source_op.name
-        optforce.emit_operation(self.source_op)
-        op = optforce.getlastop()
-        self.box = box = op
-        for index in range(len(self._items)):
-            iteritems = self._items[index].iteritems()
-            # random order is fine, except for tests
-            if not we_are_translated():
-                iteritems = list(iteritems)
-                iteritems.sort(key=lambda (x, y): x.sort_key())
-            for descr, value in iteritems:
-                subbox = value.force_box(optforce)
-                op = ResOperation(rop.SETINTERIORFIELD_GC,
-                    [box, ConstInt(index), subbox], descr=descr
-                )
-                optforce.emit_operation(op)
-
-    def _get_list_of_descrs(self):
-        descrs = []
-        for item in self._items:
-            item_descrs = item.keys()
-            sort_descrs(item_descrs)
-            descrs.append(item_descrs)
-        return descrs
-
-    def _visitor_walk_recursive(self, visitor):
-        itemdescrs = self._get_list_of_descrs()
-        itemboxes = []
-        for i in range(len(self._items)):
-            for descr in itemdescrs[i]:
-                itemboxes.append(self._items[i][descr].get_key_box())
-        visitor.register_virtual_fields(self.keybox, itemboxes)
-        for i in range(len(self._items)):
-            for descr in itemdescrs[i]:
-                self._items[i][descr].visitor_walk_recursive(visitor)
-
-    def force_at_end_of_preamble(self, already_forced, optforce):
-        if self in already_forced:
-            return self
-        already_forced[self] = self
-        for index in range(len(self._items)):
-            for descr in self._items[index].keys():
-                self._items[index][descr] = self._items[index][descr].force_at_end_of_preamble(already_forced, optforce)
-        return self
-
-    @specialize.argtype(1)
-    def _visitor_dispatch_virtual_type(self, visitor):
-        return visitor.visit_varraystruct(self.arraydescr, self._get_list_of_descrs())
-
-
-class VRawBufferInfo(AbstractVArrayInfo):
-    is_about_raw = True
-
-    def __init__(self, cpu, logops, size, source_op):
-        AbstractVirtualValue.__init__(self, source_op)
-        # note that size is unused, because we assume that the buffer is big
-        # enough to write/read everything we need. If it's not, it's undefined
-        # behavior anyway, although in theory we could probably detect such
-        # cases here
-        self.size = size
-        self.buffer = RawBuffer(cpu, logops)
-
-    def getintbound(self):
-        return IntUnbounded()
-
-    def getlength(self):
-        return len(self.buffer.values)
-
-    def get_item_value(self, i):
-        return self.buffer.values[i]
-
-    def set_item_value(self, i, newval):
-        self.buffer.values[i] = newval
-
-    def getitem_raw(self, offset, length, descr):
-        if not self.is_virtual():
-            raise InvalidRawOperation
-            # see 'test_virtual_raw_buffer_forced_but_slice_not_forced'
-            # for the test above: it's not enough to check is_virtual()
-            # on the original object, because it might be a VRawSliceValue
-            # instead.  If it is a virtual one, then we'll reach here anway.
-        return self.buffer.read_value(offset, length, descr)
-
-    def setitem_raw(self, offset, length, descr, value):
-        if not self.is_virtual():
-            raise InvalidRawOperation
-        self.buffer.write_value(offset, length, descr, value)
-
-    def _really_force(self, optforce):
-        op = self.source_op
-        assert op is not None
-        if not we_are_translated():
-            op.name = 'FORCE ' + self.source_op.name
-        optforce.emit_operation(self.source_op)
-        self.box = optforce.getlastop()
-        for i in range(len(self.buffer.offsets)):
-            # write the value
-            offset = self.buffer.offsets[i]
-            descr = self.buffer.descrs[i]
-            itemvalue = self.buffer.values[i]
-            itembox = itemvalue.force_box(optforce)
-            op = ResOperation(rop.RAW_STORE,
-                              [self.box, ConstInt(offset), itembox],
-                              descr=descr)
-            optforce.emit_operation(op)
-
-    @specialize.argtype(1)
-    def _visitor_dispatch_virtual_type(self, visitor):
-        # I *think* we need to make a copy of offsets and descrs because we
-        # want a snapshot of the virtual state right now: if we grow more
-        # elements later, we don't want them to go in this virtual state
-        return visitor.visit_vrawbuffer(self.size,
-                                        self.buffer.offsets[:],
-                                        self.buffer.descrs[:])
-
-
-class VRawSliceInfo(AbstractVirtualInfo):
-    is_about_raw = True
-
-    def __init__(self, rawbuffer_value, offset, source_op):
-        AbstractVirtualValue.__init__(self, source_op)
-        self.rawbuffer_value = rawbuffer_value
-        self.offset = offset
-
-    def getintbound(self):
-        return IntUnbounded()
-
-    def _really_force(self, optforce):
-        op = self.source_op
-        assert op is not None
-        if not we_are_translated():
-            op.name = 'FORCE ' + self.source_op.name
-        self.rawbuffer_value.force_box(optforce)
-        optforce.emit_operation(op)
-        self.box = optforce.getlastop()
-
-    def setitem_raw(self, offset, length, descr, value):
-        self.rawbuffer_value.setitem_raw(self.offset+offset, length, descr, value)
-
-    def getitem_raw(self, offset, length, descr):
-        return self.rawbuffer_value.getitem_raw(self.offset+offset, length, descr)
-
-    def _visitor_walk_recursive(self, visitor):
-        box = self.rawbuffer_value.get_key_box()
-        visitor.register_virtual_fields(self.keybox, [box])
-        self.rawbuffer_value.visitor_walk_recursive(visitor)
-
-    @specialize.argtype(1)
-    def _visitor_dispatch_virtual_type(self, visitor):
-        return visitor.visit_vrawslice(self.offset)
 
 
 class OptVirtualize(optimizer.Optimization):

From noreply at buildbot.pypy.org  Thu Jun  4 13:42:35 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Thu,  4 Jun 2015 13:42:35 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: track the length of strings
Message-ID: <20150604114235.4096F1C034D@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77854:7fb61d4136c7
Date: 2015-06-04 13:42 +0200
http://bitbucket.org/pypy/pypy/changeset/7fb61d4136c7/

Log:	track the length of strings

diff --git a/rpython/jit/metainterp/optimizeopt/intbounds.py b/rpython/jit/metainterp/optimizeopt/intbounds.py
--- a/rpython/jit/metainterp/optimizeopt/intbounds.py
+++ b/rpython/jit/metainterp/optimizeopt/intbounds.py
@@ -384,13 +384,13 @@
         self.emit_operation(op)
         self.make_nonnull_str(op.getarg(0), vstring.mode_string)
         array = self.getptrinfo(op.getarg(0))
-        #self.get_box_replacement(op).set_forwarded(array.getlenbound())
+        self.get_box_replacement(op).set_forwarded(array.getlenbound())
 
     def optimize_UNICODELEN(self, op):
         self.emit_operation(op)
         self.make_nonnull_str(op.getarg(0), vstring.mode_unicode)
         array = self.getptrinfo(op.getarg(0))
-        #self.get_box_replacement(op).set_forwarded(array.getlenbound())
+        self.get_box_replacement(op).set_forwarded(array.getlenbound())
 
     def optimize_STRGETITEM(self, op):
         self.emit_operation(op)
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -64,33 +64,6 @@
     lst6 = virt1._get_field_descr_list()
     assert lst6 is lst3
 
-def test_descrlist_dict():
-    from rpython.jit.metainterp.optimizeopt import util as optimizeutil
-    h1 = optimizeutil.descrlist_hash([])
-    h2 = optimizeutil.descrlist_hash([LLtypeMixin.valuedescr])
-    h3 = optimizeutil.descrlist_hash(
-            [LLtypeMixin.valuedescr, LLtypeMixin.nextdescr])
-    assert h1 != h2
-    assert h2 != h3
-    assert optimizeutil.descrlist_eq([], [])
-    assert not optimizeutil.descrlist_eq([], [LLtypeMixin.valuedescr])
-    assert optimizeutil.descrlist_eq([LLtypeMixin.valuedescr],
-                                     [LLtypeMixin.valuedescr])
-    assert not optimizeutil.descrlist_eq([LLtypeMixin.valuedescr],
-                                         [LLtypeMixin.nextdescr])
-    assert optimizeutil.descrlist_eq([LLtypeMixin.valuedescr, LLtypeMixin.nextdescr],
-                                     [LLtypeMixin.valuedescr, LLtypeMixin.nextdescr])
-    assert not optimizeutil.descrlist_eq([LLtypeMixin.nextdescr, LLtypeMixin.valuedescr],
-                                         [LLtypeMixin.valuedescr, LLtypeMixin.nextdescr])
-
-    # descrlist_eq should compare by identity of the descrs, not by the result
-    # of sort_key
-    class FakeDescr(object):
-        def sort_key(self):
-            return 1
-
-    assert not optimizeutil.descrlist_eq([FakeDescr()], [FakeDescr()])
-
 
 # ____________________________________________________________
 
diff --git a/rpython/jit/metainterp/optimizeopt/vstring.py b/rpython/jit/metainterp/optimizeopt/vstring.py
--- a/rpython/jit/metainterp/optimizeopt/vstring.py
+++ b/rpython/jit/metainterp/optimizeopt/vstring.py
@@ -123,6 +123,7 @@
         if not create_ops:
             return None
         lengthop = ResOperation(mode.STRLEN, [op])
+        lengthop.set_forwarded(self.getlenbound())
         self.lgtop = lengthop
         string_optimizer.emit_operation(lengthop)
         return lengthop

From noreply at buildbot.pypy.org  Thu Jun  4 14:11:57 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Thu,  4 Jun 2015 14:11:57 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: fight with spurious test failures
Message-ID: <20150604121157.4D0391C0FB8@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77855:ce07bb0bfef4
Date: 2015-06-04 14:11 +0200
http://bitbucket.org/pypy/pypy/changeset/ce07bb0bfef4/

Log:	fight with spurious test failures

diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py
--- a/rpython/jit/backend/test/runner_test.py
+++ b/rpython/jit/backend/test/runner_test.py
@@ -195,7 +195,6 @@
         assert res == 10
 
     def test_backends_dont_keep_loops_alive(self):
-        py.test.skip("don't care for now")
         import weakref, gc
         self.cpu.dont_keepalive_stuff = True
         targettoken = TargetToken()
diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -377,6 +377,7 @@
 def forget_optimization_info(lst):
     for item in lst:
         item.set_forwarded(None)
+        item.reset_value()
 
 def send_loop_to_backend(greenkey, jitdriver_sd, metainterp_sd, loop, type):
     forget_optimization_info(loop.operations)
diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py
--- a/rpython/jit/metainterp/optimizeopt/info.py
+++ b/rpython/jit/metainterp/optimizeopt/info.py
@@ -132,7 +132,8 @@
                 setfieldop = ResOperation(rop.SETFIELD_GC, [op, subbox],
                                           descr=flddescr)
                 optforce._emit_operation(setfieldop)
-                optforce.optheap.register_dirty_field(flddescr, self)
+                if optforce.optheap is not None:
+                    optforce.optheap.register_dirty_field(flddescr, self)
 
     def visitor_walk_recursive(self, instbox, visitor, optimizer):
         if visitor.already_seen_virtual(instbox):
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -1,10 +1,11 @@
+import weakref
 from rpython.rlib.objectmodel import we_are_translated, specialize
 from rpython.rlib.objectmodel import compute_identity_hash
 from rpython.rtyper.lltypesystem import lltype, llmemory
 from rpython.jit.codewriter import longlong
 
 class AbstractValue(object):
-    _repr_memo = {}
+    _repr_memo = weakref.WeakKeyDictionary()
     is_info_class = False
     _attrs_ = ()
 
@@ -35,6 +36,9 @@
             orig_op.set_forwarded(op)
         return op
 
+    def reset_value(self):
+        pass
+
 def ResOperation(opnum, args, descr=None):
     cls = opclasses[opnum]
     op = cls()
@@ -411,6 +415,9 @@
     def getref_base(self):
         return self._resref
 
+    def reset_value(self):
+        self.setref_base(lltype.nullptr(llmemory.GCREF.TO))
+
     getvalue = getref_base
 
     def forget_value(self):
@@ -476,6 +483,9 @@
     def __init__(self, r=lltype.nullptr(llmemory.GCREF.TO)):
         self.setref_base(r)
 
+    def reset_value(self):
+        self.setref_base(lltype.nullptr(llmemory.GCREF.TO))
+
     def clone_input_arg(self):
         return InputArgRef()
 
diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py
--- a/rpython/jit/metainterp/test/test_ajit.py
+++ b/rpython/jit/metainterp/test/test_ajit.py
@@ -1,7 +1,9 @@
 import sys
 
 import py
-
+import weakref
+
+from rpython.rlib import rgc
 from rpython.jit.codewriter.policy import StopAtXPolicy
 from rpython.jit.metainterp import history
 from rpython.jit.metainterp.test.support import LLJitMixin, noConst
@@ -1258,7 +1260,6 @@
 
     def test_free_object(self):
         import weakref
-        from rpython.rlib import rgc
         from rpython.rtyper.lltypesystem.lloperation import llop
         myjitdriver = JitDriver(greens = [], reds = ['n', 'x'])
         class X(object):
@@ -3983,7 +3984,6 @@
         # start with labels. I dont know which is better...
 
     def test_ll_arraycopy(self):
-        from rpython.rlib import rgc
         A = lltype.GcArray(lltype.Char)
         a = lltype.malloc(A, 10)
         for i in range(10): a[i] = chr(i)
@@ -4010,8 +4010,6 @@
         assert self.interp_operations(f, [3]) == 6
 
     def test_gc_add_memory_pressure(self):
-        from rpython.rlib import rgc
-
         def f():
             rgc.add_memory_pressure(1234)
             return 3
diff --git a/rpython/jit/metainterp/test/test_jitiface.py b/rpython/jit/metainterp/test/test_jitiface.py
--- a/rpython/jit/metainterp/test/test_jitiface.py
+++ b/rpython/jit/metainterp/test/test_jitiface.py
@@ -1,4 +1,5 @@
 
+import py
 from rpython.rlib.jit import JitDriver, JitHookInterface, Counters
 from rpython.rlib import jit_hooks
 from rpython.jit.metainterp.test.support import LLJitMixin
@@ -11,6 +12,9 @@
 class JitHookInterfaceTests(object):
     # !!!note!!! - don't subclass this from the backend. Subclass the LL
     # class later instead
+    def setup_class(cls):
+        py.test.skip("disabled")
+    
     def test_abort_quasi_immut(self):
         reasons = []
 
diff --git a/rpython/jit/metainterp/test/test_jitprof.py b/rpython/jit/metainterp/test/test_jitprof.py
--- a/rpython/jit/metainterp/test/test_jitprof.py
+++ b/rpython/jit/metainterp/test/test_jitprof.py
@@ -1,4 +1,5 @@
 
+import py
 from rpython.jit.metainterp.warmspot import ll_meta_interp
 from rpython.rlib.jit import JitDriver, dont_look_inside, elidable, Counters
 from rpython.jit.metainterp.test.support import LLJitMixin
@@ -53,6 +54,7 @@
             ]
         assert profiler.events == expected
         assert profiler.times == [2, 1]
+        py.test.skip("disabled until unrolling")
         assert profiler.counters == [1, 1, 3, 3, 2, 15, 2, 0, 0, 0, 0,
                                      0, 0, 0, 0, 0]
 
diff --git a/rpython/jit/metainterp/test/test_loop_unroll.py b/rpython/jit/metainterp/test/test_loop_unroll.py
--- a/rpython/jit/metainterp/test/test_loop_unroll.py
+++ b/rpython/jit/metainterp/test/test_loop_unroll.py
@@ -5,7 +5,7 @@
 from rpython.jit.metainterp.optimizeopt import ALL_OPTS_NAMES
 
 class LoopUnrollTest(test_loop.LoopTest):
-    enable_opts = ALL_OPTS_NAMES
+    #enable_opts = ALL_OPTS_NAMES
 
     automatic_promotion_result = {
         'int_gt': 2, 'guard_false': 2, 'jump': 1, 'int_add': 6,
diff --git a/rpython/jit/metainterp/test/test_loop_unroll_disopt.py b/rpython/jit/metainterp/test/test_loop_unroll_disopt.py
--- a/rpython/jit/metainterp/test/test_loop_unroll_disopt.py
+++ b/rpython/jit/metainterp/test/test_loop_unroll_disopt.py
@@ -5,6 +5,7 @@
 from rpython.jit.metainterp.optimizeopt import ALL_OPTS_NAMES
 
 allopts = ALL_OPTS_NAMES.split(':')
+del allopts[allopts.index('unroll')]
 for optnum in range(len(allopts)):
     myopts = allopts[:]
     del myopts[optnum]
@@ -21,5 +22,5 @@
     exec "TestLoopNo%sLLtype = TestLLtype" % (opt[0].upper() + opt[1:])
 
 del TestLLtype # No need to run the last set twice
-del TestLoopNoUnrollLLtype # This case is take care of by test_loop
+#del TestLoopNoUnrollLLtype # This case is take care of by test_loop
 

From noreply at buildbot.pypy.org  Thu Jun  4 14:14:26 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Thu,  4 Jun 2015 14:14:26 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: fix for llgraph backend
Message-ID: <20150604121426.087A61C0FB8@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77856:27601b29a154
Date: 2015-06-04 14:14 +0200
http://bitbucket.org/pypy/pypy/changeset/27601b29a154/

Log:	fix for llgraph backend

diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py
--- a/rpython/jit/backend/test/runner_test.py
+++ b/rpython/jit/backend/test/runner_test.py
@@ -1808,15 +1808,19 @@
         cpu = self.cpu
         class FakeGCCache(object):
             pass
-        
-        if not hasattr(cpu.gc_ll_descr, '_cache_gcstruct2vtable'):
-            cpu.gc_ll_descr._cache_gcstruct2vtable = {}
-        cpu.gc_ll_descr._cache_gcstruct2vtable.update({T: vtable_for_T})
-        p = T
-        while hasattr(p, 'parent'):
-            vtable_for_parent = lltype.malloc(self.MY_VTABLE, immortal=True)
-            cpu.gc_ll_descr._cache_gcstruct2vtable[p.parent] = vtable_for_parent
-            p = p.parent
+
+        if hasattr(cpu, 'gc_ll_descr'):
+            if not hasattr(cpu.gc_ll_descr, '_cache_gcstruct2vtable'):
+                cpu.gc_ll_descr._cache_gcstruct2vtable = {}
+            cpu.gc_ll_descr._cache_gcstruct2vtable.update({T: vtable_for_T})
+            p = T
+            while hasattr(p, 'parent'):
+                vtable_for_parent = lltype.malloc(self.MY_VTABLE, immortal=True)
+                cpu.gc_ll_descr._cache_gcstruct2vtable[p.parent] = vtable_for_parent
+                p = p.parent
+        else:
+            descr = cpu.sizeof(T, True)
+            descr._corresponding_vtable = vtable_for_T
         t = lltype.malloc(T)
         if T == self.T:
             t.parent.parent.typeptr = vtable_for_T

From noreply at buildbot.pypy.org  Thu Jun  4 14:16:15 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu,  4 Jun 2015 14:16:15 +0200 (CEST)
Subject: [pypy-commit] pypy default: Redo 5cf9f578ca18 (which I only noticed
 by luck was reverted...).
Message-ID: <20150604121615.738821C0FB8@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r77857:5d676752ae66
Date: 2015-06-04 13:17 +0100
http://bitbucket.org/pypy/pypy/changeset/5d676752ae66/

Log:	Redo 5cf9f578ca18 (which I only noticed by luck was reverted...). As
	shown one line before the crash, "latin-1" is the only RPython-
	sanctioned spelling, not "latin1".

diff --git a/pypy/objspace/std/newformat.py b/pypy/objspace/std/newformat.py
--- a/pypy/objspace/std/newformat.py
+++ b/pypy/objspace/std/newformat.py
@@ -528,7 +528,7 @@
 
         def _lit(self, s):
             if self.is_unicode:
-                return s.decode("ascii")
+                return s.decode("latin-1")
             else:
                 return s
 
@@ -586,8 +586,8 @@
                 thousands = ""
                 grouping = "\xFF"    # special value to mean 'stop'
             if self.is_unicode:
-                self._loc_dec = dec.decode("ascii")
-                self._loc_thousands = thousands.decode("ascii")
+                self._loc_dec = dec.decode("latin-1")
+                self._loc_thousands = thousands.decode("latin-1")
             else:
                 self._loc_dec = dec
                 self._loc_thousands = thousands
@@ -725,7 +725,7 @@
                 out.append_multiple_char(fill_char[0], spec.n_lpadding)
             if spec.n_sign:
                 if self.is_unicode:
-                    sign = spec.sign.decode("ascii")
+                    sign = spec.sign.decode("latin-1")
                 else:
                     sign = spec.sign
                 out.append(sign)
@@ -828,14 +828,14 @@
                 prefix = "0x"
             as_str = value.format(LONG_DIGITS[:base], prefix)
             if self.is_unicode:
-                return as_str.decode("ascii")
+                return as_str.decode("latin-1")
             return as_str
 
         def _int_to_base(self, base, value):
             if base == 10:
                 s = str(value)
                 if self.is_unicode:
-                    return s.decode("ascii")
+                    return s.decode("latin-1")
                 return s
             # This part is slow.
             negative = value < 0
@@ -954,7 +954,7 @@
             have_dec_point, to_remainder = self._parse_number(result, to_number)
             n_remainder = len(result) - to_remainder
             if self.is_unicode:
-                digits = result.decode("ascii")
+                digits = result.decode("latin-1")
             else:
                 digits = result
             spec = self._calc_num_width(0, sign, to_number, n_digits,
@@ -1059,8 +1059,8 @@
                                                                to_imag_number)
 
             if self.is_unicode:
-                re_num = re_num.decode("ascii")
-                im_num = im_num.decode("ascii")
+                re_num = re_num.decode("latin-1")
+                im_num = im_num.decode("latin-1")
 
             #set remainder, in CPython _parse_number sets this
             #using n_re_digits causes tests to fail

From noreply at buildbot.pypy.org  Thu Jun  4 14:23:38 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Thu,  4 Jun 2015 14:23:38 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: can't do it due to
	raise_continue_running_normally
Message-ID: <20150604122338.C17D41C0FB8@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77858:ee0c29ea36be
Date: 2015-06-04 14:23 +0200
http://bitbucket.org/pypy/pypy/changeset/ee0c29ea36be/

Log:	can't do it due to raise_continue_running_normally

diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -377,7 +377,7 @@
 def forget_optimization_info(lst):
     for item in lst:
         item.set_forwarded(None)
-        item.reset_value()
+        #item.reset_value()
 
 def send_loop_to_backend(greenkey, jitdriver_sd, metainterp_sd, loop, type):
     forget_optimization_info(loop.operations)

From noreply at buildbot.pypy.org  Thu Jun  4 14:27:20 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Thu,  4 Jun 2015 14:27:20 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Fix.
Message-ID: <20150604122720.868471C0FB8@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob <me at manueljacob.de>
Branch: py3k
Changeset: r77859:0b2fa08966dd
Date: 2015-06-04 14:09 +0200
http://bitbucket.org/pypy/pypy/changeset/0b2fa08966dd/

Log:	Fix.

diff --git a/lib_pypy/grp.py b/lib_pypy/grp.py
--- a/lib_pypy/grp.py
+++ b/lib_pypy/grp.py
@@ -43,8 +43,7 @@
 def getgrnam(name):
     if not isinstance(name, str):
         raise TypeError("expected string")
-    name = os.fsencode(name)
-    res = lib.getgrnam(name)
+    res = lib.getgrnam(os.fsencode(name))
     if not res:
         raise KeyError("'getgrnam(): name not found: %s'" % name)
     return _group_from_gstruct(res)

From noreply at buildbot.pypy.org  Thu Jun  4 14:27:21 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Thu,  4 Jun 2015 14:27:21 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Add the struct module to more tests'
 spaceconfig (the pure Python version was removed on this branch).
Message-ID: <20150604122721.BC51E1C0FB8@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob <me at manueljacob.de>
Branch: py3k
Changeset: r77860:a96f6fcb20fd
Date: 2015-06-04 14:21 +0200
http://bitbucket.org/pypy/pypy/changeset/a96f6fcb20fd/

Log:	Add the struct module to more tests' spaceconfig (the pure Python
	version was removed on this branch).

diff --git a/pypy/module/_posixsubprocess/test/test_subprocess.py b/pypy/module/_posixsubprocess/test/test_subprocess.py
--- a/pypy/module/_posixsubprocess/test/test_subprocess.py
+++ b/pypy/module/_posixsubprocess/test/test_subprocess.py
@@ -1,7 +1,7 @@
 from os.path import dirname
 
 class AppTestSubprocess:
-    spaceconfig = dict(usemodules=('_posixsubprocess', 'signal', 'fcntl', 'select'))
+    spaceconfig = dict(usemodules=('_posixsubprocess', 'signal', 'fcntl', 'select', 'struct'))
     # XXX write more tests
 
     def setup_class(cls):
diff --git a/pypy/module/_ssl/test/test_ssl.py b/pypy/module/_ssl/test/test_ssl.py
--- a/pypy/module/_ssl/test/test_ssl.py
+++ b/pypy/module/_ssl/test/test_ssl.py
@@ -2,7 +2,7 @@
 import os
 
 class AppTestSSL:
-    spaceconfig = dict(usemodules=('_ssl', '_socket', 'binascii', 'thread'))
+    spaceconfig = dict(usemodules=('_ssl', '_socket', 'struct', 'binascii', 'thread'))
 
     def setup_class(cls):
         cls.w_nullbytecert = cls.space.wrap(os.path.join(

From noreply at buildbot.pypy.org  Thu Jun  4 14:31:05 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Thu,  4 Jun 2015 14:31:05 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: whack at some tests
Message-ID: <20150604123105.7FD341C0FB8@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77861:66d113cccb50
Date: 2015-06-04 14:31 +0200
http://bitbucket.org/pypy/pypy/changeset/66d113cccb50/

Log:	whack at some tests

diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -377,6 +377,8 @@
 def forget_optimization_info(lst):
     for item in lst:
         item.set_forwarded(None)
+        # XXX we should really do it, but we need to remember the values
+        #     somehoe for ContinueRunningNormally
         #item.reset_value()
 
 def send_loop_to_backend(greenkey, jitdriver_sd, metainterp_sd, loop, type):
diff --git a/rpython/jit/metainterp/test/test_resume.py b/rpython/jit/metainterp/test/test_resume.py
--- a/rpython/jit/metainterp/test/test_resume.py
+++ b/rpython/jit/metainterp/test/test_resume.py
@@ -3,6 +3,7 @@
 import sys
 from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
 from rpython.jit.metainterp.resume import *
+from rpython.jit.metainterp.optimizeopt.info import AbstractVirtualPtrInfo
 from rpython.jit.metainterp.history import BoxInt, BoxPtr, ConstInt
 from rpython.jit.metainterp.history import ConstPtr, ConstFloat
 from rpython.jit.metainterp.optimizeopt.test.test_util import LLtypeMixin
@@ -23,15 +24,13 @@
 
 
 class FakeOptimizer(object):
-    def __init__(self, values):
-        self.values = values
-        
-    def getvalue(self, box):
-        try:
-            value = self.values[box]
-        except KeyError:
-            value = self.values[box] = OptValue(box)
-        return value
+    def get_box_replacement(self, op):
+        if not op.get_forwarded():
+            return op
+        xxx
+
+    def getrawptrinfo(self, op, create=True):
+        return op.get_forwarded()
 
 
 # ____________________________________________________________
@@ -122,11 +121,11 @@
             self.fieldnums = fieldnums
         def equals(self, fieldnums):
             return self.fieldnums == fieldnums
-    class FakeVirtualValue(AbstractVirtualValue):
+    class FakeVirtualValue(AbstractVirtualPtrInfo):
         def visitor_dispatch_virtual_type(self, *args):
             return FakeVInfo()
-    modifier = ResumeDataVirtualAdder(None, None, None)
-    v1 = FakeVirtualValue(None, None)
+    modifier = ResumeDataVirtualAdder(None, None, None, None, None)
+    v1 = FakeVirtualValue()
     vinfo1 = modifier.make_virtual_info(v1, [1, 2, 4])
     vinfo2 = modifier.make_virtual_info(v1, [1, 2, 4])
     assert vinfo1 is vinfo2
@@ -356,8 +355,8 @@
 class FakeResumeDataReader(AbstractResumeDataReader):
     VirtualCache = get_VirtualCache_class('Fake')
     
-    def allocate_with_vtable(self, known_class):
-        return FakeBuiltObject(vtable=known_class)
+    def allocate_with_vtable(self, descr):
+        return FakeBuiltObject(vtable=descr)
     def allocate_struct(self, typedescr):
         return FakeBuiltObject(typedescr=typedescr)
     def allocate_array(self, length, arraydescr, clear):
@@ -624,7 +623,7 @@
           FakeFrame("code2", 9, c3, b2)]
     capture_resumedata(fs, None, [], storage)
     memo = ResumeDataLoopMemo(FakeMetaInterpStaticData())
-    modifier = ResumeDataVirtualAdder(storage, storage, memo)
+    modifier = ResumeDataVirtualAdder(None, storage, storage, memo)
     liveboxes = modifier.finish(FakeOptimizer({}))
     metainterp = MyMetaInterp()
 
@@ -648,7 +647,7 @@
           FakeFrame("code2", 9, c3, b2)]
     capture_resumedata(fs, [b4], [], storage)
     memo = ResumeDataLoopMemo(FakeMetaInterpStaticData())
-    modifier = ResumeDataVirtualAdder(storage, memo)
+    modifier = ResumeDataVirtualAdder(None, storage, memo)
     liveboxes = modifier.finish(FakeOptimizer({}))
     metainterp = MyMetaInterp()
 
@@ -677,10 +676,10 @@
     capture_resumedata(fs, None, [], storage2)
     
     memo = ResumeDataLoopMemo(FakeMetaInterpStaticData())
-    modifier = ResumeDataVirtualAdder(storage, memo)
+    modifier = ResumeDataVirtualAdder(None, storage, memo)
     liveboxes = modifier.finish(FakeOptimizer({}))
 
-    modifier = ResumeDataVirtualAdder(storage2, memo)
+    modifier = ResumeDataVirtualAdder(None, storage2, memo)
     liveboxes2 = modifier.finish(FakeOptimizer({}))
 
     metainterp = MyMetaInterp()
@@ -1029,7 +1028,7 @@
 def test_register_virtual_fields():
     b1, b2 = InputArgInt(), InputArgInt()
     vbox = InputArgRef()
-    modifier = ResumeDataVirtualAdder(None, None, None)
+    modifier = ResumeDataVirtualAdder(None, None, None, None)
     modifier.liveboxes_from_env = {}
     modifier.liveboxes = {}
     modifier.vfieldboxes = {}
@@ -1038,7 +1037,7 @@
                                   b2: UNASSIGNED}
     assert modifier.vfieldboxes == {vbox: [b1, b2]}
 
-    modifier = ResumeDataVirtualAdder(None, None, None)
+    modifier = ResumeDataVirtualAdder(None, None, None, None)
     modifier.liveboxes_from_env = {vbox: tag(0, TAGVIRTUAL)}
     modifier.liveboxes = {}
     modifier.vfieldboxes = {}
@@ -1372,7 +1371,7 @@
     b2s, b4s = [InputArgRef(), InputArgRef()]
     storage = Storage()
     memo = ResumeDataLoopMemo(FakeMetaInterpStaticData())
-    modifier = ResumeDataVirtualAdder(storage, storage, memo)
+    modifier = ResumeDataVirtualAdder(None, storage, storage, memo)
     modifier.liveboxes_from_env = {}
     modifier.liveboxes = {}
     modifier.vfieldboxes = {}
@@ -1411,18 +1410,18 @@
     class Storage(object):
         pass
     storage = Storage()
-    modifier = ResumeDataVirtualAdder(storage, storage, None)
-    modifier._add_pending_fields([])
+    modifier = ResumeDataVirtualAdder(None, storage, storage, None)
+    modifier._add_pending_fields(None, [])
     assert not storage.rd_pendingfields
     #
     class FieldDescr(object):
         pass
     field_a = FieldDescr()
     storage = Storage()
-    modifier = ResumeDataVirtualAdder(storage, storage, None)
+    modifier = ResumeDataVirtualAdder(None, storage, storage, None)
     modifier.liveboxes_from_env = {42: rffi.cast(rffi.SHORT, 1042),
                                    61: rffi.cast(rffi.SHORT, 1061)}
-    modifier._add_pending_fields([(field_a, 42, 61, -1)])
+    modifier._add_pending_fields(FakeOptimizer(), [(field_a, 42, 61, -1)])
     pf = storage.rd_pendingfields
     assert len(pf) == 1
     assert (annlowlevel.cast_base_ptr_to_instance(FieldDescr, pf[0].lldescr)
@@ -1514,7 +1513,7 @@
     metainterp_sd = FakeMetaInterpStaticData()
     metainterp_sd.options = options
     memo = ResumeDataLoopMemo(metainterp_sd)
-    modifier = ResumeDataVirtualAdder(None, None, memo)
+    modifier = ResumeDataVirtualAdder(None, None, None, memo)
 
     for i in range(5):
         assert not modifier._invalidation_needed(5, i)

From noreply at buildbot.pypy.org  Thu Jun  4 15:43:05 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu,  4 Jun 2015 15:43:05 +0200 (CEST)
Subject: [pypy-commit] pypy.org extradoc: update the values
Message-ID: <20150604134305.D228D1C034D@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r618:d0a151845933
Date: 2015-06-04 15:43 +0200
http://bitbucket.org/pypy/pypy.org/changeset/d0a151845933/

Log:	update the values

diff --git a/don1.html b/don1.html
--- a/don1.html
+++ b/don1.html
@@ -21,7 +21,10 @@
 
   </li>
   <li>
-  <span id="donate_text">This donation will go towards supporting python 3 in PyPy</br> <a href="py3donate.html">Read proposal</a></span>
+  <span id="donate_text">This donation goes towards supporting Python 3 in PyPy.
+  Current status:
+    we have $5045 left
+  in the account.</br> <a href="py3donate.html">Read proposal</a></span>
   </li>
   <li>
   <div class="sidepic">
diff --git a/don3.html b/don3.html
--- a/don3.html
+++ b/don3.html
@@ -21,7 +21,10 @@
 
   </li>
   <li>
-  <span id="donate_text">This donation will go towards supporting NumPy in PyPy</br> <a href="numpydonate.html">Read proposal</a></span>
+  <span id="donate_text">This donation goes towards supporting NumPy in PyPy.
+  Current status:
+    we have $15999 left
+  in the account.</br> <a href="numpydonate.html">Read proposal</a></span>
   </li>
   <li>
     <div class="sidepic">
diff --git a/don4.html b/don4.html
--- a/don4.html
+++ b/don4.html
@@ -23,7 +23,10 @@
 
   </li>
   <li>
-  <span id="donate_text">This donation will go towards supporting the Transactional Memory in PyPy.</br> <a href="tmdonate2.html">Read proposal (2nd call)</a></span>
+  <span id="donate_text">This donation goes towards supporting the Transactional Memory in PyPy.
+  Current status:
+    we have $22016 left
+  in the account.</br> <a href="tmdonate2.html">Read proposal (2nd call)</a></span>
   </li>
   <li>
   <li>

From noreply at buildbot.pypy.org  Thu Jun  4 17:42:04 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Thu,  4 Jun 2015 17:42:04 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: fixes to vstring,
 we need unrolling otherwise it's chasing a wild goose :/
Message-ID: <20150604154204.EEA0D1C034D@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77862:17aa2869ed86
Date: 2015-06-04 17:42 +0200
http://bitbucket.org/pypy/pypy/changeset/17aa2869ed86/

Log:	fixes to vstring, we need unrolling otherwise it's chasing a wild
	goose :/

diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -150,6 +150,8 @@
         start_state = optimize_trace(metainterp_sd, jitdriver_sd, part,
                                      enable_opts, export_state=True)
     except InvalidLoop:
+        forget_optimization_info(part.operations)
+        forget_optimization_info(part.inputargs)
         return None
     target_token = part.operations[0].getdescr()
     assert isinstance(target_token, TargetToken)
@@ -238,6 +240,7 @@
                        jitdriver_sd.warmstate.enable_opts,
                        start_state=start_state, export_state=False)
     except InvalidLoop:
+        xxx # XXX forget optimizations
         # Fall back on jumping to preamble
         target_token = label.getdescr()
         assert isinstance(target_token, TargetToken)
@@ -250,6 +253,7 @@
                            inline_short_preamble=False, start_state=start_state,
                            export_state=False)
         except InvalidLoop:
+            xxx # XXX forget optimizations
             return None
     assert part.operations[-1].getopnum() != rop.LABEL
     target_token = label.getdescr()
@@ -374,12 +378,13 @@
                                             original_loop_token, log=log,
                                             logger=metainterp_sd.logger_ops)
 
-def forget_optimization_info(lst):
+def forget_optimization_info(lst, reset_values=False):
     for item in lst:
         item.set_forwarded(None)
         # XXX we should really do it, but we need to remember the values
         #     somehoe for ContinueRunningNormally
-        #item.reset_value()
+        if reset_values:
+            item.reset_value()
 
 def send_loop_to_backend(greenkey, jitdriver_sd, metainterp_sd, loop, type):
     forget_optimization_info(loop.operations)
@@ -916,6 +921,8 @@
                                state.enable_opts,
                                inline_short_preamble, export_state=True)
     except InvalidLoop:
+        forget_optimization_info(new_trace.operations)
+        forget_optimization_info(new_trace.inputargs)
         debug_print("compile_new_bridge: got an InvalidLoop")
         # XXX I am fairly convinced that optimize_bridge cannot actually raise
         # InvalidLoop
diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py
--- a/rpython/jit/metainterp/optimizeopt/info.py
+++ b/rpython/jit/metainterp/optimizeopt/info.py
@@ -476,6 +476,11 @@
     @specialize.arg(2)
     def get_constant_string_spec(self, optforce, mode):
         return self._unpack_str(mode)
+
+    def getlenbound(self, mode):
+        from rpython.jit.metainterp.optimizeopt.intutils import ConstIntBound
+        
+        return ConstIntBound(self.getstrlen(None, None, mode))
     
     def getstrlen(self, op, string_optimizer, mode, create_ops=True):
         from rpython.jit.metainterp.optimizeopt import vstring
diff --git a/rpython/jit/metainterp/optimizeopt/intbounds.py b/rpython/jit/metainterp/optimizeopt/intbounds.py
--- a/rpython/jit/metainterp/optimizeopt/intbounds.py
+++ b/rpython/jit/metainterp/optimizeopt/intbounds.py
@@ -384,13 +384,17 @@
         self.emit_operation(op)
         self.make_nonnull_str(op.getarg(0), vstring.mode_string)
         array = self.getptrinfo(op.getarg(0))
-        self.get_box_replacement(op).set_forwarded(array.getlenbound())
+        new_op = self.get_box_replacement(op)
+        if not new_op.is_constant():
+            new_op.set_forwarded(array.getlenbound(vstring.mode_string))
 
     def optimize_UNICODELEN(self, op):
         self.emit_operation(op)
         self.make_nonnull_str(op.getarg(0), vstring.mode_unicode)
         array = self.getptrinfo(op.getarg(0))
-        self.get_box_replacement(op).set_forwarded(array.getlenbound())
+        new_op = self.get_box_replacement(op)
+        if not new_op.is_constant():
+            new_op.set_forwarded(array.getlenbound(vstring.mode_unicode))
 
     def optimize_STRGETITEM(self, op):
         self.emit_operation(op)
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -27,229 +27,6 @@
                 self.bound.contains_bound(other.bound))
 
 
-## class OptInfo(object):
-
-##     def getlevel(self):
-##         return self._tag & 0x3
-
-##     def setlevel(self, level):
-##         self._tag = (self._tag & (~0x3)) | level
-
-##     def import_from(self, other, optimizer):
-##         if self.getlevel() == LEVEL_CONSTANT:
-##             assert other.getlevel() == LEVEL_CONSTANT
-##             assert other.box.same_constant(self.box)
-##             return
-##         assert self.getlevel() <= LEVEL_NONNULL
-##         if other.getlevel() == LEVEL_CONSTANT:
-##             self.make_constant(other.get_key_box())
-##         elif other.getlevel() == LEVEL_KNOWNCLASS:
-##             self.make_constant_class(None, other.get_known_class())
-##         else:
-##             if other.getlevel() == LEVEL_NONNULL:
-##                 self.ensure_nonnull()
-
-##     def make_guards(self, box):
-##         if self.getlevel() == LEVEL_CONSTANT:
-##             op = ResOperation(rop.GUARD_VALUE, [box, self.box], None)
-##             return [op]
-##         return []
-
-##     def copy_from(self, other_value):
-##         assert isinstance(other_value, OptValue)
-##         self.box = other_value.box
-##         self._tag = other_value._tag
-
-##     def force_box(self, optforce):
-##         xxx
-##         return self.box
-
-##     def force_at_end_of_preamble(self, already_forced, optforce):
-##         return self
-
-##     # visitor API
-
-##     def visitor_walk_recursive(self, visitor):
-##         pass
-
-##     @specialize.argtype(1)
-##     def visitor_dispatch_virtual_type(self, visitor):
-##         if self.is_virtual():
-##             return self._visitor_dispatch_virtual_type(visitor)
-##         else:
-##             return visitor.visit_not_virtual(self)
-
-##     @specialize.argtype(1)
-##     def _visitor_dispatch_virtual_type(self, visitor):
-##         assert 0, "unreachable"
-
-##     def is_constant(self):
-##         return self.getlevel() == LEVEL_CONSTANT
-
-##     def is_null(self):
-##         if self.is_constant():
-##             box = self.box
-##             assert isinstance(box, Const)
-##             return not box.nonnull()
-##         return False
-
-##     def same_value(self, other):
-##         if not other:
-##             return False
-##         if self.is_constant() and other.is_constant():
-##             return self.box.same_constant(other.box)
-##         return self is other
-
-##     def is_nonnull(self):
-##         level = self.getlevel()
-##         if level == LEVEL_NONNULL or level == LEVEL_KNOWNCLASS:
-##             return True
-##         elif level == LEVEL_CONSTANT:
-##             box = self.box
-##             assert isinstance(box, Const)
-##             return box.nonnull()
-##         else:
-##             return False
-
-##     def ensure_nonnull(self):
-##         if self.getlevel() < LEVEL_NONNULL:
-##             self.setlevel(LEVEL_NONNULL)
-
-##     def get_constant_int(self):
-##         assert self.is_constant()
-##         box = self.box
-##         assert isinstance(box, ConstInt)
-##         return box.getint()
-
-##     def is_virtual(self):
-##         return False # overwridden in VirtualInfo
-
-##     def is_forced_virtual(self):
-##         return False
-
-##     def getfield(self, ofs, default):
-##         raise NotImplementedError
-
-##     def setfield(self, ofs, value):
-##         raise NotImplementedError
-
-##     def getlength(self):
-##         raise NotImplementedError
-
-##     def getitem(self, index):
-##         raise NotImplementedError
-
-##     def setitem(self, index, value):
-##         raise NotImplementedError
-
-##     def getitem_raw(self, offset, length, descr):
-##         raise NotImplementedError
-
-##     def setitem_raw(self, offset, length, descr, value):
-##         raise NotImplementedError
-
-##     def getinteriorfield(self, index, ofs, default):
-##         raise NotImplementedError
-
-##     def setinteriorfield(self, index, ofs, value):
-##         raise NotImplementedError
-
-##     def get_missing_null_value(self):
-##         raise NotImplementedError    # only for VArrayValue
-
-##     def make_constant(self, constbox):
-##         """Replace 'self.box' with a Const box."""
-##         assert isinstance(constbox, Const)
-##         self.box = constbox
-##         self.setlevel(LEVEL_CONSTANT)
-
-##     def get_last_guard(self, optimizer):
-##         return None
-
-##     def get_known_class(self):
-##         return None
-
-##     def getlenbound(self):
-##         return None
-
-##     def getintbound(self):
-##         return None
-
-##     def get_constant_class(self, cpu):
-##         return None
-
-
-## class IntOptInfo(OptInfo):
-##     _attrs_ = ('intbound',)
-
-##     def __init__(self, level=LEVEL_UNKNOWN, known_class=None, intbound=None):
-##         OptInfo.__init__(self, level, None, None)
-##         if intbound:
-##             self.intbound = intbound
-##         else:
-##             self.intbound = IntBound(MININT, MAXINT)
-
-##     def copy_from(self, other_value):
-##         assert isinstance(other_value, IntOptValue)
-##         self.box = other_value.box
-##         self.intbound = other_value.intbound
-##         self._tag = other_value._tag
-
-##     def make_constant(self, constbox):
-##         """Replace 'self.box' with a Const box."""
-##         assert isinstance(constbox, ConstInt)
-##         self.box = constbox
-##         self.setlevel(LEVEL_CONSTANT)
-##         val = constbox.getint()
-##         self.intbound = IntBound(val, val)
-
-##     def is_nonnull(self):
-##         if OptValue.is_nonnull(self):
-##             return True
-##         if self.intbound:
-##             if self.intbound.known_gt(IntBound(0, 0)) or \
-##                self.intbound.known_lt(IntBound(0, 0)):
-##                 return True
-##         return False
-
-##     def make_nonnull(self, optimizer):
-##         assert self.getlevel() < LEVEL_NONNULL
-##         self.setlevel(LEVEL_NONNULL)
-
-##     def import_from(self, other, optimizer):
-##         OptValue.import_from(self, other, optimizer)
-##         if self.getlevel() != LEVEL_CONSTANT:
-##             if other.getintbound() is not None: # VRawBufferValue
-##                 self.intbound.intersect(other.getintbound())
-
-##     def make_guards(self, box):
-##         guards = []
-##         level = self.getlevel()
-##         if level == LEVEL_CONSTANT:
-##             op = ResOperation(rop.GUARD_VALUE, [box, self.box], None)
-##             guards.append(op)
-##         elif level == LEVEL_KNOWNCLASS:
-##             op = ResOperation(rop.GUARD_NONNULL, [box], None)
-##             guards.append(op)
-##         else:
-##             if level == LEVEL_NONNULL:
-##                 op = ResOperation(rop.GUARD_NONNULL, [box], None)
-##                 guards.append(op)
-##             self.intbound.make_guards(box, guards)
-##         return guards
-
-##     def getintbound(self):
-##         return self.intbound
-
-##     def get_last_guard(self, optimizer):
-##         return None
-
-##     def get_known_class(self):
-##         return None
-
-##     def getlenbound(self):
-##         return None
-
 
 CONST_0      = ConstInt(0)
 CONST_1      = ConstInt(1)
diff --git a/rpython/jit/metainterp/optimizeopt/vstring.py b/rpython/jit/metainterp/optimizeopt/vstring.py
--- a/rpython/jit/metainterp/optimizeopt/vstring.py
+++ b/rpython/jit/metainterp/optimizeopt/vstring.py
@@ -62,7 +62,7 @@
         self.mode = mode
         self.length = length
 
-    def getlenbound(self):
+    def getlenbound(self, mode):
         from rpython.jit.metainterp.optimizeopt import intutils
 
         if self.lenbound is None:
@@ -123,7 +123,7 @@
         if not create_ops:
             return None
         lengthop = ResOperation(mode.STRLEN, [op])
-        lengthop.set_forwarded(self.getlenbound())
+        lengthop.set_forwarded(self.getlenbound(mode))
         self.lgtop = lengthop
         string_optimizer.emit_operation(lengthop)
         return lengthop
@@ -153,6 +153,7 @@
 
     def shrink(self, length):
         assert length >= 0
+        self.length = length
         del self._chars[length:]
 
     def setup_slice(self, longerlist, start, stop):
@@ -522,16 +523,8 @@
     def _optimize_NEWSTR(self, op, mode):
         length_box = self.get_constant_box(op.getarg(0))
         if length_box and length_box.getint() <= MAX_CONST_LEN:
-            # if the original 'op' did not have a ConstInt as argument,
-            # build a new one with the ConstInt argument
-            if not isinstance(op.getarg(0), ConstInt):
-                old_op = op
-                op = op.copy_and_change(mode.NEWSTR, [length_box])
-            else:
-                old_op = None
-            vvalue = self.make_vstring_plain(op, mode, length_box.getint())
-            if old_op is not None:
-                self.optimizer.make_equal_to(old_op, vvalue)
+            assert not op.get_forwarded()
+            self.make_vstring_plain(op, mode, length_box.getint())
         else:
             self.make_nonnull_str(op, mode)
             self.emit_operation(op)
@@ -601,12 +594,12 @@
         self._optimize_STRLEN(op, mode_unicode)
 
     def _optimize_STRLEN(self, op, mode):
-        #value = self.getvalue(op.getarg(0))
-        #lengthbox = value.getstrlen(self, mode, op)
-        #if op in self.optimizer.values:
-        #    assert self.getvalue(op) is self.getvalue(lengthbox)
-        #elif op is not lengthbox:
-        #    self.make_equal_to(op, self.getvalue(lengthbox))
+        opinfo = self.getptrinfo(op.getarg(0))
+        if opinfo:
+            lgtop = opinfo.getstrlen(op, self, mode, False)
+            if lgtop is not None:
+                self.make_equal_to(op, lgtop)
+                return
         self.emit_operation(op)
 
     def optimize_COPYSTRCONTENT(self, op):
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -2887,8 +2887,7 @@
             assert i + 1 == len(self.virtualizable_boxes)
             # we're during tracing, so we should not execute it
             self.history.record(rop.SETFIELD_GC, [vbox, self.cpu.ts.CONST_NULL],
-                                self.cpu.ts.CONST_NULL.getref_base(),
-                                descr=vinfo.vable_token_descr)
+                                None, descr=vinfo.vable_token_descr)
 
     def replace_box(self, oldbox, newbox):
         for frame in self.framestack:

From noreply at buildbot.pypy.org  Thu Jun  4 18:11:25 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Thu,  4 Jun 2015 18:11:25 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: merged release-2.6.x
Message-ID: <20150604161125.E6A161C0F05@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r77863:d3969a3c7bb3
Date: 2015-06-04 14:43 +0200
http://bitbucket.org/pypy/pypy/changeset/d3969a3c7bb3/

Log:	merged release-2.6.x

diff too long, truncating to 2000 out of 40000 lines

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -10,3 +10,8 @@
 32f35069a16d819b58c1b6efb17c44e3e53397b2 release-2.3.1
 10f1b29a2bd21f837090286174a9ca030b8680b2 release-2.5.0
 9c4588d731b7fe0b08669bd732c2b676cb0a8233 release-2.5.1
+fcdb941565156385cbac04cfb891f8f4c7a92ef6 release-2.6.0
+fcdb941565156385cbac04cfb891f8f4c7a92ef6 release-2.6.0
+e03971291f3a0729ecd3ee7fae7ddb0bb82d476c release-2.6.0
+e03971291f3a0729ecd3ee7fae7ddb0bb82d476c release-2.6.0
+295ee98b69288471b0fcf2e0ede82ce5209eb90b release-2.6.0
diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -38,8 +38,8 @@
   Armin Rigo
   Maciej Fijalkowski
   Carl Friedrich Bolz
+  Amaury Forgeot d'Arc
   Antonio Cuni
-  Amaury Forgeot d'Arc
   Samuele Pedroni
   Alex Gaynor
   Brian Kearns
@@ -50,9 +50,9 @@
   Holger Krekel
   Christian Tismer
   Hakan Ardo
-  Benjamin Peterson
   Manuel Jacob
   Ronan Lamy
+  Benjamin Peterson
   Anders Chrigstrom
   Eric van Riet Paap
   Wim Lavrijsen
@@ -63,8 +63,8 @@
   Sven Hager
   Anders Lehmann
   Aurelien Campeas
+  Remi Meier
   Niklaus Haldimann
-  Remi Meier
   Camillo Bruni
   Laura Creighton
   Toon Verwaest
@@ -76,10 +76,10 @@
   David Edelsohn
   Anders Hammarquist
   Jakub Gustak
+  Gregor Wegberg
   Guido Wesdorp
   Lawrence Oluyede
   Bartosz Skowron
-  Gregor Wegberg
   Daniel Roberts
   Niko Matsakis
   Adrien Di Mascio
@@ -87,10 +87,11 @@
   Ludovic Aubry
   Jacob Hallen
   Jason Creighton
+  Richard Plangger
   Alex Martelli
   Michal Bendowski
+  stian
   Jan de Mooij
-  stian
   Tyler Wade
   Michael Foord
   Stephan Diehl
@@ -133,15 +134,15 @@
   Georg Brandl
   Bert Freudenberg
   Stian Andreassen
+  Edd Barrett
   Wanja Saatkamp
   Gerald Klix
   Mike Blume
+  Tobias Pape
   Oscar Nierstrasz
   Stefan H. Muller
-  Edd Barrett
   Jeremy Thurgood
   Rami Chowdhury
-  Tobias Pape
   Eugene Oden
   Henry Mason
   Vasily Kuznetsov
@@ -167,11 +168,13 @@
   Michael Twomey
   Lucian Branescu Mihaila
   Yichao Yu
+  Anton Gulenko
   Gabriel Lavoie
   Olivier Dormond
   Jared Grubb
   Karl Bartel
   Wouter van Heyst
+  Sebastian Pawlu&#347;
   Brian Dorsey
   Victor Stinner
   Andrews Medina
@@ -188,6 +191,7 @@
   Neil Shepperd
   Stanislaw Halik
   Mikael Sch&#246;nenberg
+  Berkin Ilbeyi
   Elmo M?ntynen
   Jonathan David Riehl
   Anders Qvist
@@ -211,11 +215,11 @@
   Carl Meyer
   Karl Ramm
   Pieter Zieschang
-  Sebastian Pawlu&#347;
   Gabriel
   Lukas Vacek
   Andrew Dalke
   Sylvain Thenault
+  Jakub Stasiak
   Nathan Taylor
   Vladimir Kryachko
   Jacek Generowicz
@@ -242,6 +246,7 @@
   Tomo Cocoa
   Toni Mattis
   Lucas Stadler
+  Julian Berman
   roberto at goyle
   Yury V. Zaytsev
   Anna Katrina Dominguez
@@ -253,6 +258,8 @@
   Ben Darnell
   Roberto De Ioris
   Juan Francisco Cantero Hurtado
+  Ruochen Huang
+  Jeong YunWon
   Godefroid Chappelle
   Joshua Gilbert
   Dan Colish
@@ -271,6 +278,7 @@
   Christian Muirhead
   Berker Peksag
   James Lan
+  Volodymyr Vladymyrov
   shoma hosaka
   Daniel Neuh&#228;user
   Ben Mather
@@ -316,6 +324,7 @@
   yasirs
   Michael Chermside
   Anna Ravencroft
+  Andrey Churin
   Dan Crosta
   Julien Phalip
   Roman Podoliaka
diff --git a/lib-python/2.7/socket.py b/lib-python/2.7/socket.py
--- a/lib-python/2.7/socket.py
+++ b/lib-python/2.7/socket.py
@@ -145,6 +145,34 @@
             name = hostname
     return name
 
+class RefCountingWarning(UserWarning):
+    pass
+
+def _do_reuse_or_drop(socket, methname):
+    try:
+        method = getattr(socket, methname)
+    except (AttributeError, TypeError):
+        warnings.warn("""'%s' object has no _reuse/_drop methods
+{{
+    You make use (or a library you are using makes use) of the internal
+    classes '_socketobject' and '_fileobject' in socket.py, initializing
+    them with custom objects.  On PyPy, these custom objects need two
+    extra methods, _reuse() and _drop(), that maintain an explicit
+    reference counter.  When _drop() has been called as many times as
+    _reuse(), then the object should be freed.
+
+    Without these methods, you get the warning here.  This is to
+    prevent the following situation: if your (or the library's) code
+    relies on reference counting for prompt closing, then on PyPy, the
+    __del__ method will be called later than on CPython.  You can
+    easily end up in a situation where you open and close a lot of
+    (high-level) '_socketobject' or '_fileobject', but the (low-level)
+    custom objects will accumulate before their __del__ are called.
+    You quickly risk running out of file descriptors, for example.
+}}""" % (socket.__class__.__name__,), RefCountingWarning, stacklevel=3)
+    else:
+        method()
+
 
 _socketmethods = (
     'bind', 'connect', 'connect_ex', 'fileno', 'listen',
@@ -182,19 +210,7 @@
         if _sock is None:
             _sock = _realsocket(family, type, proto)
         else:
-            # PyPy note about refcounting: implemented with _reuse()/_drop()
-            # on the class '_socket.socket'.  Python 3 did it differently
-            # with a reference counter on this class 'socket._socketobject'
-            # instead, but it is a less compatible change.
-            
-            # Note that a few libraries (like eventlet) poke at the
-            # private implementation of socket.py, passing custom
-            # objects to _socketobject().  These libraries need the
-            # following fix for use on PyPy: the custom objects need
-            # methods _reuse() and _drop() that maintains an explicit
-            # reference counter, starting at 0.  When it drops back to
-            # zero, close() must be called.
-            _sock._reuse()
+            _do_reuse_or_drop(_sock, '_reuse')
 
         self._sock = _sock
 
@@ -228,13 +244,13 @@
     def close(self):
         s = self._sock
         self._sock = _closedsocket()
-        s._drop()
+        _do_reuse_or_drop(s, '_drop')
     close.__doc__ = _realsocket.close.__doc__
 
     def accept(self):
         sock, addr = self._sock.accept()
         sockobj = _socketobject(_sock=sock)
-        sock._drop()    # already a copy in the _socketobject()
+        _do_reuse_or_drop(sock, '_drop') # already a copy in the _socketobject()
         return sockobj, addr
     accept.__doc__ = _realsocket.accept.__doc__
 
@@ -290,14 +306,7 @@
                  "_close"]
 
     def __init__(self, sock, mode='rb', bufsize=-1, close=False):
-        # Note that a few libraries (like eventlet) poke at the
-        # private implementation of socket.py, passing custom
-        # objects to _fileobject().  These libraries need the
-        # following fix for use on PyPy: the custom objects need
-        # methods _reuse() and _drop() that maintains an explicit
-        # reference counter, starting at 0.  When it drops back to
-        # zero, close() must be called.
-        sock._reuse()
+        _do_reuse_or_drop(sock, '_reuse')
         self._sock = sock
         self.mode = mode # Not actually used in this version
         if bufsize < 0:
@@ -338,7 +347,7 @@
                 if self._close:
                     s.close()
                 else:
-                    s._drop()
+                    _do_reuse_or_drop(s, '_drop')
 
     def __del__(self):
         try:
diff --git a/lib_pypy/_audioop_build.py b/lib_pypy/_audioop_build.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_audioop_build.py
@@ -0,0 +1,621 @@
+from cffi import FFI
+
+ffi = FFI()
+ffi.cdef("""
+typedef short PyInt16;
+
+int ratecv(char* rv, char* cp, size_t len, int size,
+           int nchannels, int inrate, int outrate,
+           int* state_d, int* prev_i, int* cur_i,
+           int weightA, int weightB);
+
+void tostereo(char* rv, char* cp, size_t len, int size,
+              double fac1, double fac2);
+void add(char* rv, char* cp1, char* cp2, size_t len1, int size);
+
+/* 2's complement (14-bit range) */
+unsigned char
+st_14linear2ulaw(PyInt16 pcm_val);
+PyInt16 st_ulaw2linear16(unsigned char);
+
+/* 2's complement (13-bit range) */
+unsigned char
+st_linear2alaw(PyInt16 pcm_val);
+PyInt16 st_alaw2linear16(unsigned char);
+
+
+void lin2adcpm(unsigned char* rv, unsigned char* cp, size_t len,
+               size_t size, int* state);
+void adcpm2lin(unsigned char* rv, unsigned char* cp, size_t len,
+               size_t size, int* state);
+""")
+
+# This code is directly copied from CPython file: Modules/audioop.c
+_AUDIOOP_C_MODULE = r"""
+typedef short PyInt16;
+typedef int Py_Int32;
+
+/* Code shamelessly stolen from sox, 12.17.7, g711.c
+** (c) Craig Reese, Joe Campbell and Jeff Poskanzer 1989 */
+
+/* From g711.c:
+ *
+ * December 30, 1994:
+ * Functions linear2alaw, linear2ulaw have been updated to correctly
+ * convert unquantized 16 bit values.
+ * Tables for direct u- to A-law and A- to u-law conversions have been
+ * corrected.
+ * Borge Lindberg, Center for PersonKommunikation, Aalborg University.
+ * bli at cpk.auc.dk
+ *
+ */
+#define BIAS 0x84   /* define the add-in bias for 16 bit samples */
+#define CLIP 32635
+#define SIGN_BIT        (0x80)          /* Sign bit for a A-law byte. */
+#define QUANT_MASK      (0xf)           /* Quantization field mask. */
+#define SEG_SHIFT       (4)             /* Left shift for segment number. */
+#define SEG_MASK        (0x70)          /* Segment field mask. */
+
+static PyInt16 seg_aend[8] = {0x1F, 0x3F, 0x7F, 0xFF,
+                              0x1FF, 0x3FF, 0x7FF, 0xFFF};
+static PyInt16 seg_uend[8] = {0x3F, 0x7F, 0xFF, 0x1FF,
+                              0x3FF, 0x7FF, 0xFFF, 0x1FFF};
+
+static PyInt16
+search(PyInt16 val, PyInt16 *table, int size)
+{
+    int i;
+
+    for (i = 0; i < size; i++) {
+        if (val <= *table++)
+            return (i);
+    }
+    return (size);
+}
+#define st_ulaw2linear16(uc) (_st_ulaw2linear16[uc])
+#define st_alaw2linear16(uc) (_st_alaw2linear16[uc])
+
+static PyInt16 _st_ulaw2linear16[256] = {
+    -32124,  -31100,  -30076,  -29052,  -28028,  -27004,  -25980,
+    -24956,  -23932,  -22908,  -21884,  -20860,  -19836,  -18812,
+    -17788,  -16764,  -15996,  -15484,  -14972,  -14460,  -13948,
+    -13436,  -12924,  -12412,  -11900,  -11388,  -10876,  -10364,
+     -9852,   -9340,   -8828,   -8316,   -7932,   -7676,   -7420,
+     -7164,   -6908,   -6652,   -6396,   -6140,   -5884,   -5628,
+     -5372,   -5116,   -4860,   -4604,   -4348,   -4092,   -3900,
+     -3772,   -3644,   -3516,   -3388,   -3260,   -3132,   -3004,
+     -2876,   -2748,   -2620,   -2492,   -2364,   -2236,   -2108,
+     -1980,   -1884,   -1820,   -1756,   -1692,   -1628,   -1564,
+     -1500,   -1436,   -1372,   -1308,   -1244,   -1180,   -1116,
+     -1052,    -988,    -924,    -876,    -844,    -812,    -780,
+      -748,    -716,    -684,    -652,    -620,    -588,    -556,
+      -524,    -492,    -460,    -428,    -396,    -372,    -356,
+      -340,    -324,    -308,    -292,    -276,    -260,    -244,
+      -228,    -212,    -196,    -180,    -164,    -148,    -132,
+      -120,    -112,    -104,     -96,     -88,     -80,     -72,
+       -64,     -56,     -48,     -40,     -32,     -24,     -16,
+    -8,       0,   32124,   31100,   30076,   29052,   28028,
+     27004,   25980,   24956,   23932,   22908,   21884,   20860,
+     19836,   18812,   17788,   16764,   15996,   15484,   14972,
+     14460,   13948,   13436,   12924,   12412,   11900,   11388,
+     10876,   10364,    9852,    9340,    8828,    8316,    7932,
+      7676,    7420,    7164,    6908,    6652,    6396,    6140,
+      5884,    5628,    5372,    5116,    4860,    4604,    4348,
+      4092,    3900,    3772,    3644,    3516,    3388,    3260,
+      3132,    3004,    2876,    2748,    2620,    2492,    2364,
+      2236,    2108,    1980,    1884,    1820,    1756,    1692,
+      1628,    1564,    1500,    1436,    1372,    1308,    1244,
+      1180,    1116,    1052,     988,     924,     876,     844,
+       812,     780,     748,     716,     684,     652,     620,
+       588,     556,     524,     492,     460,     428,     396,
+       372,     356,     340,     324,     308,     292,     276,
+       260,     244,     228,     212,     196,     180,     164,
+       148,     132,     120,     112,     104,      96,      88,
+    80,      72,      64,      56,      48,      40,      32,
+    24,      16,       8,       0
+};
+
+/*
+ * linear2ulaw() accepts a 14-bit signed integer and encodes it as u-law data
+ * stored in a unsigned char.  This function should only be called with
+ * the data shifted such that it only contains information in the lower
+ * 14-bits.
+ *
+ * In order to simplify the encoding process, the original linear magnitude
+ * is biased by adding 33 which shifts the encoding range from (0 - 8158) to
+ * (33 - 8191). The result can be seen in the following encoding table:
+ *
+ *      Biased Linear Input Code        Compressed Code
+ *      ------------------------        ---------------
+ *      00000001wxyza                   000wxyz
+ *      0000001wxyzab                   001wxyz
+ *      000001wxyzabc                   010wxyz
+ *      00001wxyzabcd                   011wxyz
+ *      0001wxyzabcde                   100wxyz
+ *      001wxyzabcdef                   101wxyz
+ *      01wxyzabcdefg                   110wxyz
+ *      1wxyzabcdefgh                   111wxyz
+ *
+ * Each biased linear code has a leading 1 which identifies the segment
+ * number. The value of the segment number is equal to 7 minus the number
+ * of leading 0's. The quantization interval is directly available as the
+ * four bits wxyz.  * The trailing bits (a - h) are ignored.
+ *
+ * Ordinarily the complement of the resulting code word is used for
+ * transmission, and so the code word is complemented before it is returned.
+ *
+ * For further information see John C. Bellamy's Digital Telephony, 1982,
+ * John Wiley & Sons, pps 98-111 and 472-476.
+ */
+static unsigned char
+st_14linear2ulaw(PyInt16 pcm_val)       /* 2's complement (14-bit range) */
+{
+    PyInt16         mask;
+    PyInt16         seg;
+    unsigned char   uval;
+
+    /* The original sox code does this in the calling function, not here */
+    pcm_val = pcm_val >> 2;
+
+    /* u-law inverts all bits */
+    /* Get the sign and the magnitude of the value. */
+    if (pcm_val < 0) {
+        pcm_val = -pcm_val;
+        mask = 0x7F;
+    } else {
+        mask = 0xFF;
+    }
+    if ( pcm_val > CLIP ) pcm_val = CLIP;           /* clip the magnitude */
+    pcm_val += (BIAS >> 2);
+
+    /* Convert the scaled magnitude to segment number. */
+    seg = search(pcm_val, seg_uend, 8);
+
+    /*
+     * Combine the sign, segment, quantization bits;
+     * and complement the code word.
+     */
+    if (seg >= 8)           /* out of range, return maximum value. */
+        return (unsigned char) (0x7F ^ mask);
+    else {
+        uval = (unsigned char) (seg << 4) | ((pcm_val >> (seg + 1)) & 0xF);
+        return (uval ^ mask);
+    }
+
+}
+
+static PyInt16 _st_alaw2linear16[256] = {
+     -5504,   -5248,   -6016,   -5760,   -4480,   -4224,   -4992,
+     -4736,   -7552,   -7296,   -8064,   -7808,   -6528,   -6272,
+     -7040,   -6784,   -2752,   -2624,   -3008,   -2880,   -2240,
+     -2112,   -2496,   -2368,   -3776,   -3648,   -4032,   -3904,
+     -3264,   -3136,   -3520,   -3392,  -22016,  -20992,  -24064,
+    -23040,  -17920,  -16896,  -19968,  -18944,  -30208,  -29184,
+    -32256,  -31232,  -26112,  -25088,  -28160,  -27136,  -11008,
+    -10496,  -12032,  -11520,   -8960,   -8448,   -9984,   -9472,
+    -15104,  -14592,  -16128,  -15616,  -13056,  -12544,  -14080,
+    -13568,    -344,    -328,    -376,    -360,    -280,    -264,
+      -312,    -296,    -472,    -456,    -504,    -488,    -408,
+      -392,    -440,    -424,     -88,     -72,    -120,    -104,
+       -24,      -8,     -56,     -40,    -216,    -200,    -248,
+      -232,    -152,    -136,    -184,    -168,   -1376,   -1312,
+     -1504,   -1440,   -1120,   -1056,   -1248,   -1184,   -1888,
+     -1824,   -2016,   -1952,   -1632,   -1568,   -1760,   -1696,
+      -688,    -656,    -752,    -720,    -560,    -528,    -624,
+      -592,    -944,    -912,   -1008,    -976,    -816,    -784,
+      -880,    -848,    5504,    5248,    6016,    5760,    4480,
+      4224,    4992,    4736,    7552,    7296,    8064,    7808,
+      6528,    6272,    7040,    6784,    2752,    2624,    3008,
+      2880,    2240,    2112,    2496,    2368,    3776,    3648,
+      4032,    3904,    3264,    3136,    3520,    3392,   22016,
+     20992,   24064,   23040,   17920,   16896,   19968,   18944,
+     30208,   29184,   32256,   31232,   26112,   25088,   28160,
+     27136,   11008,   10496,   12032,   11520,    8960,    8448,
+      9984,    9472,   15104,   14592,   16128,   15616,   13056,
+     12544,   14080,   13568,     344,     328,     376,     360,
+       280,     264,     312,     296,     472,     456,     504,
+       488,     408,     392,     440,     424,      88,      72,
+       120,     104,      24,       8,      56,      40,     216,
+       200,     248,     232,     152,     136,     184,     168,
+      1376,    1312,    1504,    1440,    1120,    1056,    1248,
+      1184,    1888,    1824,    2016,    1952,    1632,    1568,
+      1760,    1696,     688,     656,     752,     720,     560,
+       528,     624,     592,     944,     912,    1008,     976,
+       816,     784,     880,     848
+};
+
+/*
+ * linear2alaw() accepts an 13-bit signed integer and encodes it as A-law data
+ * stored in a unsigned char.  This function should only be called with
+ * the data shifted such that it only contains information in the lower
+ * 13-bits.
+ *
+ *              Linear Input Code       Compressed Code
+ *      ------------------------        ---------------
+ *      0000000wxyza                    000wxyz
+ *      0000001wxyza                    001wxyz
+ *      000001wxyzab                    010wxyz
+ *      00001wxyzabc                    011wxyz
+ *      0001wxyzabcd                    100wxyz
+ *      001wxyzabcde                    101wxyz
+ *      01wxyzabcdef                    110wxyz
+ *      1wxyzabcdefg                    111wxyz
+ *
+ * For further information see John C. Bellamy's Digital Telephony, 1982,
+ * John Wiley & Sons, pps 98-111 and 472-476.
+ */
+static unsigned char
+st_linear2alaw(PyInt16 pcm_val) /* 2's complement (13-bit range) */
+{
+    PyInt16         mask;
+    short           seg;
+    unsigned char   aval;
+
+    /* The original sox code does this in the calling function, not here */
+    pcm_val = pcm_val >> 3;
+
+    /* A-law using even bit inversion */
+    if (pcm_val >= 0) {
+        mask = 0xD5;            /* sign (7th) bit = 1 */
+    } else {
+        mask = 0x55;            /* sign bit = 0 */
+        pcm_val = -pcm_val - 1;
+    }
+
+    /* Convert the scaled magnitude to segment number. */
+    seg = search(pcm_val, seg_aend, 8);
+
+    /* Combine the sign, segment, and quantization bits. */
+
+    if (seg >= 8)           /* out of range, return maximum value. */
+        return (unsigned char) (0x7F ^ mask);
+    else {
+        aval = (unsigned char) seg << SEG_SHIFT;
+        if (seg < 2)
+            aval |= (pcm_val >> 1) & QUANT_MASK;
+        else
+            aval |= (pcm_val >> seg) & QUANT_MASK;
+        return (aval ^ mask);
+    }
+}
+/* End of code taken from sox */
+
+/* Intel ADPCM step variation table */
+static int indexTable[16] = {
+    -1, -1, -1, -1, 2, 4, 6, 8,
+    -1, -1, -1, -1, 2, 4, 6, 8,
+};
+
+static int stepsizeTable[89] = {
+    7, 8, 9, 10, 11, 12, 13, 14, 16, 17,
+    19, 21, 23, 25, 28, 31, 34, 37, 41, 45,
+    50, 55, 60, 66, 73, 80, 88, 97, 107, 118,
+    130, 143, 157, 173, 190, 209, 230, 253, 279, 307,
+    337, 371, 408, 449, 494, 544, 598, 658, 724, 796,
+    876, 963, 1060, 1166, 1282, 1411, 1552, 1707, 1878, 2066,
+    2272, 2499, 2749, 3024, 3327, 3660, 4026, 4428, 4871, 5358,
+    5894, 6484, 7132, 7845, 8630, 9493, 10442, 11487, 12635, 13899,
+    15289, 16818, 18500, 20350, 22385, 24623, 27086, 29794, 32767
+};
+
+#define CHARP(cp, i) ((signed char *)(cp+i))
+#define SHORTP(cp, i) ((short *)(cp+i))
+#define LONGP(cp, i) ((Py_Int32 *)(cp+i))
+"""
+
+C_SOURCE = _AUDIOOP_C_MODULE + r"""
+#include <math.h>
+
+static const int maxvals[] = {0, 0x7F, 0x7FFF, 0x7FFFFF, 0x7FFFFFFF};
+/* -1 trick is needed on Windows to support -0x80000000 without a warning */
+static const int minvals[] = {0, -0x80, -0x8000, -0x800000, -0x7FFFFFFF-1};
+
+static int
+fbound(double val, double minval, double maxval)
+{
+    if (val > maxval)
+        val = maxval;
+    else if (val < minval + 1)
+        val = minval;
+    return val;
+}
+
+static int
+gcd(int a, int b)
+{
+    while (b > 0) {
+        int tmp = a % b;
+        a = b;
+        b = tmp;
+    }
+    return a;
+}
+
+int ratecv(char* rv, char* cp, size_t len, int size,
+           int nchannels, int inrate, int outrate,
+           int* state_d, int* prev_i, int* cur_i,
+           int weightA, int weightB)
+{
+    char *ncp = rv;
+    int d, chan;
+
+    /* divide inrate and outrate by their greatest common divisor */
+    d = gcd(inrate, outrate);
+    inrate /= d;
+    outrate /= d;
+    /* divide weightA and weightB by their greatest common divisor */
+    d = gcd(weightA, weightB);
+    weightA /= d;
+    weightA /= d;
+
+    d = *state_d;
+
+    for (;;) {
+        while (d < 0) {
+            if (len == 0) {
+                *state_d = d;
+                return ncp - rv;
+            }
+            for (chan = 0; chan < nchannels; chan++) {
+                prev_i[chan] = cur_i[chan];
+                if (size == 1)
+                    cur_i[chan] = ((int)*CHARP(cp, 0)) << 24;
+                else if (size == 2)
+                    cur_i[chan] = ((int)*SHORTP(cp, 0)) << 16;
+                else if (size == 4)
+                    cur_i[chan] = (int)*LONGP(cp, 0);
+                cp += size;
+                /* implements a simple digital filter */
+                cur_i[chan] = (int)(
+                    ((double)weightA * (double)cur_i[chan] +
+                     (double)weightB * (double)prev_i[chan]) /
+                    ((double)weightA + (double)weightB));
+            }
+            len--;
+            d += outrate;
+        }
+        while (d >= 0) {
+            for (chan = 0; chan < nchannels; chan++) {
+                int cur_o;
+                cur_o = (int)(((double)prev_i[chan] * (double)d +
+                         (double)cur_i[chan] * (double)(outrate - d)) /
+                    (double)outrate);
+                if (size == 1)
+                    *CHARP(ncp, 0) = (signed char)(cur_o >> 24);
+                else if (size == 2)
+                    *SHORTP(ncp, 0) = (short)(cur_o >> 16);
+                else if (size == 4)
+                    *LONGP(ncp, 0) = (Py_Int32)(cur_o);
+                ncp += size;
+            }
+            d -= inrate;
+        }
+    }
+}
+
+void tostereo(char* rv, char* cp, size_t len, int size,
+              double fac1, double fac2)
+{
+    int val1, val2, val = 0;
+    double fval, maxval, minval;
+    char *ncp = rv;
+    int i;
+
+    maxval = (double) maxvals[size];
+    minval = (double) minvals[size];
+
+    for ( i=0; i < len; i += size ) {
+        if ( size == 1 )      val = (int)*CHARP(cp, i);
+        else if ( size == 2 ) val = (int)*SHORTP(cp, i);
+        else if ( size == 4 ) val = (int)*LONGP(cp, i);
+
+        fval = (double)val*fac1;
+        val1 = (int)floor(fbound(fval, minval, maxval));
+
+        fval = (double)val*fac2;
+        val2 = (int)floor(fbound(fval, minval, maxval));
+
+        if ( size == 1 )      *CHARP(ncp, i*2) = (signed char)val1;
+        else if ( size == 2 ) *SHORTP(ncp, i*2) = (short)val1;
+        else if ( size == 4 ) *LONGP(ncp, i*2) = (Py_Int32)val1;
+
+        if ( size == 1 )      *CHARP(ncp, i*2+1) = (signed char)val2;
+        else if ( size == 2 ) *SHORTP(ncp, i*2+2) = (short)val2;
+        else if ( size == 4 ) *LONGP(ncp, i*2+4) = (Py_Int32)val2;
+    }
+}
+
+void add(char* rv, char* cp1, char* cp2, size_t len1, int size)
+{
+    int i;
+    int val1 = 0, val2 = 0, minval, maxval, newval;
+    char* ncp = rv;
+
+    maxval = maxvals[size];
+    minval = minvals[size];
+
+    for ( i=0; i < len1; i += size ) {
+        if ( size == 1 )      val1 = (int)*CHARP(cp1, i);
+        else if ( size == 2 ) val1 = (int)*SHORTP(cp1, i);
+        else if ( size == 4 ) val1 = (int)*LONGP(cp1, i);
+
+        if ( size == 1 )      val2 = (int)*CHARP(cp2, i);
+        else if ( size == 2 ) val2 = (int)*SHORTP(cp2, i);
+        else if ( size == 4 ) val2 = (int)*LONGP(cp2, i);
+
+        if (size < 4) {
+            newval = val1 + val2;
+            /* truncate in case of overflow */
+            if (newval > maxval)
+                newval = maxval;
+            else if (newval < minval)
+                newval = minval;
+        }
+        else {
+            double fval = (double)val1 + (double)val2;
+            /* truncate in case of overflow */
+            newval = (int)floor(fbound(fval, minval, maxval));
+        }
+
+        if ( size == 1 )      *CHARP(ncp, i) = (signed char)newval;
+        else if ( size == 2 ) *SHORTP(ncp, i) = (short)newval;
+        else if ( size == 4 ) *LONGP(ncp, i) = (Py_Int32)newval;
+    }
+}
+
+void lin2adcpm(unsigned char* ncp, unsigned char* cp, size_t len,
+               size_t size, int* state)
+{
+    int step, outputbuffer = 0, bufferstep;
+    int val = 0;
+    int diff, vpdiff, sign, delta;
+    size_t i;
+    int valpred = state[0];
+    int index = state[1];
+
+    step = stepsizeTable[index];
+    bufferstep = 1;
+
+    for ( i=0; i < len; i += size ) {
+        if ( size == 1 )      val = ((int)*CHARP(cp, i)) << 8;
+        else if ( size == 2 ) val = (int)*SHORTP(cp, i);
+        else if ( size == 4 ) val = ((int)*LONGP(cp, i)) >> 16;
+
+        /* Step 1 - compute difference with previous value */
+        diff = val - valpred;
+        sign = (diff < 0) ? 8 : 0;
+        if ( sign ) diff = (-diff);
+
+        /* Step 2 - Divide and clamp */
+        /* Note:
+        ** This code *approximately* computes:
+        **    delta = diff*4/step;
+        **    vpdiff = (delta+0.5)*step/4;
+        ** but in shift step bits are dropped. The net result of this
+        ** is that even if you have fast mul/div hardware you cannot
+        ** put it to good use since the fixup would be too expensive.
+        */
+        delta = 0;
+        vpdiff = (step >> 3);
+
+        if ( diff >= step ) {
+            delta = 4;
+            diff -= step;
+            vpdiff += step;
+        }
+        step >>= 1;
+        if ( diff >= step  ) {
+            delta |= 2;
+            diff -= step;
+            vpdiff += step;
+        }
+        step >>= 1;
+        if ( diff >= step ) {
+            delta |= 1;
+            vpdiff += step;
+        }
+
+        /* Step 3 - Update previous value */
+        if ( sign )
+            valpred -= vpdiff;
+        else
+            valpred += vpdiff;
+
+        /* Step 4 - Clamp previous value to 16 bits */
+        if ( valpred > 32767 )
+            valpred = 32767;
+        else if ( valpred < -32768 )
+            valpred = -32768;
+
+        /* Step 5 - Assemble value, update index and step values */
+        delta |= sign;
+
+        index += indexTable[delta];
+        if ( index < 0 ) index = 0;
+        if ( index > 88 ) index = 88;
+        step = stepsizeTable[index];
+
+        /* Step 6 - Output value */
+        if ( bufferstep ) {
+            outputbuffer = (delta << 4) & 0xf0;
+        } else {
+            *ncp++ = (delta & 0x0f) | outputbuffer;
+        }
+        bufferstep = !bufferstep;
+    }
+    state[0] = valpred;
+    state[1] = index;
+}
+
+
+void adcpm2lin(unsigned char* ncp, unsigned char* cp, size_t len,
+               size_t size, int* state)
+{
+    int step, inputbuffer = 0, bufferstep;
+    int val = 0;
+    int diff, vpdiff, sign, delta;
+    size_t i;
+    int valpred = state[0];
+    int index = state[1];
+
+    step = stepsizeTable[index];
+    bufferstep = 0;
+
+    for ( i=0; i < len*size*2; i += size ) {
+        /* Step 1 - get the delta value and compute next index */
+        if ( bufferstep ) {
+            delta = inputbuffer & 0xf;
+        } else {
+            inputbuffer = *cp++;
+            delta = (inputbuffer >> 4) & 0xf;
+        }
+
+        bufferstep = !bufferstep;
+
+        /* Step 2 - Find new index value (for later) */
+        index += indexTable[delta];
+        if ( index < 0 ) index = 0;
+        if ( index > 88 ) index = 88;
+
+        /* Step 3 - Separate sign and magnitude */
+        sign = delta & 8;
+        delta = delta & 7;
+
+        /* Step 4 - Compute difference and new predicted value */
+        /*
+        ** Computes 'vpdiff = (delta+0.5)*step/4', but see comment
+        ** in adpcm_coder.
+        */
+        vpdiff = step >> 3;
+        if ( delta & 4 ) vpdiff += step;
+        if ( delta & 2 ) vpdiff += step>>1;
+        if ( delta & 1 ) vpdiff += step>>2;
+
+        if ( sign )
+            valpred -= vpdiff;
+        else
+            valpred += vpdiff;
+
+        /* Step 5 - clamp output value */
+        if ( valpred > 32767 )
+            valpred = 32767;
+        else if ( valpred < -32768 )
+            valpred = -32768;
+
+        /* Step 6 - Update step value */
+        step = stepsizeTable[index];
+
+        /* Step 6 - Output value */
+        if ( size == 1 ) *CHARP(ncp, i) = (signed char)(valpred >> 8);
+        else if ( size == 2 ) *SHORTP(ncp, i) = (short)(valpred);
+        else if ( size == 4 ) *LONGP(ncp, i) = (Py_Int32)(valpred<<16);
+    }
+    state[0] = valpred;
+    state[1] = index;
+}
+"""
+
+ffi.set_source("_audioop_cffi", C_SOURCE)
+
+if __name__ == "__main__":
+    ffi.compile()
diff --git a/lib_pypy/_ctypes/function.py b/lib_pypy/_ctypes/function.py
--- a/lib_pypy/_ctypes/function.py
+++ b/lib_pypy/_ctypes/function.py
@@ -276,7 +276,11 @@
             if argtypes:
                 args = [argtype._CData_retval(argtype.from_address(arg)._buffer)
                         for argtype, arg in zip(argtypes, args)]
-            return to_call(*args)
+            try:
+                return to_call(*args)
+            except SystemExit as e:
+                handle_system_exit(e)
+                raise
         return f
 
     def __call__(self, *args, **kwargs):
@@ -302,14 +306,16 @@
 
             try:
                 newargs = self._convert_args_for_callback(argtypes, args)
-            except (UnicodeError, TypeError, ValueError), e:
+            except (UnicodeError, TypeError, ValueError) as e:
                 raise ArgumentError(str(e))
             try:
-                res = self.callable(*newargs)
+                try:
+                    res = self.callable(*newargs)
+                except SystemExit as e:
+                    handle_system_exit(e)
+                    raise
             except:
                 exc_info = sys.exc_info()
-                if issubclass(exc_info[0], SystemExit):
-                    exc_info = handle_system_exit(exc_info)
                 traceback.print_tb(exc_info[2], file=sys.stderr)
                 print >>sys.stderr, "%s: %s" % (exc_info[0].__name__, exc_info[1])
                 return 0
@@ -569,7 +575,7 @@
             for i, argtype in enumerate(argtypes):
                 try:
                     keepalive, newarg, newargtype = self._conv_param(argtype, args[i])
-                except (UnicodeError, TypeError, ValueError), e:
+                except (UnicodeError, TypeError, ValueError) as e:
                     raise ArgumentError(str(e))
                 keepalives.append(keepalive)
                 newargs.append(newarg)
@@ -580,7 +586,7 @@
             for i, arg in enumerate(extra):
                 try:
                     keepalive, newarg, newargtype = self._conv_param(None, arg)
-                except (UnicodeError, TypeError, ValueError), e:
+                except (UnicodeError, TypeError, ValueError) as e:
                     raise ArgumentError(str(e))
                 keepalives.append(keepalive)
                 newargs.append(newarg)
@@ -719,14 +725,13 @@
 make_fastpath_subclass.memo = {}
 
 
-def handle_system_exit(exc_info):
+def handle_system_exit(e):
     # issue #1194: if we get SystemExit here, then exit the interpreter.
     # Highly obscure imho but some people seem to depend on it.
-    try:
-        if sys.flags.inspect:
-            return exc_info   # Don't exit if -i flag was given.
-
-        code = exc_info[1].code
+    if sys.flags.inspect:
+        return   # Don't exit if -i flag was given.
+    else:
+        code = e.code
         if isinstance(code, int):
             exitcode = code
         else:
@@ -737,6 +742,3 @@
             exitcode = 1
 
         _rawffi.exit(exitcode)
-
-    except:
-        return sys.exc_info()
diff --git a/lib_pypy/_curses.py b/lib_pypy/_curses.py
--- a/lib_pypy/_curses.py
+++ b/lib_pypy/_curses.py
@@ -6,326 +6,7 @@
     raise ImportError('No module named _curses')
 from functools import wraps
 
-from cffi import FFI
-
-ffi = FFI()
-
-ffi.cdef("""
-typedef ... WINDOW;
-typedef ... SCREEN;
-typedef unsigned long mmask_t;
-typedef unsigned char bool;
-typedef unsigned long chtype;
-typedef chtype attr_t;
-
-typedef struct
-{
-    short id;           /* ID to distinguish multiple devices */
-    int x, y, z;        /* event coordinates (character-cell) */
-    mmask_t bstate;     /* button state bits */
-}
-MEVENT;
-
-static const int ERR, OK;
-static const int TRUE, FALSE;
-static const int KEY_MIN, KEY_MAX;
-
-static const int COLOR_BLACK;
-static const int COLOR_RED;
-static const int COLOR_GREEN;
-static const int COLOR_YELLOW;
-static const int COLOR_BLUE;
-static const int COLOR_MAGENTA;
-static const int COLOR_CYAN;
-static const int COLOR_WHITE;
-
-static const chtype A_ATTRIBUTES;
-static const chtype A_NORMAL;
-static const chtype A_STANDOUT;
-static const chtype A_UNDERLINE;
-static const chtype A_REVERSE;
-static const chtype A_BLINK;
-static const chtype A_DIM;
-static const chtype A_BOLD;
-static const chtype A_ALTCHARSET;
-static const chtype A_INVIS;
-static const chtype A_PROTECT;
-static const chtype A_CHARTEXT;
-static const chtype A_COLOR;
-
-static const int BUTTON1_RELEASED;
-static const int BUTTON1_PRESSED;
-static const int BUTTON1_CLICKED;
-static const int BUTTON1_DOUBLE_CLICKED;
-static const int BUTTON1_TRIPLE_CLICKED;
-static const int BUTTON2_RELEASED;
-static const int BUTTON2_PRESSED;
-static const int BUTTON2_CLICKED;
-static const int BUTTON2_DOUBLE_CLICKED;
-static const int BUTTON2_TRIPLE_CLICKED;
-static const int BUTTON3_RELEASED;
-static const int BUTTON3_PRESSED;
-static const int BUTTON3_CLICKED;
-static const int BUTTON3_DOUBLE_CLICKED;
-static const int BUTTON3_TRIPLE_CLICKED;
-static const int BUTTON4_RELEASED;
-static const int BUTTON4_PRESSED;
-static const int BUTTON4_CLICKED;
-static const int BUTTON4_DOUBLE_CLICKED;
-static const int BUTTON4_TRIPLE_CLICKED;
-static const int BUTTON_SHIFT;
-static const int BUTTON_CTRL;
-static const int BUTTON_ALT;
-static const int ALL_MOUSE_EVENTS;
-static const int REPORT_MOUSE_POSITION;
-
-int setupterm(char *, int, int *);
-
-WINDOW *stdscr;
-int COLORS;
-int COLOR_PAIRS;
-int COLS;
-int LINES;
-
-int baudrate(void);
-int beep(void);
-int box(WINDOW *, chtype, chtype);
-bool can_change_color(void);
-int cbreak(void);
-int clearok(WINDOW *, bool);
-int color_content(short, short*, short*, short*);
-int copywin(const WINDOW*, WINDOW*, int, int, int, int, int, int, int);
-int curs_set(int);
-int def_prog_mode(void);
-int def_shell_mode(void);
-int delay_output(int);
-int delwin(WINDOW *);
-WINDOW * derwin(WINDOW *, int, int, int, int);
-int doupdate(void);
-int echo(void);
-int endwin(void);
-char erasechar(void);
-void filter(void);
-int flash(void);
-int flushinp(void);
-chtype getbkgd(WINDOW *);
-WINDOW * getwin(FILE *);
-int halfdelay(int);
-bool has_colors(void);
-bool has_ic(void);
-bool has_il(void);
-void idcok(WINDOW *, bool);
-int idlok(WINDOW *, bool);
-void immedok(WINDOW *, bool);
-WINDOW * initscr(void);
-int init_color(short, short, short, short);
-int init_pair(short, short, short);
-int intrflush(WINDOW *, bool);
-bool isendwin(void);
-bool is_linetouched(WINDOW *, int);
-bool is_wintouched(WINDOW *);
-const char * keyname(int);
-int keypad(WINDOW *, bool);
-char killchar(void);
-int leaveok(WINDOW *, bool);
-char * longname(void);
-int meta(WINDOW *, bool);
-int mvderwin(WINDOW *, int, int);
-int mvwaddch(WINDOW *, int, int, const chtype);
-int mvwaddnstr(WINDOW *, int, int, const char *, int);
-int mvwaddstr(WINDOW *, int, int, const char *);
-int mvwchgat(WINDOW *, int, int, int, attr_t, short, const void *);
-int mvwdelch(WINDOW *, int, int);
-int mvwgetch(WINDOW *, int, int);
-int mvwgetnstr(WINDOW *, int, int, char *, int);
-int mvwin(WINDOW *, int, int);
-chtype mvwinch(WINDOW *, int, int);
-int mvwinnstr(WINDOW *, int, int, char *, int);
-int mvwinsch(WINDOW *, int, int, chtype);
-int mvwinsnstr(WINDOW *, int, int, const char *, int);
-int mvwinsstr(WINDOW *, int, int, const char *);
-int napms(int);
-WINDOW * newpad(int, int);
-WINDOW * newwin(int, int, int, int);
-int nl(void);
-int nocbreak(void);
-int nodelay(WINDOW *, bool);
-int noecho(void);
-int nonl(void);
-void noqiflush(void);
-int noraw(void);
-int notimeout(WINDOW *, bool);
-int overlay(const WINDOW*, WINDOW *);
-int overwrite(const WINDOW*, WINDOW *);
-int pair_content(short, short*, short*);
-int pechochar(WINDOW *, const chtype);
-int pnoutrefresh(WINDOW*, int, int, int, int, int, int);
-int prefresh(WINDOW *, int, int, int, int, int, int);
-int putwin(WINDOW *, FILE *);
-void qiflush(void);
-int raw(void);
-int redrawwin(WINDOW *);
-int resetty(void);
-int reset_prog_mode(void);
-int reset_shell_mode(void);
-int savetty(void);
-int scroll(WINDOW *);
-int scrollok(WINDOW *, bool);
-int start_color(void);
-WINDOW * subpad(WINDOW *, int, int, int, int);
-WINDOW * subwin(WINDOW *, int, int, int, int);
-int syncok(WINDOW *, bool);
-chtype termattrs(void);
-char * termname(void);
-int touchline(WINDOW *, int, int);
-int touchwin(WINDOW *);
-int typeahead(int);
-int ungetch(int);
-int untouchwin(WINDOW *);
-void use_env(bool);
-int waddch(WINDOW *, const chtype);
-int waddnstr(WINDOW *, const char *, int);
-int waddstr(WINDOW *, const char *);
-int wattron(WINDOW *, int);
-int wattroff(WINDOW *, int);
-int wattrset(WINDOW *, int);
-int wbkgd(WINDOW *, chtype);
-void wbkgdset(WINDOW *, chtype);
-int wborder(WINDOW *, chtype, chtype, chtype, chtype,
-            chtype, chtype, chtype, chtype);
-int wchgat(WINDOW *, int, attr_t, short, const void *);
-int wclear(WINDOW *);
-int wclrtobot(WINDOW *);
-int wclrtoeol(WINDOW *);
-void wcursyncup(WINDOW *);
-int wdelch(WINDOW *);
-int wdeleteln(WINDOW *);
-int wechochar(WINDOW *, const chtype);
-int werase(WINDOW *);
-int wgetch(WINDOW *);
-int wgetnstr(WINDOW *, char *, int);
-int whline(WINDOW *, chtype, int);
-chtype winch(WINDOW *);
-int winnstr(WINDOW *, char *, int);
-int winsch(WINDOW *, chtype);
-int winsdelln(WINDOW *, int);
-int winsertln(WINDOW *);
-int winsnstr(WINDOW *, const char *, int);
-int winsstr(WINDOW *, const char *);
-int wmove(WINDOW *, int, int);
-int wresize(WINDOW *, int, int);
-int wnoutrefresh(WINDOW *);
-int wredrawln(WINDOW *, int, int);
-int wrefresh(WINDOW *);
-int wscrl(WINDOW *, int);
-int wsetscrreg(WINDOW *, int, int);
-int wstandout(WINDOW *);
-int wstandend(WINDOW *);
-void wsyncdown(WINDOW *);
-void wsyncup(WINDOW *);
-void wtimeout(WINDOW *, int);
-int wtouchln(WINDOW *, int, int, int);
-int wvline(WINDOW *, chtype, int);
-int tigetflag(char *);
-int tigetnum(char *);
-char * tigetstr(char *);
-int putp(const char *);
-char * tparm(const char *, ...);
-int getattrs(const WINDOW *);
-int getcurx(const WINDOW *);
-int getcury(const WINDOW *);
-int getbegx(const WINDOW *);
-int getbegy(const WINDOW *);
-int getmaxx(const WINDOW *);
-int getmaxy(const WINDOW *);
-int getparx(const WINDOW *);
-int getpary(const WINDOW *);
-
-int getmouse(MEVENT *);
-int ungetmouse(MEVENT *);
-mmask_t mousemask(mmask_t, mmask_t *);
-bool wenclose(const WINDOW *, int, int);
-int mouseinterval(int);
-
-void setsyx(int y, int x);
-const char *unctrl(chtype);
-int use_default_colors(void);
-
-int has_key(int);
-bool is_term_resized(int, int);
-
-#define _m_STRICT_SYSV_CURSES ...
-#define _m_NCURSES_MOUSE_VERSION ...
-#define _m_NetBSD ...
-int _m_ispad(WINDOW *);
-
-chtype acs_map[];
-
-// For _curses_panel:
-
-typedef ... PANEL;
-
-WINDOW *panel_window(const PANEL *);
-void update_panels(void);
-int hide_panel(PANEL *);
-int show_panel(PANEL *);
-int del_panel(PANEL *);
-int top_panel(PANEL *);
-int bottom_panel(PANEL *);
-PANEL *new_panel(WINDOW *);
-PANEL *panel_above(const PANEL *);
-PANEL *panel_below(const PANEL *);
-int set_panel_userptr(PANEL *, void *);
-const void *panel_userptr(const PANEL *);
-int move_panel(PANEL *, int, int);
-int replace_panel(PANEL *,WINDOW *);
-int panel_hidden(const PANEL *);
-
-void _m_getsyx(int *yx);
-""")
-
-
-lib = ffi.verify("""
-#ifdef __APPLE__
-/* the following define is necessary for OS X 10.6+; without it, the
-   Apple-supplied ncurses.h sets NCURSES_OPAQUE to 1, and then Python
-   can't get at the WINDOW flags field. */
-#define NCURSES_OPAQUE 0
-#endif
-
-#include <ncurses.h>
-#include <panel.h>
-#include <term.h>
-
-#if defined STRICT_SYSV_CURSES
-#define _m_STRICT_SYSV_CURSES TRUE
-#else
-#define _m_STRICT_SYSV_CURSES FALSE
-#endif
-
-#if defined NCURSES_MOUSE_VERSION
-#define _m_NCURSES_MOUSE_VERSION TRUE
-#else
-#define _m_NCURSES_MOUSE_VERSION FALSE
-#endif
-
-#if defined __NetBSD__
-#define _m_NetBSD TRUE
-#else
-#define _m_NetBSD FALSE
-#endif
-
-int _m_ispad(WINDOW *win) {
-    // <curses.h> may not have _flags (and possibly _ISPAD),
-    // but for now let's assume that <ncurses.h> always has it
-    return (win->_flags & _ISPAD);
-}
-
-void _m_getsyx(int *yx) {
-    getsyx(yx[0], yx[1]);
-}
-""", libraries=['ncurses', 'panel'])
-
+from _curses_cffi import ffi, lib
 
 def _copy_to_globals(name):
     globals()[name] = getattr(lib, name)
diff --git a/lib_pypy/_curses_build.py b/lib_pypy/_curses_build.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_curses_build.py
@@ -0,0 +1,323 @@
+from cffi import FFI
+
+ffi = FFI()
+
+ffi.set_source("_curses_cffi", """
+#ifdef __APPLE__
+/* the following define is necessary for OS X 10.6+; without it, the
+   Apple-supplied ncurses.h sets NCURSES_OPAQUE to 1, and then Python
+   can't get at the WINDOW flags field. */
+#define NCURSES_OPAQUE 0
+#endif
+
+#include <ncurses.h>
+#include <panel.h>
+#include <term.h>
+
+#if defined STRICT_SYSV_CURSES
+#define _m_STRICT_SYSV_CURSES TRUE
+#else
+#define _m_STRICT_SYSV_CURSES FALSE
+#endif
+
+#if defined NCURSES_MOUSE_VERSION
+#define _m_NCURSES_MOUSE_VERSION TRUE
+#else
+#define _m_NCURSES_MOUSE_VERSION FALSE
+#endif
+
+#if defined __NetBSD__
+#define _m_NetBSD TRUE
+#else
+#define _m_NetBSD FALSE
+#endif
+
+int _m_ispad(WINDOW *win) {
+    // <curses.h> may not have _flags (and possibly _ISPAD),
+    // but for now let's assume that <ncurses.h> always has it
+    return (win->_flags & _ISPAD);
+}
+
+void _m_getsyx(int *yx) {
+    getsyx(yx[0], yx[1]);
+}
+""", libraries=['ncurses', 'panel'])
+
+
+ffi.cdef("""
+typedef ... WINDOW;
+typedef ... SCREEN;
+typedef unsigned long... mmask_t;
+typedef unsigned char bool;
+typedef unsigned long... chtype;
+typedef chtype attr_t;
+
+typedef struct
+{
+    short id;           /* ID to distinguish multiple devices */
+    int x, y, z;        /* event coordinates (character-cell) */
+    mmask_t bstate;     /* button state bits */
+}
+MEVENT;
+
+static const int ERR, OK;
+static const int TRUE, FALSE;
+static const int KEY_MIN, KEY_MAX;
+
+static const int COLOR_BLACK;
+static const int COLOR_RED;
+static const int COLOR_GREEN;
+static const int COLOR_YELLOW;
+static const int COLOR_BLUE;
+static const int COLOR_MAGENTA;
+static const int COLOR_CYAN;
+static const int COLOR_WHITE;
+
+static const chtype A_ATTRIBUTES;
+static const chtype A_NORMAL;
+static const chtype A_STANDOUT;
+static const chtype A_UNDERLINE;
+static const chtype A_REVERSE;
+static const chtype A_BLINK;
+static const chtype A_DIM;
+static const chtype A_BOLD;
+static const chtype A_ALTCHARSET;
+static const chtype A_INVIS;
+static const chtype A_PROTECT;
+static const chtype A_CHARTEXT;
+static const chtype A_COLOR;
+
+static const int BUTTON1_RELEASED;
+static const int BUTTON1_PRESSED;
+static const int BUTTON1_CLICKED;
+static const int BUTTON1_DOUBLE_CLICKED;
+static const int BUTTON1_TRIPLE_CLICKED;
+static const int BUTTON2_RELEASED;
+static const int BUTTON2_PRESSED;
+static const int BUTTON2_CLICKED;
+static const int BUTTON2_DOUBLE_CLICKED;
+static const int BUTTON2_TRIPLE_CLICKED;
+static const int BUTTON3_RELEASED;
+static const int BUTTON3_PRESSED;
+static const int BUTTON3_CLICKED;
+static const int BUTTON3_DOUBLE_CLICKED;
+static const int BUTTON3_TRIPLE_CLICKED;
+static const int BUTTON4_RELEASED;
+static const int BUTTON4_PRESSED;
+static const int BUTTON4_CLICKED;
+static const int BUTTON4_DOUBLE_CLICKED;
+static const int BUTTON4_TRIPLE_CLICKED;
+static const int BUTTON_SHIFT;
+static const int BUTTON_CTRL;
+static const int BUTTON_ALT;
+static const int ALL_MOUSE_EVENTS;
+static const int REPORT_MOUSE_POSITION;
+
+int setupterm(char *, int, int *);
+
+WINDOW *stdscr;
+int COLORS;
+int COLOR_PAIRS;
+int COLS;
+int LINES;
+
+int baudrate(void);
+int beep(void);
+int box(WINDOW *, chtype, chtype);
+bool can_change_color(void);
+int cbreak(void);
+int clearok(WINDOW *, bool);
+int color_content(short, short*, short*, short*);
+int copywin(const WINDOW*, WINDOW*, int, int, int, int, int, int, int);
+int curs_set(int);
+int def_prog_mode(void);
+int def_shell_mode(void);
+int delay_output(int);
+int delwin(WINDOW *);
+WINDOW * derwin(WINDOW *, int, int, int, int);
+int doupdate(void);
+int echo(void);
+int endwin(void);
+char erasechar(void);
+void filter(void);
+int flash(void);
+int flushinp(void);
+chtype getbkgd(WINDOW *);
+WINDOW * getwin(FILE *);
+int halfdelay(int);
+bool has_colors(void);
+bool has_ic(void);
+bool has_il(void);
+void idcok(WINDOW *, bool);
+int idlok(WINDOW *, bool);
+void immedok(WINDOW *, bool);
+WINDOW * initscr(void);
+int init_color(short, short, short, short);
+int init_pair(short, short, short);
+int intrflush(WINDOW *, bool);
+bool isendwin(void);
+bool is_linetouched(WINDOW *, int);
+bool is_wintouched(WINDOW *);
+const char * keyname(int);
+int keypad(WINDOW *, bool);
+char killchar(void);
+int leaveok(WINDOW *, bool);
+char * longname(void);
+int meta(WINDOW *, bool);
+int mvderwin(WINDOW *, int, int);
+int mvwaddch(WINDOW *, int, int, const chtype);
+int mvwaddnstr(WINDOW *, int, int, const char *, int);
+int mvwaddstr(WINDOW *, int, int, const char *);
+int mvwchgat(WINDOW *, int, int, int, attr_t, short, const void *);
+int mvwdelch(WINDOW *, int, int);
+int mvwgetch(WINDOW *, int, int);
+int mvwgetnstr(WINDOW *, int, int, char *, int);
+int mvwin(WINDOW *, int, int);
+chtype mvwinch(WINDOW *, int, int);
+int mvwinnstr(WINDOW *, int, int, char *, int);
+int mvwinsch(WINDOW *, int, int, chtype);
+int mvwinsnstr(WINDOW *, int, int, const char *, int);
+int mvwinsstr(WINDOW *, int, int, const char *);
+int napms(int);
+WINDOW * newpad(int, int);
+WINDOW * newwin(int, int, int, int);
+int nl(void);
+int nocbreak(void);
+int nodelay(WINDOW *, bool);
+int noecho(void);
+int nonl(void);
+void noqiflush(void);
+int noraw(void);
+int notimeout(WINDOW *, bool);
+int overlay(const WINDOW*, WINDOW *);
+int overwrite(const WINDOW*, WINDOW *);
+int pair_content(short, short*, short*);
+int pechochar(WINDOW *, const chtype);
+int pnoutrefresh(WINDOW*, int, int, int, int, int, int);
+int prefresh(WINDOW *, int, int, int, int, int, int);
+int putwin(WINDOW *, FILE *);
+void qiflush(void);
+int raw(void);
+int redrawwin(WINDOW *);
+int resetty(void);
+int reset_prog_mode(void);
+int reset_shell_mode(void);
+int savetty(void);
+int scroll(WINDOW *);
+int scrollok(WINDOW *, bool);
+int start_color(void);
+WINDOW * subpad(WINDOW *, int, int, int, int);
+WINDOW * subwin(WINDOW *, int, int, int, int);
+int syncok(WINDOW *, bool);
+chtype termattrs(void);
+char * termname(void);
+int touchline(WINDOW *, int, int);
+int touchwin(WINDOW *);
+int typeahead(int);
+int ungetch(int);
+int untouchwin(WINDOW *);
+void use_env(bool);
+int waddch(WINDOW *, const chtype);
+int waddnstr(WINDOW *, const char *, int);
+int waddstr(WINDOW *, const char *);
+int wattron(WINDOW *, int);
+int wattroff(WINDOW *, int);
+int wattrset(WINDOW *, int);
+int wbkgd(WINDOW *, chtype);
+void wbkgdset(WINDOW *, chtype);
+int wborder(WINDOW *, chtype, chtype, chtype, chtype,
+            chtype, chtype, chtype, chtype);
+int wchgat(WINDOW *, int, attr_t, short, const void *);
+int wclear(WINDOW *);
+int wclrtobot(WINDOW *);
+int wclrtoeol(WINDOW *);
+void wcursyncup(WINDOW *);
+int wdelch(WINDOW *);
+int wdeleteln(WINDOW *);
+int wechochar(WINDOW *, const chtype);
+int werase(WINDOW *);
+int wgetch(WINDOW *);
+int wgetnstr(WINDOW *, char *, int);
+int whline(WINDOW *, chtype, int);
+chtype winch(WINDOW *);
+int winnstr(WINDOW *, char *, int);
+int winsch(WINDOW *, chtype);
+int winsdelln(WINDOW *, int);
+int winsertln(WINDOW *);
+int winsnstr(WINDOW *, const char *, int);
+int winsstr(WINDOW *, const char *);
+int wmove(WINDOW *, int, int);
+int wresize(WINDOW *, int, int);
+int wnoutrefresh(WINDOW *);
+int wredrawln(WINDOW *, int, int);
+int wrefresh(WINDOW *);
+int wscrl(WINDOW *, int);
+int wsetscrreg(WINDOW *, int, int);
+int wstandout(WINDOW *);
+int wstandend(WINDOW *);
+void wsyncdown(WINDOW *);
+void wsyncup(WINDOW *);
+void wtimeout(WINDOW *, int);
+int wtouchln(WINDOW *, int, int, int);
+int wvline(WINDOW *, chtype, int);
+int tigetflag(char *);
+int tigetnum(char *);
+char * tigetstr(char *);
+int putp(const char *);
+char * tparm(const char *, ...);
+int getattrs(const WINDOW *);
+int getcurx(const WINDOW *);
+int getcury(const WINDOW *);
+int getbegx(const WINDOW *);
+int getbegy(const WINDOW *);
+int getmaxx(const WINDOW *);
+int getmaxy(const WINDOW *);
+int getparx(const WINDOW *);
+int getpary(const WINDOW *);
+
+int getmouse(MEVENT *);
+int ungetmouse(MEVENT *);
+mmask_t mousemask(mmask_t, mmask_t *);
+bool wenclose(const WINDOW *, int, int);
+int mouseinterval(int);
+
+void setsyx(int y, int x);
+const char *unctrl(chtype);
+int use_default_colors(void);
+
+int has_key(int);
+bool is_term_resized(int, int);
+
+#define _m_STRICT_SYSV_CURSES ...
+#define _m_NCURSES_MOUSE_VERSION ...
+#define _m_NetBSD ...
+int _m_ispad(WINDOW *);
+
+chtype acs_map[];
+
+// For _curses_panel:
+
+typedef ... PANEL;
+
+WINDOW *panel_window(const PANEL *);
+void update_panels(void);
+int hide_panel(PANEL *);
+int show_panel(PANEL *);
+int del_panel(PANEL *);
+int top_panel(PANEL *);
+int bottom_panel(PANEL *);
+PANEL *new_panel(WINDOW *);
+PANEL *panel_above(const PANEL *);
+PANEL *panel_below(const PANEL *);
+int set_panel_userptr(PANEL *, void *);
+const void *panel_userptr(const PANEL *);
+int move_panel(PANEL *, int, int);
+int replace_panel(PANEL *,WINDOW *);
+int panel_hidden(const PANEL *);
+
+void _m_getsyx(int *yx);
+""")
+
+
+if __name__ == "__main__":
+    ffi.compile()
diff --git a/lib_pypy/_functools.py b/lib_pypy/_functools.py
--- a/lib_pypy/_functools.py
+++ b/lib_pypy/_functools.py
@@ -8,16 +8,16 @@
     partial(func, *args, **keywords) - new function with partial application
     of the given arguments and keywords.
     """
-
-    def __init__(self, *args, **keywords):
-        if not args:
-            raise TypeError('__init__() takes at least 2 arguments (1 given)')
-        func, args = args[0], args[1:]
+    def __init__(*args, **keywords):
+        if len(args) < 2:
+            raise TypeError('__init__() takes at least 2 arguments (%d given)'
+                            % len(args))
+        self, func, args = args[0], args[1], args[2:]
         if not callable(func):
             raise TypeError("the first argument must be callable")
         self._func = func
         self._args = args
-        self._keywords = keywords or None
+        self._keywords = keywords
 
     def __delattr__(self, key):
         if key == '__dict__':
@@ -37,19 +37,22 @@
         return self._keywords
 
     def __call__(self, *fargs, **fkeywords):
-        if self.keywords is not None:
-            fkeywords = dict(self.keywords, **fkeywords)
-        return self.func(*(self.args + fargs), **fkeywords)
+        if self._keywords:
+            fkeywords = dict(self._keywords, **fkeywords)
+        return self._func(*(self._args + fargs), **fkeywords)
 
     def __reduce__(self):
         d = dict((k, v) for k, v in self.__dict__.iteritems() if k not in
                 ('_func', '_args', '_keywords'))
         if len(d) == 0:
             d = None
-        return (type(self), (self.func,),
-                (self.func, self.args, self.keywords, d))
+        return (type(self), (self._func,),
+                (self._func, self._args, self._keywords, d))
 
     def __setstate__(self, state):
-        self._func, self._args, self._keywords, d = state
+        func, args, keywords, d = state
         if d is not None:
             self.__dict__.update(d)
+        self._func = func
+        self._args = args
+        self._keywords = keywords
diff --git a/lib_pypy/_gdbm_build.py b/lib_pypy/_gdbm_build.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_gdbm_build.py
@@ -0,0 +1,65 @@
+import cffi, os, sys
+
+ffi = cffi.FFI()
+ffi.cdef('''
+#define GDBM_READER ...
+#define GDBM_WRITER ...
+#define GDBM_WRCREAT ...
+#define GDBM_NEWDB ...
+#define GDBM_FAST ...
+#define GDBM_SYNC ...
+#define GDBM_NOLOCK ...
+#define GDBM_REPLACE ...
+
+void* gdbm_open(char *, int, int, int, void (*)());
+void gdbm_close(void*);
+
+typedef struct {
+    char *dptr;
+    int   dsize;
+} datum;
+
+datum gdbm_fetch(void*, datum);
+datum pygdbm_fetch(void*, char*, int);
+int gdbm_delete(void*, datum);
+int gdbm_store(void*, datum, datum, int);
+int gdbm_exists(void*, datum);
+int pygdbm_exists(void*, char*, int);
+
+int gdbm_reorganize(void*);
+
+datum gdbm_firstkey(void*);
+datum gdbm_nextkey(void*, datum);
+void gdbm_sync(void*);
+
+char* gdbm_strerror(int);
+int gdbm_errno;
+
+void free(void*);
+''')
+
+
+kwds = {}
+if sys.platform.startswith('freebsd'):
+    _localbase = os.environ.get('LOCALBASE', '/usr/local')
+    kwds['include_dirs'] = [os.path.join(_localbase, 'include')]
+    kwds['library_dirs'] = [os.path.join(_localbase, 'lib')]
+
+ffi.set_source("_gdbm_cffi", '''
+#include <stdlib.h>
+#include "gdbm.h"
+
+static datum pygdbm_fetch(GDBM_FILE gdbm_file, char *dptr, int dsize) {
+    datum key = {dptr, dsize};
+    return gdbm_fetch(gdbm_file, key);
+}
+
+static int pygdbm_exists(GDBM_FILE gdbm_file, char *dptr, int dsize) {
+    datum key = {dptr, dsize};
+    return gdbm_exists(gdbm_file, key);
+}
+''', libraries=['gdbm'], **kwds)
+
+
+if __name__ == '__main__':
+    ffi.compile()
diff --git a/lib_pypy/_pwdgrp_build.py b/lib_pypy/_pwdgrp_build.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_pwdgrp_build.py
@@ -0,0 +1,53 @@
+from cffi import FFI
+
+ffi = FFI()
+
+ffi.set_source("_pwdgrp_cffi", """
+#include <sys/types.h>
+#include <pwd.h>
+#include <grp.h>
+""")
+
+
+ffi.cdef("""
+
+typedef int... uid_t;
+typedef int... gid_t;
+
+struct passwd {
+    char *pw_name;
+    char *pw_passwd;
+    uid_t pw_uid;
+    gid_t pw_gid;
+    char *pw_gecos;
+    char *pw_dir;
+    char *pw_shell;
+    ...;
+};
+
+struct group {
+    char *gr_name;       /* group name */
+    char *gr_passwd;     /* group password */
+    gid_t gr_gid;        /* group ID */
+    char **gr_mem;        /* group members */
+};
+
+struct passwd *getpwuid(uid_t uid);
+struct passwd *getpwnam(const char *name);
+
+struct passwd *getpwent(void);
+void setpwent(void);
+void endpwent(void);
+
+struct group *getgrgid(gid_t gid);
+struct group *getgrnam(const char *name);
+
+struct group *getgrent(void);
+void setgrent(void);
+void endgrent(void);
+
+""")
+
+
+if __name__ == "__main__":
+    ffi.compile()
diff --git a/lib_pypy/_sqlite3.py b/lib_pypy/_sqlite3.py
--- a/lib_pypy/_sqlite3.py
+++ b/lib_pypy/_sqlite3.py
@@ -47,243 +47,7 @@
 else:
     _BLOB_TYPE = buffer
 
-from cffi import FFI as _FFI
-
-_ffi = _FFI()
-
-_ffi.cdef("""
-#define SQLITE_OK ...
-#define SQLITE_ERROR ...
-#define SQLITE_INTERNAL ...
-#define SQLITE_PERM ...
-#define SQLITE_ABORT ...
-#define SQLITE_BUSY ...
-#define SQLITE_LOCKED ...
-#define SQLITE_NOMEM ...
-#define SQLITE_READONLY ...
-#define SQLITE_INTERRUPT ...
-#define SQLITE_IOERR ...
-#define SQLITE_CORRUPT ...
-#define SQLITE_NOTFOUND ...
-#define SQLITE_FULL ...
-#define SQLITE_CANTOPEN ...
-#define SQLITE_PROTOCOL ...
-#define SQLITE_EMPTY ...
-#define SQLITE_SCHEMA ...
-#define SQLITE_TOOBIG ...
-#define SQLITE_CONSTRAINT ...
-#define SQLITE_MISMATCH ...
-#define SQLITE_MISUSE ...
-#define SQLITE_NOLFS ...
-#define SQLITE_AUTH ...
-#define SQLITE_FORMAT ...
-#define SQLITE_RANGE ...
-#define SQLITE_NOTADB ...
-#define SQLITE_ROW ...
-#define SQLITE_DONE ...
-#define SQLITE_INTEGER ...
-#define SQLITE_FLOAT ...
-#define SQLITE_BLOB ...
-#define SQLITE_NULL ...
-#define SQLITE_TEXT ...
-#define SQLITE3_TEXT ...
-
-#define SQLITE_TRANSIENT ...
-#define SQLITE_UTF8 ...
-
-#define SQLITE_DENY ...
-#define SQLITE_IGNORE ...
-
-#define SQLITE_CREATE_INDEX ...
-#define SQLITE_CREATE_TABLE ...
-#define SQLITE_CREATE_TEMP_INDEX ...
-#define SQLITE_CREATE_TEMP_TABLE ...
-#define SQLITE_CREATE_TEMP_TRIGGER ...
-#define SQLITE_CREATE_TEMP_VIEW ...
-#define SQLITE_CREATE_TRIGGER ...
-#define SQLITE_CREATE_VIEW ...
-#define SQLITE_DELETE ...
-#define SQLITE_DROP_INDEX ...
-#define SQLITE_DROP_TABLE ...
-#define SQLITE_DROP_TEMP_INDEX ...
-#define SQLITE_DROP_TEMP_TABLE ...
-#define SQLITE_DROP_TEMP_TRIGGER ...
-#define SQLITE_DROP_TEMP_VIEW ...
-#define SQLITE_DROP_TRIGGER ...
-#define SQLITE_DROP_VIEW ...
-#define SQLITE_INSERT ...
-#define SQLITE_PRAGMA ...
-#define SQLITE_READ ...
-#define SQLITE_SELECT ...
-#define SQLITE_TRANSACTION ...
-#define SQLITE_UPDATE ...
-#define SQLITE_ATTACH ...
-#define SQLITE_DETACH ...
-#define SQLITE_ALTER_TABLE ...
-#define SQLITE_REINDEX ...
-#define SQLITE_ANALYZE ...
-#define SQLITE_CREATE_VTABLE ...
-#define SQLITE_DROP_VTABLE ...
-#define SQLITE_FUNCTION ...
-
-const char *sqlite3_libversion(void);
-
-typedef ... sqlite3;
-typedef ... sqlite3_stmt;
-typedef ... sqlite3_context;
-typedef ... sqlite3_value;
-typedef int64_t sqlite3_int64;
-typedef uint64_t sqlite3_uint64;
-
-int sqlite3_open(
-    const char *filename,   /* Database filename (UTF-8) */
-    sqlite3 **ppDb          /* OUT: SQLite db handle */
-);
-
-int sqlite3_close(sqlite3 *);
-
-int sqlite3_busy_timeout(sqlite3*, int ms);
-int sqlite3_prepare_v2(
-    sqlite3 *db,            /* Database handle */
-    const char *zSql,       /* SQL statement, UTF-8 encoded */
-    int nByte,              /* Maximum length of zSql in bytes. */
-    sqlite3_stmt **ppStmt,  /* OUT: Statement handle */
-    const char **pzTail     /* OUT: Pointer to unused portion of zSql */
-);
-int sqlite3_finalize(sqlite3_stmt *pStmt);
-int sqlite3_data_count(sqlite3_stmt *pStmt);
-int sqlite3_column_count(sqlite3_stmt *pStmt);
-const char *sqlite3_column_name(sqlite3_stmt*, int N);
-int sqlite3_get_autocommit(sqlite3*);
-int sqlite3_reset(sqlite3_stmt *pStmt);
-int sqlite3_step(sqlite3_stmt*);
-int sqlite3_errcode(sqlite3 *db);
-const char *sqlite3_errmsg(sqlite3*);
-int sqlite3_changes(sqlite3*);
-
-int sqlite3_bind_blob(sqlite3_stmt*, int, const void*, int n, void(*)(void*));
-int sqlite3_bind_double(sqlite3_stmt*, int, double);
-int sqlite3_bind_int(sqlite3_stmt*, int, int);
-int sqlite3_bind_int64(sqlite3_stmt*, int, sqlite3_int64);
-int sqlite3_bind_null(sqlite3_stmt*, int);
-int sqlite3_bind_text(sqlite3_stmt*, int, const char*, int n, void(*)(void*));
-int sqlite3_bind_text16(sqlite3_stmt*, int, const void*, int, void(*)(void*));
-int sqlite3_bind_value(sqlite3_stmt*, int, const sqlite3_value*);
-int sqlite3_bind_zeroblob(sqlite3_stmt*, int, int n);
-
-const void *sqlite3_column_blob(sqlite3_stmt*, int iCol);
-int sqlite3_column_bytes(sqlite3_stmt*, int iCol);
-double sqlite3_column_double(sqlite3_stmt*, int iCol);
-int sqlite3_column_int(sqlite3_stmt*, int iCol);
-sqlite3_int64 sqlite3_column_int64(sqlite3_stmt*, int iCol);
-const unsigned char *sqlite3_column_text(sqlite3_stmt*, int iCol);
-const void *sqlite3_column_text16(sqlite3_stmt*, int iCol);
-int sqlite3_column_type(sqlite3_stmt*, int iCol);
-const char *sqlite3_column_decltype(sqlite3_stmt*,int);
-
-void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*);
-int sqlite3_create_collation(
-    sqlite3*,
-    const char *zName,
-    int eTextRep,
-    void*,
-    int(*xCompare)(void*,int,const void*,int,const void*)
-);
-int sqlite3_set_authorizer(
-    sqlite3*,
-    int (*xAuth)(void*,int,const char*,const char*,const char*,const char*),
-    void *pUserData
-);
-int sqlite3_create_function(
-    sqlite3 *db,
-    const char *zFunctionName,
-    int nArg,
-    int eTextRep,
-    void *pApp,
-    void (*xFunc)(sqlite3_context*,int,sqlite3_value**),
-    void (*xStep)(sqlite3_context*,int,sqlite3_value**),
-    void (*xFinal)(sqlite3_context*)
-);
-void *sqlite3_aggregate_context(sqlite3_context*, int nBytes);
-
-sqlite3_int64 sqlite3_last_insert_rowid(sqlite3*);
-int sqlite3_bind_parameter_count(sqlite3_stmt*);
-const char *sqlite3_bind_parameter_name(sqlite3_stmt*, int);
-int sqlite3_total_changes(sqlite3*);
-
-int sqlite3_prepare(
-    sqlite3 *db,            /* Database handle */
-    const char *zSql,       /* SQL statement, UTF-8 encoded */
-    int nByte,              /* Maximum length of zSql in bytes. */
-    sqlite3_stmt **ppStmt,  /* OUT: Statement handle */
-    const char **pzTail     /* OUT: Pointer to unused portion of zSql */
-);
-
-void sqlite3_result_blob(sqlite3_context*, const void*, int, void(*)(void*));
-void sqlite3_result_double(sqlite3_context*, double);
-void sqlite3_result_error(sqlite3_context*, const char*, int);
-void sqlite3_result_error16(sqlite3_context*, const void*, int);
-void sqlite3_result_error_toobig(sqlite3_context*);
-void sqlite3_result_error_nomem(sqlite3_context*);
-void sqlite3_result_error_code(sqlite3_context*, int);
-void sqlite3_result_int(sqlite3_context*, int);
-void sqlite3_result_int64(sqlite3_context*, sqlite3_int64);
-void sqlite3_result_null(sqlite3_context*);
-void sqlite3_result_text(sqlite3_context*, const char*, int, void(*)(void*));
-void sqlite3_result_text16(sqlite3_context*, const void*, int, void(*)(void*));
-void sqlite3_result_text16le(sqlite3_context*,const void*, int,void(*)(void*));
-void sqlite3_result_text16be(sqlite3_context*,const void*, int,void(*)(void*));

From noreply at buildbot.pypy.org  Thu Jun  4 18:11:27 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Thu,  4 Jun 2015 18:11:27 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: added missing vec counters to jitprof
	test
Message-ID: <20150604161127.7A3EA1C0F05@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r77864:b41e4cb3e026
Date: 2015-06-04 14:53 +0200
http://bitbucket.org/pypy/pypy/changeset/b41e4cb3e026/

Log:	added missing vec counters to jitprof test

diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -1632,6 +1632,17 @@
         size = op.result.getsize()
         self.perform(op, [resloc, imm(size)], resloc)
 
+    def consider_vec_int_expand(self, op):
+        arg = op.getarg(0)
+        if isinstance(arg, Const):
+            resloc = self.xrm.expand_int(op.result, arg)
+            return
+        args = op.getarglist()
+        resloc = self.xrm.force_result_in_reg(op.result, arg, args)
+        assert isinstance(op.result, BoxVector)
+        size = op.result.getsize()
+        self.perform(op, [resloc, imm(size)], resloc)
+
     def consider_vec_int_signext(self, op):
         args = op.getarglist()
         resloc = self.xrm.force_result_in_reg(op.result, op.getarg(0), args)
diff --git a/rpython/jit/metainterp/test/test_jitprof.py b/rpython/jit/metainterp/test/test_jitprof.py
--- a/rpython/jit/metainterp/test/test_jitprof.py
+++ b/rpython/jit/metainterp/test/test_jitprof.py
@@ -54,7 +54,7 @@
         assert profiler.events == expected
         assert profiler.times == [2, 1]
         assert profiler.counters == [1, 1, 3, 3, 2, 15, 2, 0, 0, 0, 0,
-                                     0, 0, 0, 0, 0]
+                                     0, 0, 0, 0, 0, 0, 0]
 
     def test_simple_loop_with_call(self):
         @dont_look_inside

From noreply at buildbot.pypy.org  Thu Jun  4 18:11:28 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Thu,  4 Jun 2015 18:11:28 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: adapted test_zjit to changes in micro
	numpy
Message-ID: <20150604161128.E1CFF1C0F05@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r77865:a0cf34db417d
Date: 2015-06-04 18:10 +0200
http://bitbucket.org/pypy/pypy/changeset/a0cf34db417d/

Log:	adapted test_zjit to changes in micro numpy added a stub for int
	expansion

diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py
--- a/pypy/module/micronumpy/compile.py
+++ b/pypy/module/micronumpy/compile.py
@@ -61,6 +61,8 @@
     w_AttributeError = W_TypeObject("AttributeError")
     w_StopIteration = W_TypeObject("StopIteration")
     w_KeyError = W_TypeObject("KeyError")
+    w_SystemExit = W_TypeObject("SystemExit")
+    w_KeyboardInterrupt = W_TypeObject("KeyboardInterrupt")
     w_None = None
 
     w_bool = W_TypeObject("bool")
@@ -342,8 +344,19 @@
                 return FloatObject(float(int(w_dtype.value)))
             if isinstance(w_dtype, boxes.W_Int32Box):
                 return FloatObject(float(int(w_dtype.value)))
+            if isinstance(w_dtype, boxes.W_Int16Box):
+                return FloatObject(float(int(w_dtype.value)))
+            if isinstance(w_dtype, boxes.W_Int8Box):
+                return FloatObject(float(int(w_dtype.value)))
+            if isinstance(w_dtype, IntObject):
+                return FloatObject(float(w_dtype.intval))
+        if tp is self.w_int:
+            if isinstance(w_dtype, FloatObject):
+                return IntObject(int(w_dtype.floatval))
+
         return w_dtype
 
+    @specialize.arg(2)
     def call_method(self, w_obj, s, *args):
         # XXX even the hacks have hacks
         return getattr(w_obj, 'descr_' + s)(self, *args)
@@ -732,10 +745,10 @@
                 w_res = logical_xor.reduce(interp.space, arr, None)
             elif self.name == "unegative":
                 neg = ufuncs.get(interp.space).negative
-                w_res = neg.call(interp.space, [arr], None, None, None)
+                w_res = neg.call(interp.space, [arr], None, 'unsafe', None)
             elif self.name == "cos":
                 cos = ufuncs.get(interp.space).cos
-                w_res = cos.call(interp.space, [arr], None, None, None)
+                w_res = cos.call(interp.space, [arr], None, 'unsafe', None)
             elif self.name == "flat":
                 w_res = arr.descr_get_flatiter(interp.space)
             elif self.name == "argsort":
diff --git a/pypy/module/micronumpy/loop.py b/pypy/module/micronumpy/loop.py
--- a/pypy/module/micronumpy/loop.py
+++ b/pypy/module/micronumpy/loop.py
@@ -16,7 +16,7 @@
 call2_driver = jit.JitDriver(
     name='numpy_call2',
     greens=['shapelen', 'func', 'calc_dtype', 'res_dtype'],
-    reds='auto')
+    reds='auto', vectorize=True)
 
 def call2(space, shape, func, calc_dtype, w_lhs, w_rhs, out):
     if w_lhs.get_size() == 1:
@@ -50,9 +50,9 @@
         w_out = func(calc_dtype, w_left, w_right)
         out_iter.setitem(out_state, w_out.convert_to(space, res_dtype))
         out_state = out_iter.next(out_state)
-        # if not set to None, the values will be loop carried, forcing
-        # the vectorization to unpack the vector registers at the end
-        # of the loop
+        # if not set to None, the values will be loop carried
+        # (for the var,var case), forcing the vectorization to unpack
+        # the vector registers at the end of the loop
         if left_iter:
             w_left = None
         if right_iter:
diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -2636,6 +2636,11 @@
         elif size == 8:
             self.mc.MOVDDUP(resloc, srcloc)
 
+    def genop_vec_int_expand(self, op, arglocs, resloc):
+        srcloc, sizeloc = arglocs
+        size = sizeloc.value
+        raise NotImplementedError
+
     def genop_vec_int_pack(self, op, arglocs, resloc):
         resultloc, sourceloc, residxloc, srcidxloc, countloc, sizeloc = arglocs
         assert isinstance(resultloc, RegLoc)
diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -81,6 +81,20 @@
         rffi.cast(rffi.CArrayPtr(longlong.FLOATSTORAGE), adr)[1] = y
         return ConstFloatLoc(adr)
 
+    def expand_int(self, var, const):
+        assert isinstance(var, BoxVector)
+        if var.getsize() == 4:
+            loc = self.expand_single_float(const)
+        else:
+            loc = self.expand_double_float(const)
+        adr = self.assembler.datablockwrapper.malloc_aligned(16, 16)
+        x = c.getfloatstorage()
+        y = longlong.ZEROF
+        rffi.cast(rffi.CArrayPtr(longlong.FLOATSTORAGE), adr)[0] = x
+        rffi.cast(rffi.CArrayPtr(longlong.FLOATSTORAGE), adr)[1] = y
+        self.reg_bindings[var] = loc
+        return loc
+
     def expand_float(self, var, const):
         assert isinstance(var, BoxVector)
         if var.getsize() == 4:

From noreply at buildbot.pypy.org  Thu Jun  4 18:26:15 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Thu,  4 Jun 2015 18:26:15 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Try to make tests with threads and signals
	more reliable.
Message-ID: <20150604162615.6F74B1C1017@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob <me at manueljacob.de>
Branch: py3k
Changeset: r77866:906fd02db149
Date: 2015-06-04 17:27 +0200
http://bitbucket.org/pypy/pypy/changeset/906fd02db149/

Log:	Try to make tests with threads and signals more reliable.

diff --git a/pypy/module/thread/test/test_lock.py b/pypy/module/thread/test/test_lock.py
--- a/pypy/module/thread/test/test_lock.py
+++ b/pypy/module/thread/test/test_lock.py
@@ -207,6 +207,8 @@
             assert result
         finally:
             signal.signal(signal.SIGUSR1, old_handler)
+            for i in range(50):
+                time.sleep(0.1)
 
     def test_lock_acquire_retries_on_intr(self):
         import _thread

From noreply at buildbot.pypy.org  Thu Jun  4 18:26:16 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Thu,  4 Jun 2015 18:26:16 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Add workaround for when host interpreter
 has slightly different errnos than interpreter under test.
Message-ID: <20150604162616.99C7F1C1017@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob <me at manueljacob.de>
Branch: py3k
Changeset: r77867:e6c53e141773
Date: 2015-06-04 18:26 +0200
http://bitbucket.org/pypy/pypy/changeset/e6c53e141773/

Log:	Add workaround for when host interpreter has slightly different
	errnos than interpreter under test.

	See comment for details.

diff --git a/pypy/module/errno/test/test_errno.py b/pypy/module/errno/test/test_errno.py
--- a/pypy/module/errno/test/test_errno.py
+++ b/pypy/module/errno/test/test_errno.py
@@ -11,8 +11,28 @@
         assert not hasattr(self.errno, '__file__')
 
     def test_constants(self):
-        for code, name in self.errorcode.items():
+        host_errorcode = self.errorcode.copy()
+        # On some systems, ENOTSUP is an alias to EOPNOTSUPP.  Adjust the
+        # host_errorcode dictionary in case the host interpreter has slightly
+        # different errorcodes than the interpreter under test
+        if ('ENOTSUP' not in host_errorcode.values() and
+            'ENOTSUP' in self.errno.errorcode.values()):
+            host_errorcode[self.errno.ENOTSUP] = 'ENOTSUP'
+        if ('EOPNOTSUPP' not in host_errorcode.values() and
+            'EOPNOTSUPP' in self.errno.errorcode.values()):
+            host_errorcode[self.errno.EOPNOTSUPP] = 'EOPNOTSUPP'
+        for code, name in host_errorcode.items():
             assert getattr(self.errno, name) == code
 
     def test_errorcode(self):
-        assert self.errorcode == self.errno.errorcode
+        host_errorcode = self.errorcode.copy()
+        # On some systems, ENOTSUP is an alias to EOPNOTSUPP.  Adjust the
+        # host_errorcode dictionary in case the host interpreter has slightly
+        # different errorcodes than the interpreter under test
+        if ('ENOTSUP' not in host_errorcode.values() and
+            'ENOTSUP' in self.errno.errorcode.values()):
+            host_errorcode[self.errno.ENOTSUP] = 'ENOTSUP'
+        if ('EOPNOTSUPP' not in host_errorcode.values() and
+            'EOPNOTSUPP' in self.errno.errorcode.values()):
+            host_errorcode[self.errno.EOPNOTSUPP] = 'EOPNOTSUPP'
+        assert host_errorcode == self.errno.errorcode

From noreply at buildbot.pypy.org  Thu Jun  4 18:37:32 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Thu,  4 Jun 2015 18:37:32 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: int expand 64 bit working
Message-ID: <20150604163732.215981C12E8@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r77868:812b3caa438c
Date: 2015-06-04 18:37 +0200
http://bitbucket.org/pypy/pypy/changeset/812b3caa438c/

Log:	int expand 64 bit working

diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -196,6 +196,21 @@
         assert int(result) == 7+1+8+1+11+2+12+2
         self.check_vectorized(2, 2)
 
+    def define_int_expand():
+        return """
+        a = astype(|30|, int)
+        c = astype(|1|, int)
+        c[0] = 16
+        b = a + c
+        x1 = b -> 7
+        x2 = b -> 8
+        x1 + x2
+        """
+    def test_int_expand(self):
+        result = self.run("int_expand")
+        assert int(result) == 7+16+8+16
+        self.check_vectorized(2, 2)
+
     def define_int32_add_const():
         return """
         a = astype(|30|, int32)
diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -2638,8 +2638,13 @@
 
     def genop_vec_int_expand(self, op, arglocs, resloc):
         srcloc, sizeloc = arglocs
+        assert not srcloc.is_xmm
         size = sizeloc.value
-        raise NotImplementedError
+        if size == 8:
+            self.mc.PINSRQ_xri(resloc.value, srcloc.value, 0)
+            self.mc.PINSRQ_xri(resloc.value, srcloc.value, 1)
+        else:
+            raise NotImplementedError("missing size %d for int expand" % (size,))
 
     def genop_vec_int_pack(self, op, arglocs, resloc):
         resultloc, sourceloc, residxloc, srcidxloc, countloc, sizeloc = arglocs
diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -81,20 +81,6 @@
         rffi.cast(rffi.CArrayPtr(longlong.FLOATSTORAGE), adr)[1] = y
         return ConstFloatLoc(adr)
 
-    def expand_int(self, var, const):
-        assert isinstance(var, BoxVector)
-        if var.getsize() == 4:
-            loc = self.expand_single_float(const)
-        else:
-            loc = self.expand_double_float(const)
-        adr = self.assembler.datablockwrapper.malloc_aligned(16, 16)
-        x = c.getfloatstorage()
-        y = longlong.ZEROF
-        rffi.cast(rffi.CArrayPtr(longlong.FLOATSTORAGE), adr)[0] = x
-        rffi.cast(rffi.CArrayPtr(longlong.FLOATSTORAGE), adr)[1] = y
-        self.reg_bindings[var] = loc
-        return loc
-
     def expand_float(self, var, const):
         assert isinstance(var, BoxVector)
         if var.getsize() == 4:
@@ -1639,6 +1625,7 @@
         arg = op.getarg(0)
         if isinstance(arg, Const):
             resloc = self.xrm.expand_float(op.result, arg)
+            # TODO consider this
             return
         args = op.getarglist()
         resloc = self.xrm.force_result_in_reg(op.result, arg, args)
@@ -1649,13 +1636,14 @@
     def consider_vec_int_expand(self, op):
         arg = op.getarg(0)
         if isinstance(arg, Const):
-            resloc = self.xrm.expand_int(op.result, arg)
-            return
-        args = op.getarglist()
-        resloc = self.xrm.force_result_in_reg(op.result, arg, args)
+            srcloc = self.rm.convert_to_imm(arg)
+        else:
+            args = op.getarglist()
+            srcloc = self.make_sure_var_in_reg(arg, args)
+        resloc = self.xrm.force_allocate_reg(op.result, args)
         assert isinstance(op.result, BoxVector)
         size = op.result.getsize()
-        self.perform(op, [resloc, imm(size)], resloc)
+        self.perform(op, [srcloc, imm(size)], resloc)
 
     def consider_vec_int_signext(self, op):
         args = op.getarglist()

From noreply at buildbot.pypy.org  Thu Jun  4 18:38:17 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Thu,  4 Jun 2015 18:38:17 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: start killing things
Message-ID: <20150604163817.4C3631C12E8@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77869:e525465d156c
Date: 2015-06-04 18:22 +0200
http://bitbucket.org/pypy/pypy/changeset/e525465d156c/

Log:	start killing things

diff --git a/rpython/jit/backend/llsupport/gc.py b/rpython/jit/backend/llsupport/gc.py
--- a/rpython/jit/backend/llsupport/gc.py
+++ b/rpython/jit/backend/llsupport/gc.py
@@ -127,8 +127,9 @@
     def gc_malloc_unicode(self, num_elem):
         return self._bh_malloc_array(num_elem, self.unicode_descr)
 
-    def _record_constptrs(self, op, gcrefs_output_list, ops_with_movable_const_ptr,
-            changeable_const_pointers):
+    def _record_constptrs(self, op, gcrefs_output_list,
+                          ops_with_movable_const_ptr,
+                          changeable_const_pointers):
         ops_with_movable_const_ptr[op] = []
         for i in range(op.numargs()):
             v = op.getarg(i)
@@ -152,7 +153,6 @@
     def _rewrite_changeable_constptrs(self, op, ops_with_movable_const_ptr, moving_obj_tracker):
         newops = []
         for arg_i in ops_with_movable_const_ptr[op]:
-            raise Exception("implement me")
             v = op.getarg(arg_i)
             # assert to make sure we got what we expected
             assert isinstance(v, ConstPtr)
@@ -418,6 +418,7 @@
     DEBUG = False    # forced to True by x86/test/test_zrpy_gc.py
     kind = 'framework'
     round_up = True
+    layoutbuilder = None
 
     def is_shadow_stack(self):
         return self.gcrootmap.is_shadow_stack
diff --git a/rpython/jit/backend/llsupport/test/test_descr.py b/rpython/jit/backend/llsupport/test/test_descr.py
--- a/rpython/jit/backend/llsupport/test/test_descr.py
+++ b/rpython/jit/backend/llsupport/test/test_descr.py
@@ -13,24 +13,24 @@
     T = lltype.GcStruct('T')
     S = lltype.GcStruct('S', ('x', lltype.Char),
                              ('y', lltype.Ptr(T)))
-    descr_s = get_size_descr(None, c0, S, False)
-    descr_t = get_size_descr(None, c0, T, False)
+    descr_s = get_size_descr(c0, S, False)
+    descr_t = get_size_descr(c0, T, False)
     assert descr_s.size == symbolic.get_size(S, False)
     assert descr_t.size == symbolic.get_size(T, False)
     assert descr_s.count_fields_if_immutable() == -1
     assert descr_t.count_fields_if_immutable() == -1
     assert descr_t.gc_fielddescrs == []
     assert len(descr_s.gc_fielddescrs) == 1
-    assert descr_s == get_size_descr(None, c0, S, False)
-    assert descr_s != get_size_descr(None, c1, S, False)
+    assert descr_s == get_size_descr(c0, S, False)
+    assert descr_s != get_size_descr(c1, S, False)
     #
-    descr_s = get_size_descr(None, c1, S, False)
+    descr_s = get_size_descr(c1, S, False)
     assert isinstance(descr_s.size, Symbolic)
     assert descr_s.count_fields_if_immutable() == -1
 
     PARENT = lltype.Struct('P', ('x', lltype.Ptr(T)))
     STRUCT = lltype.GcStruct('S', ('parent', PARENT), ('y', lltype.Ptr(T)))
-    descr_struct = get_size_descr(None, c0, STRUCT, False)
+    descr_struct = get_size_descr(c0, STRUCT, False)
     assert len(descr_struct.gc_fielddescrs) == 2
 
 def test_get_size_descr_immut():
@@ -49,7 +49,7 @@
     for STRUCT, expected in [(S, 0), (T, 1), (U, 3), (V, 3)]:
         for translated in [False, True]:
             c0 = GcCache(translated)
-            descr_s = get_size_descr(None, c0, STRUCT, False)
+            descr_s = get_size_descr(c0, STRUCT, False)
             assert descr_s.count_fields_if_immutable() == expected
 
 def test_get_field_descr():
@@ -329,7 +329,7 @@
     S = lltype.GcStruct('S', ('x', lltype.Char),
                              ('y', lltype.Ptr(T)),
                              ('z', lltype.Ptr(T)))
-    descr1 = get_size_descr(None, c0, S, False)
+    descr1 = get_size_descr(c0, S, False)
     s = symbolic.get_size(S, False)
     assert repr_of_descr(descr1) == '<SizeDescr %d>' % s
     #
diff --git a/rpython/jit/backend/llsupport/test/test_gc.py b/rpython/jit/backend/llsupport/test/test_gc.py
--- a/rpython/jit/backend/llsupport/test/test_gc.py
+++ b/rpython/jit/backend/llsupport/test/test_gc.py
@@ -23,7 +23,7 @@
     #
     # ---------- gc_malloc ----------
     S = lltype.GcStruct('S', ('x', lltype.Signed))
-    sizedescr = descr.get_size_descr(None, gc_ll_descr, S, False)
+    sizedescr = descr.get_size_descr(gc_ll_descr, S, False)
     p = gc_ll_descr.gc_malloc(sizedescr)
     assert record == [(sizedescr.size, p)]
     del record[:]
@@ -143,7 +143,7 @@
 
     def test_gc_malloc(self):
         S = lltype.GcStruct('S', ('x', lltype.Signed))
-        sizedescr = descr.get_size_descr(None, self.gc_ll_descr, S, False)
+        sizedescr = descr.get_size_descr(self.gc_ll_descr, S, False)
         p = self.gc_ll_descr.gc_malloc(sizedescr)
         assert lltype.typeOf(p) == llmemory.GCREF
         assert self.llop1.record == [("fixedsize", repr(sizedescr.size),
diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py
--- a/rpython/jit/metainterp/history.py
+++ b/rpython/jit/metainterp/history.py
@@ -351,161 +351,6 @@
 
 CONST_NULL = ConstPtr(ConstPtr.value)
 
-class Box(AbstractValue):
-    __slots__ = ()
-    _extended_display = True
-    _counter = 0
-    is_box = True  # hint that we want to make links in graphviz from this
-
-    @staticmethod
-    def _new(x):
-        "NOT_RPYTHON"
-        kind = getkind(lltype.typeOf(x))
-        if kind == "int":
-            intval = lltype.cast_primitive(lltype.Signed, x)
-            return BoxInt(intval)
-        elif kind == "ref":
-            ptrval = lltype.cast_opaque_ptr(llmemory.GCREF, x)
-            return BoxPtr(ptrval)
-        elif kind == "float":
-            return BoxFloat(longlong.getfloatstorage(x))
-        else:
-            raise NotImplementedError(kind)
-
-    def nonconstbox(self):
-        return self
-
-    def __repr__(self):
-        result = str(self)
-        if self._extended_display:
-            result += '(%s)' % self._getrepr_()
-        return result
-
-    def __str__(self):
-        if not hasattr(self, '_str'):
-            try:
-                if self.type == INT:
-                    t = 'i'
-                elif self.type == FLOAT:
-                    t = 'f'
-                else:
-                    t = 'p'
-            except AttributeError:
-                t = 'b'
-            self._str = '%s%d' % (t, Box._counter)
-            Box._counter += 1
-        return self._str
-
-    def _get_str(self):    # for debugging only
-        return self.constbox()._get_str()
-
-    def forget_value(self):
-        raise NotImplementedError
-
-class BoxInt(Box):
-    type = INT
-    _attrs_ = ('value',)
-
-    def __init__(self, value=0):
-        raise Exception("boxes no longer supported")
-        if not we_are_translated():
-            if is_valid_int(value):
-                value = int(value)    # bool -> int
-            else:
-                assert lltype.typeOf(value) == lltype.Signed
-        self.value = value
-
-    def forget_value(self):
-        self.value = 0
-
-    def constbox(self):
-        return ConstInt(self.value)
-
-    def getint(self):
-        return self.value
-
-    def getaddr(self):
-        return heaptracker.int2adr(self.value)
-
-    def _get_hash_(self):
-        return make_hashable_int(self.value)
-
-    def nonnull(self):
-        return self.value != 0
-
-    def _getrepr_(self):
-        return self.value
-
-    def repr_rpython(self):
-        return repr_rpython(self, 'bi')
-
-class BoxFloat(Box):
-    type = FLOAT
-    _attrs_ = ('value',)
-
-    def __init__(self, valuestorage=longlong.ZEROF):
-        xxxx
-        assert lltype.typeOf(valuestorage) is longlong.FLOATSTORAGE
-        self.value = valuestorage
-
-    def forget_value(self):
-        self.value = longlong.ZEROF
-
-    def constbox(self):
-        return ConstFloat(self.value)
-
-    def getfloatstorage(self):
-        return self.value
-
-    def _get_hash_(self):
-        return longlong.gethash(self.value)
-
-    def nonnull(self):
-        return bool(longlong.extract_bits(self.value))
-
-    def _getrepr_(self):
-        return self.getfloat()
-
-    def repr_rpython(self):
-        return repr_rpython(self, 'bf')
-
-class BoxPtr(Box):
-    type = REF
-    _attrs_ = ('value',)
-
-    def __init__(self, value=lltype.nullptr(llmemory.GCREF.TO)):
-        raise Exception("boxes no longer supported")
-        assert lltype.typeOf(value) == llmemory.GCREF
-        self.value = value
-
-    def forget_value(self):
-        self.value = lltype.nullptr(llmemory.GCREF.TO)
-
-    def constbox(self):
-        return ConstPtr(self.value)
-
-    def getref_base(self):
-        return self.value
-
-    def getref(self, PTR):
-        return lltype.cast_opaque_ptr(PTR, self.getref_base())
-    getref._annspecialcase_ = 'specialize:arg(1)'
-
-    def _get_hash_(self):
-        if self.value:
-            return lltype.identityhash(self.value)
-        else:
-            return 0
-
-    def nonnull(self):
-        return bool(self.value)
-
-    def repr_rpython(self):
-        return repr_rpython(self, 'bp')
-
-    _getrepr_ = repr_pointer
-
-
 # ____________________________________________________________
 
 
diff --git a/rpython/jit/metainterp/optimizeopt/__init__.py b/rpython/jit/metainterp/optimizeopt/__init__.py
--- a/rpython/jit/metainterp/optimizeopt/__init__.py
+++ b/rpython/jit/metainterp/optimizeopt/__init__.py
@@ -33,14 +33,14 @@
 
 def build_opt_chain(metainterp_sd, enable_opts):
     optimizations = []
-    unroll = False # 'unroll' in enable_opts    # 'enable_opts' is normally a dict
+    unroll = 'unroll' in enable_opts    # 'enable_opts' is normally a dict
     for name, opt in unroll_all_opts:
         if name in enable_opts:
             if opt is not None:
                 o = opt()
                 optimizations.append(o)
 
-    if 1 or ('rewrite' not in enable_opts or 'virtualize' not in enable_opts
+    if ('rewrite' not in enable_opts or 'virtualize' not in enable_opts
         or 'heap' not in enable_opts or 'unroll' not in enable_opts
         or 'pure' not in enable_opts):
         optimizations.append(OptSimplify(unroll))
@@ -59,7 +59,6 @@
                                                           loop.operations)
         optimizations, unroll = build_opt_chain(metainterp_sd, enable_opts)
         if unroll:
-            raise Exception("unrolling disabled")
             return optimize_unroll(metainterp_sd, jitdriver_sd, loop,
                                    optimizations,
                                    inline_short_preamble, start_state,
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -3,7 +3,7 @@
 from rpython.jit.metainterp.logger import LogOperations
 from rpython.jit.metainterp.history import Const, ConstInt, REF, ConstPtr
 from rpython.jit.metainterp.optimizeopt.intutils import IntBound,\
-     IntUnbounded, ConstIntBound, MININT, MAXINT
+     ConstIntBound, MININT, MAXINT
 from rpython.jit.metainterp.optimizeopt.util import make_dispatcher_method
 from rpython.jit.metainterp.resoperation import rop, AbstractResOp, GuardResOp
 from rpython.jit.metainterp.optimizeopt import info
@@ -11,21 +11,6 @@
 from rpython.rlib.objectmodel import specialize, we_are_translated
 
 
-class LenBound(object):
-    def __init__(self, mode, descr, bound):
-        self.mode = mode
-        self.descr = descr
-        self.bound = bound
-
-    def clone(self):
-        return LenBound(self.mode, self.descr, self.bound.clone())
-
-    def generalization_of(self, other):
-        return (other is not None and
-                self.mode == other.mode and
-                self.descr == other.descr and
-                self.bound.contains_bound(other.bound))
-
 
 
 CONST_0      = ConstInt(0)
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -106,6 +106,8 @@
         escape_n(f)
         jump()
         """
+        import pdb
+        pdb.set_trace()
         self.optimize_loop(ops, ops)
 
     def test_constant_propagate(self):
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_util.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py
@@ -421,10 +421,7 @@
 
         jump_args = jumpop.getarglist()[:]
         operations = operations[:-1]
-        memo = compile.Memo(inputargs, jump_args)
-        cloned_operations = [op.clone(memo) for op in operations]
-        for op in cloned_operations:
-            op.is_source_op = True
+        xxx
 
         preamble = TreeLoop('preamble')
         preamble.inputargs = inputargs
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -390,8 +390,6 @@
         assert short[-1].getopnum() == rop.JUMP
         target_token = start_label.getdescr()
         assert isinstance(target_token, TargetToken)
-
-        xxx
         # Turn guards into conditional jumps to the preamble
         #for i in range(len(short)):
         #    op = short[i]
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -134,16 +134,6 @@
         if self.type != 'v':
             newop.copy_value_from(self)
         return newop
-            
-    def clone(self, memo):
-        args = [memo.get(arg, arg) for arg in self.getarglist()]
-        descr = self.getdescr()
-        op = ResOperation(self.getopnum(), args[:], descr)
-        if not we_are_translated():
-            op.name = self.name
-            op.pc = self.pc
-        memo.set(self, op)
-        return op
 
     def repr(self, memo, graytext=False):
         # RPython-friendly version
@@ -344,14 +334,6 @@
         newop.rd_frame_info_list = self.rd_frame_info_list
         return newop
 
-    def clone(self, memo):
-        newop = AbstractResOp.clone(self, memo)
-        assert isinstance(newop, GuardResOp)
-        newop.setfailargs(self.getfailargs())
-        newop.rd_snapshot = self.rd_snapshot
-        newop.rd_frame_info_list = self.rd_frame_info_list
-        return newop
-
 # ===========
 # type mixins
 # ===========
@@ -377,9 +359,6 @@
     def nonnull(self):
         return self._resint != 0
 
-    def clone_input_arg(self):
-        return InputArgInt()
-
 class FloatOp(object):
     _mixin_ = True
 
@@ -402,9 +381,6 @@
     def nonnull(self):
         return bool(longlong.extract_bits(self._resfloat))
 
-    def clone_input_arg(self):
-        return InputArgFloat()
-
 class RefOp(object):
     _mixin_ = True
 
@@ -436,9 +412,6 @@
     def nonnull(self):
         return bool(self._resref)
 
-    def clone_input_arg(self):
-        return InputArgRef()
-
 class AbstractInputArg(AbstractValue):
     _forwarded = None
 
@@ -469,16 +442,10 @@
     def __init__(self, intval=0):
         self.setint(intval)
 
-    def clone_input_arg(self):
-        return InputArgInt()
-
 class InputArgFloat(FloatOp, AbstractInputArg):
     def __init__(self, f=0.0):
         self.setfloatstorage(f)
 
-    def clone_input_arg(self):
-        return InputArgFloat()
-
 class InputArgRef(RefOp, AbstractInputArg):
     def __init__(self, r=lltype.nullptr(llmemory.GCREF.TO)):
         self.setref_base(r)
@@ -486,9 +453,6 @@
     def reset_value(self):
         self.setref_base(lltype.nullptr(llmemory.GCREF.TO))
 
-    def clone_input_arg(self):
-        return InputArgRef()
-
 # ============
 # arity mixins
 # ============

From noreply at buildbot.pypy.org  Thu Jun  4 18:38:18 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Thu,  4 Jun 2015 18:38:18 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: kill kill kill
Message-ID: <20150604163818.948011C12E8@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77870:ef10515df039
Date: 2015-06-04 18:23 +0200
http://bitbucket.org/pypy/pypy/changeset/ef10515df039/

Log:	kill kill kill

diff --git a/rpython/jit/metainterp/logger.py b/rpython/jit/metainterp/logger.py
--- a/rpython/jit/metainterp/logger.py
+++ b/rpython/jit/metainterp/logger.py
@@ -1,5 +1,4 @@
-from rpython.jit.metainterp.history import (ConstInt, BoxInt, ConstFloat,
-    BoxFloat, TargetToken)
+from rpython.jit.metainterp.history import ConstInt, ConstFloat
 from rpython.jit.metainterp.resoperation import rop, AbstractInputArg
 from rpython.rlib.debug import (have_debug_prints, debug_start, debug_stop,
     debug_print)
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -106,8 +106,6 @@
         escape_n(f)
         jump()
         """
-        import pdb
-        pdb.set_trace()
         self.optimize_loop(ops, ops)
 
     def test_constant_propagate(self):
diff --git a/rpython/jit/metainterp/optimizeopt/vstring.py b/rpython/jit/metainterp/optimizeopt/vstring.py
--- a/rpython/jit/metainterp/optimizeopt/vstring.py
+++ b/rpython/jit/metainterp/optimizeopt/vstring.py
@@ -1,8 +1,8 @@
 from rpython.jit.codewriter.effectinfo import EffectInfo
-from rpython.jit.metainterp.history import (BoxInt, Const, ConstInt, ConstPtr,
-    get_const_ptr_for_string, get_const_ptr_for_unicode, BoxPtr, REF, INT,
+from rpython.jit.metainterp.history import (Const, ConstInt, ConstPtr,
+    get_const_ptr_for_string, get_const_ptr_for_unicode, REF, INT,
     DONT_CHANGE)
-from rpython.jit.metainterp.optimizeopt import optimizer, virtualize
+from rpython.jit.metainterp.optimizeopt import optimizer
 from rpython.jit.metainterp.optimizeopt.optimizer import CONST_0, CONST_1
 from rpython.jit.metainterp.optimizeopt.optimizer import llhelper, REMOVED
 from rpython.jit.metainterp.optimizeopt.util import make_dispatcher_method
diff --git a/rpython/jit/metainterp/test/test_resume.py b/rpython/jit/metainterp/test/test_resume.py
--- a/rpython/jit/metainterp/test/test_resume.py
+++ b/rpython/jit/metainterp/test/test_resume.py
@@ -4,7 +4,7 @@
 from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
 from rpython.jit.metainterp.resume import *
 from rpython.jit.metainterp.optimizeopt.info import AbstractVirtualPtrInfo
-from rpython.jit.metainterp.history import BoxInt, BoxPtr, ConstInt
+from rpython.jit.metainterp.history import ConstInt
 from rpython.jit.metainterp.history import ConstPtr, ConstFloat
 from rpython.jit.metainterp.optimizeopt.test.test_util import LLtypeMixin
 from rpython.jit.metainterp import executor
diff --git a/rpython/jit/metainterp/typesystem.py b/rpython/jit/metainterp/typesystem.py
--- a/rpython/jit/metainterp/typesystem.py
+++ b/rpython/jit/metainterp/typesystem.py
@@ -33,7 +33,6 @@
     nullptr = staticmethod(lltype.nullptr)
     cast_instance_to_base_ref = staticmethod(cast_instance_to_base_ptr)
     BASETYPE = llmemory.GCREF
-    BoxRef = history.BoxPtr
     ConstRef = history.ConstPtr
     loops_done_with_this_frame_ref = None # patched by compile.py
     NULLREF = history.ConstPtr.value
diff --git a/rpython/jit/tool/oparser_model.py b/rpython/jit/tool/oparser_model.py
--- a/rpython/jit/tool/oparser_model.py
+++ b/rpython/jit/tool/oparser_model.py
@@ -4,7 +4,6 @@
 def get_real_model():
     class LoopModel(object):
         from rpython.jit.metainterp.history import TreeLoop, JitCellToken
-        from rpython.jit.metainterp.history import Box, BoxInt, BoxFloat
         from rpython.jit.metainterp.history import ConstInt, ConstPtr, ConstFloat
         from rpython.jit.metainterp.history import BasicFailDescr, BasicFinalDescr, TargetToken
         from rpython.jit.metainterp.typesystem import llhelper

From noreply at buildbot.pypy.org  Thu Jun  4 18:38:19 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Thu,  4 Jun 2015 18:38:19 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: this is I think necessary
Message-ID: <20150604163819.C0BBB1C12E8@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77871:96fb95477afe
Date: 2015-06-04 18:38 +0200
http://bitbucket.org/pypy/pypy/changeset/96fb95477afe/

Log:	this is I think necessary

diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -488,7 +488,7 @@
         op = self.replace_op_with(op, op.getopnum())
         # XXX look in C and maybe specialize on number of args
         for i in range(op.numargs()):
-            arg = self.force_box(op.getarg(i))
+            arg = self.get_box_replacement(self.force_box(op.getarg(i)))
             #self.ensure_imported(value)
             #    newbox = value.force_box(self)
             op.setarg(i, arg)

From noreply at buildbot.pypy.org  Thu Jun  4 18:39:39 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Thu,  4 Jun 2015 18:39:39 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: no,
	we already call this from force_box
Message-ID: <20150604163939.D38D01C034D@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77872:d3db30dc2194
Date: 2015-06-04 18:39 +0200
http://bitbucket.org/pypy/pypy/changeset/d3db30dc2194/

Log:	no, we already call this from force_box

diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -488,7 +488,7 @@
         op = self.replace_op_with(op, op.getopnum())
         # XXX look in C and maybe specialize on number of args
         for i in range(op.numargs()):
-            arg = self.get_box_replacement(self.force_box(op.getarg(i)))
+            arg = self.force_box(op.getarg(i))
             #self.ensure_imported(value)
             #    newbox = value.force_box(self)
             op.setarg(i, arg)

From noreply at buildbot.pypy.org  Thu Jun  4 18:47:09 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Thu,  4 Jun 2015 18:47:09 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: silenced rpython complaints
Message-ID: <20150604164709.04BE71C034D@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r77873:e5d7788d191b
Date: 2015-06-04 18:47 +0200
http://bitbucket.org/pypy/pypy/changeset/e5d7788d191b/

Log:	silenced rpython complaints

diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -2638,6 +2638,7 @@
 
     def genop_vec_int_expand(self, op, arglocs, resloc):
         srcloc, sizeloc = arglocs
+        assert isinstance(srcloc, RegLoc)
         assert not srcloc.is_xmm
         size = sizeloc.value
         if size == 8:
diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -1635,10 +1635,10 @@
 
     def consider_vec_int_expand(self, op):
         arg = op.getarg(0)
+        args = op.getarglist()
         if isinstance(arg, Const):
             srcloc = self.rm.convert_to_imm(arg)
         else:
-            args = op.getarglist()
             srcloc = self.make_sure_var_in_reg(arg, args)
         resloc = self.xrm.force_allocate_reg(op.result, args)
         assert isinstance(op.result, BoxVector)

From noreply at buildbot.pypy.org  Thu Jun  4 23:10:01 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Thu,  4 Jun 2015 23:10:01 +0200 (CEST)
Subject: [pypy-commit] pypy default: assign to self.flags later in __init__
 and fewer times, seems to help immutablility
Message-ID: <20150604211001.82DC41C0987@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: 
Changeset: r77874:f183f9e36f8d
Date: 2015-06-05 00:10 +0300
http://bitbucket.org/pypy/pypy/changeset/f183f9e36f8d/

Log:	assign to self.flags later in __init__ and fewer times, seems to
	help immutablility

diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py
--- a/pypy/module/micronumpy/concrete.py
+++ b/pypy/module/micronumpy/concrete.py
@@ -438,7 +438,7 @@
     def __init__(self, shape, dtype, order, strides, backstrides,
                  storage=lltype.nullptr(RAW_STORAGE), zero=True):
         gcstruct = V_OBJECTSTORE
-        self.flags = NPY.ARRAY_ALIGNED | NPY.ARRAY_WRITEABLE
+        flags = NPY.ARRAY_ALIGNED | NPY.ARRAY_WRITEABLE
         if storage == lltype.nullptr(RAW_STORAGE):
             length = support.product(shape) 
             if dtype.num == NPY.OBJECT:
@@ -446,15 +446,16 @@
                 gcstruct = _create_objectstore(storage, length, dtype.elsize)
             else:
                 storage = dtype.itemtype.malloc(length * dtype.elsize, zero=zero)
-            self.flags |= NPY.ARRAY_OWNDATA
+            flags |= NPY.ARRAY_OWNDATA
         start = calc_start(shape, strides)
         ConcreteArrayNotOwning.__init__(self, shape, dtype, order, strides, backstrides,
                                         storage, start=start)
         self.gcstruct = gcstruct
         if is_c_contiguous(self):
-            self.flags |= NPY.ARRAY_C_CONTIGUOUS
+            flags |= NPY.ARRAY_C_CONTIGUOUS
         if is_f_contiguous(self):
-            self.flags |= NPY.ARRAY_F_CONTIGUOUS
+            flags |= NPY.ARRAY_F_CONTIGUOUS
+        self.flags = flags
 
     def __del__(self):
         if self.gcstruct:
@@ -469,14 +470,15 @@
                                         strides, backstrides, storage, start)
         self.orig_base = orig_base
         if isinstance(orig_base, W_NumpyObject):
-            self.flags = orig_base.get_flags() & NPY.ARRAY_ALIGNED
-            self.flags |=  orig_base.get_flags() & NPY.ARRAY_WRITEABLE
+            flags = orig_base.get_flags() & NPY.ARRAY_ALIGNED
+            flags |=  orig_base.get_flags() & NPY.ARRAY_WRITEABLE
         else:
-            self.flags = 0
+            flags = 0
         if is_c_contiguous(self):
-            self.flags |= NPY.ARRAY_C_CONTIGUOUS
+            flags |= NPY.ARRAY_C_CONTIGUOUS
         if is_f_contiguous(self):
-            self.flags |= NPY.ARRAY_F_CONTIGUOUS
+            flags |= NPY.ARRAY_F_CONTIGUOUS
+        self.flags = flags
 
     def base(self):
         return self.orig_base
@@ -524,12 +526,13 @@
         self.size = support.product(shape) * self.dtype.elsize
         self.start = start
         self.orig_arr = orig_arr
-        self.flags = parent.flags & NPY.ARRAY_ALIGNED
-        self.flags |= parent.flags & NPY.ARRAY_WRITEABLE
+        flags = parent.flags & NPY.ARRAY_ALIGNED
+        flags |= parent.flags & NPY.ARRAY_WRITEABLE
         if is_c_contiguous(self):
-            self.flags |= NPY.ARRAY_C_CONTIGUOUS
+            flags |= NPY.ARRAY_C_CONTIGUOUS
         if is_f_contiguous(self):
-            self.flags |= NPY.ARRAY_F_CONTIGUOUS
+            flags |= NPY.ARRAY_F_CONTIGUOUS
+        self.flags = flags
 
     def base(self):
         return self.orig_arr

From noreply at buildbot.pypy.org  Fri Jun  5 05:56:14 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Fri,  5 Jun 2015 05:56:14 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Fix test.
Message-ID: <20150605035614.E1E281C0FD4@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob <me at manueljacob.de>
Branch: py3k
Changeset: r77875:8655efcabf66
Date: 2015-06-05 05:55 +0200
http://bitbucket.org/pypy/pypy/changeset/8655efcabf66/

Log:	Fix test.

diff --git a/pypy/module/test_lib_pypy/test_grp_extra.py b/pypy/module/test_lib_pypy/test_grp_extra.py
--- a/pypy/module/test_lib_pypy/test_grp_extra.py
+++ b/pypy/module/test_lib_pypy/test_grp_extra.py
@@ -20,7 +20,7 @@
             assert g.gr_gid == 0
             assert 'root' in g.gr_mem or g.gr_mem == []
             assert g.gr_name == name
-            assert isinstance(g.gr_passwd, bytes)    # usually just 'x', don't hope :-)
+            assert isinstance(g.gr_passwd, str)    # usually just 'x', don't hope :-)
             break
         else:
             raise

From noreply at buildbot.pypy.org  Fri Jun  5 08:56:02 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Fri,  5 Jun 2015 08:56:02 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: problem with float abs (expanded
 variable but did not remove second use),
 andps constant pointed to neg constant
Message-ID: <20150605065602.6C13C1C1038@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r77876:062f2c00c3f0
Date: 2015-06-05 08:56 +0200
http://bitbucket.org/pypy/pypy/changeset/062f2c00c3f0/

Log:	problem with float abs (expanded variable but did not remove second
	use), andps constant pointed to neg constant added a test for int32
	expand

diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -211,6 +211,21 @@
         assert int(result) == 7+16+8+16
         self.check_vectorized(2, 2)
 
+    def define_int32_expand():
+        return """
+        a = astype(|30|, int32)
+        c = astype(|1|, int32)
+        c[0] = 16i
+        b = a + c
+        x1 = b -> 7
+        x2 = b -> 8
+        x1 + x2
+        """
+    def test_int32_expand(self):
+        result = self.run("int32_expand")
+        assert int(result) == 7+16+8+16
+        self.check_vectorized(2, 2)
+
     def define_int32_add_const():
         return """
         a = astype(|30|, int32)
diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -103,7 +103,7 @@
         # 0x80000000800000008000000080000000
         single_neg_const = '\x00\x00\x00\x80\x00\x00\x00\x80\x00\x00\x00\x80\x00\x00\x00\x80'
         #
-        data = neg_const + neg_const + abs_const + abs_const + \
+        data = neg_const + abs_const + \
                single_neg_const + single_abs_const
         datablockwrapper = MachineDataBlockWrapper(self.cpu.asmmemmgr, [])
         float_constants = datablockwrapper.malloc_aligned(len(data), alignment=16)

From noreply at buildbot.pypy.org  Fri Jun  5 09:36:05 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Fri,  5 Jun 2015 09:36:05 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: start fighting with unrolling
Message-ID: <20150605073605.136401C1027@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77877:06323de370d7
Date: 2015-06-05 09:18 +0200
http://bitbucket.org/pypy/pypy/changeset/06323de370d7/

Log:	start fighting with unrolling

diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -144,7 +144,7 @@
         self.optimize_loop(ops, expected)
 
     def test_constfold_all(self):
-        from rpython.jit.metainterp.executor import _execute_nonspec
+        from rpython.jit.metainterp.executor import _execute_arglist
         import random
         for opnum in range(rop.INT_ADD, rop.SAME_AS_I+1):
             try:
@@ -168,7 +168,7 @@
             jump()
             """ % (op.lower(), ', '.join(map(str, args)))
             argboxes = [InputArgInt(a) for a in args]
-            expected_value = _execute_nonspec(self.cpu, None, opnum, argboxes)
+            expected_value = _execute_arglist(self.cpu, None, opnum, argboxes)
             expected = """
             []
             escape_n(%d)
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_util.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py
@@ -395,6 +395,8 @@
         from rpython.jit.metainterp.optimizeopt.util import args_dict
 
         self.loop = loop
+        operations = loop.operations
+        inputargs = loop.inputargs
         loop.call_pure_results = args_dict()
         if call_pure_results is not None:
             for k, v in call_pure_results.items():
@@ -405,10 +407,13 @@
         if hasattr(self, 'callinfocollection'):
             metainterp_sd.callinfocollection = self.callinfocollection
         #
-        return optimize_trace(metainterp_sd, None, loop,
+        state = optimize_trace(metainterp_sd, None, loop,
                               self.enable_opts,
                               start_state=start_state,
                               export_state=export_state)
+        compile.forget_optimization_info(operations)
+        compile.forget_optimization_info(inputargs)
+        return state
 
     def unroll_and_optimize(self, loop, call_pure_results=None):
         metainterp_sd = FakeMetaInterpStaticData(self.cpu)
@@ -421,7 +426,6 @@
 
         jump_args = jumpop.getarglist()[:]
         operations = operations[:-1]
-        xxx
 
         preamble = TreeLoop('preamble')
         preamble.inputargs = inputargs
@@ -436,10 +440,9 @@
         assert preamble.operations[-1].getopnum() == rop.LABEL
 
         loop.operations = [preamble.operations[-1]] + \
-                          cloned_operations + \
-                          [ResOperation(rop.JUMP, [memo.get(a, a) for a in jump_args],
+                          operations + \
+                          [ResOperation(rop.JUMP, jump_args[:],
                                         descr=token)]
-                          #[inliner.inline_op(jumpop)]
         
         assert loop.operations[-1].getopnum() == rop.JUMP
         assert loop.operations[0].getopnum() == rop.LABEL
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -175,11 +175,11 @@
 
     def export_state(self, targetop):
         original_jump_args = targetop.getarglist()
-        jump_args = [self.getvalue(a).get_key_box() for a in original_jump_args]
+        jump_args = [self.get_box_replacement(a) for a in original_jump_args]
 
         virtual_state = self.get_virtual_state(jump_args)
 
-        values = [self.getvalue(arg) for arg in jump_args]
+        values = [self.getinfo(arg) for arg in jump_args]
         inputargs = virtual_state.make_inputargs(values, self.optimizer)
         short_inputargs = virtual_state.make_inputargs(values, self.optimizer, keyboxes=True)
 
@@ -401,6 +401,7 @@
         #        short[i] = op
 
         # Clone ops and boxes to get private versions and
+        return
         short_inputargs = short[0].getarglist()
         boxmap = {}
         newargs = [None] * len(short_inputargs)
@@ -411,8 +412,8 @@
             else:
                 newargs[i] = a.clone_input_arg()
                 boxmap[a] = newargs[i]
-        memo = Memo(short_inputargs, newargs)
-        target_token.assumed_classes = {}
+        #memo = Memo(short_inputargs, newargs)
+        #target_token.assumed_classes = {}
         for i in range(len(short)):
             op = short[i]
             newop = op.clone(memo)
diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py
@@ -457,7 +457,8 @@
         self.info_counter = -1
         self.numnotvirtuals = 0
         for s in state:
-            s.enum(self)
+            if s:
+                s.enum(self)
 
     def generalization_of(self, other, bad=None, cpu=None):
         state = GenerateGuardState(cpu=cpu, bad=bad)
@@ -528,6 +529,9 @@
         return self.optimizer.getvalue(box)
 
     def state(self, box):
+        if box.type == 'r':
+            xxxx
+        return None
         value = self.getvalue(box)
         box = value.get_key_box()
         try:
@@ -551,12 +555,15 @@
             opt = self.optimizer.optearlyforce
         else:
             opt = self.optimizer
-        values = [self.getvalue(box).force_at_end_of_preamble(already_forced,
-                                                              opt)
-                  for box in jump_args]
+        for box in jump_args:
+            if box.type == 'r':
+                zxsadsadsa
+        #values = [self.getvalue(box).force_at_end_of_preamble(already_forced,
+        #                                                      opt)
+        #          for box in jump_args]
 
-        for value in values:
-            value.visitor_walk_recursive(self)
+        #for value in values:
+        #    value.visitor_walk_recursive(self)
         return VirtualState([self.state(box) for box in jump_args])
 
     def visit_not_virtual(self, value):
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -122,6 +122,7 @@
 
     def copy_and_change(self, opnum, args=None, descr=None):
         "shallow copy: the returned operation is meant to be used in place of self"
+        # XXX specialize
         from rpython.jit.metainterp.history import DONT_CHANGE
         
         if args is None:
diff --git a/rpython/jit/tool/oparser.py b/rpython/jit/tool/oparser.py
--- a/rpython/jit/tool/oparser.py
+++ b/rpython/jit/tool/oparser.py
@@ -28,19 +28,10 @@
         if args is not None:
             op.initarglist(args)
         else:
-            op.initarglist(self._args)
+            op.initarglist(self._args[:])
         assert descr is None
         return op
 
-    def clone(self, memo):
-        op = self.__class__()
-        op.initarglist([memo.get(arg, arg) for arg in self.getarglist()])
-        memo.set(self, op)
-        return op
-
-    def clone_input_arg(self):
-        assert self.type == 'r'
-        return InputArgRef()
 
 class ESCAPE_OP_I(ESCAPE_OP):
     type = 'i'
@@ -76,11 +67,6 @@
     def getopname(self):
         return 'force_spill'
 
-    def clone(self):
-        op = FORCE_SPILL()
-        op.initarglist(self.getarglist()[:])
-        return op
-
     def copy_and_change(self, opnum, args=None, descr=None):
         assert opnum == self.OPNUM
         newop = FORCE_SPILL()

From noreply at buildbot.pypy.org  Fri Jun  5 09:36:06 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Fri,  5 Jun 2015 09:36:06 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: redisable unrolling - let's get
 everything else going first
Message-ID: <20150605073606.5A2041C1027@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77878:e48479cbf6f8
Date: 2015-06-05 09:19 +0200
http://bitbucket.org/pypy/pypy/changeset/e48479cbf6f8/

Log:	redisable unrolling - let's get everything else going first

diff --git a/rpython/jit/metainterp/optimizeopt/__init__.py b/rpython/jit/metainterp/optimizeopt/__init__.py
--- a/rpython/jit/metainterp/optimizeopt/__init__.py
+++ b/rpython/jit/metainterp/optimizeopt/__init__.py
@@ -33,14 +33,14 @@
 
 def build_opt_chain(metainterp_sd, enable_opts):
     optimizations = []
-    unroll = 'unroll' in enable_opts    # 'enable_opts' is normally a dict
+    unroll = False # 'unroll' in enable_opts    # 'enable_opts' is normally a dict
     for name, opt in unroll_all_opts:
         if name in enable_opts:
             if opt is not None:
                 o = opt()
                 optimizations.append(o)
 
-    if ('rewrite' not in enable_opts or 'virtualize' not in enable_opts
+    if 1 or ('rewrite' not in enable_opts or 'virtualize' not in enable_opts
         or 'heap' not in enable_opts or 'unroll' not in enable_opts
         or 'pure' not in enable_opts):
         optimizations.append(OptSimplify(unroll))

From noreply at buildbot.pypy.org  Fri Jun  5 09:36:07 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Fri,  5 Jun 2015 09:36:07 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: a test and a fix
Message-ID: <20150605073607.808501C1027@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77879:8b21abbf1e11
Date: 2015-06-05 09:32 +0200
http://bitbucket.org/pypy/pypy/changeset/8b21abbf1e11/

Log:	a test and a fix

diff --git a/rpython/jit/metainterp/optimizeopt/TODO b/rpython/jit/metainterp/optimizeopt/TODO
--- a/rpython/jit/metainterp/optimizeopt/TODO
+++ b/rpython/jit/metainterp/optimizeopt/TODO
@@ -1,4 +1,5 @@
 * certain cases where VirtualArray or VirtualStructArray is forced (but
   heap.py is not notified about fields being dirty)
-* arraylen_gc is not handling length bound optimization at all
-* mark_opaque_pointer is ignored
+* arraylen_gc is not handling length bound optimization at all (we need to
+  wait till unrolling for tests)
+* mark_opaque_pointer is ignored (which is fine until unrolling)
diff --git a/rpython/jit/metainterp/optimizeopt/heap.py b/rpython/jit/metainterp/optimizeopt/heap.py
--- a/rpython/jit/metainterp/optimizeopt/heap.py
+++ b/rpython/jit/metainterp/optimizeopt/heap.py
@@ -223,10 +223,16 @@
     def register_dirty_field(self, descr, info):
         self.field_cache(descr).register_dirty_field(info)
 
+    def register_dirty_array_field(self, arraydescr, index, info):
+        self.arrayitem_cache(arraydescr, index).register_dirty_field(info)
+
     def clean_caches(self):
         del self._lazy_setfields_and_arrayitems[:]
         for descr, cf in self.cached_fields.items():
             cf.invalidate(descr)
+        for submap in self.cached_arrayitems.itervalues():
+            for index, cf in submap.iteritems():
+                cf.invalidate(None)
         self.cached_arrayitems.clear()
         self.cached_dict_reads.clear()
 
diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py
--- a/rpython/jit/metainterp/optimizeopt/info.py
+++ b/rpython/jit/metainterp/optimizeopt/info.py
@@ -225,11 +225,7 @@
     def visitor_walk_recursive(self, op, visitor, optimizer):
         itemboxes = self.buffer.values
         visitor.register_virtual_fields(op, itemboxes)
-        #for itembox in itemboxes:
-        #    xxx
-        #    itemvalue = self.get_item_value(i)
-        #    if itemvalue is not None:
-        #        itemvalue.visitor_walk_recursive(visitor)
+        # there can be no virtuals stored in raw buffer
 
     @specialize.argtype(1)
     def visitor_dispatch_virtual_type(self, visitor):
@@ -306,7 +302,9 @@
                                      [op, ConstInt(i), subbox],
                                       descr=arraydescr)
                 optforce._emit_operation(setop)
-                # xxxx optforce.optheap
+                if optforce.optheap is not None:
+                    optforce.optheap.register_dirty_array_field(
+                        arraydescr, i, self)
 
     def setitem(self, index, op, cf=None, optheap=None):
         if self._items is None:
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -5559,6 +5559,17 @@
         """
         self.optimize_loop(ops, expected)
 
+    def test_dirty_array_field_after_force(self):
+        ops = """
+        []
+        p0 = new_array(1, descr=arraydescr)
+        setarrayitem_gc(p0, 0, 0, descr=arraydescr)
+        escape_n(p0) # force
+        call_may_force_n(1, descr=mayforcevirtdescr)
+        i1 = getarrayitem_gc_i(p0, 0, descr=arraydescr)
+        finish(i1)
+        """
+        self.optimize_loop(ops, ops)
 
 class TestLLtype(BaseTestOptimizeBasic, LLtypeMixin):
     pass

From noreply at buildbot.pypy.org  Fri Jun  5 09:36:08 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Fri,  5 Jun 2015 09:36:08 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: a test, explanation and update TODO
Message-ID: <20150605073608.A80821C1027@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77880:93f24053fbe7
Date: 2015-06-05 09:35 +0200
http://bitbucket.org/pypy/pypy/changeset/93f24053fbe7/

Log:	a test, explanation and update TODO

diff --git a/rpython/jit/metainterp/optimizeopt/TODO b/rpython/jit/metainterp/optimizeopt/TODO
--- a/rpython/jit/metainterp/optimizeopt/TODO
+++ b/rpython/jit/metainterp/optimizeopt/TODO
@@ -1,5 +1,3 @@
-* certain cases where VirtualArray or VirtualStructArray is forced (but
-  heap.py is not notified about fields being dirty)
 * arraylen_gc is not handling length bound optimization at all (we need to
   wait till unrolling for tests)
 * mark_opaque_pointer is ignored (which is fine until unrolling)
diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py
--- a/rpython/jit/metainterp/optimizeopt/info.py
+++ b/rpython/jit/metainterp/optimizeopt/info.py
@@ -374,7 +374,8 @@
                                               [op, ConstInt(index), subbox],
                                               descr=flddescr)
                     optforce._emit_operation(setfieldop)
-                    # XXX optforce.optheap
+                    # heapcache does not work for interiorfields
+                    # if it does, we would need a fix here
                 i += 1
 
     def visitor_walk_recursive(self, instbox, visitor, optimizer):
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -5571,5 +5571,18 @@
         """
         self.optimize_loop(ops, ops)
 
+    def test_dirty_array_of_structs_field_after_force(self):
+        ops = """
+        []
+        p0 = new_array_clear(1, descr=complexarraydescr)
+        setinteriorfield_gc(p0, 0, 0.0, descr=complexrealdescr)
+        setinteriorfield_gc(p0, 0, 0.0, descr=compleximagdescr)
+        escape_n(p0) # force
+        call_may_force_n(1, descr=mayforcevirtdescr)
+        f1 = getinteriorfield_gc_f(p0, 0, descr=compleximagdescr)
+        finish(f1)
+        """
+        self.optimize_loop(ops, ops)
+
 class TestLLtype(BaseTestOptimizeBasic, LLtypeMixin):
     pass

From noreply at buildbot.pypy.org  Fri Jun  5 09:49:54 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Fri,  5 Jun 2015 09:49:54 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: give up on integrating
 rewrite_consptr. we allow stuff to be modified in place in this simple case
Message-ID: <20150605074954.49F1A1C1038@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77881:e7a2ace43aac
Date: 2015-06-05 09:49 +0200
http://bitbucket.org/pypy/pypy/changeset/e7a2ace43aac/

Log:	give up on integrating rewrite_consptr. we allow stuff to be
	modified in place in this simple case

diff --git a/rpython/jit/backend/llsupport/gc.py b/rpython/jit/backend/llsupport/gc.py
--- a/rpython/jit/backend/llsupport/gc.py
+++ b/rpython/jit/backend/llsupport/gc.py
@@ -156,15 +156,13 @@
             v = op.getarg(arg_i)
             # assert to make sure we got what we expected
             assert isinstance(v, ConstPtr)
-            result_ptr = BoxPtr()
             array_index = moving_obj_tracker.get_array_index(v)
             load_op = ResOperation(rop.GETARRAYITEM_GC,
                     [moving_obj_tracker.const_ptr_gcref_array,
                         ConstInt(array_index)],
-                    result_ptr,
                     descr=moving_obj_tracker.ptr_array_descr)
             newops.append(load_op)
-            op.setarg(arg_i, result_ptr)
+            op.setarg(arg_i, load_op)
         #
         newops.append(op)
         return newops

From noreply at buildbot.pypy.org  Fri Jun  5 09:54:10 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Fri,  5 Jun 2015 09:54:10 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: fix all the llsupport tests
Message-ID: <20150605075410.2BF401C1038@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77882:d2d6134391d3
Date: 2015-06-05 09:54 +0200
http://bitbucket.org/pypy/pypy/changeset/d2d6134391d3/

Log:	fix all the llsupport tests

diff --git a/rpython/jit/backend/llsupport/assembler.py b/rpython/jit/backend/llsupport/assembler.py
--- a/rpython/jit/backend/llsupport/assembler.py
+++ b/rpython/jit/backend/llsupport/assembler.py
@@ -2,7 +2,7 @@
 from rpython.jit.backend.llsupport.memcpy import memcpy_fn, memset_fn
 from rpython.jit.backend.llsupport.symbolic import WORD
 from rpython.jit.metainterp.history import (INT, REF, FLOAT, JitCellToken,
-    ConstInt, BoxInt, AbstractFailDescr)
+    ConstInt, AbstractFailDescr)
 from rpython.jit.metainterp.resoperation import ResOperation, rop
 from rpython.rlib import rgc
 from rpython.rlib.debug import (debug_start, debug_stop, have_debug_prints,
diff --git a/rpython/jit/backend/llsupport/gc.py b/rpython/jit/backend/llsupport/gc.py
--- a/rpython/jit/backend/llsupport/gc.py
+++ b/rpython/jit/backend/llsupport/gc.py
@@ -9,7 +9,7 @@
 from rpython.rtyper.annlowlevel import llhelper, cast_instance_to_gcref
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
 from rpython.jit.codewriter import heaptracker
-from rpython.jit.metainterp.history import ConstPtr, AbstractDescr, BoxPtr, ConstInt
+from rpython.jit.metainterp.history import ConstPtr, AbstractDescr, ConstInt
 from rpython.jit.metainterp.resoperation import rop, ResOperation
 from rpython.jit.backend.llsupport import symbolic, jitframe
 from rpython.jit.backend.llsupport.symbolic import WORD
@@ -157,7 +157,7 @@
             # assert to make sure we got what we expected
             assert isinstance(v, ConstPtr)
             array_index = moving_obj_tracker.get_array_index(v)
-            load_op = ResOperation(rop.GETARRAYITEM_GC,
+            load_op = ResOperation(rop.GETARRAYITEM_GC_R,
                     [moving_obj_tracker.const_ptr_gcref_array,
                         ConstInt(array_index)],
                     descr=moving_obj_tracker.ptr_array_descr)
diff --git a/rpython/jit/backend/llsupport/test/test_pinned_object_rewrite.py b/rpython/jit/backend/llsupport/test/test_pinned_object_rewrite.py
--- a/rpython/jit/backend/llsupport/test/test_pinned_object_rewrite.py
+++ b/rpython/jit/backend/llsupport/test/test_pinned_object_rewrite.py
@@ -137,15 +137,15 @@
     def test_simple_getfield_twice(self):
         self.check_rewrite("""
             []
-            i0 = getfield_gc(ConstPtr(pinned_obj_gcref), descr=pinned_obj_my_int_descr)
-            i1 = getfield_gc(ConstPtr(notpinned_obj_gcref), descr=notpinned_obj_my_int_descr)
-            i2 = getfield_gc(ConstPtr(pinned_obj_gcref), descr=pinned_obj_my_int_descr)
+            i0 = getfield_gc_i(ConstPtr(pinned_obj_gcref), descr=pinned_obj_my_int_descr)
+            i1 = getfield_gc_i(ConstPtr(notpinned_obj_gcref), descr=notpinned_obj_my_int_descr)
+            i2 = getfield_gc_i(ConstPtr(pinned_obj_gcref), descr=pinned_obj_my_int_descr)
             """, """
             []
-            p1 = getarrayitem_gc(ConstPtr(ptr_array_gcref), 0, descr=ptr_array_descr)
-            i0 = getfield_gc(p1, descr=pinned_obj_my_int_descr)
-            i1 = getfield_gc(ConstPtr(notpinned_obj_gcref), descr=notpinned_obj_my_int_descr)
-            p2 = getarrayitem_gc(ConstPtr(ptr_array_gcref), 1, descr=ptr_array_descr)
-            i2 = getfield_gc(p2, descr=pinned_obj_my_int_descr)
+            p1 = getarrayitem_gc_r(ConstPtr(ptr_array_gcref), 0, descr=ptr_array_descr)
+            i0 = getfield_gc_i(p1, descr=pinned_obj_my_int_descr)
+            i1 = getfield_gc_i(ConstPtr(notpinned_obj_gcref), descr=notpinned_obj_my_int_descr)
+            p2 = getarrayitem_gc_r(ConstPtr(ptr_array_gcref), 1, descr=ptr_array_descr)
+            i2 = getfield_gc_i(p2, descr=pinned_obj_my_int_descr)
             """)
         assert len(self.gc_ll_descr.last_moving_obj_tracker._indexes) == 2
diff --git a/rpython/jit/backend/llsupport/test/test_regalloc_integration.py b/rpython/jit/backend/llsupport/test/test_regalloc_integration.py
--- a/rpython/jit/backend/llsupport/test/test_regalloc_integration.py
+++ b/rpython/jit/backend/llsupport/test/test_regalloc_integration.py
@@ -3,14 +3,13 @@
 """
 
 import py
-from rpython.jit.metainterp.history import BoxInt, ConstInt,\
-     BoxPtr, ConstPtr, BasicFailDescr, JitCellToken, TargetToken
-from rpython.jit.metainterp.resoperation import rop, ResOperation
-from rpython.jit.backend.llsupport.descr import GcCache
+from rpython.jit.metainterp.history import BasicFailDescr, JitCellToken,\
+     TargetToken
+from rpython.jit.metainterp.resoperation import rop
 from rpython.jit.backend.detect_cpu import getcpuclass
 from rpython.jit.backend.llsupport.regalloc import is_comparison_or_ovf_op
 from rpython.jit.tool.oparser import parse
-from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
+from rpython.rtyper.lltypesystem import lltype, llmemory
 from rpython.rtyper.annlowlevel import llhelper
 from rpython.rtyper.lltypesystem import rstr
 from rpython.rtyper import rclass
diff --git a/rpython/jit/backend/llsupport/test/test_rewrite.py b/rpython/jit/backend/llsupport/test/test_rewrite.py
--- a/rpython/jit/backend/llsupport/test/test_rewrite.py
+++ b/rpython/jit/backend/llsupport/test/test_rewrite.py
@@ -30,13 +30,13 @@
     def check_rewrite(self, frm_operations, to_operations, **namespace):
         S = lltype.GcStruct('S', ('x', lltype.Signed),
                                  ('y', lltype.Signed))
-        sdescr = get_size_descr(self.cpu, self.gc_ll_descr, S, False)
+        sdescr = get_size_descr(self.gc_ll_descr, S, False)
         sdescr.tid = 1234
         #
         T = lltype.GcStruct('T', ('y', lltype.Signed),
                                  ('z', lltype.Ptr(S)),
                                  ('t', lltype.Signed))
-        tdescr = get_size_descr(self.cpu, self.gc_ll_descr, T, False)
+        tdescr = get_size_descr(self.gc_ll_descr, T, False)
         tdescr.tid = 5678
         tzdescr = get_field_descr(self.gc_ll_descr, T, 'z')
         #
@@ -56,7 +56,7 @@
         clendescr = cdescr.lendescr
         #
         E = lltype.GcStruct('Empty')
-        edescr = get_size_descr(self.cpu, self.gc_ll_descr, E, False)
+        edescr = get_size_descr(self.gc_ll_descr, E, False)
         edescr.tid = 9000
         #
         vtable_descr = self.gc_ll_descr.fielddescr_vtable
diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -6,7 +6,7 @@
                                                 DEBUG_COUNTER, debug_bridge)
 from rpython.jit.backend.llsupport.asmmemmgr import MachineDataBlockWrapper
 from rpython.jit.backend.llsupport.gcmap import allocate_gcmap
-from rpython.jit.metainterp.history import Const, Box, VOID
+from rpython.jit.metainterp.history import Const, VOID
 from rpython.jit.metainterp.history import AbstractFailDescr, INT, REF, FLOAT
 from rpython.rtyper.lltypesystem import lltype, rffi, rstr, llmemory
 from rpython.rtyper.lltypesystem.lloperation import llop
@@ -829,13 +829,8 @@
     def dump(self, text):
         if not self.verbose:
             return
-        _prev = Box._extended_display
-        try:
-            Box._extended_display = False
-            pos = self.mc.get_relative_pos()
-            print >> sys.stderr, ' 0x%x  %s' % (pos, text)
-        finally:
-            Box._extended_display = _prev
+        pos = self.mc.get_relative_pos()
+        print >> sys.stderr, ' 0x%x  %s' % (pos, text)
 
     # ------------------------------------------------------------
 
diff --git a/rpython/jit/metainterp/virtualizable.py b/rpython/jit/metainterp/virtualizable.py
--- a/rpython/jit/metainterp/virtualizable.py
+++ b/rpython/jit/metainterp/virtualizable.py
@@ -15,7 +15,6 @@
         self.warmrunnerdesc = warmrunnerdesc
         cpu = warmrunnerdesc.cpu
         self.cpu = cpu
-        self.BoxArray = cpu.ts.BoxRef
         #
         VTYPEPTR1 = VTYPEPTR
         while 'virtualizable_accessor' not in deref(VTYPEPTR)._hints:

From noreply at buildbot.pypy.org  Fri Jun  5 10:08:03 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Fri,  5 Jun 2015 10:08:03 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: fix fix fix
Message-ID: <20150605080803.877241C11B6@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77883:6745a522e616
Date: 2015-06-05 10:07 +0200
http://bitbucket.org/pypy/pypy/changeset/6745a522e616/

Log:	fix fix fix

diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py
--- a/rpython/jit/metainterp/optimizeopt/info.py
+++ b/rpython/jit/metainterp/optimizeopt/info.py
@@ -479,7 +479,7 @@
     def getlenbound(self, mode):
         from rpython.jit.metainterp.optimizeopt.intutils import ConstIntBound
         
-        return ConstIntBound(self.getstrlen(None, None, mode))
+        return ConstIntBound(self.getstrlen(None, None, mode).getint())
     
     def getstrlen(self, op, string_optimizer, mode, create_ops=True):
         from rpython.jit.metainterp.optimizeopt import vstring
diff --git a/rpython/jit/metainterp/test/test_resume.py b/rpython/jit/metainterp/test/test_resume.py
--- a/rpython/jit/metainterp/test/test_resume.py
+++ b/rpython/jit/metainterp/test/test_resume.py
@@ -3,7 +3,8 @@
 import sys
 from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
 from rpython.jit.metainterp.resume import *
-from rpython.jit.metainterp.optimizeopt.info import AbstractVirtualPtrInfo
+from rpython.jit.metainterp.optimizeopt.info import AbstractInfo,\
+     AbstractVirtualPtrInfo
 from rpython.jit.metainterp.history import ConstInt
 from rpython.jit.metainterp.history import ConstPtr, ConstFloat
 from rpython.jit.metainterp.optimizeopt.test.test_util import LLtypeMixin
@@ -25,13 +26,19 @@
 
 class FakeOptimizer(object):
     def get_box_replacement(self, op):
-        if not op.get_forwarded():
-            return op
-        xxx
+        while (op.get_forwarded() is not None and
+               not isinstance(op.get_forwarded(), AbstractInfo)):
+            op = op.get_forwarded()
+        return op
 
     def getrawptrinfo(self, op, create=True):
+        op = self.get_box_replacement(op)
         return op.get_forwarded()
 
+    def getptrinfo(self, op, create=True):
+        op = self.get_box_replacement(op)
+        return op.get_forwarded()        
+
 
 # ____________________________________________________________
 
@@ -124,7 +131,7 @@
     class FakeVirtualValue(AbstractVirtualPtrInfo):
         def visitor_dispatch_virtual_type(self, *args):
             return FakeVInfo()
-    modifier = ResumeDataVirtualAdder(None, None, None, None, None)
+    modifier = ResumeDataVirtualAdder(None, None, None, None)
     v1 = FakeVirtualValue()
     vinfo1 = modifier.make_virtual_info(v1, [1, 2, 4])
     vinfo2 = modifier.make_virtual_info(v1, [1, 2, 4])
@@ -624,7 +631,7 @@
     capture_resumedata(fs, None, [], storage)
     memo = ResumeDataLoopMemo(FakeMetaInterpStaticData())
     modifier = ResumeDataVirtualAdder(None, storage, storage, memo)
-    liveboxes = modifier.finish(FakeOptimizer({}))
+    liveboxes = modifier.finish(FakeOptimizer())
     metainterp = MyMetaInterp()
 
     b1t, b2t, b3t = [BoxInt(), InputArgRef(), BoxInt()]
@@ -901,7 +908,7 @@
 
     memo = ResumeDataLoopMemo(FakeMetaInterpStaticData())
 
-    numb, liveboxes, v = memo.number(FakeOptimizer({}), snap1)
+    numb, liveboxes, v = memo.number(FakeOptimizer(), snap1)
     assert v == 0
 
     assert liveboxes == {b1: tag(0, TAGBOX), b2: tag(1, TAGBOX),
@@ -913,7 +920,7 @@
                                     tag(0, TAGBOX), tag(2, TAGINT)]
     assert not numb.prev.prev
 
-    numb2, liveboxes2, v = memo.number(FakeOptimizer({}), snap2)
+    numb2, liveboxes2, v = memo.number(FakeOptimizer(), snap2)
     assert v == 0
     
     assert liveboxes2 == {b1: tag(0, TAGBOX), b2: tag(1, TAGBOX),
@@ -926,20 +933,16 @@
     env3 = [c3, b3, b1, c3]
     snap3 = Snapshot(snap, env3)
 
-    class FakeValue(object):
-        def __init__(self, virt, box):
+    class FakeVirtualInfo(AbstractInfo):
+        def __init__(self, virt):
             self.virt = virt
-            self.valuebox = box
-
-        def get_key_box(self):
-            return self.valuebox
 
         def is_virtual(self):
             return self.virt
 
     # renamed
-    numb3, liveboxes3, v = memo.number(FakeOptimizer({b3: FakeValue(False, c4)}),
-                                       snap3)
+    b3.set_forwarded(c4)
+    numb3, liveboxes3, v = memo.number(FakeOptimizer(), snap3)
     assert v == 0
     
     assert liveboxes3 == {b1: tag(0, TAGBOX), b2: tag(1, TAGBOX)}
@@ -949,10 +952,10 @@
 
     # virtual
     env4 = [c3, b4, b1, c3]
-    snap4 = Snapshot(snap, env4)    
+    snap4 = Snapshot(snap, env4)
 
-    numb4, liveboxes4, v = memo.number(FakeOptimizer({b4: FakeValue(True, b4)}),
-                                       snap4)
+    b4.set_forwarded(FakeVirtualInfo(True))
+    numb4, liveboxes4, v = memo.number(FakeOptimizer(), snap4)
     assert v == 1
     
     assert liveboxes4 == {b1: tag(0, TAGBOX), b2: tag(1, TAGBOX),
@@ -962,11 +965,11 @@
     assert numb4.prev == numb.prev
 
     env5 = [b1, b4, b5]
-    snap5 = Snapshot(snap4, env5)    
+    snap5 = Snapshot(snap4, env5)
 
-    numb5, liveboxes5, v = memo.number(FakeOptimizer({b4: FakeValue(True, b4),
-                                                      b5: FakeValue(True, b5)}),
-                                       snap5)
+    b4.set_forwarded(FakeVirtualInfo(True))
+    b5.set_forwarded(FakeVirtualInfo(True))
+    numb5, liveboxes5, v = memo.number(FakeOptimizer(), snap5)
     assert v == 2
     
     assert liveboxes5 == {b1: tag(0, TAGBOX), b2: tag(1, TAGBOX),
@@ -1028,7 +1031,7 @@
 def test_register_virtual_fields():
     b1, b2 = InputArgInt(), InputArgInt()
     vbox = InputArgRef()
-    modifier = ResumeDataVirtualAdder(None, None, None, None)
+    modifier = ResumeDataVirtualAdder(FakeOptimizer(), None, None, None)
     modifier.liveboxes_from_env = {}
     modifier.liveboxes = {}
     modifier.vfieldboxes = {}
@@ -1037,7 +1040,7 @@
                                   b2: UNASSIGNED}
     assert modifier.vfieldboxes == {vbox: [b1, b2]}
 
-    modifier = ResumeDataVirtualAdder(None, None, None, None)
+    modifier = ResumeDataVirtualAdder(FakeOptimizer(), None, None, None)
     modifier.liveboxes_from_env = {vbox: tag(0, TAGVIRTUAL)}
     modifier.liveboxes = {}
     modifier.vfieldboxes = {}
@@ -1067,8 +1070,8 @@
     b1s, b2s, b3s = [ConstInt(sys.maxint), ConstInt(2**16), ConstInt(-65)]
     storage = make_storage(b1s, b2s, b3s)
     memo = ResumeDataLoopMemo(FakeMetaInterpStaticData())    
-    modifier = ResumeDataVirtualAdder(storage, storage, memo)
-    liveboxes = modifier.finish(FakeOptimizer({}))
+    modifier = ResumeDataVirtualAdder(FakeOptimizer(), storage, storage, memo)
+    liveboxes = modifier.finish(FakeOptimizer())
     assert storage.rd_snapshot is None
     cpu = MyCPU([])
     reader = ResumeDataDirectReader(MyMetaInterp(cpu), storage, "deadframe")
@@ -1081,15 +1084,16 @@
     b1s, b2s, b3s = [ConstInt(sys.maxint), ConstInt(2**16), ConstInt(-65)]
     storage = make_storage(b1s, b2s, b3s)
     memo = ResumeDataLoopMemo(FakeMetaInterpStaticData())
-    modifier = ResumeDataVirtualAdder(storage, storage, memo)
-    modifier.finish(FakeOptimizer({}))
+    modifier = ResumeDataVirtualAdder(FakeOptimizer(), storage, storage, memo)
+    modifier.finish(FakeOptimizer())
     assert len(memo.consts) == 2
     assert storage.rd_consts is memo.consts
 
     b1s, b2s, b3s = [ConstInt(sys.maxint), ConstInt(2**17), ConstInt(-65)]
     storage2 = make_storage(b1s, b2s, b3s)
-    modifier2 = ResumeDataVirtualAdder(storage2, storage2, memo)
-    modifier2.finish(FakeOptimizer({}))
+    modifier2 = ResumeDataVirtualAdder(FakeOptimizer(), storage2, storage2,
+                                       memo)
+    modifier2.finish(FakeOptimizer())
     assert len(memo.consts) == 3    
     assert storage2.rd_consts is memo.consts
 

From noreply at buildbot.pypy.org  Fri Jun  5 10:43:39 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Fri,  5 Jun 2015 10:43:39 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: whack at test_resume until it passes
Message-ID: <20150605084339.2A1091C033F@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77884:143291527023
Date: 2015-06-05 10:43 +0200
http://bitbucket.org/pypy/pypy/changeset/143291527023/

Log:	whack at test_resume until it passes

diff --git a/rpython/jit/metainterp/resume.py b/rpython/jit/metainterp/resume.py
--- a/rpython/jit/metainterp/resume.py
+++ b/rpython/jit/metainterp/resume.py
@@ -513,9 +513,8 @@
                 num = self._gettagged(box)
                 fieldnum = self._gettagged(fieldbox)
                 # the index is limited to 2147483647 (64-bit machines only)
-                #if itemindex > 2147483647:
-                #    raise TagOverflow
-                #itemindex = rffi.cast(rffi.INT, itemindex)
+                if itemindex > 2147483647:
+                    raise TagOverflow
                 #
                 rd_pendingfields[i].lldescr = lldescr
                 rd_pendingfields[i].num = num
diff --git a/rpython/jit/metainterp/test/test_resume.py b/rpython/jit/metainterp/test/test_resume.py
--- a/rpython/jit/metainterp/test/test_resume.py
+++ b/rpython/jit/metainterp/test/test_resume.py
@@ -2,16 +2,23 @@
 import py
 import sys
 from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
-from rpython.jit.metainterp.resume import *
-from rpython.jit.metainterp.optimizeopt.info import AbstractInfo,\
-     AbstractVirtualPtrInfo
-from rpython.jit.metainterp.history import ConstInt
+from rpython.jit.metainterp.resume import ResumeDataVirtualAdder,\
+     AbstractResumeDataReader, get_VirtualCache_class, ResumeDataBoxReader,\
+     tag, TagOverflow, untag, tagged_eq, UNASSIGNED, TAGBOX, TAGVIRTUAL,\
+     tagged_list_eq, AbstractVirtualInfo, NUMBERING, TAGCONST, NULLREF,\
+     ResumeDataDirectReader, TAGINT, REF, VirtualInfo, VStructInfo,\
+     VArrayInfoNotClear, VStrPlainInfo, VStrConcatInfo, VStrSliceInfo,\
+     VUniPlainInfo, VUniConcatInfo, VUniSliceInfo, Snapshot, FrameInfo,\
+     capture_resumedata, ResumeDataLoopMemo, UNASSIGNEDVIRTUAL, INT,\
+     annlowlevel, PENDINGFIELDSP
+from rpython.jit.metainterp.optimizeopt import info
+from rpython.jit.metainterp.history import ConstInt, Const, AbstractDescr
 from rpython.jit.metainterp.history import ConstPtr, ConstFloat
 from rpython.jit.metainterp.optimizeopt.test.test_util import LLtypeMixin
 from rpython.jit.metainterp import executor
 from rpython.jit.codewriter import heaptracker, longlong
 from rpython.jit.metainterp.resoperation import ResOperation, InputArgInt,\
-     InputArgRef, InputArgFloat
+     InputArgRef, rop
 from rpython.rlib.debug import debug_start, debug_stop, debug_print,\
      have_debug_prints
 
@@ -27,7 +34,7 @@
 class FakeOptimizer(object):
     def get_box_replacement(self, op):
         while (op.get_forwarded() is not None and
-               not isinstance(op.get_forwarded(), AbstractInfo)):
+               not isinstance(op.get_forwarded(), info.AbstractInfo)):
             op = op.get_forwarded()
         return op
 
@@ -128,7 +135,7 @@
             self.fieldnums = fieldnums
         def equals(self, fieldnums):
             return self.fieldnums == fieldnums
-    class FakeVirtualValue(AbstractVirtualPtrInfo):
+    class FakeVirtualValue(info.AbstractVirtualPtrInfo):
         def visitor_dispatch_virtual_type(self, *args):
             return FakeVInfo()
     modifier = ResumeDataVirtualAdder(None, None, None, None)
@@ -933,7 +940,7 @@
     env3 = [c3, b3, b1, c3]
     snap3 = Snapshot(snap, env3)
 
-    class FakeVirtualInfo(AbstractInfo):
+    class FakeVirtualInfo(info.AbstractInfo):
         def __init__(self, virt):
             self.virt = virt
 
@@ -1148,18 +1155,11 @@
     storage = make_storage(b1s, b2s, b3s)
     memo = ResumeDataLoopMemo(FakeMetaInterpStaticData())
     b1_2 = InputArgInt()
-    modifier = ResumeDataVirtualAdder(storage, storage, memo)
-    class FakeValue(object):
+    modifier = ResumeDataVirtualAdder(FakeOptimizer(), storage, storage, memo)
 
-        def is_virtual(self):
-            return False
-
-        def get_key_box(self):
-            return b1_2
-
-    val = FakeValue()
-    values = {b1s: val, b2s: val}  
-    liveboxes = modifier.finish(FakeOptimizer(values))
+    b1s.set_forwarded(b1_2)
+    b2s.set_forwarded(b1_2)
+    liveboxes = modifier.finish(FakeOptimizer())
     assert storage.rd_snapshot is None
     b1t, b3t = [InputArgInt(11), InputArgInt(33)]
     newboxes = _resume_remap(liveboxes, [b1_2, b3s], b1t, b3t)
@@ -1179,8 +1179,8 @@
     b1s = ConstInt(111)
     storage = make_storage(b1s, b2s, b3s)
     memo = ResumeDataLoopMemo(FakeMetaInterpStaticData())        
-    modifier = ResumeDataVirtualAdder(storage, storage, memo)
-    liveboxes = modifier.finish(FakeOptimizer({}))
+    modifier = ResumeDataVirtualAdder(FakeOptimizer(), storage, storage, memo)
+    liveboxes = modifier.finish(FakeOptimizer())
     b2t, b3t = [InputArgRef(demo55o), InputArgInt(33)]
     newboxes = _resume_remap(liveboxes, [b2s, b3s], b2t, b3t)
     metainterp = MyMetaInterp()
@@ -1196,34 +1196,34 @@
 
 
 def test_virtual_adder_make_virtual():
-    b2s, b3s, b4s, b5s = [InputArgRef(), InputArgInt(3), InputArgRef(), InputArgRef()]  
+    b2s, b3s, b4s, b5s = [InputArgRef(), InputArgInt(3), InputArgRef(),
+                          InputArgRef()]  
     c1s = ConstInt(111)
     storage = Storage()
     memo = ResumeDataLoopMemo(FakeMetaInterpStaticData())
-    modifier = ResumeDataVirtualAdder(storage, storage, memo)
+    modifier = ResumeDataVirtualAdder(FakeOptimizer(), storage, storage, memo)
     modifier.liveboxes_from_env = {}
     modifier.liveboxes = {}
     modifier.vfieldboxes = {}
 
-    v4 = VirtualValue(fakeoptimizer, ConstAddr(LLtypeMixin.node_vtable_adr2,
-                                                LLtypeMixin.cpu), b4s)
-    v4.setfield(LLtypeMixin.nextdescr, OptValue(b2s))
-    v4.setfield(LLtypeMixin.valuedescr, OptValue(b3s))
-    v4.setfield(LLtypeMixin.otherdescr, OptValue(b5s))
-    v4._cached_sorted_fields = [LLtypeMixin.nextdescr, LLtypeMixin.valuedescr,
-                                LLtypeMixin.otherdescr]
-    v2 = VirtualValue(fakeoptimizer, ConstAddr(LLtypeMixin.node_vtable_adr,
-                                                LLtypeMixin.cpu), b2s)
-    v2.setfield(LLtypeMixin.nextdescr, v4)
-    v2.setfield(LLtypeMixin.valuedescr, OptValue(c1s))
-    v2._cached_sorted_fields = [LLtypeMixin.nextdescr, LLtypeMixin.valuedescr]
+    vdescr = LLtypeMixin.nodesize2
+    v4 = info.InstancePtrInfo(ConstAddr(LLtypeMixin.node_vtable_adr2,
+                                        LLtypeMixin.cpu), vdescr)
+    b4s.set_forwarded(v4)
+    v4.setfield(LLtypeMixin.nextdescr, b2s)
+    v4.setfield(LLtypeMixin.valuedescr, b3s)
+    v4.setfield(LLtypeMixin.otherdescr, b5s)
+    v2 = info.InstancePtrInfo(ConstAddr(LLtypeMixin.node_vtable_adr,
+                                        LLtypeMixin.cpu), LLtypeMixin.nodesize)
+    v2.setfield(LLtypeMixin.nextdescr, b4s)
+    v2.setfield(LLtypeMixin.valuedescr, c1s)
+    b2s.set_forwarded(v2)
 
-    modifier.register_virtual_fields(b2s, [b4s, c1s])
-    modifier.register_virtual_fields(b4s, [b2s, b3s, b5s])
-    values = {b2s: v2, b4s: v4}
+    modifier.register_virtual_fields(b2s, [c1s, None, None, b4s])
+    modifier.register_virtual_fields(b4s, [b3s, None, None, b2s, b5s])
 
     liveboxes = []
-    modifier._number_virtuals(liveboxes, FakeOptimizer(values), 0)
+    modifier._number_virtuals(liveboxes, FakeOptimizer(), 0)
     storage.rd_consts = memo.consts[:]
     storage.rd_numb = None
     # resume
@@ -1241,12 +1241,8 @@
     b2t = reader.decode_ref(modifier._gettagged(b2s))
     b4t = reader.decode_ref(modifier._gettagged(b4s))
     trace = metainterp.trace
-    b2new = (rop.NEW_WITH_VTABLE, [ConstAddr(LLtypeMixin.node_vtable_adr,
-                                         LLtypeMixin.cpu)],
-                              b2t.getref_base(), None)
-    b4new = (rop.NEW_WITH_VTABLE, [ConstAddr(LLtypeMixin.node_vtable_adr2,
-                                         LLtypeMixin.cpu)],
-                              b4t.getref_base(), None)
+    b2new = (rop.NEW_WITH_VTABLE, [], b2t.getref_base(), LLtypeMixin.nodesize)
+    b4new = (rop.NEW_WITH_VTABLE, [], b4t.getref_base(), LLtypeMixin.nodesize2)
     b2set = [(rop.SETFIELD_GC, [b2t, b4t],      None, LLtypeMixin.nextdescr),
              (rop.SETFIELD_GC, [b2t, c1s],      None, LLtypeMixin.valuedescr)]
     b4set = [(rop.SETFIELD_GC, [b4t, b2t],     None, LLtypeMixin.nextdescr),
@@ -1257,6 +1253,7 @@
     # check that we get the operations in 'expected', in a possibly different
     # order.
     assert len(trace) == len(expected)
+    orig = trace[:]
     with CompareableConsts():
         for x in trace:
             assert x in expected
@@ -1283,17 +1280,17 @@
     c1s = ConstInt(111)
     storage = Storage()
     memo = ResumeDataLoopMemo(FakeMetaInterpStaticData())
-    modifier = ResumeDataVirtualAdder(storage, storage, memo)
+    modifier = ResumeDataVirtualAdder(FakeOptimizer(), storage, storage, memo)
     modifier.liveboxes_from_env = {}
     modifier.liveboxes = {}
     modifier.vfieldboxes = {}
 
-    v2 = VArrayValue(LLtypeMixin.arraydescr, None, 2, b2s)
+    v2 = info.ArrayPtrInfo(vdescr=LLtypeMixin.arraydescr, size=2)
+    b2s.set_forwarded(v2)
     v2._items = [b4s, c1s]
     modifier.register_virtual_fields(b2s, [b4s, c1s])
     liveboxes = []
-    values = {b2s: v2}
-    modifier._number_virtuals(liveboxes, FakeOptimizer(values), 0)
+    modifier._number_virtuals(liveboxes, FakeOptimizer(), 0)
     dump_storage(storage, liveboxes)
     storage.rd_consts = memo.consts[:]
     storage.rd_numb = None
@@ -1332,16 +1329,17 @@
     c1s = ConstInt(111)
     storage = Storage()
     memo = ResumeDataLoopMemo(FakeMetaInterpStaticData())
-    modifier = ResumeDataVirtualAdder(storage, storage, memo)
+    modifier = ResumeDataVirtualAdder(FakeOptimizer(), storage, storage, memo)
     modifier.liveboxes_from_env = {}
     modifier.liveboxes = {}
     modifier.vfieldboxes = {}
-    v2 = VStructValue(fakeoptimizer, LLtypeMixin.ssize, b2s)
-    v2.setfield(LLtypeMixin.adescr, OptValue(c1s))
-    v2.setfield(LLtypeMixin.bdescr, OptValue(b4s))
+    v2 = info.StructPtrInfo(LLtypeMixin.ssize)
+    b2s.set_forwarded(v2)
+    v2.setfield(LLtypeMixin.adescr, c1s)
+    v2.setfield(LLtypeMixin.bdescr, b4s)
     modifier.register_virtual_fields(b2s, [c1s, b4s])
     liveboxes = []
-    modifier._number_virtuals(liveboxes, FakeOptimizer({b2s: v2}), 0)
+    modifier._number_virtuals(liveboxes, FakeOptimizer(), 0)
     dump_storage(storage, liveboxes)
     storage.rd_consts = memo.consts[:]
     storage.rd_numb = None
@@ -1380,16 +1378,14 @@
     modifier.liveboxes = {}
     modifier.vfieldboxes = {}
 
-    v2 = OptValue(b2s)
-    v4 = OptValue(b4s)
     modifier.register_box(b2s)
     modifier.register_box(b4s)
 
-    values = {b4s: v4, b2s: v2}
     liveboxes = []
-    modifier._number_virtuals(liveboxes, FakeOptimizer(values), 0)
+    modifier._number_virtuals(liveboxes, FakeOptimizer(), 0)
     assert liveboxes == [b2s, b4s] or liveboxes == [b4s, b2s]
-    modifier._add_pending_fields([(LLtypeMixin.nextdescr, b2s, b4s, -1)])
+    modifier._add_pending_fields(FakeOptimizer(), [
+        ResOperation(rop.SETFIELD_GC, [b2s, b4s], descr=LLtypeMixin.nextdescr)])
     storage.rd_consts = memo.consts[:]
     storage.rd_numb = None
     # resume
@@ -1418,14 +1414,18 @@
     modifier._add_pending_fields(None, [])
     assert not storage.rd_pendingfields
     #
-    class FieldDescr(object):
+    class FieldDescr(AbstractDescr):
         pass
     field_a = FieldDescr()
     storage = Storage()
     modifier = ResumeDataVirtualAdder(None, storage, storage, None)
-    modifier.liveboxes_from_env = {42: rffi.cast(rffi.SHORT, 1042),
-                                   61: rffi.cast(rffi.SHORT, 1061)}
-    modifier._add_pending_fields(FakeOptimizer(), [(field_a, 42, 61, -1)])
+    a = InputArgInt()
+    b = InputArgInt()
+    modifier.liveboxes_from_env = {a: rffi.cast(rffi.SHORT, 1042),
+                                   b: rffi.cast(rffi.SHORT, 1061)}
+    modifier._add_pending_fields(FakeOptimizer(), [
+        ResOperation(rop.SETFIELD_GC, [a, b],
+                     descr=field_a)])
     pf = storage.rd_pendingfields
     assert len(pf) == 1
     assert (annlowlevel.cast_base_ptr_to_instance(FieldDescr, pf[0].lldescr)
@@ -1436,13 +1436,20 @@
     #
     array_a = FieldDescr()
     storage = Storage()
-    modifier = ResumeDataVirtualAdder(storage, storage, None)
-    modifier.liveboxes_from_env = {42: rffi.cast(rffi.SHORT, 1042),
-                                   61: rffi.cast(rffi.SHORT, 1061),
-                                   62: rffi.cast(rffi.SHORT, 1062),
-                                   63: rffi.cast(rffi.SHORT, 1063)}
-    modifier._add_pending_fields([(array_a, 42, 61, 0),
-                                  (array_a, 42, 62, 2147483647)])
+    modifier = ResumeDataVirtualAdder(None, storage, storage, None)
+    a42 = InputArgInt()
+    a61 = InputArgInt()
+    a62 = InputArgInt()
+    a63 = InputArgInt()
+    modifier.liveboxes_from_env = {a42: rffi.cast(rffi.SHORT, 1042),
+                                   a61: rffi.cast(rffi.SHORT, 1061),
+                                   a62: rffi.cast(rffi.SHORT, 1062),
+                                   a63: rffi.cast(rffi.SHORT, 1063)}
+    modifier._add_pending_fields(FakeOptimizer(), [
+        ResOperation(rop.SETARRAYITEM_GC, [a42, ConstInt(0), a61],
+                     descr=array_a),
+        ResOperation(rop.SETARRAYITEM_GC, [a42, ConstInt(2147483647), a62],
+                     descr=array_a)])
     pf = storage.rd_pendingfields
     assert len(pf) == 2
     assert (annlowlevel.cast_base_ptr_to_instance(FieldDescr, pf[0].lldescr)
@@ -1456,8 +1463,10 @@
     assert rffi.cast(lltype.Signed, pf[1].fieldnum) == 1062
     assert rffi.cast(lltype.Signed, pf[1].itemindex) == 2147483647
     #
-    py.test.raises(TagOverflow, modifier._add_pending_fields,
-                   [(array_a, 42, 63, 2147483648)])
+    py.test.raises(TagOverflow, modifier._add_pending_fields, FakeOptimizer(),
+                   [ResOperation(rop.SETARRAYITEM_GC,
+                                 [a42, ConstInt(2147483648), a63],
+                       descr=array_a)])
 
 def test_resume_reader_fields_and_arrayitems():
     class ResumeReader(AbstractResumeDataReader):

From noreply at buildbot.pypy.org  Fri Jun  5 10:45:24 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Fri,  5 Jun 2015 10:45:24 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: comment out unused function
Message-ID: <20150605084524.835501C033F@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77885:817653b602e3
Date: 2015-06-05 10:45 +0200
http://bitbucket.org/pypy/pypy/changeset/817653b602e3/

Log:	comment out unused function

diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py
--- a/rpython/jit/metainterp/optimizeopt/info.py
+++ b/rpython/jit/metainterp/optimizeopt/info.py
@@ -279,11 +279,11 @@
             self._init_items(const, size, clear)
         self._clear = clear
 
-    def getlenbound(self):
-        if self.lenbound is None:
-            raise Exception("implement me - lenbound")
-            xxx
-        return self.lenbound
+    #def getlenbound(self):
+    #    if self.lenbound is None:
+    #        raise Exception("implement me - lenbound")
+    #        xxx
+    #    return self.lenbound
 
     def _init_items(self, const, size, clear):
         self.length = size

From noreply at buildbot.pypy.org  Fri Jun  5 10:47:15 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Fri,  5 Jun 2015 10:47:15 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: fix
Message-ID: <20150605084715.D31531C048F@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77886:630f68685f3e
Date: 2015-06-05 10:47 +0200
http://bitbucket.org/pypy/pypy/changeset/630f68685f3e/

Log:	fix

diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -879,7 +879,7 @@
                 eqbox = self.implement_guard_value(eqbox, pc)
                 isstandard = eqbox.getint()
                 if isstandard:
-                    if isinstance(box, history.BoxPtr):
+                    if box.type == 'r':
                         self.metainterp.replace_box(box, standard_box)
                     return False
         if not self.metainterp.heapcache.is_unescaped(box):

From noreply at buildbot.pypy.org  Fri Jun  5 10:48:35 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Fri,  5 Jun 2015 10:48:35 +0200 (CEST)
Subject: [pypy-commit] pypy default: Add and use a helper module that delays
 recursive calls, to turn them
Message-ID: <20150605084835.14E9B1C048F@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r77887:0c7bf923862f
Date: 2015-06-05 10:36 +0200
http://bitbucket.org/pypy/pypy/changeset/0c7bf923862f/

Log:	Add and use a helper module that delays recursive calls, to turn
	them non-recursive.

diff --git a/rpython/annotator/bookkeeper.py b/rpython/annotator/bookkeeper.py
--- a/rpython/annotator/bookkeeper.py
+++ b/rpython/annotator/bookkeeper.py
@@ -21,6 +21,7 @@
 from rpython.annotator.argument import simple_args
 from rpython.rlib.objectmodel import r_dict, r_ordereddict, Symbolic
 from rpython.tool.algo.unionfind import UnionFind
+from rpython.tool.flattenrec import FlattenRecursion
 from rpython.rtyper import extregistry
 
 
@@ -425,6 +426,8 @@
             self.methoddescs[key] = result
             return result
 
+    _see_mutable_flattenrec = FlattenRecursion()
+
     def see_mutable(self, x):
         key = (x.__class__, x)
         if key in self.seen_mutable:
@@ -433,8 +436,11 @@
         self.seen_mutable[key] = True
         self.event('mutable', x)
         source = InstanceSource(self, x)
-        for attr in source.all_instance_attributes():
-            clsdef.add_source_for_attribute(attr, source) # can trigger reflowing
+        def delayed():
+            for attr in source.all_instance_attributes():
+                clsdef.add_source_for_attribute(attr, source)
+                # ^^^ can trigger reflowing
+        self._see_mutable_flattenrec(delayed)
 
     def valueoftype(self, t):
         return annotationoftype(t, self)
diff --git a/rpython/rtyper/rclass.py b/rpython/rtyper/rclass.py
--- a/rpython/rtyper/rclass.py
+++ b/rpython/rtyper/rclass.py
@@ -7,6 +7,7 @@
 from rpython.rlib.objectmodel import UnboxedValue
 from rpython.tool.pairtype import pairtype, pair
 from rpython.tool.identity_dict import identity_dict
+from rpython.tool.flattenrec import FlattenRecursion
 from rpython.rtyper.extregistry import ExtRegistryEntry
 from rpython.rtyper.error import TyperError
 from rpython.rtyper.lltypesystem import lltype
@@ -767,11 +768,14 @@
             self.initialize_prebuilt_data(Ellipsis, self.classdef, result)
             return result
 
+    _initialize_data_flattenrec = FlattenRecursion()
+
     def initialize_prebuilt_instance(self, value, classdef, result):
         # must fill in the hash cache before the other ones
         # (see test_circular_hash_initialization)
         self.initialize_prebuilt_hash(value, result)
-        self.initialize_prebuilt_data(value, classdef, result)
+        self._initialize_data_flattenrec(self.initialize_prebuilt_data,
+                                         value, classdef, result)
 
     def get_ll_hash_function(self):
         return ll_inst_hash
diff --git a/rpython/rtyper/test/test_rclass.py b/rpython/rtyper/test/test_rclass.py
--- a/rpython/rtyper/test/test_rclass.py
+++ b/rpython/rtyper/test/test_rclass.py
@@ -1279,3 +1279,16 @@
             return cls[k](a, b).b
 
         assert self.interpret(f, [1, 4, 7]) == 7
+
+    def test_flatten_convert_const(self):
+        # check that we can convert_const() a chain of more than 1000
+        # instances
+        class A(object):
+            def __init__(self, next):
+                self.next = next
+        a = None
+        for i in range(1500):
+            a = A(a)
+        def f():
+            return a.next.next.next.next is not None
+        assert self.interpret(f, []) == True
diff --git a/rpython/tool/flattenrec.py b/rpython/tool/flattenrec.py
new file mode 100644
--- /dev/null
+++ b/rpython/tool/flattenrec.py
@@ -0,0 +1,25 @@
+"""
+A general way to flatten deeply recursive algorithms by delaying some
+parts until later.
+"""
+
+
+class FlattenRecursion(object):
+
+    def __init__(self):
+        self.later = None
+
+    def __call__(self, func, *args, **kwds):
+        """Call func(*args, **kwds), either now, or, if we're recursing,
+        then the call will be done later by the first level.
+        """
+        if self.later is not None:
+            self.later.append((func, args, kwds))
+        else:
+            self.later = lst = []
+            try:
+                func(*args, **kwds)
+                for func, args, kwds in lst:
+                    func(*args, **kwds)
+            finally:
+                self.later = None
diff --git a/rpython/tool/test/test_flattenrec.py b/rpython/tool/test/test_flattenrec.py
new file mode 100644
--- /dev/null
+++ b/rpython/tool/test/test_flattenrec.py
@@ -0,0 +1,13 @@
+from rpython.tool.flattenrec import FlattenRecursion
+
+def test_flattenrec():
+    r = FlattenRecursion()
+    seen = set()
+
+    def rec(n):
+        if n > 0:
+            r(rec, n-1)
+        seen.add(n)
+
+    rec(10000)
+    assert seen == set(range(10001))

From noreply at buildbot.pypy.org  Fri Jun  5 10:48:36 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Fri,  5 Jun 2015 10:48:36 +0200 (CEST)
Subject: [pypy-commit] pypy default: Turn this class into a thread-local
 storage, to fix issues in case
Message-ID: <20150605084836.5994F1C048F@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r77888:741893a5757c
Date: 2015-06-05 10:39 +0200
http://bitbucket.org/pypy/pypy/changeset/741893a5757c/

Log:	Turn this class into a thread-local storage, to fix issues in case
	it's used by multiple threads

diff --git a/rpython/tool/flattenrec.py b/rpython/tool/flattenrec.py
--- a/rpython/tool/flattenrec.py
+++ b/rpython/tool/flattenrec.py
@@ -3,8 +3,14 @@
 parts until later.
 """
 
+try:
+    from thread import _local as TlsClass
+except ImportError:
+    class TlsClass(object):
+        pass
 
-class FlattenRecursion(object):
+
+class FlattenRecursion(TlsClass):
 
     def __init__(self):
         self.later = None

From noreply at buildbot.pypy.org  Fri Jun  5 10:48:37 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Fri,  5 Jun 2015 10:48:37 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge heads
Message-ID: <20150605084837.8D7501C048F@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r77889:7c18d4c2934f
Date: 2015-06-05 09:49 +0100
http://bitbucket.org/pypy/pypy/changeset/7c18d4c2934f/

Log:	merge heads

diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py
--- a/pypy/module/micronumpy/concrete.py
+++ b/pypy/module/micronumpy/concrete.py
@@ -438,7 +438,7 @@
     def __init__(self, shape, dtype, order, strides, backstrides,
                  storage=lltype.nullptr(RAW_STORAGE), zero=True):
         gcstruct = V_OBJECTSTORE
-        self.flags = NPY.ARRAY_ALIGNED | NPY.ARRAY_WRITEABLE
+        flags = NPY.ARRAY_ALIGNED | NPY.ARRAY_WRITEABLE
         if storage == lltype.nullptr(RAW_STORAGE):
             length = support.product(shape) 
             if dtype.num == NPY.OBJECT:
@@ -446,15 +446,16 @@
                 gcstruct = _create_objectstore(storage, length, dtype.elsize)
             else:
                 storage = dtype.itemtype.malloc(length * dtype.elsize, zero=zero)
-            self.flags |= NPY.ARRAY_OWNDATA
+            flags |= NPY.ARRAY_OWNDATA
         start = calc_start(shape, strides)
         ConcreteArrayNotOwning.__init__(self, shape, dtype, order, strides, backstrides,
                                         storage, start=start)
         self.gcstruct = gcstruct
         if is_c_contiguous(self):
-            self.flags |= NPY.ARRAY_C_CONTIGUOUS
+            flags |= NPY.ARRAY_C_CONTIGUOUS
         if is_f_contiguous(self):
-            self.flags |= NPY.ARRAY_F_CONTIGUOUS
+            flags |= NPY.ARRAY_F_CONTIGUOUS
+        self.flags = flags
 
     def __del__(self):
         if self.gcstruct:
@@ -469,14 +470,15 @@
                                         strides, backstrides, storage, start)
         self.orig_base = orig_base
         if isinstance(orig_base, W_NumpyObject):
-            self.flags = orig_base.get_flags() & NPY.ARRAY_ALIGNED
-            self.flags |=  orig_base.get_flags() & NPY.ARRAY_WRITEABLE
+            flags = orig_base.get_flags() & NPY.ARRAY_ALIGNED
+            flags |=  orig_base.get_flags() & NPY.ARRAY_WRITEABLE
         else:
-            self.flags = 0
+            flags = 0
         if is_c_contiguous(self):
-            self.flags |= NPY.ARRAY_C_CONTIGUOUS
+            flags |= NPY.ARRAY_C_CONTIGUOUS
         if is_f_contiguous(self):
-            self.flags |= NPY.ARRAY_F_CONTIGUOUS
+            flags |= NPY.ARRAY_F_CONTIGUOUS
+        self.flags = flags
 
     def base(self):
         return self.orig_base
@@ -524,12 +526,13 @@
         self.size = support.product(shape) * self.dtype.elsize
         self.start = start
         self.orig_arr = orig_arr
-        self.flags = parent.flags & NPY.ARRAY_ALIGNED
-        self.flags |= parent.flags & NPY.ARRAY_WRITEABLE
+        flags = parent.flags & NPY.ARRAY_ALIGNED
+        flags |= parent.flags & NPY.ARRAY_WRITEABLE
         if is_c_contiguous(self):
-            self.flags |= NPY.ARRAY_C_CONTIGUOUS
+            flags |= NPY.ARRAY_C_CONTIGUOUS
         if is_f_contiguous(self):
-            self.flags |= NPY.ARRAY_F_CONTIGUOUS
+            flags |= NPY.ARRAY_F_CONTIGUOUS
+        self.flags = flags
 
     def base(self):
         return self.orig_arr
diff --git a/pypy/objspace/std/newformat.py b/pypy/objspace/std/newformat.py
--- a/pypy/objspace/std/newformat.py
+++ b/pypy/objspace/std/newformat.py
@@ -528,7 +528,7 @@
 
         def _lit(self, s):
             if self.is_unicode:
-                return s.decode("ascii")
+                return s.decode("latin-1")
             else:
                 return s
 
@@ -586,8 +586,8 @@
                 thousands = ""
                 grouping = "\xFF"    # special value to mean 'stop'
             if self.is_unicode:
-                self._loc_dec = dec.decode("ascii")
-                self._loc_thousands = thousands.decode("ascii")
+                self._loc_dec = dec.decode("latin-1")
+                self._loc_thousands = thousands.decode("latin-1")
             else:
                 self._loc_dec = dec
                 self._loc_thousands = thousands
@@ -725,7 +725,7 @@
                 out.append_multiple_char(fill_char[0], spec.n_lpadding)
             if spec.n_sign:
                 if self.is_unicode:
-                    sign = spec.sign.decode("ascii")
+                    sign = spec.sign.decode("latin-1")
                 else:
                     sign = spec.sign
                 out.append(sign)
@@ -828,14 +828,14 @@
                 prefix = "0x"
             as_str = value.format(LONG_DIGITS[:base], prefix)
             if self.is_unicode:
-                return as_str.decode("ascii")
+                return as_str.decode("latin-1")
             return as_str
 
         def _int_to_base(self, base, value):
             if base == 10:
                 s = str(value)
                 if self.is_unicode:
-                    return s.decode("ascii")
+                    return s.decode("latin-1")
                 return s
             # This part is slow.
             negative = value < 0
@@ -954,7 +954,7 @@
             have_dec_point, to_remainder = self._parse_number(result, to_number)
             n_remainder = len(result) - to_remainder
             if self.is_unicode:
-                digits = result.decode("ascii")
+                digits = result.decode("latin-1")
             else:
                 digits = result
             spec = self._calc_num_width(0, sign, to_number, n_digits,
@@ -1059,8 +1059,8 @@
                                                                to_imag_number)
 
             if self.is_unicode:
-                re_num = re_num.decode("ascii")
-                im_num = im_num.decode("ascii")
+                re_num = re_num.decode("latin-1")
+                im_num = im_num.decode("latin-1")
 
             #set remainder, in CPython _parse_number sets this
             #using n_re_digits causes tests to fail

From noreply at buildbot.pypy.org  Fri Jun  5 10:51:24 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Fri,  5 Jun 2015 10:51:24 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: Add and use a helper module that
 delays recursive calls, to turn them
Message-ID: <20150605085124.69FE81C048F@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: optresult
Changeset: r77890:b3cb86b784a3
Date: 2015-06-05 10:36 +0200
http://bitbucket.org/pypy/pypy/changeset/b3cb86b784a3/

Log:	Add and use a helper module that delays recursive calls, to turn
	them non-recursive.

diff --git a/rpython/annotator/bookkeeper.py b/rpython/annotator/bookkeeper.py
--- a/rpython/annotator/bookkeeper.py
+++ b/rpython/annotator/bookkeeper.py
@@ -21,6 +21,7 @@
 from rpython.annotator.argument import simple_args
 from rpython.rlib.objectmodel import r_dict, r_ordereddict, Symbolic
 from rpython.tool.algo.unionfind import UnionFind
+from rpython.tool.flattenrec import FlattenRecursion
 from rpython.rtyper import extregistry
 
 
@@ -426,6 +427,8 @@
             self.methoddescs[key] = result
             return result
 
+    _see_mutable_flattenrec = FlattenRecursion()
+
     def see_mutable(self, x):
         key = (x.__class__, x)
         if key in self.seen_mutable:
@@ -434,8 +437,11 @@
         self.seen_mutable[key] = True
         self.event('mutable', x)
         source = InstanceSource(self, x)
-        for attr in source.all_instance_attributes():
-            clsdef.add_source_for_attribute(attr, source) # can trigger reflowing
+        def delayed():
+            for attr in source.all_instance_attributes():
+                clsdef.add_source_for_attribute(attr, source)
+                # ^^^ can trigger reflowing
+        self._see_mutable_flattenrec(delayed)
 
     def valueoftype(self, t):
         return annotationoftype(t, self)
diff --git a/rpython/rtyper/rclass.py b/rpython/rtyper/rclass.py
--- a/rpython/rtyper/rclass.py
+++ b/rpython/rtyper/rclass.py
@@ -7,6 +7,7 @@
 from rpython.rlib.objectmodel import UnboxedValue
 from rpython.tool.pairtype import pairtype, pair
 from rpython.tool.identity_dict import identity_dict
+from rpython.tool.flattenrec import FlattenRecursion
 from rpython.rtyper.extregistry import ExtRegistryEntry
 from rpython.rtyper.error import TyperError
 from rpython.rtyper.lltypesystem import lltype
@@ -767,11 +768,14 @@
             self.initialize_prebuilt_data(Ellipsis, self.classdef, result)
             return result
 
+    _initialize_data_flattenrec = FlattenRecursion()
+
     def initialize_prebuilt_instance(self, value, classdef, result):
         # must fill in the hash cache before the other ones
         # (see test_circular_hash_initialization)
         self.initialize_prebuilt_hash(value, result)
-        self.initialize_prebuilt_data(value, classdef, result)
+        self._initialize_data_flattenrec(self.initialize_prebuilt_data,
+                                         value, classdef, result)
 
     def get_ll_hash_function(self):
         return ll_inst_hash
diff --git a/rpython/rtyper/test/test_rclass.py b/rpython/rtyper/test/test_rclass.py
--- a/rpython/rtyper/test/test_rclass.py
+++ b/rpython/rtyper/test/test_rclass.py
@@ -1279,3 +1279,16 @@
             return cls[k](a, b).b
 
         assert self.interpret(f, [1, 4, 7]) == 7
+
+    def test_flatten_convert_const(self):
+        # check that we can convert_const() a chain of more than 1000
+        # instances
+        class A(object):
+            def __init__(self, next):
+                self.next = next
+        a = None
+        for i in range(1500):
+            a = A(a)
+        def f():
+            return a.next.next.next.next is not None
+        assert self.interpret(f, []) == True
diff --git a/rpython/tool/flattenrec.py b/rpython/tool/flattenrec.py
new file mode 100644
--- /dev/null
+++ b/rpython/tool/flattenrec.py
@@ -0,0 +1,25 @@
+"""
+A general way to flatten deeply recursive algorithms by delaying some
+parts until later.
+"""
+
+
+class FlattenRecursion(object):
+
+    def __init__(self):
+        self.later = None
+
+    def __call__(self, func, *args, **kwds):
+        """Call func(*args, **kwds), either now, or, if we're recursing,
+        then the call will be done later by the first level.
+        """
+        if self.later is not None:
+            self.later.append((func, args, kwds))
+        else:
+            self.later = lst = []
+            try:
+                func(*args, **kwds)
+                for func, args, kwds in lst:
+                    func(*args, **kwds)
+            finally:
+                self.later = None
diff --git a/rpython/tool/test/test_flattenrec.py b/rpython/tool/test/test_flattenrec.py
new file mode 100644
--- /dev/null
+++ b/rpython/tool/test/test_flattenrec.py
@@ -0,0 +1,13 @@
+from rpython.tool.flattenrec import FlattenRecursion
+
+def test_flattenrec():
+    r = FlattenRecursion()
+    seen = set()
+
+    def rec(n):
+        if n > 0:
+            r(rec, n-1)
+        seen.add(n)
+
+    rec(10000)
+    assert seen == set(range(10001))

From noreply at buildbot.pypy.org  Fri Jun  5 10:51:25 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Fri,  5 Jun 2015 10:51:25 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: Turn this class into a thread-local
 storage, to fix issues in case
Message-ID: <20150605085125.9897E1C048F@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: optresult
Changeset: r77891:02ec14154667
Date: 2015-06-05 10:39 +0200
http://bitbucket.org/pypy/pypy/changeset/02ec14154667/

Log:	Turn this class into a thread-local storage, to fix issues in case
	it's used by multiple threads

diff --git a/rpython/tool/flattenrec.py b/rpython/tool/flattenrec.py
--- a/rpython/tool/flattenrec.py
+++ b/rpython/tool/flattenrec.py
@@ -3,8 +3,14 @@
 parts until later.
 """
 
+try:
+    from thread import _local as TlsClass
+except ImportError:
+    class TlsClass(object):
+        pass
 
-class FlattenRecursion(object):
+
+class FlattenRecursion(TlsClass):
 
     def __init__(self):
         self.later = None

From noreply at buildbot.pypy.org  Fri Jun  5 10:54:52 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Fri,  5 Jun 2015 10:54:52 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: try to collect a few more times to
 see if it's a problem
Message-ID: <20150605085452.7BE8C1C048F@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77892:5a8c401e9d0b
Date: 2015-06-05 10:54 +0200
http://bitbucket.org/pypy/pypy/changeset/5a8c401e9d0b/

Log:	try to collect a few more times to see if it's a problem

diff --git a/rpython/jit/backend/llsupport/test/zrpy_gc_test.py b/rpython/jit/backend/llsupport/test/zrpy_gc_test.py
--- a/rpython/jit/backend/llsupport/test/zrpy_gc_test.py
+++ b/rpython/jit/backend/llsupport/test/zrpy_gc_test.py
@@ -51,7 +51,8 @@
             r = g(name, n)
             r_list.append(r)
             rgc.collect()
-        rgc.collect(); rgc.collect()
+        for i in range(3):
+            rgc.collect(); rgc.collect()
         freed = 0
         for r in r_list:
             if r() is None:

From noreply at buildbot.pypy.org  Fri Jun  5 11:04:06 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Fri,  5 Jun 2015 11:04:06 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: did not help, revert
Message-ID: <20150605090406.8163A1C048F@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77893:b2df1768946d
Date: 2015-06-05 11:04 +0200
http://bitbucket.org/pypy/pypy/changeset/b2df1768946d/

Log:	did not help, revert

diff --git a/rpython/jit/backend/llsupport/test/zrpy_gc_test.py b/rpython/jit/backend/llsupport/test/zrpy_gc_test.py
--- a/rpython/jit/backend/llsupport/test/zrpy_gc_test.py
+++ b/rpython/jit/backend/llsupport/test/zrpy_gc_test.py
@@ -51,8 +51,7 @@
             r = g(name, n)
             r_list.append(r)
             rgc.collect()
-        for i in range(3):
-            rgc.collect(); rgc.collect()
+        rgc.collect(); rgc.collect()
         freed = 0
         for r in r_list:
             if r() is None:

From noreply at buildbot.pypy.org  Fri Jun  5 11:04:53 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Fri,  5 Jun 2015 11:04:53 +0200 (CEST)
Subject: [pypy-commit] pypy default: Add a test similar to
 test_rclass.test_flatten_convert_const, passes
Message-ID: <20150605090453.6637B1C048F@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r77894:f296c1fbe753
Date: 2015-06-05 10:05 +0100
http://bitbucket.org/pypy/pypy/changeset/f296c1fbe753/

Log:	Add a test similar to test_rclass.test_flatten_convert_const, passes

diff --git a/rpython/translator/c/test/test_newgc.py b/rpython/translator/c/test/test_newgc.py
--- a/rpython/translator/c/test/test_newgc.py
+++ b/rpython/translator/c/test/test_newgc.py
@@ -1246,6 +1246,28 @@
     def test_check_zero_works(self):
         self.run("check_zero_works")
 
+    def define_long_chain_of_instances(self):
+        class A(object):
+            def __init__(self, next):
+                self.next = next
+        a = None
+        for i in range(1500):
+            a = A(a)
+
+        def fn():
+            i = 0
+            x = a
+            while x is not None:
+                i += 1
+                x = x.next
+            return i
+        return fn
+
+    def test_long_chain_of_instances(self):
+        res = self.run("long_chain_of_instances")
+        assert res == 1500
+        
+
 class TestSemiSpaceGC(UsingFrameworkTest, snippet.SemiSpaceGCTestDefines):
     gcpolicy = "semispace"
     should_be_moving = True

From noreply at buildbot.pypy.org  Fri Jun  5 11:59:51 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Fri,  5 Jun 2015 11:59:51 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: random fix
Message-ID: <20150605095951.1F9CA1C033F@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77895:99069ececc7c
Date: 2015-06-05 11:59 +0200
http://bitbucket.org/pypy/pypy/changeset/99069ececc7c/

Log:	random fix

diff --git a/rpython/jit/codewriter/heaptracker.py b/rpython/jit/codewriter/heaptracker.py
--- a/rpython/jit/codewriter/heaptracker.py
+++ b/rpython/jit/codewriter/heaptracker.py
@@ -153,6 +153,8 @@
         FIELD = getattr(STRUCT, name)
         if FIELD is lltype.Void:
             continue
+        if name.startswith('_pad'):
+            continue
         if name == 'typeptr':
             continue # dealt otherwise
         elif isinstance(FIELD, lltype.Struct):

From noreply at buildbot.pypy.org  Fri Jun  5 12:13:38 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Fri,  5 Jun 2015 12:13:38 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: try to fix annotation
Message-ID: <20150605101338.95B721C048F@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77896:4c25326069a6
Date: 2015-06-05 12:13 +0200
http://bitbucket.org/pypy/pypy/changeset/4c25326069a6/

Log:	try to fix annotation

diff --git a/rpython/jit/backend/llsupport/descr.py b/rpython/jit/backend/llsupport/descr.py
--- a/rpython/jit/backend/llsupport/descr.py
+++ b/rpython/jit/backend/llsupport/descr.py
@@ -35,6 +35,7 @@
 class SizeDescr(AbstractDescr):
     size = 0      # help translation
     tid = llop.combine_ushort(lltype.Signed, 0, 0)
+    _corresponding_vtable = lltype.nullptr(rclass.OBJECT_VTABLE)
 
     def __init__(self, size, count_fields_if_immut=-1,
                  gc_fielddescrs=None, all_fielddescrs=None,

From noreply at buildbot.pypy.org  Fri Jun  5 12:17:16 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Fri,  5 Jun 2015 12:17:16 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: do it that way
Message-ID: <20150605101716.9278E1C048F@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77897:b16250fb7832
Date: 2015-06-05 12:17 +0200
http://bitbucket.org/pypy/pypy/changeset/b16250fb7832/

Log:	do it that way

diff --git a/rpython/jit/codewriter/heaptracker.py b/rpython/jit/codewriter/heaptracker.py
--- a/rpython/jit/codewriter/heaptracker.py
+++ b/rpython/jit/codewriter/heaptracker.py
@@ -93,11 +93,9 @@
     # register the correspondance 'vtable' <-> 'STRUCT' in the cpu
     sizedescr = cpu.sizeof(STRUCT, has_gcstruct_a_vtable(STRUCT))
     assert sizedescr.as_vtable_size_descr() is sizedescr
-    try:
+    if sizedescr._corresponding_vtable:
         assert sizedescr._corresponding_vtable == vtable
         return
-    except AttributeError:
-        pass
     assert lltype.typeOf(vtable) == VTABLETYPE
     if not hasattr(cpu.tracker, '_all_size_descrs_with_vtable'):
         cpu.tracker._all_size_descrs_with_vtable = []

From noreply at buildbot.pypy.org  Fri Jun  5 12:28:56 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Fri,  5 Jun 2015 12:28:56 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: a bit blindly implement
	direct_call_release_gil
Message-ID: <20150605102856.381FB1C048F@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77898:bbf72f83ccb5
Date: 2015-06-05 12:29 +0200
http://bitbucket.org/pypy/pypy/changeset/bbf72f83ccb5/

Log:	a bit blindly implement direct_call_release_gil

diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -1590,6 +1590,9 @@
                 if effectinfo.oopspecindex == effectinfo.OS_LIBFFI_CALL:
                     resbox = self.metainterp.direct_libffi_call(allboxes, descr,
                                                                 tp)
+                elif effectinfo.is_call_release_gil():
+                    resbox = self.metainterp.direct_call_release_gil(allboxes,
+                                                                descr, tp)
                 elif tp == 'i':
                     resbox = self.metainterp.execute_and_record_varargs(
                         rop.CALL_MAY_FORCE_I, allboxes, descr=descr)
@@ -1604,8 +1607,6 @@
                         rop.CALL_MAY_FORCE_N, allboxes, descr=descr)
                 else:
                     assert False
-                if effectinfo.is_call_release_gil():
-                    self.metainterp.direct_call_release_gil()
                 self.metainterp.vrefs_after_residual_call()
                 vablebox = None
                 if assembler_call:
@@ -3041,6 +3042,13 @@
             box_result = self.history.record(
                 rop.CALL_RELEASE_GIL_F, [c_saveall, argboxes[2]] + arg_boxes,
                 value, descr=calldescr)
+        elif tp == 'v':
+            value = executor.execute_varargs(self.cpu, self,
+                                             rop.CALL_MAY_FORCE_N,
+                                             argboxes, orig_calldescr)
+            box_result = self.history.record(
+                rop.CALL_RELEASE_GIL_N, [c_saveall, argboxes[2]] + arg_boxes,
+                value, descr=calldescr)
         else:
             assert False
         #
@@ -3048,20 +3056,38 @@
         # special op libffi_save_result_{int,float}
         return box_result
 
-    def direct_call_release_gil(self):
-        op = self.history.operations.pop()
-        assert op.is_call_may_force()
-        descr = op.getdescr()
-        effectinfo = descr.get_extra_info()
+    def direct_call_release_gil(self, argboxes, calldescr, tp):
+        effectinfo = calldescr.get_extra_info()
         realfuncaddr, saveerr = effectinfo.call_release_gil_target
         funcbox = ConstInt(heaptracker.adr2int(realfuncaddr))
         savebox = ConstInt(saveerr)
-        assert False, "not yet"
-        self.history.record(rop.CALL_RELEASE_GIL,
-                            [savebox, funcbox] + op.getarglist()[1:],
-                            op.result, descr)
+        if tp == 'i':
+            value = executor.execute_varargs(self.cpu, self,
+                                             rop.CALL_MAY_FORCE_I,
+                                             argboxes, calldescr)
+            resbox = self.history.record(rop.CALL_RELEASE_GIL_I,
+                                         [savebox, funcbox] + argboxes[1:],
+                                         value, calldescr)
+        elif tp == 'f':
+            value = executor.execute_varargs(self.cpu, self,
+                                             rop.CALL_MAY_FORCE_F,
+                                             argboxes, calldescr)
+            resbox = self.history.record(rop.CALL_RELEASE_GIL_F,
+                                         [savebox, funcbox] + argboxes[1:],
+                                         value, calldescr)
+        elif tp == 'v':
+            value = executor.execute_varargs(self.cpu, self,
+                                             rop.CALL_MAY_FORCE_N,
+                                             argboxes, calldescr)
+            resbox = self.history.record(rop.CALL_RELEASE_GIL_N,
+                                         [savebox, funcbox] + argboxes[1:],
+                                         value, calldescr)
+        else:
+            assert False, "no CALL_RELEASE_GIL_R"
+            
         if not we_are_translated():       # for llgraph
-            descr._original_func_ = op.getarg(0).value
+            calldescr._original_func_ = argboxes[0].getint()
+        return resbox
 
     def do_not_in_trace_call(self, allboxes, descr):
         self.clear_exception()

From noreply at buildbot.pypy.org  Fri Jun  5 12:38:38 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Fri,  5 Jun 2015 12:38:38 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: fix
Message-ID: <20150605103838.E6BDC1C048F@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77899:c19969e54850
Date: 2015-06-05 12:38 +0200
http://bitbucket.org/pypy/pypy/changeset/c19969e54850/

Log:	fix

diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -3043,12 +3043,13 @@
                 rop.CALL_RELEASE_GIL_F, [c_saveall, argboxes[2]] + arg_boxes,
                 value, descr=calldescr)
         elif tp == 'v':
-            value = executor.execute_varargs(self.cpu, self,
+            executor.execute_varargs(self.cpu, self,
                                              rop.CALL_MAY_FORCE_N,
                                              argboxes, orig_calldescr)
-            box_result = self.history.record(
+            self.history.record(
                 rop.CALL_RELEASE_GIL_N, [c_saveall, argboxes[2]] + arg_boxes,
-                value, descr=calldescr)
+                None, descr=calldescr)
+            box_result = None
         else:
             assert False
         #
@@ -3076,12 +3077,13 @@
                                          [savebox, funcbox] + argboxes[1:],
                                          value, calldescr)
         elif tp == 'v':
-            value = executor.execute_varargs(self.cpu, self,
+            executor.execute_varargs(self.cpu, self,
                                              rop.CALL_MAY_FORCE_N,
                                              argboxes, calldescr)
-            resbox = self.history.record(rop.CALL_RELEASE_GIL_N,
+            self.history.record(rop.CALL_RELEASE_GIL_N,
                                          [savebox, funcbox] + argboxes[1:],
-                                         value, calldescr)
+                                         calldescr)
+            resbox = None
         else:
             assert False, "no CALL_RELEASE_GIL_R"
             

From noreply at buildbot.pypy.org  Fri Jun  5 12:43:44 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Fri,  5 Jun 2015 12:43:44 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: fix the spelling
Message-ID: <20150605104344.04E1B1C1027@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77900:ef3f82932eea
Date: 2015-06-05 12:43 +0200
http://bitbucket.org/pypy/pypy/changeset/ef3f82932eea/

Log:	fix the spelling

diff --git a/rpython/jit/codewriter/heaptracker.py b/rpython/jit/codewriter/heaptracker.py
--- a/rpython/jit/codewriter/heaptracker.py
+++ b/rpython/jit/codewriter/heaptracker.py
@@ -151,7 +151,7 @@
         FIELD = getattr(STRUCT, name)
         if FIELD is lltype.Void:
             continue
-        if name.startswith('_pad'):
+        if name.startswith('c__pad'):
             continue
         if name == 'typeptr':
             continue # dealt otherwise

From noreply at buildbot.pypy.org  Fri Jun  5 12:45:56 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Fri,  5 Jun 2015 12:45:56 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: missed a None here
Message-ID: <20150605104556.245D11C1027@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77901:b4b7db4c6656
Date: 2015-06-05 12:46 +0200
http://bitbucket.org/pypy/pypy/changeset/b4b7db4c6656/

Log:	missed a None here

diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -3082,7 +3082,7 @@
                                              argboxes, calldescr)
             self.history.record(rop.CALL_RELEASE_GIL_N,
                                          [savebox, funcbox] + argboxes[1:],
-                                         calldescr)
+                                         None, calldescr)
             resbox = None
         else:
             assert False, "no CALL_RELEASE_GIL_R"

From noreply at buildbot.pypy.org  Fri Jun  5 12:59:22 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Fri,  5 Jun 2015 12:59:22 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: int expansion for int16 and int8 added,
 int32/16 test added first already passes
Message-ID: <20150605105922.7DDF11C1038@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r77902:eb3cc9cf75f4
Date: 2015-06-05 12:59 +0200
http://bitbucket.org/pypy/pypy/changeset/eb3cc9cf75f4/

Log:	int expansion for int16 and int8 added, int32/16 test added first
	already passes

diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -226,6 +226,32 @@
         assert int(result) == 7+16+8+16
         self.check_vectorized(2, 2)
 
+    def define_int16_expand():
+        return """
+        a = astype(|30|, int16)
+        c = astype(|1|, int16)
+        c[0] = 16i
+        b = a + c
+        sum(b -> 7:14)
+        """
+    def test_int16_expand(self):
+        result = self.run("int16_expand")
+        assert int(result) == 8*16 + sum(range(7,15))
+        self.check_vectorized(2, 2)
+
+    def define_int8_expand():
+        return """
+        a = astype(|30|, int16)
+        c = astype(|1|, int16)
+        c[0] = 8i
+        b = a + c
+        sum(b -> 0:17)
+        """
+    def test_int16_expand(self):
+        result = self.run("int16_expand")
+        assert int(result) == 16*8 + sum(range(0,17))
+        self.check_vectorized(2, 2)
+
     def define_int32_add_const():
         return """
         a = astype(|30|, int32)
diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -54,6 +54,7 @@
         self.float_const_abs_addr = 0
         self.single_float_const_neg_addr = 0
         self.single_float_const_abs_addr = 0
+        self.expand_byte_mask_addr = 0
         self.malloc_slowpath = 0
         self.malloc_slowpath_varsize = 0
         self.wb_slowpath = [0, 0, 0, 0, 0]
@@ -102,9 +103,11 @@
         single_abs_const = '\xFF\xFF\xFF\x7F\xFF\xFF\xFF\x7F\xFF\xFF\xFF\x7F\xFF\xFF\xFF\x7F'
         # 0x80000000800000008000000080000000
         single_neg_const = '\x00\x00\x00\x80\x00\x00\x00\x80\x00\x00\x00\x80\x00\x00\x00\x80'
+        zero_const = '\x00' * 16
         #
         data = neg_const + abs_const + \
-               single_neg_const + single_abs_const
+               single_neg_const + single_abs_const + \
+               zero_const
         datablockwrapper = MachineDataBlockWrapper(self.cpu.asmmemmgr, [])
         float_constants = datablockwrapper.malloc_aligned(len(data), alignment=16)
         datablockwrapper.done()
@@ -115,6 +118,7 @@
         self.float_const_abs_addr = float_constants + 16
         self.single_float_const_neg_addr = float_constants + 32
         self.single_float_const_abs_addr = float_constants + 48
+        self.expand_byte_mask_addr = float_constants + 64
 
     def set_extra_stack_depth(self, mc, value):
         if self._is_asmgcc():
@@ -2641,7 +2645,18 @@
         assert isinstance(srcloc, RegLoc)
         assert not srcloc.is_xmm
         size = sizeloc.value
-        if size == 8:
+        if size == 1:
+            self.mc.PINSRB_xri(resloc.value, srcloc.value, 0)
+            self.mc.PSHUFB(resloc, heap(self.expand_byte_mask_addr))
+        elif size == 2:
+            self.mc.PINSRW_xri(resloc.value, srcloc.value, 0)
+            self.mc.PINSRW_xri(resloc.value, srcloc.value, 4)
+            self.mc.PSHUFLW_xxi(resloc.value, resloc.value, 0)
+            self.mc.PSHUFHW_xxi(resloc.value, resloc.value, 0)
+        elif size == 4:
+            self.mc.PINSRD_xri(resloc.value, srcloc.value, 0)
+            self.mc.PSHUFD_xxi(resloc.value, resloc.value, 0)
+        elif size == 8:
             self.mc.PINSRQ_xri(resloc.value, srcloc.value, 0)
             self.mc.PINSRQ_xri(resloc.value, srcloc.value, 1)
         else:
diff --git a/rpython/jit/backend/x86/regloc.py b/rpython/jit/backend/x86/regloc.py
--- a/rpython/jit/backend/x86/regloc.py
+++ b/rpython/jit/backend/x86/regloc.py
@@ -715,6 +715,8 @@
     PUNPCKLDQ = _binaryop('PUNPCKLDQ')
     PUNPCKHDQ = _binaryop('PUNPCKHDQ')
 
+    PSHUFB = _binaryop('PSHUFB')
+
     CALL = _relative_unaryop('CALL')
     JMP = _relative_unaryop('JMP')
 
diff --git a/rpython/jit/backend/x86/rx86.py b/rpython/jit/backend/x86/rx86.py
--- a/rpython/jit/backend/x86/rx86.py
+++ b/rpython/jit/backend/x86/rx86.py
@@ -743,6 +743,11 @@
     SHUFPD_xxi = xmminsn('\x66', rex_nw, '\x0F\xC6', register(1,8), register(2), '\xC0', immediate(3, 'b'))
 
     PSHUFD_xxi = xmminsn('\x66', rex_nw, '\x0F\x70', register(1,8), register(2), '\xC0', immediate(3, 'b'))
+    PSHUFHW_xxi = xmminsn('\xF3', rex_nw, '\x0F\x70', register(1,8), register(2), '\xC0', immediate(3, 'b'))
+    PSHUFLW_xxi = xmminsn('\xF2', rex_nw, '\x0F\x70', register(1,8), register(2), '\xC0', immediate(3, 'b'))
+    PSHUFB_xx = xmminsn('\x66', rex_nw, '\x0F\x38\x00', register(1,8), register(2), '\xC0')
+    PSHUFB_xm = xmminsn('\x66', rex_nw, '\x0F\x38\x00', register(1,8), mem_reg_plus_const(2))
+
 
     # following require SSE4_1
     PEXTRQ_rxi = xmminsn('\x66', rex_w, '\x0F\x3A\x16', register(2,8), register(1), '\xC0', immediate(3, 'b'))
diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py
--- a/rpython/jit/metainterp/history.py
+++ b/rpython/jit/metainterp/history.py
@@ -395,7 +395,7 @@
                 t = 'b'
             self._str = '%s%d' % (t, Box._counter)
             if self.type == VECTOR:
-                self._str = '%s%d[%s%d#%d]' % (t, Box._counter, self.item_type,
+                self._str = '%s%d[%s%d|%d]' % (t, Box._counter, self.item_type,
                                                self.item_size * 8, self.item_count)
             Box._counter += 1
         return self._str
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_schedule.py b/rpython/jit/metainterp/optimizeopt/test/test_schedule.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_schedule.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_schedule.py
@@ -22,7 +22,7 @@
             'long': self.intarraydescr,
             'int': self.int32arraydescr,
         }
-        loop = opparse("        [p0,p1,p2,p3,p4,p5,i0,i1,i2,i3,i4,i5,i6,i7,i8,i9,f0,f1,f2,f3,f4,f5]\n" + source + \
+        loop = opparse("        [p0,p1,p2,p3,p4,p5,i0,i1,i2,i3,i4,i5,i6,i7,i8,i9,f0,f1,f2,f3,f4,f5,v103204[i32|4]]\n" + source + \
                        "\n        jump(p0,p1,p2,p3,p4,p5,i0,i1,i2,i3,i4,i5,i6,i7,i8,i9,f0,f1,f2,f3,f4,f5)",
                        cpu=self.cpu,
                        namespace=ns)
@@ -39,13 +39,15 @@
     def pack(self, loop, l, r):
         return [Node(op,1+l+i) for i,op in enumerate(loop.operations[1+l:1+r])]
 
-    def schedule(self, loop_orig, packs, vec_reg_size=16, prepend_invariant=False):
+    def schedule(self, loop_orig, packs, vec_reg_size=16, prepend_invariant=False, getvboxfunc=None):
         loop = get_model(False).ExtendedTreeLoop("loop")
         loop.original_jitcell_token = loop_orig.original_jitcell_token
         loop.inputargs = loop_orig.inputargs
 
         ops = []
         vsd = VecScheduleData(vec_reg_size)
+        if getvboxfunc is not None:
+            vsd.getvector_of_box = getvboxfunc
         for pack in packs:
             if len(pack) == 1:
                 ops.append(pack[0].getoperation())
@@ -73,7 +75,7 @@
         pack1 = self.pack(loop1, 0, 6)
         loop2 = self.schedule(loop1, [pack1])
         loop3 = self.parse("""
-        v1[i32#4] = vec_raw_load(p0, i0, 4, descr=float)
+        v10[i32|4] = vec_raw_load(p0, i0, 4, descr=float)
         i14 = raw_load(p0, i4, descr=float)
         i15 = raw_load(p0, i5, descr=float)
         """, False)
@@ -90,9 +92,9 @@
         pack2 = self.pack(loop1, 2, 4)
         loop2 = self.schedule(loop1, [pack1, pack2])
         loop3 = self.parse("""
-        v1[i64#2] = vec_raw_load(p0, i0, 2, descr=long)
-        v2[i32#2] = vec_int_signext(v1[i64#2], 4)
-        v3[f64#2] = vec_cast_int_to_float(v2[i32#2])
+        v10[i64|2] = vec_raw_load(p0, i0, 2, descr=long)
+        v20[i32|2] = vec_int_signext(v10[i64|2], 4)
+        v30[f64|2] = vec_cast_int_to_float(v20[i32|2])
         """, False)
         self.assert_equal(loop2, loop3)
 
@@ -104,12 +106,12 @@
         pack1 = self.pack(loop1, 0, 2)
         loop2 = self.schedule(loop1, [pack1], prepend_invariant=True)
         loop3 = self.parse("""
-        v1[i64#2] = vec_box(2)
-        v2[i64#2] = vec_int_pack(v1[i64#2], i0, 0, 1)
-        v3[i64#2] = vec_int_pack(v2[i64#2], i1, 1, 1)
-        v4[i64#2] = vec_int_expand(73)
+        v10[i64|2] = vec_box(2)
+        v20[i64|2] = vec_int_pack(v10[i64|2], i0, 0, 1)
+        v30[i64|2] = vec_int_pack(v20[i64|2], i1, 1, 1)
+        v40[i64|2] = vec_int_expand(73)
         #
-        v5[i64#2] = vec_int_add(v3[i64#2], v4[i64#2])
+        v50[i64|2] = vec_int_add(v30[i64|2], v40[i64|2])
         """, False)
         self.assert_equal(loop2, loop3)
 
@@ -120,12 +122,12 @@
         pack1 = self.pack(loop1, 0, 2)
         loop2 = self.schedule(loop1, [pack1], prepend_invariant=True)
         loop3 = self.parse("""
-        v1[f64#2] = vec_box(2)
-        v2[f64#2] = vec_float_pack(v1[f64#2], f0, 0, 1)
-        v3[f64#2] = vec_float_pack(v2[f64#2], f1, 1, 1)
-        v4[f64#2] = vec_float_expand(73.0)
+        v10[f64|2] = vec_box(2)
+        v20[f64|2] = vec_float_pack(v10[f64|2], f0, 0, 1)
+        v30[f64|2] = vec_float_pack(v20[f64|2], f1, 1, 1)
+        v40[f64|2] = vec_float_expand(73.0)
         #
-        v5[f64#2] = vec_float_add(v3[f64#2], v4[f64#2])
+        v50[f64|2] = vec_float_add(v30[f64|2], v40[f64|2])
         """, False)
         self.assert_equal(loop2, loop3)
 
@@ -140,12 +142,35 @@
         pack2 = self.pack(loop1, 2, 4)
         loop2 = self.schedule(loop1, [pack1, pack2], prepend_invariant=True)
         loop3 = self.parse("""
-        v1[f64#2] = vec_box(2)
-        v2[f64#2] = vec_float_pack(v1[f64#2], f0, 0, 1)
-        v3[f64#2] = vec_float_pack(v2[f64#2], f1, 1, 1)
-        v4[f64#2] = vec_float_expand(f5) # only expaned once
+        v10[f64|2] = vec_box(2)
+        v20[f64|2] = vec_float_pack(v10[f64|2], f0, 0, 1)
+        v30[f64|2] = vec_float_pack(v20[f64|2], f1, 1, 1)
+        v40[f64|2] = vec_float_expand(f5) | only expaned once
         #
-        v5[f64#2] = vec_float_add(v3[f64#2], v4[f64#2])
-        v6[f64#2] = vec_float_add(v5[f64#2], v4[f64#2])
+        v50[f64|2] = vec_float_add(v30[f64|2], v40[f64|2])
+        v60[f64|2] = vec_float_add(v50[f64|2], v40[f64|2])
         """, False)
         self.assert_equal(loop2, loop3)
+
+    def find_input_arg(self, name, loop):
+        for arg in loop.inputargs:
+            if str(arg).startswith(name):
+                return arg
+        raise Exception("could not find %s in args %s" % (name, loop.inputargs))
+
+    def test_signext_int16(self):
+        loop1 = self.parse("""
+        i10 = int_signext(i1, 2)
+        i11 = int_signext(i1, 2)
+        i12 = int_signext(i1, 2)
+        i13 = int_signext(i1, 2)
+        """)
+        pack1 = self.pack(loop1, 0, 4)
+        v103204 = self.find_input_arg('v103204', loop1)
+        def i1inv103204(var):
+            return 0, v103204
+        loop2 = self.schedule(loop1, [pack1], prepend_invariant=True, getvboxfunc=i1inv103204)
+        loop3 = self.parse("""
+        v11[i16|4] = vec_int_signext(v103204[i32|4], 2)
+        """, False)
+        self.assert_equal(loop2, loop3)
diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/vectorize.py
@@ -776,7 +776,7 @@
 class PackType(object):
     UNKNOWN_TYPE = '-'
 
-    def __init__(self, type, size, signed, count=-1, scalar_cost=1, vector_cost=1):
+    def __init__(self, type, size, signed, count=-1):
         assert type in (FLOAT, INT, PackType.UNKNOWN_TYPE)
         self.type = type
         self.size = size
@@ -826,7 +826,6 @@
     def clone(self):
         return PackType(self.type, self.size, self.signed, self.count)
 
-
 class OpToVectorOp(object):
     def __init__(self, arg_ptypes, result_ptype):
         self.arg_ptypes = [a for a in arg_ptypes] # do not use a tuple. rpython cannot union
@@ -837,6 +836,9 @@
         self.input_type = None
         self.output_type = None
 
+    def clone_vbox_set_count(self, box, count):
+        return BoxVector(box.item_type, count, box.item_size, box.item_signed)
+
     def is_vector_arg(self, i):
         if i < 0 or i >= len(self.arg_ptypes):
             return False
@@ -985,8 +987,7 @@
         return vbox_cloned
 
     def unpack(self, vbox, index, count, arg_ptype):
-        vbox_cloned = vbox.clonebox()
-        vbox_cloned.item_count = count
+        vbox_cloned = self.clone_vbox_set_count(vbox, count)
         opnum = rop.VEC_FLOAT_UNPACK
         if vbox.item_type == INT:
             opnum = rop.VEC_INT_UNPACK
@@ -1012,8 +1013,8 @@
             if pos == -1:
                 i += 1
                 continue
-            new_box = tgt_box.clonebox()
-            new_box.item_count += src_box.item_count
+            count = tgt_box.item_count + src_box.item_count
+            new_box = self.clone_vbox_set_count(tgt_box, count)
             op = ResOperation(opnum, [tgt_box, src_box, ConstInt(i),
                                       ConstInt(src_box.item_count)], new_box)
             self.preamble_ops.append(op)
diff --git a/rpython/jit/tool/oparser.py b/rpython/jit/tool/oparser.py
--- a/rpython/jit/tool/oparser.py
+++ b/rpython/jit/tool/oparser.py
@@ -123,12 +123,12 @@
             box = ts.BoxRef()
             _box_counter_more_than(self.model, elem[1:])
         elif elem.startswith('v'):
-            pattern = re.compile('.*\[(u?)(i|f)(\d+)#(\d+)\]')
+            pattern = re.compile('.*\[(u?)(i|f)(\d+)(#|\|)(\d+)\]')
             match = pattern.match(elem)
             if match:
                 item_type = match.group(2)[0]
                 item_size = int(match.group(3)) // 8
-                item_count = int(match.group(4))
+                item_count = int(match.group(5))
                 item_signed = not (match.group(1) == 'u')
                 box = self.model.BoxVector(item_type, item_count, item_size, item_signed)
                 lbracket = elem.find('[')

From noreply at buildbot.pypy.org  Fri Jun  5 13:14:22 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Fri,  5 Jun 2015 13:14:22 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: updating the input type. when items are
 packed their count changes, thus the input and output type must be updated
Message-ID: <20150605111422.485F31C048F@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r77903:9f502085659b
Date: 2015-06-05 13:14 +0200
http://bitbucket.org/pypy/pypy/changeset/9f502085659b/

Log:	updating the input type. when items are packed their count changes,
	thus the input and output type must be updated

diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/vectorize.py
@@ -864,12 +864,16 @@
     def determine_output_type(self, op):
         return self.determine_input_type(op)
 
+    def update_input_output(self, pack):
+        op0 = pack.operations[0].getoperation()
+        self.input_type = self.determine_input_type(op0)
+        self.output_type = self.determine_output_type(op0)
+
     def as_vector_operation(self, pack, sched_data, oplist):
         self.sched_data = sched_data
         self.preamble_ops = oplist
-        op0 = pack.operations[0].getoperation()
-        self.input_type = self.determine_input_type(op0)
-        self.output_type = self.determine_output_type(op0)
+        self.update_input_output(pack)
+
 
         off = 0
         stride = self.split_pack(pack)
@@ -956,14 +960,17 @@
             # the argument is scattered along different vector boxes
             args = [op.getoperation().getarg(argidx) for op in ops]
             vbox = self._pack(vbox, packed, args, packable)
+            self.update_input_output(self.pack)
         elif packed > packable:
             # the argument has more items than the operation is able to process!
             vbox = self.unpack(vbox, off, packable, self.input_type)
+            self.update_input_output(self.pack)
         #
         if off != 0 and box_pos != 0:
             # The original box is at a position != 0 but it
             # is required to be at position 0. Unpack it!
             vbox = self.unpack(vbox, off, len(ops), self.input_type)
+            self.update_input_output(self.pack)
         # convert size i64 -> i32, i32 -> i64, ...
         if self.input_type.getsize() > 0 and \
            self.input_type.getsize() != vbox.getsize():

From noreply at buildbot.pypy.org  Fri Jun  5 13:34:57 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Fri,  5 Jun 2015 13:34:57 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: one potential stupid segfault
Message-ID: <20150605113457.AFFDF1C11B6@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77904:40cba7408802
Date: 2015-06-05 13:35 +0200
http://bitbucket.org/pypy/pypy/changeset/40cba7408802/

Log:	one potential stupid segfault

diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py
--- a/rpython/jit/metainterp/optimizeopt/rewrite.py
+++ b/rpython/jit/metainterp/optimizeopt/rewrite.py
@@ -481,7 +481,10 @@
                 else:
                     cls0 = info0.get_known_class(self.optimizer.cpu)
                 if cls0 is not None:
-                    cls1 = info1.get_known_class(self.optimizer.cpu)
+                    if info1 is None:
+                        cls1 = None
+                    else:
+                        cls1 = info1.get_known_class(self.optimizer.cpu)
                     if cls1 is not None and not cls0.same_constant(cls1):
                         # cannot be the same object, as we know that their
                         # class is different

From noreply at buildbot.pypy.org  Fri Jun  5 13:56:34 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Fri,  5 Jun 2015 13:56:34 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: an experiment in specialization
Message-ID: <20150605115634.14F091C1027@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77905:96d560350883
Date: 2015-06-05 13:56 +0200
http://bitbucket.org/pypy/pypy/changeset/96d560350883/

Log:	an experiment in specialization

diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py
--- a/rpython/jit/backend/llsupport/rewrite.py
+++ b/rpython/jit/backend/llsupport/rewrite.py
@@ -86,25 +86,12 @@
         return op
 
     def emit_op(self, op):
-        op = self.get_box_replacement(op)
-        orig_op = op
-        # XXX specialize on number of args
-        replaced = False
-        for i in range(op.numargs()):
-            orig_arg = op.getarg(i)
-            arg = self.get_box_replacement(orig_arg)
-            if orig_arg is not arg:
-                if not replaced:
-                    op = op.copy_and_change(op.getopnum())
-                    orig_op.set_forwarded(op)
-                    replaced = True
-                op.setarg(i, arg)
         if op.is_guard():
-            if not replaced:
-                op = op.copy_and_change(op.getopnum())
-                orig_op.set_forwarded(op)
-            op.setfailargs([self.get_box_replacement(a, True)
-                            for a in op.getfailargs()])
+            op = op.copy_and_change(op.getopnum())
+            op = op.get_replacement_for_rewrite()
+            op.setfailargs([arg.get_replacement_for_rewrite() for arg in op.getfailargs()])
+        else:
+            op = op.get_replacement_for_rewrite()
         self._newops.append(op)
 
     def replace_op_with(self, op, newop):
diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py
--- a/rpython/jit/metainterp/history.py
+++ b/rpython/jit/metainterp/history.py
@@ -204,6 +204,9 @@
     def constbox(self):
         return self
 
+    def get_replacement_for_rewrite(self):
+        return self
+
     def same_box(self, other):
         return self.same_constant(other)
 
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -24,6 +24,9 @@
     def get_forwarded(self):
         return None
 
+    def get_replacement(self):
+        return self
+
     def set_forwarded(self, forwarded_to):
         raise Exception("oups")
 
@@ -74,6 +77,11 @@
     def get_forwarded(self):
         return self._forwarded
 
+    def get_replacement(self):
+        if self._forwarded:
+            return self._forwarded
+        return self
+
     def set_forwarded(self, forwarded_to):
         self._forwarded = forwarded_to
 
@@ -419,6 +427,12 @@
     def get_forwarded(self):
         return self._forwarded
 
+    # this is for rewrite.py, we can have several versions depending on
+    # invariants
+    def get_replacement_for_rewrite(self):
+        assert self._forwarded is None
+        return self
+
     def set_forwarded(self, forwarded_to):
         self._forwarded = forwarded_to
 
@@ -464,6 +478,11 @@
     def initarglist(self, args):
         assert len(args) == 0
 
+    def get_replacement_for_rewrite(self):
+        if self._forwarded:
+            return self._forwarded
+        return self
+
     def getarglist(self):
         return []
 
@@ -485,6 +504,18 @@
         assert len(args) == 1
         self._arg0, = args
 
+    def get_replacement_for_rewrite(self):
+        if self._forwarded:
+            return self._forwarded.get_replacement_for_rewrite()
+        arg0 = self._arg0.get_replacement()
+        if arg0 is not self._arg0:
+            op = self.__class__()
+            if isinstance(self, ResOpWithDescr):
+                op.setdescr(self.getdescr())
+            op._arg0 = arg0
+            return op
+        return self
+
     def getarglist(self):
         return [self._arg0]
 
@@ -532,6 +563,20 @@
         else:
             raise IndexError
 
+    def get_replacement_for_rewrite(self):
+        if self._forwarded:
+            return self._forwarded.get_replacement_for_rewrite()
+        arg0 = self._arg0.get_replacement()
+        arg1 = self._arg1.get_replacement()
+        if arg0 is not self._arg0 or arg1 is not self._arg1:
+            op = self.__class__()
+            if isinstance(self, ResOpWithDescr):
+                op.setdescr(self.getdescr())
+            op._arg0 = arg0
+            op._arg1 = arg1
+            return op
+        return self
+
     def getarglist(self):
         return [self._arg0, self._arg1]
 
@@ -562,6 +607,23 @@
         else:
             raise IndexError
 
+    def get_replacement_for_rewrite(self):
+        if self._forwarded:
+            return self._forwarded.get_replacement_for_rewrite()
+        arg0 = self._arg0.get_replacement()
+        arg1 = self._arg1.get_replacement()
+        arg2 = self._arg2.get_replacement()
+        if (arg0 is not self._arg0 or arg1 is not self._arg1 or
+            arg2 is not self._arg2):
+            op = self.__class__()
+            if isinstance(self, ResOpWithDescr):
+                op.setdescr(self.getdescr())
+            op._arg0 = arg0
+            op._arg1 = arg1
+            op._arg2 = arg2
+            return op
+        return self
+
     def setarg(self, i, box):
         if i == 0:
             self._arg0 = box
@@ -583,6 +645,20 @@
                self.__class__.__name__.startswith('FINISH'):   # XXX remove me
             assert len(args) <= 1      # FINISH operations take 0 or 1 arg now
 
+    def get_replacement_for_rewrite(self):
+        if self._forwarded:
+            return self._forwarded.get_replacement_for_rewrite()
+        for arg in self._args:
+            if arg is not arg.get_replacement():
+                break
+        else:
+            return self
+        op = self.__class__()
+        op._args = [arg.get_replacement() for arg in self._args]
+        if isinstance(self, ResOpWithDescr):
+            op.setdescr(self.getdescr())
+        return op
+
     def getarglist(self):
         return self._args
 

From noreply at buildbot.pypy.org  Fri Jun  5 14:11:39 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Fri,  5 Jun 2015 14:11:39 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: fix for running tests
Message-ID: <20150605121139.B31231C048F@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77906:3bbfaffdc9ba
Date: 2015-06-05 14:11 +0200
http://bitbucket.org/pypy/pypy/changeset/3bbfaffdc9ba/

Log:	fix for running tests

diff --git a/rpython/jit/codewriter/heaptracker.py b/rpython/jit/codewriter/heaptracker.py
--- a/rpython/jit/codewriter/heaptracker.py
+++ b/rpython/jit/codewriter/heaptracker.py
@@ -93,7 +93,7 @@
     # register the correspondance 'vtable' <-> 'STRUCT' in the cpu
     sizedescr = cpu.sizeof(STRUCT, has_gcstruct_a_vtable(STRUCT))
     assert sizedescr.as_vtable_size_descr() is sizedescr
-    if sizedescr._corresponding_vtable:
+    if getattr(sizedescr, '_corresponding_vtable', None):
         assert sizedescr._corresponding_vtable == vtable
         return
     assert lltype.typeOf(vtable) == VTABLETYPE

From noreply at buildbot.pypy.org  Fri Jun  5 14:16:45 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Fri,  5 Jun 2015 14:16:45 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: oops
Message-ID: <20150605121645.0F4CA1C048F@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77907:7f75733a01dd
Date: 2015-06-05 14:16 +0200
http://bitbucket.org/pypy/pypy/changeset/7f75733a01dd/

Log:	oops

diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -79,7 +79,7 @@
 
     def get_replacement(self):
         if self._forwarded:
-            return self._forwarded
+            return self._forwarded.get_replacement()
         return self
 
     def set_forwarded(self, forwarded_to):
@@ -479,9 +479,7 @@
         assert len(args) == 0
 
     def get_replacement_for_rewrite(self):
-        if self._forwarded:
-            return self._forwarded
-        return self
+        return self.get_replacement()
 
     def getarglist(self):
         return []
@@ -513,6 +511,7 @@
             if isinstance(self, ResOpWithDescr):
                 op.setdescr(self.getdescr())
             op._arg0 = arg0
+            self.set_forwarded(op)
             return op
         return self
 
@@ -574,6 +573,7 @@
                 op.setdescr(self.getdescr())
             op._arg0 = arg0
             op._arg1 = arg1
+            self.set_forwarded(op)
             return op
         return self
 
@@ -621,6 +621,7 @@
             op._arg0 = arg0
             op._arg1 = arg1
             op._arg2 = arg2
+            self.set_forwarded(op)
             return op
         return self
 
@@ -654,6 +655,7 @@
         else:
             return self
         op = self.__class__()
+        self.set_forwarded(op)
         op._args = [arg.get_replacement() for arg in self._args]
         if isinstance(self, ResOpWithDescr):
             op.setdescr(self.getdescr())

From noreply at buildbot.pypy.org  Fri Jun  5 14:20:37 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Fri,  5 Jun 2015 14:20:37 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: pfff
Message-ID: <20150605122037.7AAF11C1038@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77908:3d5076890948
Date: 2015-06-05 14:20 +0200
http://bitbucket.org/pypy/pypy/changeset/3d5076890948/

Log:	pfff

diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py
--- a/rpython/jit/backend/llsupport/rewrite.py
+++ b/rpython/jit/backend/llsupport/rewrite.py
@@ -88,8 +88,14 @@
     def emit_op(self, op):
         if op.is_guard():
             op = op.copy_and_change(op.getopnum())
-            op = op.get_replacement_for_rewrite()
-            op.setfailargs([arg.get_replacement_for_rewrite() for arg in op.getfailargs()])
+            newop = op.get_replacement_for_rewrite()
+            _newfailargs = []
+            for arg in op.getfailargs():
+                if arg is not None:
+                    arg = arg.get_replacement_for_rewrite()
+                _newfailargs.append(arg)
+            newop.setfailargs(_newfailargs)
+            op = newop
         else:
             op = op.get_replacement_for_rewrite()
         self._newops.append(op)

From noreply at buildbot.pypy.org  Fri Jun  5 14:27:26 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Fri,  5 Jun 2015 14:27:26 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: another oops
Message-ID: <20150605122726.E83D81C1038@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77909:106ab253738e
Date: 2015-06-05 14:27 +0200
http://bitbucket.org/pypy/pypy/changeset/106ab253738e/

Log:	another oops

diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py
--- a/rpython/jit/backend/llsupport/rewrite.py
+++ b/rpython/jit/backend/llsupport/rewrite.py
@@ -87,8 +87,10 @@
 
     def emit_op(self, op):
         if op.is_guard():
-            op = op.copy_and_change(op.getopnum())
             newop = op.get_replacement_for_rewrite()
+            if newop is op:
+                newop = op.copy_and_change(op.getopnum())
+                op.set_forwarded(newop)
             _newfailargs = []
             for arg in op.getfailargs():
                 if arg is not None:

From noreply at buildbot.pypy.org  Fri Jun  5 14:46:57 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Fri,  5 Jun 2015 14:46:57 +0200 (CEST)
Subject: [pypy-commit] stmgc c8-gil-like: typos
Message-ID: <20150605124657.243231C033F@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: c8-gil-like
Changeset: r1791:98218b34a5ee
Date: 2015-06-05 14:47 +0200
http://bitbucket.org/pypy/stmgc/changeset/98218b34a5ee/

Log:	typos

diff --git a/c8/CALL_RELEASE_GIL b/c8/CALL_RELEASE_GIL
--- a/c8/CALL_RELEASE_GIL
+++ b/c8/CALL_RELEASE_GIL
@@ -73,7 +73,7 @@
   Otherwise it calls a helper, _stm_detach_noninevitable_transaction().
 
 - _stm_reattach_transaction(old): called with the old value from
-  stm_detach_inevitable_transaction (which was swapped to be NULL just
+  stm_detached_inevitable_from_thread (which was swapped to be NULL just
   now).  If old != NULL, this swap had the effect that we took over
   the inevitable transaction originally detached from a different
   thread; we need to fix a few things like the shadowstack and %gs but
@@ -107,7 +107,7 @@
   return, unlike stm_become_inevitable() which must continue running
   the existing transaction.
 
-- the commit logic of a non-inevitable transaction waits if there is
+- commit logic of a non-inevitable transaction: we wait if there is
   an inevitable transaction.  Here too, if the inevitable transaction
   is found to be detached, we could just commit it now.  Or, a better
   approach: if we find a detached inevitable transaction we grab it

From noreply at buildbot.pypy.org  Fri Jun  5 15:12:56 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Fri,  5 Jun 2015 15:12:56 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: the specialization seems to be an
	overkill
Message-ID: <20150605131256.7F3C01C048F@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77910:f03f53422028
Date: 2015-06-05 15:12 +0200
http://bitbucket.org/pypy/pypy/changeset/f03f53422028/

Log:	the specialization seems to be an overkill

diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py
--- a/rpython/jit/backend/llsupport/rewrite.py
+++ b/rpython/jit/backend/llsupport/rewrite.py
@@ -86,20 +86,24 @@
         return op
 
     def emit_op(self, op):
+        op = self.get_box_replacement(op)
+        orig_op = op
+        replaced = False
+        for i in range(op.numargs()):
+            orig_arg = op.getarg(i)
+            arg = self.get_box_replacement(orig_arg)
+            if orig_arg is not arg:
+                if not replaced:
+                    op = op.copy_and_change(op.getopnum())
+                    orig_op.set_forwarded(op)
+                    replaced = True
+                op.setarg(i, arg)
         if op.is_guard():
-            newop = op.get_replacement_for_rewrite()
-            if newop is op:
-                newop = op.copy_and_change(op.getopnum())
-                op.set_forwarded(newop)
-            _newfailargs = []
-            for arg in op.getfailargs():
-                if arg is not None:
-                    arg = arg.get_replacement_for_rewrite()
-                _newfailargs.append(arg)
-            newop.setfailargs(_newfailargs)
-            op = newop
-        else:
-            op = op.get_replacement_for_rewrite()
+            if not replaced:
+                op = op.copy_and_change(op.getopnum())
+                orig_op.set_forwarded(op)
+            op.setfailargs([self.get_box_replacement(a, True)
+                            for a in op.getfailargs()])
         self._newops.append(op)
 
     def replace_op_with(self, op, newop):
diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py
--- a/rpython/jit/metainterp/history.py
+++ b/rpython/jit/metainterp/history.py
@@ -204,9 +204,6 @@
     def constbox(self):
         return self
 
-    def get_replacement_for_rewrite(self):
-        return self
-
     def same_box(self, other):
         return self.same_constant(other)
 
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -24,9 +24,6 @@
     def get_forwarded(self):
         return None
 
-    def get_replacement(self):
-        return self
-
     def set_forwarded(self, forwarded_to):
         raise Exception("oups")
 
@@ -77,11 +74,6 @@
     def get_forwarded(self):
         return self._forwarded
 
-    def get_replacement(self):
-        if self._forwarded:
-            return self._forwarded.get_replacement()
-        return self
-
     def set_forwarded(self, forwarded_to):
         self._forwarded = forwarded_to
 
@@ -427,12 +419,6 @@
     def get_forwarded(self):
         return self._forwarded
 
-    # this is for rewrite.py, we can have several versions depending on
-    # invariants
-    def get_replacement_for_rewrite(self):
-        assert self._forwarded is None
-        return self
-
     def set_forwarded(self, forwarded_to):
         self._forwarded = forwarded_to
 
@@ -478,9 +464,6 @@
     def initarglist(self, args):
         assert len(args) == 0
 
-    def get_replacement_for_rewrite(self):
-        return self.get_replacement()
-
     def getarglist(self):
         return []
 
@@ -502,19 +485,6 @@
         assert len(args) == 1
         self._arg0, = args
 
-    def get_replacement_for_rewrite(self):
-        if self._forwarded:
-            return self._forwarded.get_replacement_for_rewrite()
-        arg0 = self._arg0.get_replacement()
-        if arg0 is not self._arg0:
-            op = self.__class__()
-            if isinstance(self, ResOpWithDescr):
-                op.setdescr(self.getdescr())
-            op._arg0 = arg0
-            self.set_forwarded(op)
-            return op
-        return self
-
     def getarglist(self):
         return [self._arg0]
 
@@ -562,21 +532,6 @@
         else:
             raise IndexError
 
-    def get_replacement_for_rewrite(self):
-        if self._forwarded:
-            return self._forwarded.get_replacement_for_rewrite()
-        arg0 = self._arg0.get_replacement()
-        arg1 = self._arg1.get_replacement()
-        if arg0 is not self._arg0 or arg1 is not self._arg1:
-            op = self.__class__()
-            if isinstance(self, ResOpWithDescr):
-                op.setdescr(self.getdescr())
-            op._arg0 = arg0
-            op._arg1 = arg1
-            self.set_forwarded(op)
-            return op
-        return self
-
     def getarglist(self):
         return [self._arg0, self._arg1]
 
@@ -607,24 +562,6 @@
         else:
             raise IndexError
 
-    def get_replacement_for_rewrite(self):
-        if self._forwarded:
-            return self._forwarded.get_replacement_for_rewrite()
-        arg0 = self._arg0.get_replacement()
-        arg1 = self._arg1.get_replacement()
-        arg2 = self._arg2.get_replacement()
-        if (arg0 is not self._arg0 or arg1 is not self._arg1 or
-            arg2 is not self._arg2):
-            op = self.__class__()
-            if isinstance(self, ResOpWithDescr):
-                op.setdescr(self.getdescr())
-            op._arg0 = arg0
-            op._arg1 = arg1
-            op._arg2 = arg2
-            self.set_forwarded(op)
-            return op
-        return self
-
     def setarg(self, i, box):
         if i == 0:
             self._arg0 = box
@@ -646,21 +583,6 @@
                self.__class__.__name__.startswith('FINISH'):   # XXX remove me
             assert len(args) <= 1      # FINISH operations take 0 or 1 arg now
 
-    def get_replacement_for_rewrite(self):
-        if self._forwarded:
-            return self._forwarded.get_replacement_for_rewrite()
-        for arg in self._args:
-            if arg is not arg.get_replacement():
-                break
-        else:
-            return self
-        op = self.__class__()
-        self.set_forwarded(op)
-        op._args = [arg.get_replacement() for arg in self._args]
-        if isinstance(self, ResOpWithDescr):
-            op.setdescr(self.getdescr())
-        return op
-
     def getarglist(self):
         return self._args
 

From noreply at buildbot.pypy.org  Fri Jun  5 16:37:48 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Fri,  5 Jun 2015 16:37:48 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: of course
Message-ID: <20150605143748.2FDD21C048F@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77912:301b8b6b16f3
Date: 2015-06-05 16:37 +0200
http://bitbucket.org/pypy/pypy/changeset/301b8b6b16f3/

Log:	of course

diff --git a/rpython/jit/metainterp/heapcache.py b/rpython/jit/metainterp/heapcache.py
--- a/rpython/jit/metainterp/heapcache.py
+++ b/rpython/jit/metainterp/heapcache.py
@@ -1,7 +1,7 @@
 from rpython.jit.metainterp.history import Const, ConstInt
-from rpython.jit.metainterp.resoperation import rop, OpHelpers
+from rpython.jit.metainterp.resoperation import rop, OpHelpers, AbstractValue
 
-class HeapCacheValue(object):
+class HeapCacheValue(AbstractValue):
     def __init__(self, box):
         self.box = box
         self.likely_virtual = False

From noreply at buildbot.pypy.org  Fri Jun  5 16:35:26 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Fri,  5 Jun 2015 16:35:26 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: an attempt to make heapcache use new
 mechanism of tracking stuff
Message-ID: <20150605143526.8EFE11C048F@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77911:d217ec5f41a7
Date: 2015-06-05 16:35 +0200
http://bitbucket.org/pypy/pypy/changeset/d217ec5f41a7/

Log:	an attempt to make heapcache use new mechanism of tracking stuff

diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -129,6 +129,8 @@
     metainterp_sd = metainterp.staticdata
     jitdriver_sd = metainterp.jitdriver_sd
     history = metainterp.history
+    forget_optimization_info(inputargs)
+    forget_optimization_info(history.operations)
 
     enable_opts = jitdriver_sd.warmstate.enable_opts
     if try_disabling_unroll:
@@ -905,6 +907,8 @@
     #
     # Attempt to use optimize_bridge().  This may return None in case
     # it does not work -- i.e. none of the existing old_loop_tokens match.
+    forget_optimization_info(metainterp.history.inputargs)
+    forget_optimization_info(metainterp.history.operations)
     new_trace = create_empty_loop(metainterp)
     new_trace.inputargs = metainterp.history.inputargs[:]
 
diff --git a/rpython/jit/metainterp/heapcache.py b/rpython/jit/metainterp/heapcache.py
--- a/rpython/jit/metainterp/heapcache.py
+++ b/rpython/jit/metainterp/heapcache.py
@@ -1,4 +1,4 @@
-from rpython.jit.metainterp.history import ConstInt
+from rpython.jit.metainterp.history import Const, ConstInt
 from rpython.jit.metainterp.resoperation import rop, OpHelpers
 
 class HeapCacheValue(object):
@@ -62,11 +62,19 @@
 
 class HeapCache(object):
     def __init__(self):
-        self.reset()
+        self.list_of_operations = []
+        self._reset(None)
 
     def reset(self):
+        self._reset(self.list_of_operations)
+
+    def _reset(self, lst):
+        if lst is not None:
+            for i in range(len(lst)):
+                lst[i].set_forwarded(None)
+        self.const_cache = {}
         # maps boxes to values
-        self.values = {}
+        #self.values = {}
         # store the boxes that contain newly allocated objects, this maps the
         # boxes to a bool, the bool indicates whether or not the object has
         # escaped the trace or not (True means the box never escaped, False
@@ -93,16 +101,29 @@
         self.heap_array_cache = {}
 
     def reset_keep_likely_virtuals(self):
-        for value in self.values.itervalues():
-            value.reset_keep_likely_virtual()
+        for elem in self.list_of_operations:
+            value = self.getvalue(elem, False)
+            if value is not None:
+                assert isinstance(value, HeapCacheValue)
+                value.reset_keep_likely_virtual()
         self.heap_cache = {}
         self.heap_array_cache = {}
 
-    def getvalue(self, box):
-        value = self.values.get(box, None)
-        if not value:
-            value = self.values[box] = HeapCacheValue(box)
-        return value
+    def getvalue(self, box, create=True):
+        if isinstance(box, Const):
+            v = self.const_cache.get(box, None)
+            if v is None:
+                self.const_cache[box] = v = HeapCacheValue(box)
+            return v
+        v = box.get_forwarded()
+        if v is None:
+            if not create:
+                return None
+            v = HeapCacheValue(box)
+            self.list_of_operations.append(box)
+            box.set_forwarded(v)
+        assert isinstance(v, HeapCacheValue)
+        return v
 
     def getvalues(self, boxes):
         return [self.getvalue(box) for box in boxes]
@@ -158,7 +179,7 @@
                 self._escape_box(box)
 
     def _escape_box(self, box):
-        value = self.values.get(box, None)
+        value = self.getvalue(box, False)
         if not value:
             return
         self._escape(value)
@@ -267,31 +288,31 @@
         self.reset_keep_likely_virtuals()
 
     def is_class_known(self, box):
-        value = self.values.get(box, None)
-        if value:
-            return value.known_class
+        v = self.getvalue(box, False)
+        if v:
+            return v.known_class
         return False
 
     def class_now_known(self, box):
         self.getvalue(box).known_class = True
 
     def is_nonstandard_virtualizable(self, box):
-        value = self.values.get(box, None)
-        if value:
-            return value.nonstandard_virtualizable
+        v = self.getvalue(box, False)
+        if v:
+            return v.nonstandard_virtualizable
         return False
 
     def nonstandard_virtualizables_now_known(self, box):
         self.getvalue(box).nonstandard_virtualizable = True
 
     def is_unescaped(self, box):
-        value = self.values.get(box, None)
+        value = self.getvalue(box, False)
         if value:
             return value.is_unescaped
         return False
 
     def is_likely_virtual(self, box):
-        value = self.values.get(box, None)
+        value = self.getvalue(box, False)
         if value:
             return value.likely_virtual
         return False
@@ -307,11 +328,11 @@
         self.arraylen_now_known(box, lengthbox)
 
     def getfield(self, box, descr):
-        value = self.values.get(box, None)
-        if value:
+        v = self.getvalue(box, False)
+        if v:
             cache = self.heap_cache.get(descr, None)
             if cache:
-                tovalue = cache.read(value)
+                tovalue = cache.read(v)
                 if tovalue:
                     return tovalue.box
         return None
@@ -335,7 +356,7 @@
     def getarrayitem(self, box, indexbox, descr):
         if not isinstance(indexbox, ConstInt):
             return None
-        value = self.values.get(box, None)
+        value = self.getvalue(box, False)
         if value is None:
             return None
         index = indexbox.getint()
@@ -379,7 +400,7 @@
             indexcache.do_write_with_aliasing(value, fieldvalue)
 
     def arraylen(self, box):
-        value = self.values.get(box, None)
+        value = self.getvalue(box, False)
         if value and value.length:
             return value.length.box
         return None
@@ -389,8 +410,11 @@
         value.length = self.getvalue(lengthbox)
 
     def replace_box(self, oldbox, newbox):
-        value = self.values.get(oldbox, None)
+        value = self.getvalue(oldbox, False)
         if value is None:
             return
         value.box = newbox
-        self.values[newbox] = value
+        if isinstance(newbox, Const):
+            self.const_cache[newbox] = value
+        else:
+            newbox.set_forwarded(value)
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -1907,6 +1907,7 @@
         self.current_call_id = 0
 
     def retrace_needed(self, trace, exported_state):
+        raise Exception("I dont want that function to exist")
         self.partial_trace = trace
         self.retracing_from = len(self.history.operations) - 1
         self.exported_state = exported_state
diff --git a/rpython/jit/metainterp/test/test_heapcache.py b/rpython/jit/metainterp/test/test_heapcache.py
--- a/rpython/jit/metainterp/test/test_heapcache.py
+++ b/rpython/jit/metainterp/test/test_heapcache.py
@@ -2,14 +2,6 @@
 from rpython.jit.metainterp.resoperation import rop, InputArgInt
 from rpython.jit.metainterp.history import ConstInt, BasicFailDescr
 
-box1 = "box1"
-box2 = "box2"
-box3 = "box3"
-box4 = "box4"
-box5 = "box5"
-lengthbox1 = object()
-lengthbox2 = object()
-lengthbox3 = object()
 descr1 = object()
 descr2 = object()
 descr3 = object()
@@ -58,29 +50,37 @@
 class TestHeapCache(object):
     def test_known_class_box(self):
         h = HeapCache()
-        assert not h.is_class_known(1)
-        assert not h.is_class_known(2)
-        h.class_now_known(1)
-        assert h.is_class_known(1)
-        assert not h.is_class_known(2)
+        i0 = InputArgInt(1)
+        i1 = InputArgInt(2)
+        assert not h.is_class_known(i0)
+        assert not h.is_class_known(i1)
+        h.class_now_known(i0)
+        assert h.is_class_known(i0)
+        assert not h.is_class_known(i1)
 
         h.reset()
-        assert not h.is_class_known(1)
-        assert not h.is_class_known(2)
+        assert not h.is_class_known(i0)
+        assert not h.is_class_known(i1)
 
     def test_nonstandard_virtualizable(self):
         h = HeapCache()
-        assert not h.is_nonstandard_virtualizable(1)
-        assert not h.is_nonstandard_virtualizable(2)
-        h.nonstandard_virtualizables_now_known(1)
-        assert h.is_nonstandard_virtualizable(1)
-        assert not h.is_nonstandard_virtualizable(2)
+        i0 = InputArgInt(1)
+        i1 = InputArgInt(2)
+        assert not h.is_nonstandard_virtualizable(i0)
+        assert not h.is_nonstandard_virtualizable(i1)
+        h.nonstandard_virtualizables_now_known(i0)
+        assert h.is_nonstandard_virtualizable(i0)
+        assert not h.is_nonstandard_virtualizable(i1)
 
         h.reset()
-        assert not h.is_nonstandard_virtualizable(1)
-        assert not h.is_nonstandard_virtualizable(2)
+        assert not h.is_nonstandard_virtualizable(i0)
+        assert not h.is_nonstandard_virtualizable(i1)
 
     def test_heapcache_fields(self):
+
+        box1 = InputArgInt(0)
+        box2 = InputArgInt(1)
+        box3 = InputArgInt(2)
         h = HeapCache()
         assert h.getfield(box1, descr1) is None
         assert h.getfield(box1, descr2) is None
@@ -105,6 +105,11 @@
 
     def test_heapcache_read_fields_multiple(self):
         h = HeapCache()
+
+        box1 = InputArgInt(0)
+        box2 = InputArgInt(1)
+        box3 = InputArgInt(2)
+        box4 = InputArgInt(3)
         h.getfield_now_known(box1, descr1, box2)
         h.getfield_now_known(box3, descr1, box4)
         assert h.getfield(box1, descr1) is box2
@@ -120,6 +125,10 @@
 
     def test_heapcache_write_fields_multiple(self):
         h = HeapCache()
+        box1 = InputArgInt(0)
+        box2 = InputArgInt(1)
+        box3 = InputArgInt(2)
+        box4 = InputArgInt(3)
         h.setfield(box1, box2, descr1)
         assert h.getfield(box1, descr1) is box2
         h.setfield(box3, box4, descr1)
@@ -148,6 +157,10 @@
 
     def test_heapcache_arrays(self):
         h = HeapCache()
+        box1 = InputArgInt(0)
+        box2 = InputArgInt(1)
+        box3 = InputArgInt(2)
+        box4 = InputArgInt(3)
         assert h.getarrayitem(box1, index1, descr1) is None
         assert h.getarrayitem(box1, index1, descr2) is None
         assert h.getarrayitem(box1, index2, descr1) is None
@@ -190,6 +203,10 @@
 
     def test_heapcache_array_nonconst_index(self):
         h = HeapCache()
+        box1 = InputArgInt(0)
+        box2 = InputArgInt(1)
+        box3 = InputArgInt(2)
+        box4 = InputArgInt(3)
         h.setarrayitem(box1, index1, box2, descr1)
         h.setarrayitem(box1, index2, box4, descr1)
         assert h.getarrayitem(box1, index1, descr1) is box2
@@ -200,6 +217,10 @@
 
     def test_heapcache_read_fields_multiple_array(self):
         h = HeapCache()
+        box1 = InputArgInt(0)
+        box2 = InputArgInt(1)
+        box3 = InputArgInt(2)
+        box4 = InputArgInt(3)
         h.getarrayitem_now_known(box1, index1, box2, descr1)
         h.getarrayitem_now_known(box3, index1, box4, descr1)
         assert h.getarrayitem(box1, index1, descr1) is box2
@@ -215,6 +236,10 @@
 
     def test_heapcache_write_fields_multiple_array(self):
         h = HeapCache()
+        box1 = InputArgInt(0)
+        box2 = InputArgInt(1)
+        box3 = InputArgInt(2)
+        box4 = InputArgInt(3)
         h.setarrayitem(box1, index1, box2, descr1)
         assert h.getarrayitem(box1, index1, descr1) is box2
         h.setarrayitem(box3, index1, box4, descr1)
@@ -243,6 +268,10 @@
 
     def test_length_cache(self):
         h = HeapCache()
+        box1 = InputArgInt(0)
+        box2 = InputArgInt(1)
+        lengthbox1 = InputArgInt(2)
+        lengthbox2 = InputArgInt(3)
         h.new_array(box1, lengthbox1)
         assert h.arraylen(box1) is lengthbox1
 
@@ -253,6 +282,9 @@
 
     def test_invalidate_cache(self):
         h = HeapCache()
+        box1 = InputArgInt(0)
+        box2 = InputArgInt(1)
+        box4 = InputArgInt(3)
         h.setfield(box1, box2, descr1)
         h.setarrayitem(box1, index1, box2, descr1)
         h.setarrayitem(box1, index2, box4, descr1)
@@ -286,6 +318,10 @@
 
     def test_replace_box(self):
         h = HeapCache()
+        box1 = InputArgInt(0)
+        box2 = InputArgInt(1)
+        box3 = InputArgInt(2)
+        box4 = InputArgInt(3)
         h.setfield(box1, box2, descr1)
         h.setfield(box1, box3, descr2)
         h.setfield(box2, box3, descr3)
@@ -307,6 +343,11 @@
 
     def test_replace_box_twice(self):
         h = HeapCache()
+        box1 = InputArgInt(0)
+        box2 = InputArgInt(1)
+        box3 = InputArgInt(2)
+        box4 = InputArgInt(3)
+        box5 = InputArgInt(4)    
         h.setfield(box1, box2, descr1)
         h.setfield(box1, box3, descr2)
         h.setfield(box2, box3, descr3)
@@ -330,6 +371,12 @@
 
     def test_replace_box_array(self):
         h = HeapCache()
+        box1 = InputArgInt(0)
+        box2 = InputArgInt(1)
+        box3 = InputArgInt(2)
+        box4 = InputArgInt(3)
+        lengthbox1 = InputArgInt(0)
+        lengthbox2 = InputArgInt(2)
         h.setarrayitem(box1, index1, box2, descr1)
         h.setarrayitem(box1, index1, box3, descr2)
         h.arraylen_now_known(box1, lengthbox1)
@@ -349,6 +396,15 @@
 
     def test_replace_box_array_twice(self):
         h = HeapCache()
+        box1 = InputArgInt(0)
+        box2 = InputArgInt(1)
+        box3 = InputArgInt(2)
+        box4 = InputArgInt(3)
+        box5 = InputArgInt(4)
+        lengthbox1 = InputArgInt(0)
+        lengthbox2 = InputArgInt(1)
+        lengthbox3 = InputArgInt(2)
+        
         h.setarrayitem(box1, index1, box2, descr1)
         h.setarrayitem(box1, index1, box3, descr2)
         h.arraylen_now_known(box1, lengthbox1)
@@ -370,6 +426,12 @@
 
     def test_ll_arraycopy(self):
         h = HeapCache()
+        box1 = InputArgInt(0)
+        box2 = InputArgInt(1)
+        box3 = InputArgInt(2)
+        box4 = InputArgInt(3)
+        box5 = InputArgInt(4)
+        lengthbox1 = InputArgInt(0)
         h.new_array(box1, lengthbox1)
         h.setarrayitem(box1, index1, box2, descr1)
         h.new_array(box2, lengthbox1)
@@ -398,49 +460,68 @@
 
     def test_ll_arraycopy_differing_descrs(self):
         h = HeapCache()
+        box1 = InputArgInt(0)
+        box2 = InputArgInt(1)
+        box3 = InputArgInt(2)
+        lengthbox2 = InputArgInt(1)
         h.setarrayitem(box1, index1, box2, descr2)
         assert h.getarrayitem(box1, index1, descr2) is box2
         h.new_array(box2, lengthbox2)
         h.invalidate_caches(
             rop.CALL_N,
             arraycopydescr1,
-            [None, box3, box2, index1, index1, index2]
+            [ConstInt(123), box3, box2, index1, index1, index2]
         )
         assert h.getarrayitem(box1, index1, descr2) is box2
 
     def test_ll_arraycopy_differing_descrs_nonconst_index(self):
         h = HeapCache()
+        box1 = InputArgInt(0)
+        box2 = InputArgInt(1)
+        box3 = InputArgInt(2)
         h.setarrayitem(box1, index1, box2, descr2)
         assert h.getarrayitem(box1, index1, descr2) is box2
         h.invalidate_caches(
             rop.CALL_N,
             arraycopydescr1,
-            [None, box3, box2, index1, index1, InputArgInt()]
+            [ConstInt(123), box3, box2, index1, index1, InputArgInt()]
         )
         assert h.getarrayitem(box1, index1, descr2) is box2
 
     def test_ll_arraycopy_result_propogated(self):
         h = HeapCache()
+        box1 = InputArgInt(0)
+        box2 = InputArgInt(1)
+        box3 = InputArgInt(2)
         h.setarrayitem(box1, index1, box2, descr1)
         h.invalidate_caches(
             rop.CALL_N,
             arraycopydescr1,
-            [None, box1, box3, index1, index1, index2]
+            [ConstInt(13), box1, box3, index1, index1, index2]
         )
         assert h.getarrayitem(box3, index1, descr1) is box2
 
     def test_ll_arraycopy_dest_new(self):
         h = HeapCache()
+        box1 = InputArgInt(0)
+        box2 = InputArgInt(1)
+        box3 = InputArgInt(2)
+        box4 = InputArgInt(3)
+        lengthbox1 = InputArgInt(0)
         h.new_array(box1, lengthbox1)
         h.setarrayitem(box3, index1, box4, descr1)
         h.invalidate_caches(
             rop.CALL_N,
             arraycopydescr1,
-            [None, box2, box1, index1, index1, index2]
+            [ConstInt(13), box2, box1, index1, index1, index2]
         )
 
     def test_ll_arraycopy_doesnt_escape_arrays(self):
         h = HeapCache()
+        box1 = InputArgInt(0)
+        box2 = InputArgInt(1)
+        lengthbox1 = InputArgInt(1)
+        lengthbox2 = InputArgInt(2)
         h.new_array(box1, lengthbox1)
         h.new_array(box2, lengthbox2)
         h.invalidate_caches(
@@ -453,13 +534,15 @@
         h.invalidate_caches(
             rop.CALL_N,
             arraycopydescr1,
-            [None, box2, box1, index1, index1, InputArgInt()]
+            [ConstInt(123), box2, box1, index1, index1, InputArgInt()]
         )
         assert not h.is_unescaped(box1)
         assert not h.is_unescaped(box2)
 
     def test_unescaped(self):
         h = HeapCache()
+        box1 = InputArgInt(0)
+        box2 = InputArgInt(1)
         assert not h.is_unescaped(box1)
         h.new(box2)
         assert h.is_unescaped(box2)
@@ -470,6 +553,9 @@
 
     def test_unescaped_testing(self):
         h = HeapCache()
+        box1 = InputArgInt(0)
+        box2 = InputArgInt(1)
+        box3 = InputArgInt(2)
         h.new(box1)
         h.new(box2)
         assert h.is_unescaped(box1)
@@ -488,6 +574,8 @@
 
     def test_ops_dont_escape(self):
         h = HeapCache()
+        box1 = InputArgInt(0)
+        box2 = InputArgInt(1)
         h.new(box1)
         h.new(box2)
         assert h.is_unescaped(box1)
@@ -501,6 +589,9 @@
 
     def test_circular_virtuals(self):
         h = HeapCache()
+        box1 = InputArgInt(0)
+        box2 = InputArgInt(1)
+        box3 = InputArgInt(2)
         h.new(box1)
         h.new(box2)
         h.invalidate_caches(rop.SETFIELD_GC, None, [box1, box2])
@@ -509,6 +600,10 @@
 
     def test_unescaped_array(self):
         h = HeapCache()
+        box1 = InputArgInt(0)
+        box2 = InputArgInt(1)
+        lengthbox1 = InputArgInt(0)
+        lengthbox2 = InputArgInt(1)
         h.new_array(box1, lengthbox1)
         assert h.is_unescaped(box1)
         h.invalidate_caches(rop.SETARRAYITEM_GC, None, [box1, index1, box2])
@@ -532,6 +627,9 @@
 
     def test_call_doesnt_invalidate_unescaped_boxes(self):
         h = HeapCache()
+        box1 = InputArgInt(0)
+        box2 = InputArgInt(1)
+
         h.new(box1)
         assert h.is_unescaped(box1)
         h.setfield(box1, box2, descr1)
@@ -543,6 +641,9 @@
 
     def test_call_doesnt_invalidate_unescaped_array_boxes(self):
         h = HeapCache()
+        box1 = InputArgInt(0)
+        lengthbox1 = InputArgInt(2)
+        box3 = InputArgInt(1)
         h.new_array(box1, lengthbox1)
         assert h.is_unescaped(box1)
         h.setarrayitem(box1, index1, box3, descr1)
@@ -554,6 +655,8 @@
 
     def test_bug_missing_ignored_operations(self):
         h = HeapCache()
+        box1 = InputArgInt(0)
+        box2 = InputArgInt(1)
         h.new(box1)
         h.new(box2)
         h.setfield(box1, box2, descr1)
@@ -576,6 +679,8 @@
         # calling some residual code that changes the values on box3: then
         # the content of box2 is still cached at the old value.
         h = HeapCache()
+        box1 = InputArgInt(0)
+        box2 = InputArgInt(1)
         h.new(box1)
         h.new(box2)
         h.setfield(box1, box2, descr1)
@@ -588,6 +693,8 @@
 
     def test_bug_heap_cache_is_cleared_but_not_is_unescaped_2(self):
         h = HeapCache()
+        box1 = InputArgInt(0)
+        box2 = InputArgInt(1)
         h.new(box1)
         h.new(box2)
         h.setfield(box1, box2, descr1)
@@ -609,6 +716,8 @@
 
     def test_is_likely_virtual(self):
         h = HeapCache()
+        box1 = InputArgInt(0)
+
         h.new(box1)
         assert h.is_unescaped(box1)
         assert h.is_likely_virtual(box1)

From noreply at buildbot.pypy.org  Fri Jun  5 16:42:43 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Fri,  5 Jun 2015 16:42:43 +0200 (CEST)
Subject: [pypy-commit] cffi default: Add comments in the C source that will
 hopefully be printed by the C
Message-ID: <20150605144243.8A3E81C033F@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2162:e33302548e71
Date: 2015-06-05 16:43 +0200
http://bitbucket.org/cffi/cffi/changeset/e33302548e71/

Log:	Add comments in the C source that will hopefully be printed by the C
	compiler in case something is wrong on that line

diff --git a/cffi/recompiler.py b/cffi/recompiler.py
--- a/cffi/recompiler.py
+++ b/cffi/recompiler.py
@@ -775,7 +775,8 @@
             try:
                 if ftype.is_integer_type() or fbitsize >= 0:
                     # accept all integers, but complain on float or double
-                    prnt('  (void)((p->%s) << 1);' % fname)
+                    prnt("  (void)((p->%s) << 1);  /* check that '%s.%s' is "
+                         "an integer */" % (fname, cname, fname))
                     continue
                 # only accept exactly the type declared, except that '[]'
                 # is interpreted as a '*' and so will match any array length.
@@ -949,7 +950,7 @@
             prnt('{')
             prnt('  int n = (%s) <= 0;' % (name,))
             prnt('  *o = (unsigned long long)((%s) << 0);'
-                 '  /* check that we get an integer */' % (name,))
+                 '  /* check that %s is an integer */' % (name, name))
             if check_value is not None:
                 if check_value > 0:
                     check_value = '%dU' % (check_value,)
@@ -1088,8 +1089,9 @@
         self.cffi_types[index] = CffiOp(OP_PRIMITIVE, prim_index)
 
     def _emit_bytecode_UnknownIntegerType(self, tp, index):
-        s = '_cffi_prim_int(sizeof(%s), (((%s)-1) << 0) <= 0)' % (
-            tp.name, tp.name)
+        s = ('_cffi_prim_int(sizeof(%s), (\n'
+             '           ((%s)-1) << 0 /* check that %s is an integer type */\n'
+             '         ) <= 0)' % (tp.name, tp.name, tp.name))
         self.cffi_types[index] = CffiOp(OP_PRIMITIVE, s)
 
     def _emit_bytecode_RawFunctionType(self, tp, index):

From noreply at buildbot.pypy.org  Fri Jun  5 16:52:34 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Fri,  5 Jun 2015 16:52:34 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: disable those features in favor of
 having a working --fork-before for now
Message-ID: <20150605145234.314531C033F@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77913:61b5d0427757
Date: 2015-06-05 16:52 +0200
http://bitbucket.org/pypy/pypy/changeset/61b5d0427757/

Log:	disable those features in favor of having a working --fork-before
	for now

diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py
--- a/pypy/module/__pypy__/__init__.py
+++ b/pypy/module/__pypy__/__init__.py
@@ -111,17 +111,17 @@
         PYC_MAGIC = get_pyc_magic(self.space)
         self.extra_interpdef('PYC_MAGIC', 'space.wrap(%d)' % PYC_MAGIC)
         #
-        try:
-            from rpython.jit.backend import detect_cpu
-            model = detect_cpu.autodetect()
-            self.extra_interpdef('cpumodel', 'space.wrap(%r)' % model)
-        except Exception:
-            if self.space.config.translation.jit:
-                raise
-            else:
-                pass   # ok fine to ignore in this case
+        #try:
+        #    from rpython.jit.backend import detect_cpu
+        #    model = detect_cpu.autodetect()
+        #    self.extra_interpdef('cpumodel', 'space.wrap(%r)' % model)
+        #except Exception:
+        #    if self.space.config.translation.jit:
+        #        raise
+        #    else:
+        #        pass   # ok fine to ignore in this case
         #
-        if self.space.config.translation.jit:
-            features = detect_cpu.getcpufeatures(model)
-            self.extra_interpdef('jit_backend_features',
-                                    'space.wrap(%r)' % features)
+        #if self.space.config.translation.jit:
+        ##    features = detect_cpu.getcpufeatures(model)
+        #    self.extra_interpdef('jit_backend_features',
+        #                            'space.wrap(%r)' % features)

From noreply at buildbot.pypy.org  Fri Jun  5 16:57:38 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Fri,  5 Jun 2015 16:57:38 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: write a comment
Message-ID: <20150605145738.192F31C033F@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77914:ebc5c19f748d
Date: 2015-06-05 16:57 +0200
http://bitbucket.org/pypy/pypy/changeset/ebc5c19f748d/

Log:	write a comment

diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py
--- a/pypy/module/__pypy__/__init__.py
+++ b/pypy/module/__pypy__/__init__.py
@@ -110,6 +110,10 @@
                                      'interp_magic.mapdict_cache_counter')
         PYC_MAGIC = get_pyc_magic(self.space)
         self.extra_interpdef('PYC_MAGIC', 'space.wrap(%d)' % PYC_MAGIC)
+        # XXX
+        # the following code prevents --fork-before=pyjitpl from working,
+        # proper fix would be to use some llop that is only rendered by the
+        # JIT
         #
         #try:
         #    from rpython.jit.backend import detect_cpu

From noreply at buildbot.pypy.org  Fri Jun  5 17:25:13 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Fri,  5 Jun 2015 17:25:13 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: backout an attempt on heapcache,
	does not seem to help
Message-ID: <20150605152513.C94251C048F@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77915:078c342333aa
Date: 2015-06-05 17:25 +0200
http://bitbucket.org/pypy/pypy/changeset/078c342333aa/

Log:	backout an attempt on heapcache, does not seem to help

diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -129,8 +129,6 @@
     metainterp_sd = metainterp.staticdata
     jitdriver_sd = metainterp.jitdriver_sd
     history = metainterp.history
-    forget_optimization_info(inputargs)
-    forget_optimization_info(history.operations)
 
     enable_opts = jitdriver_sd.warmstate.enable_opts
     if try_disabling_unroll:
@@ -907,8 +905,6 @@
     #
     # Attempt to use optimize_bridge().  This may return None in case
     # it does not work -- i.e. none of the existing old_loop_tokens match.
-    forget_optimization_info(metainterp.history.inputargs)
-    forget_optimization_info(metainterp.history.operations)
     new_trace = create_empty_loop(metainterp)
     new_trace.inputargs = metainterp.history.inputargs[:]
 
diff --git a/rpython/jit/metainterp/heapcache.py b/rpython/jit/metainterp/heapcache.py
--- a/rpython/jit/metainterp/heapcache.py
+++ b/rpython/jit/metainterp/heapcache.py
@@ -1,7 +1,7 @@
-from rpython.jit.metainterp.history import Const, ConstInt
-from rpython.jit.metainterp.resoperation import rop, OpHelpers, AbstractValue
+from rpython.jit.metainterp.history import ConstInt
+from rpython.jit.metainterp.resoperation import rop, OpHelpers
 
-class HeapCacheValue(AbstractValue):
+class HeapCacheValue(object):
     def __init__(self, box):
         self.box = box
         self.likely_virtual = False
@@ -62,19 +62,11 @@
 
 class HeapCache(object):
     def __init__(self):
-        self.list_of_operations = []
-        self._reset(None)
+        self.reset()
 
     def reset(self):
-        self._reset(self.list_of_operations)
-
-    def _reset(self, lst):
-        if lst is not None:
-            for i in range(len(lst)):
-                lst[i].set_forwarded(None)
-        self.const_cache = {}
         # maps boxes to values
-        #self.values = {}
+        self.values = {}
         # store the boxes that contain newly allocated objects, this maps the
         # boxes to a bool, the bool indicates whether or not the object has
         # escaped the trace or not (True means the box never escaped, False
@@ -101,29 +93,16 @@
         self.heap_array_cache = {}
 
     def reset_keep_likely_virtuals(self):
-        for elem in self.list_of_operations:
-            value = self.getvalue(elem, False)
-            if value is not None:
-                assert isinstance(value, HeapCacheValue)
-                value.reset_keep_likely_virtual()
+        for value in self.values.itervalues():
+            value.reset_keep_likely_virtual()
         self.heap_cache = {}
         self.heap_array_cache = {}
 
-    def getvalue(self, box, create=True):
-        if isinstance(box, Const):
-            v = self.const_cache.get(box, None)
-            if v is None:
-                self.const_cache[box] = v = HeapCacheValue(box)
-            return v
-        v = box.get_forwarded()
-        if v is None:
-            if not create:
-                return None
-            v = HeapCacheValue(box)
-            self.list_of_operations.append(box)
-            box.set_forwarded(v)
-        assert isinstance(v, HeapCacheValue)
-        return v
+    def getvalue(self, box):
+        value = self.values.get(box, None)
+        if not value:
+            value = self.values[box] = HeapCacheValue(box)
+        return value
 
     def getvalues(self, boxes):
         return [self.getvalue(box) for box in boxes]
@@ -179,7 +158,7 @@
                 self._escape_box(box)
 
     def _escape_box(self, box):
-        value = self.getvalue(box, False)
+        value = self.values.get(box, None)
         if not value:
             return
         self._escape(value)
@@ -288,31 +267,31 @@
         self.reset_keep_likely_virtuals()
 
     def is_class_known(self, box):
-        v = self.getvalue(box, False)
-        if v:
-            return v.known_class
+        value = self.values.get(box, None)
+        if value:
+            return value.known_class
         return False
 
     def class_now_known(self, box):
         self.getvalue(box).known_class = True
 
     def is_nonstandard_virtualizable(self, box):
-        v = self.getvalue(box, False)
-        if v:
-            return v.nonstandard_virtualizable
+        value = self.values.get(box, None)
+        if value:
+            return value.nonstandard_virtualizable
         return False
 
     def nonstandard_virtualizables_now_known(self, box):
         self.getvalue(box).nonstandard_virtualizable = True
 
     def is_unescaped(self, box):
-        value = self.getvalue(box, False)
+        value = self.values.get(box, None)
         if value:
             return value.is_unescaped
         return False
 
     def is_likely_virtual(self, box):
-        value = self.getvalue(box, False)
+        value = self.values.get(box, None)
         if value:
             return value.likely_virtual
         return False
@@ -328,11 +307,11 @@
         self.arraylen_now_known(box, lengthbox)
 
     def getfield(self, box, descr):
-        v = self.getvalue(box, False)
-        if v:
+        value = self.values.get(box, None)
+        if value:
             cache = self.heap_cache.get(descr, None)
             if cache:
-                tovalue = cache.read(v)
+                tovalue = cache.read(value)
                 if tovalue:
                     return tovalue.box
         return None
@@ -356,7 +335,7 @@
     def getarrayitem(self, box, indexbox, descr):
         if not isinstance(indexbox, ConstInt):
             return None
-        value = self.getvalue(box, False)
+        value = self.values.get(box, None)
         if value is None:
             return None
         index = indexbox.getint()
@@ -400,7 +379,7 @@
             indexcache.do_write_with_aliasing(value, fieldvalue)
 
     def arraylen(self, box):
-        value = self.getvalue(box, False)
+        value = self.values.get(box, None)
         if value and value.length:
             return value.length.box
         return None
@@ -410,11 +389,8 @@
         value.length = self.getvalue(lengthbox)
 
     def replace_box(self, oldbox, newbox):
-        value = self.getvalue(oldbox, False)
+        value = self.values.get(oldbox, None)
         if value is None:
             return
         value.box = newbox
-        if isinstance(newbox, Const):
-            self.const_cache[newbox] = value
-        else:
-            newbox.set_forwarded(value)
+        self.values[newbox] = value
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -1907,7 +1907,6 @@
         self.current_call_id = 0
 
     def retrace_needed(self, trace, exported_state):
-        raise Exception("I dont want that function to exist")
         self.partial_trace = trace
         self.retracing_from = len(self.history.operations) - 1
         self.exported_state = exported_state
diff --git a/rpython/jit/metainterp/test/test_heapcache.py b/rpython/jit/metainterp/test/test_heapcache.py
--- a/rpython/jit/metainterp/test/test_heapcache.py
+++ b/rpython/jit/metainterp/test/test_heapcache.py
@@ -2,6 +2,14 @@
 from rpython.jit.metainterp.resoperation import rop, InputArgInt
 from rpython.jit.metainterp.history import ConstInt, BasicFailDescr
 
+box1 = "box1"
+box2 = "box2"
+box3 = "box3"
+box4 = "box4"
+box5 = "box5"
+lengthbox1 = object()
+lengthbox2 = object()
+lengthbox3 = object()
 descr1 = object()
 descr2 = object()
 descr3 = object()
@@ -50,37 +58,29 @@
 class TestHeapCache(object):
     def test_known_class_box(self):
         h = HeapCache()
-        i0 = InputArgInt(1)
-        i1 = InputArgInt(2)
-        assert not h.is_class_known(i0)
-        assert not h.is_class_known(i1)
-        h.class_now_known(i0)
-        assert h.is_class_known(i0)
-        assert not h.is_class_known(i1)
+        assert not h.is_class_known(1)
+        assert not h.is_class_known(2)
+        h.class_now_known(1)
+        assert h.is_class_known(1)
+        assert not h.is_class_known(2)
 
         h.reset()
-        assert not h.is_class_known(i0)
-        assert not h.is_class_known(i1)
+        assert not h.is_class_known(1)
+        assert not h.is_class_known(2)
 
     def test_nonstandard_virtualizable(self):
         h = HeapCache()
-        i0 = InputArgInt(1)
-        i1 = InputArgInt(2)
-        assert not h.is_nonstandard_virtualizable(i0)
-        assert not h.is_nonstandard_virtualizable(i1)
-        h.nonstandard_virtualizables_now_known(i0)
-        assert h.is_nonstandard_virtualizable(i0)
-        assert not h.is_nonstandard_virtualizable(i1)
+        assert not h.is_nonstandard_virtualizable(1)
+        assert not h.is_nonstandard_virtualizable(2)
+        h.nonstandard_virtualizables_now_known(1)
+        assert h.is_nonstandard_virtualizable(1)
+        assert not h.is_nonstandard_virtualizable(2)
 
         h.reset()
-        assert not h.is_nonstandard_virtualizable(i0)
-        assert not h.is_nonstandard_virtualizable(i1)
+        assert not h.is_nonstandard_virtualizable(1)
+        assert not h.is_nonstandard_virtualizable(2)
 
     def test_heapcache_fields(self):
-
-        box1 = InputArgInt(0)
-        box2 = InputArgInt(1)
-        box3 = InputArgInt(2)
         h = HeapCache()
         assert h.getfield(box1, descr1) is None
         assert h.getfield(box1, descr2) is None
@@ -105,11 +105,6 @@
 
     def test_heapcache_read_fields_multiple(self):
         h = HeapCache()
-
-        box1 = InputArgInt(0)
-        box2 = InputArgInt(1)
-        box3 = InputArgInt(2)
-        box4 = InputArgInt(3)
         h.getfield_now_known(box1, descr1, box2)
         h.getfield_now_known(box3, descr1, box4)
         assert h.getfield(box1, descr1) is box2
@@ -125,10 +120,6 @@
 
     def test_heapcache_write_fields_multiple(self):
         h = HeapCache()
-        box1 = InputArgInt(0)
-        box2 = InputArgInt(1)
-        box3 = InputArgInt(2)
-        box4 = InputArgInt(3)
         h.setfield(box1, box2, descr1)
         assert h.getfield(box1, descr1) is box2
         h.setfield(box3, box4, descr1)
@@ -157,10 +148,6 @@
 
     def test_heapcache_arrays(self):
         h = HeapCache()
-        box1 = InputArgInt(0)
-        box2 = InputArgInt(1)
-        box3 = InputArgInt(2)
-        box4 = InputArgInt(3)
         assert h.getarrayitem(box1, index1, descr1) is None
         assert h.getarrayitem(box1, index1, descr2) is None
         assert h.getarrayitem(box1, index2, descr1) is None
@@ -203,10 +190,6 @@
 
     def test_heapcache_array_nonconst_index(self):
         h = HeapCache()
-        box1 = InputArgInt(0)
-        box2 = InputArgInt(1)
-        box3 = InputArgInt(2)
-        box4 = InputArgInt(3)
         h.setarrayitem(box1, index1, box2, descr1)
         h.setarrayitem(box1, index2, box4, descr1)
         assert h.getarrayitem(box1, index1, descr1) is box2
@@ -217,10 +200,6 @@
 
     def test_heapcache_read_fields_multiple_array(self):
         h = HeapCache()
-        box1 = InputArgInt(0)
-        box2 = InputArgInt(1)
-        box3 = InputArgInt(2)
-        box4 = InputArgInt(3)
         h.getarrayitem_now_known(box1, index1, box2, descr1)
         h.getarrayitem_now_known(box3, index1, box4, descr1)
         assert h.getarrayitem(box1, index1, descr1) is box2
@@ -236,10 +215,6 @@
 
     def test_heapcache_write_fields_multiple_array(self):
         h = HeapCache()
-        box1 = InputArgInt(0)
-        box2 = InputArgInt(1)
-        box3 = InputArgInt(2)
-        box4 = InputArgInt(3)
         h.setarrayitem(box1, index1, box2, descr1)
         assert h.getarrayitem(box1, index1, descr1) is box2
         h.setarrayitem(box3, index1, box4, descr1)
@@ -268,10 +243,6 @@
 
     def test_length_cache(self):
         h = HeapCache()
-        box1 = InputArgInt(0)
-        box2 = InputArgInt(1)
-        lengthbox1 = InputArgInt(2)
-        lengthbox2 = InputArgInt(3)
         h.new_array(box1, lengthbox1)
         assert h.arraylen(box1) is lengthbox1
 
@@ -282,9 +253,6 @@
 
     def test_invalidate_cache(self):
         h = HeapCache()
-        box1 = InputArgInt(0)
-        box2 = InputArgInt(1)
-        box4 = InputArgInt(3)
         h.setfield(box1, box2, descr1)
         h.setarrayitem(box1, index1, box2, descr1)
         h.setarrayitem(box1, index2, box4, descr1)
@@ -318,10 +286,6 @@
 
     def test_replace_box(self):
         h = HeapCache()
-        box1 = InputArgInt(0)
-        box2 = InputArgInt(1)
-        box3 = InputArgInt(2)
-        box4 = InputArgInt(3)
         h.setfield(box1, box2, descr1)
         h.setfield(box1, box3, descr2)
         h.setfield(box2, box3, descr3)
@@ -343,11 +307,6 @@
 
     def test_replace_box_twice(self):
         h = HeapCache()
-        box1 = InputArgInt(0)
-        box2 = InputArgInt(1)
-        box3 = InputArgInt(2)
-        box4 = InputArgInt(3)
-        box5 = InputArgInt(4)    
         h.setfield(box1, box2, descr1)
         h.setfield(box1, box3, descr2)
         h.setfield(box2, box3, descr3)
@@ -371,12 +330,6 @@
 
     def test_replace_box_array(self):
         h = HeapCache()
-        box1 = InputArgInt(0)
-        box2 = InputArgInt(1)
-        box3 = InputArgInt(2)
-        box4 = InputArgInt(3)
-        lengthbox1 = InputArgInt(0)
-        lengthbox2 = InputArgInt(2)
         h.setarrayitem(box1, index1, box2, descr1)
         h.setarrayitem(box1, index1, box3, descr2)
         h.arraylen_now_known(box1, lengthbox1)
@@ -396,15 +349,6 @@
 
     def test_replace_box_array_twice(self):
         h = HeapCache()
-        box1 = InputArgInt(0)
-        box2 = InputArgInt(1)
-        box3 = InputArgInt(2)
-        box4 = InputArgInt(3)
-        box5 = InputArgInt(4)
-        lengthbox1 = InputArgInt(0)
-        lengthbox2 = InputArgInt(1)
-        lengthbox3 = InputArgInt(2)
-        
         h.setarrayitem(box1, index1, box2, descr1)
         h.setarrayitem(box1, index1, box3, descr2)
         h.arraylen_now_known(box1, lengthbox1)
@@ -426,12 +370,6 @@
 
     def test_ll_arraycopy(self):
         h = HeapCache()
-        box1 = InputArgInt(0)
-        box2 = InputArgInt(1)
-        box3 = InputArgInt(2)
-        box4 = InputArgInt(3)
-        box5 = InputArgInt(4)
-        lengthbox1 = InputArgInt(0)
         h.new_array(box1, lengthbox1)
         h.setarrayitem(box1, index1, box2, descr1)
         h.new_array(box2, lengthbox1)
@@ -460,68 +398,49 @@
 
     def test_ll_arraycopy_differing_descrs(self):
         h = HeapCache()
-        box1 = InputArgInt(0)
-        box2 = InputArgInt(1)
-        box3 = InputArgInt(2)
-        lengthbox2 = InputArgInt(1)
         h.setarrayitem(box1, index1, box2, descr2)
         assert h.getarrayitem(box1, index1, descr2) is box2
         h.new_array(box2, lengthbox2)
         h.invalidate_caches(
             rop.CALL_N,
             arraycopydescr1,
-            [ConstInt(123), box3, box2, index1, index1, index2]
+            [None, box3, box2, index1, index1, index2]
         )
         assert h.getarrayitem(box1, index1, descr2) is box2
 
     def test_ll_arraycopy_differing_descrs_nonconst_index(self):
         h = HeapCache()
-        box1 = InputArgInt(0)
-        box2 = InputArgInt(1)
-        box3 = InputArgInt(2)
         h.setarrayitem(box1, index1, box2, descr2)
         assert h.getarrayitem(box1, index1, descr2) is box2
         h.invalidate_caches(
             rop.CALL_N,
             arraycopydescr1,
-            [ConstInt(123), box3, box2, index1, index1, InputArgInt()]
+            [None, box3, box2, index1, index1, InputArgInt()]
         )
         assert h.getarrayitem(box1, index1, descr2) is box2
 
     def test_ll_arraycopy_result_propogated(self):
         h = HeapCache()
-        box1 = InputArgInt(0)
-        box2 = InputArgInt(1)
-        box3 = InputArgInt(2)
         h.setarrayitem(box1, index1, box2, descr1)
         h.invalidate_caches(
             rop.CALL_N,
             arraycopydescr1,
-            [ConstInt(13), box1, box3, index1, index1, index2]
+            [None, box1, box3, index1, index1, index2]
         )
         assert h.getarrayitem(box3, index1, descr1) is box2
 
     def test_ll_arraycopy_dest_new(self):
         h = HeapCache()
-        box1 = InputArgInt(0)
-        box2 = InputArgInt(1)
-        box3 = InputArgInt(2)
-        box4 = InputArgInt(3)
-        lengthbox1 = InputArgInt(0)
         h.new_array(box1, lengthbox1)
         h.setarrayitem(box3, index1, box4, descr1)
         h.invalidate_caches(
             rop.CALL_N,
             arraycopydescr1,
-            [ConstInt(13), box2, box1, index1, index1, index2]
+            [None, box2, box1, index1, index1, index2]
         )
 
     def test_ll_arraycopy_doesnt_escape_arrays(self):
         h = HeapCache()
-        box1 = InputArgInt(0)
-        box2 = InputArgInt(1)
-        lengthbox1 = InputArgInt(1)
-        lengthbox2 = InputArgInt(2)
         h.new_array(box1, lengthbox1)
         h.new_array(box2, lengthbox2)
         h.invalidate_caches(
@@ -534,15 +453,13 @@
         h.invalidate_caches(
             rop.CALL_N,
             arraycopydescr1,
-            [ConstInt(123), box2, box1, index1, index1, InputArgInt()]
+            [None, box2, box1, index1, index1, InputArgInt()]
         )
         assert not h.is_unescaped(box1)
         assert not h.is_unescaped(box2)
 
     def test_unescaped(self):
         h = HeapCache()
-        box1 = InputArgInt(0)
-        box2 = InputArgInt(1)
         assert not h.is_unescaped(box1)
         h.new(box2)
         assert h.is_unescaped(box2)
@@ -553,9 +470,6 @@
 
     def test_unescaped_testing(self):
         h = HeapCache()
-        box1 = InputArgInt(0)
-        box2 = InputArgInt(1)
-        box3 = InputArgInt(2)
         h.new(box1)
         h.new(box2)
         assert h.is_unescaped(box1)
@@ -574,8 +488,6 @@
 
     def test_ops_dont_escape(self):
         h = HeapCache()
-        box1 = InputArgInt(0)
-        box2 = InputArgInt(1)
         h.new(box1)
         h.new(box2)
         assert h.is_unescaped(box1)
@@ -589,9 +501,6 @@
 
     def test_circular_virtuals(self):
         h = HeapCache()
-        box1 = InputArgInt(0)
-        box2 = InputArgInt(1)
-        box3 = InputArgInt(2)
         h.new(box1)
         h.new(box2)
         h.invalidate_caches(rop.SETFIELD_GC, None, [box1, box2])
@@ -600,10 +509,6 @@
 
     def test_unescaped_array(self):
         h = HeapCache()
-        box1 = InputArgInt(0)
-        box2 = InputArgInt(1)
-        lengthbox1 = InputArgInt(0)
-        lengthbox2 = InputArgInt(1)
         h.new_array(box1, lengthbox1)
         assert h.is_unescaped(box1)
         h.invalidate_caches(rop.SETARRAYITEM_GC, None, [box1, index1, box2])
@@ -627,9 +532,6 @@
 
     def test_call_doesnt_invalidate_unescaped_boxes(self):
         h = HeapCache()
-        box1 = InputArgInt(0)
-        box2 = InputArgInt(1)
-
         h.new(box1)
         assert h.is_unescaped(box1)
         h.setfield(box1, box2, descr1)
@@ -641,9 +543,6 @@
 
     def test_call_doesnt_invalidate_unescaped_array_boxes(self):
         h = HeapCache()
-        box1 = InputArgInt(0)
-        lengthbox1 = InputArgInt(2)
-        box3 = InputArgInt(1)
         h.new_array(box1, lengthbox1)
         assert h.is_unescaped(box1)
         h.setarrayitem(box1, index1, box3, descr1)
@@ -655,8 +554,6 @@
 
     def test_bug_missing_ignored_operations(self):
         h = HeapCache()
-        box1 = InputArgInt(0)
-        box2 = InputArgInt(1)
         h.new(box1)
         h.new(box2)
         h.setfield(box1, box2, descr1)
@@ -679,8 +576,6 @@
         # calling some residual code that changes the values on box3: then
         # the content of box2 is still cached at the old value.
         h = HeapCache()
-        box1 = InputArgInt(0)
-        box2 = InputArgInt(1)
         h.new(box1)
         h.new(box2)
         h.setfield(box1, box2, descr1)
@@ -693,8 +588,6 @@
 
     def test_bug_heap_cache_is_cleared_but_not_is_unescaped_2(self):
         h = HeapCache()
-        box1 = InputArgInt(0)
-        box2 = InputArgInt(1)
         h.new(box1)
         h.new(box2)
         h.setfield(box1, box2, descr1)
@@ -716,8 +609,6 @@
 
     def test_is_likely_virtual(self):
         h = HeapCache()
-        box1 = InputArgInt(0)
-
         h.new(box1)
         assert h.is_unescaped(box1)
         assert h.is_likely_virtual(box1)

From noreply at buildbot.pypy.org  Fri Jun  5 17:45:12 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Fri,  5 Jun 2015 17:45:12 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Make this test,
 which involves threads and signals, more likely to pass.
Message-ID: <20150605154512.0450A1C033F@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob <me at manueljacob.de>
Branch: py3k
Changeset: r77916:585e0e6ae1c2
Date: 2015-06-05 17:45 +0200
http://bitbucket.org/pypy/pypy/changeset/585e0e6ae1c2/

Log:	Make this test, which involves threads and signals, more likely to
	pass.

diff --git a/pypy/module/thread/test/test_thread.py b/pypy/module/thread/test/test_thread.py
--- a/pypy/module/thread/test/test_thread.py
+++ b/pypy/module/thread/test/test_thread.py
@@ -244,7 +244,7 @@
 
         def busy_wait():
             waiting.append(None)
-            for x in range(10):
+            for x in range(100):
                 print('tick...', x)  # <-force the GIL to be released, as
                 time.sleep(0.1)    #   time.sleep doesn't do non-translated
             waiting.pop()
@@ -252,7 +252,7 @@
         # This is normally called by app_main.py
         signal.signal(signal.SIGINT, signal.default_int_handler)
 
-        for i in range(100):
+        for i in range(10):
             print()
             print("loop", i)
             waiting = []

From noreply at buildbot.pypy.org  Fri Jun  5 18:42:46 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Fri,  5 Jun 2015 18:42:46 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Expose the __class__ cell to Python code.
Message-ID: <20150605164246.27B171C033F@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob <me at manueljacob.de>
Branch: py3k
Changeset: r77917:070a2053d3fe
Date: 2015-06-05 18:42 +0200
http://bitbucket.org/pypy/pypy/changeset/070a2053d3fe/

Log:	Expose the __class__ cell to Python code.

	This fixes issue #1930.

diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -1339,10 +1339,10 @@
         # compile the body proper
         self._handle_body(cls.body)
         # return the (empty) __class__ cell
-        scope = self.scope.lookup("@__class__")
+        scope = self.scope.lookup("__class__")
         if scope == symtable.SCOPE_CELL:
             # Return the cell where to store __class__
-            self.emit_op_arg(ops.LOAD_CLOSURE, self.cell_vars["@__class__"])
+            self.emit_op_arg(ops.LOAD_CLOSURE, self.cell_vars["__class__"])
         else:
             # This happens when nobody references the cell
             self.load_const(self.space.w_None)
diff --git a/pypy/interpreter/astcompiler/symtable.py b/pypy/interpreter/astcompiler/symtable.py
--- a/pypy/interpreter/astcompiler/symtable.py
+++ b/pypy/interpreter/astcompiler/symtable.py
@@ -242,7 +242,7 @@
     def note_symbol(self, identifier, role):
         # Special-case super: it counts as a use of __class__
         if role == SYM_USED and identifier == 'super':
-            self.note_symbol('@__class__', SYM_USED)
+            self.note_symbol('__class__', SYM_USED)
         return Scope.note_symbol(self, identifier, role)
 
     def note_yield(self, yield_node):
@@ -300,12 +300,12 @@
         return misc.mangle(name, self.name)
 
     def _pass_special_names(self, local, new_bound):
-        assert '@__class__' in local
-        new_bound['@__class__'] = None
+        assert '__class__' in local
+        new_bound['__class__'] = None
 
     def _finalize_cells(self, free):
         for name, role in self.symbols.iteritems():
-            if role == SCOPE_LOCAL and name in free and name == '@__class__':
+            if role == SCOPE_LOCAL and name in free and name == '__class__':
                 self.symbols[name] = SCOPE_CELL
                 del free[name]
 
@@ -394,7 +394,7 @@
             clsdef.kwargs.walkabout(self)
         self.visit_sequence(clsdef.decorator_list)
         self.push_scope(ClassScope(clsdef), clsdef)
-        self.note_symbol('@__class__', SYM_ASSIGNED)
+        self.note_symbol('__class__', SYM_ASSIGNED)
         self.note_symbol('__locals__', SYM_PARAM)
         self.visit_sequence(clsdef.body)
         self.pop_scope()
diff --git a/pypy/interpreter/test/test_class.py b/pypy/interpreter/test/test_class.py
--- a/pypy/interpreter/test/test_class.py
+++ b/pypy/interpreter/test/test_class.py
@@ -109,3 +109,9 @@
         c = C()
         assert c.one == "two"
         raises(AttributeError, getattr, c, "two")
+
+    def test___class__(self):
+        class C(object):
+            def get_class(self):
+                return __class__
+        assert C().get_class()
diff --git a/pypy/module/__builtin__/descriptor.py b/pypy/module/__builtin__/descriptor.py
--- a/pypy/module/__builtin__/descriptor.py
+++ b/pypy/module/__builtin__/descriptor.py
@@ -67,7 +67,7 @@
                     "super(): arg[0] deleted"))
         index = 0
         for name in code.co_freevars:
-            if name == "@__class__":
+            if name == "__class__":
                 break
             index += 1
         else:

From noreply at buildbot.pypy.org  Fri Jun  5 18:57:00 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Fri,  5 Jun 2015 18:57:00 +0200 (CEST)
Subject: [pypy-commit] pypy disable-unroll-for-short-loops: an attempt to
 disable unrolling if we hit a certain threshold
Message-ID: <20150605165700.727151C033F@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: disable-unroll-for-short-loops
Changeset: r77918:bdf385bc3c7c
Date: 2015-06-05 18:57 +0200
http://bitbucket.org/pypy/pypy/changeset/bdf385bc3c7c/

Log:	an attempt to disable unrolling if we hit a certain threshold

diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -103,6 +103,35 @@
 
 # ____________________________________________________________
 
+
+def disable_unrolling_if_loop_too_long(loop, metainterp):
+    threshold = metainterp.jitdriver_sd.warmstate.disable_unrolling_threshold
+    if len(loop.operations) < threshold:
+        return False
+    # now we need to patch label, like in simplify.py
+    last_op = loop.operations[-1]
+    descr = last_op.getdescr()
+    assert isinstance(descr, TargetToken)
+    descr = descr.targeting_jitcell_token
+    assert isinstance (descr, JitCellToken)
+    last_op = last_op.copy_and_change(rop.JUMP)
+    if not descr.target_tokens:
+        # fish the first label
+        for op in loop.operations:
+            if op.getopnum() == rop.LABEL:
+                target_token = op.getdescr()
+                break
+        else:
+            assert False, "can't find a label"
+        assert isinstance(target_token, TargetToken)
+        assert target_token.targeting_jitcell_token is descr
+        last_op.setdescr(target_token)
+    else:
+        assert len(descr.target_tokens) == 1
+        last_op.setdescr(descr.target_tokens[0])
+    loop.operations[-1] = last_op
+    return True
+
 def compile_loop(metainterp, greenkey, start,
                  inputargs, jumpargs,
                  full_preamble_needed=True,
@@ -148,27 +177,28 @@
     if part.quasi_immutable_deps:
         loop.quasi_immutable_deps.update(part.quasi_immutable_deps)
     if part.operations[-1].getopnum() == rop.LABEL:
-        inliner = Inliner(inputargs, jumpargs)
-        part.quasi_immutable_deps = None
-        part.operations = [part.operations[-1]] + \
-                          [inliner.inline_op(h_ops[i]) for i in range(start, len(h_ops))] + \
-                          [ResOperation(rop.JUMP, [inliner.inline_arg(a) for a in jumpargs],
-                                        None, descr=jitcell_token)]
-        target_token = part.operations[0].getdescr()
-        assert isinstance(target_token, TargetToken)
-        all_target_tokens.append(target_token)
-        inputargs = jumpargs
-        jumpargs = part.operations[-1].getarglist()
+        if not disable_unrolling_if_loop_too_long(part, metainterp):
+            inliner = Inliner(inputargs, jumpargs)
+            part.quasi_immutable_deps = None
+            part.operations = [part.operations[-1]] + \
+                              [inliner.inline_op(h_ops[i]) for i in range(start, len(h_ops))] + \
+                              [ResOperation(rop.JUMP, [inliner.inline_arg(a) for a in jumpargs],
+                                            None, descr=jitcell_token)]
+            target_token = part.operations[0].getdescr()
+            assert isinstance(target_token, TargetToken)
+            all_target_tokens.append(target_token)
+            inputargs = jumpargs
+            jumpargs = part.operations[-1].getarglist()
 
-        try:
-            optimize_trace(metainterp_sd, jitdriver_sd, part, enable_opts,
-                           start_state=start_state, export_state=False)
-        except InvalidLoop:
-            return None
+            try:
+                optimize_trace(metainterp_sd, jitdriver_sd, part, enable_opts,
+                               start_state=start_state, export_state=False)
+            except InvalidLoop:
+                return None
 
-        loop.operations = loop.operations[:-1] + part.operations
-        if part.quasi_immutable_deps:
-            loop.quasi_immutable_deps.update(part.quasi_immutable_deps)
+            loop.operations = loop.operations[:-1] + part.operations
+            if part.quasi_immutable_deps:
+                loop.quasi_immutable_deps.update(part.quasi_immutable_deps)
     assert part.operations[-1].getopnum() != rop.LABEL
 
     if not loop.quasi_immutable_deps:
diff --git a/rpython/jit/metainterp/warmstate.py b/rpython/jit/metainterp/warmstate.py
--- a/rpython/jit/metainterp/warmstate.py
+++ b/rpython/jit/metainterp/warmstate.py
@@ -256,6 +256,9 @@
     def set_param_inlining(self, value):
         self.inlining = value
 
+    def set_param_disable_unrolling(self, value):
+        self.disable_unrolling_threshold = value
+
     def set_param_enable_opts(self, value):
         from rpython.jit.metainterp.optimizeopt import ALL_OPTS_DICT, ALL_OPTS_NAMES
 
diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py
--- a/rpython/rlib/jit.py
+++ b/rpython/rlib/jit.py
@@ -549,6 +549,7 @@
     'retrace_limit': 'how many times we can try retracing before giving up',
     'max_retrace_guards': 'number of extra guards a retrace can cause',
     'max_unroll_loops': 'number of extra unrollings a loop can cause',
+    'disable_unrolling': 'after how many operations we should not unroll',
     'enable_opts': 'INTERNAL USE ONLY (MAY NOT WORK OR LEAD TO CRASHES): '
                    'optimizations to enable, or all = %s' % ENABLE_ALL_OPTS,
     'max_unroll_recursion': 'how many levels deep to unroll a recursive function'
@@ -564,6 +565,7 @@
               'retrace_limit': 5,
               'max_retrace_guards': 15,
               'max_unroll_loops': 0,
+              'disable_unrolling': 1000,
               'enable_opts': 'all',
               'max_unroll_recursion': 7,
               }

From noreply at buildbot.pypy.org  Fri Jun  5 19:30:00 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Fri,  5 Jun 2015 19:30:00 +0200 (CEST)
Subject: [pypy-commit] pypy unicode-dtype: Enable np.unicode_
Message-ID: <20150605173000.651B31C1038@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: unicode-dtype
Changeset: r77919:9d0ebecde2ab
Date: 2015-06-05 01:03 +0100
http://bitbucket.org/pypy/pypy/changeset/9d0ebecde2ab/

Log:	Enable np.unicode_

diff --git a/pypy/module/micronumpy/boxes.py b/pypy/module/micronumpy/boxes.py
--- a/pypy/module/micronumpy/boxes.py
+++ b/pypy/module/micronumpy/boxes.py
@@ -607,7 +607,6 @@
 
 class W_UnicodeBox(W_CharacterBox):
     def descr__new__unicode_box(space, w_subtype, w_arg):
-        raise oefmt(space.w_NotImplementedError, "Unicode is not supported yet")
         from pypy.module.micronumpy.descriptor import new_unicode_dtype
         arg = space.unicode_w(space.unicode_from_object(w_arg))
         # XXX size computations, we need tests anyway
diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py
--- a/pypy/module/micronumpy/test/test_dtypes.py
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -1059,12 +1059,8 @@
     def test_unicode_boxes(self):
         from numpy import unicode_
         import sys
-        if '__pypy__' in sys.builtin_module_names:
-            exc = raises(NotImplementedError, unicode_, 3)
-            assert exc.value.message.find('not supported yet') >= 0
-        else:
-            u = unicode_(3)
-            assert isinstance(u, unicode)
+        u = unicode_(3)
+        assert isinstance(u, unicode)
 
     def test_character_dtype(self):
         import numpy as np
@@ -1133,7 +1129,7 @@
 
     def test_array_from_record(self):
         import numpy as np
-        a = np.array(('???', -999, -12345678.9), 
+        a = np.array(('???', -999, -12345678.9),
                      dtype=[('c', '|S3'), ('a', '<i8'), ('b', '<f8')])
         # Change the order of the keys
         b = np.array(a, dtype=[('a', '<i8'), ('b', '<f8'), ('c', '|S3')])
@@ -1141,7 +1137,7 @@
         assert b.dtype.fields['a'][1] == 0
         assert b['a'] == -999
         a = np.array(('N/A', 1e+20, 1e+20, 999999),
-                     dtype=[('name', '|S4'), ('x', '<f8'), 
+                     dtype=[('name', '|S4'), ('x', '<f8'),
                             ('y', '<f8'), ('block', '<i8', (2, 3))])
         assert (a['block'] == 999999).all()
 

From noreply at buildbot.pypy.org  Fri Jun  5 19:30:01 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Fri,  5 Jun 2015 19:30:01 +0200 (CEST)
Subject: [pypy-commit] pypy unicode-dtype: Reimplement W_UnicodeBox as a
 simple wrapper around an interp-level unicode object
Message-ID: <20150605173001.A67A51C11B6@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: unicode-dtype
Changeset: r77920:fdc675061202
Date: 2015-06-05 18:29 +0100
http://bitbucket.org/pypy/pypy/changeset/fdc675061202/

Log:	Reimplement W_UnicodeBox as a simple wrapper around an interp-level
	unicode object

diff --git a/pypy/module/micronumpy/boxes.py b/pypy/module/micronumpy/boxes.py
--- a/pypy/module/micronumpy/boxes.py
+++ b/pypy/module/micronumpy/boxes.py
@@ -606,15 +606,25 @@
         return W_StringBox(arr, 0, arr.dtype)
 
 class W_UnicodeBox(W_CharacterBox):
+    def __init__(self, value):
+        self._value = value
+
+    def convert_to(self, space, dtype):
+        if dtype.is_unicode():
+            return self
+        elif dtype.is_object():
+            return W_ObjectBox(space.wrap(self._value))
+        else:
+            raise oefmt(space.w_NotImplementedError,
+                        "Conversion from unicode not implemented yet")
+
+    def get_dtype(self, space):
+        from pypy.module.micronumpy.descriptor import new_unicode_dtype
+        return new_unicode_dtype(space, len(self._value))
+
     def descr__new__unicode_box(space, w_subtype, w_arg):
-        from pypy.module.micronumpy.descriptor import new_unicode_dtype
-        arg = space.unicode_w(space.unicode_from_object(w_arg))
-        # XXX size computations, we need tests anyway
-        arr = VoidBoxStorage(len(arg), new_unicode_dtype(space, len(arg)))
-        # XXX not this way, we need store
-        #for i in range(len(arg)):
-        #    arr.storage[i] = arg[i]
-        return W_UnicodeBox(arr, 0, arr.dtype)
+        value = space.unicode_w(space.unicode_from_object(w_arg))
+        return W_UnicodeBox(value)
 
 class W_ObjectBox(W_GenericBox):
     descr__new__, _get_dtype, descr_reduce = new_dtype_getter(NPY.OBJECT)
diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py
--- a/pypy/module/micronumpy/test/test_dtypes.py
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -1052,16 +1052,6 @@
         assert d.name == "unicode256"
         assert d.num == 19
 
-    def test_string_boxes(self):
-        from numpy import str_
-        assert isinstance(str_(3), str_)
-
-    def test_unicode_boxes(self):
-        from numpy import unicode_
-        import sys
-        u = unicode_(3)
-        assert isinstance(u, unicode)
-
     def test_character_dtype(self):
         import numpy as np
         from numpy import array, character
diff --git a/pypy/module/micronumpy/test/test_scalar.py b/pypy/module/micronumpy/test/test_scalar.py
--- a/pypy/module/micronumpy/test/test_scalar.py
+++ b/pypy/module/micronumpy/test/test_scalar.py
@@ -457,3 +457,14 @@
 
         for t in complex64, complex128:
             _do_test(t, 17j, -17j)
+
+    def test_string_boxes(self):
+        from numpy import str_
+        assert isinstance(str_(3), str_)
+        assert str_(3) == '3'
+
+    def test_unicode_boxes(self):
+        from numpy import unicode_
+        u = unicode_(3)
+        assert isinstance(u, unicode)
+        assert u == u'3'

From noreply at buildbot.pypy.org  Fri Jun  5 19:40:22 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Fri,  5 Jun 2015 19:40:22 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: extract instructions sse4 had some wrong
 parameters in assembler
Message-ID: <20150605174022.BCC341C033F@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r77921:3c733c6463df
Date: 2015-06-05 19:40 +0200
http://bitbucket.org/pypy/pypy/changeset/3c733c6463df/

Log:	extract instructions sse4 had some wrong parameters in assembler
	added test case for that

diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -232,11 +232,13 @@
         c = astype(|1|, int16)
         c[0] = 16i
         b = a + c
-        sum(b -> 7:14)
+        d = b -> 7:9
+        sum(d)
         """
     def test_int16_expand(self):
         result = self.run("int16_expand")
-        assert int(result) == 8*16 + sum(range(7,15))
+        i = 2
+        assert int(result) == i*16 + sum(range(7,7+i))
         self.check_vectorized(2, 2)
 
     def define_int8_expand():
@@ -245,10 +247,11 @@
         c = astype(|1|, int16)
         c[0] = 8i
         b = a + c
-        sum(b -> 0:17)
+        d = b -> 0:17
+        sum(d)
         """
-    def test_int16_expand(self):
-        result = self.run("int16_expand")
+    def test_int8_expand(self):
+        result = self.run("int8_expand")
         assert int(result) == 16*8 + sum(range(0,17))
         self.check_vectorized(2, 2)
 
diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -2613,7 +2613,14 @@
         tosize = tosizeloc.value
         if size == tosize:
             return # already the right size
-        if size == 4 and tosize == 8:
+        if size == 4 and tosize == 2:
+            scratch = X86_64_SCRATCH_REG
+            self.mc.PSHUFLW_xxi(resloc.value, srcloc.value, 0b11111000)
+            self.mc.PEXTRW_rxi(scratch.value, srcloc.value, 4)
+            self.mc.PINSRW_xri(resloc.value, scratch.value, 2)
+            self.mc.PEXTRW_rxi(scratch.value, srcloc.value, 6)
+            self.mc.PINSRW_xri(resloc.value, scratch.value, 3)
+        elif size == 4 and tosize == 8:
             scratch = X86_64_SCRATCH_REG.value
             self.mc.PEXTRD_rxi(scratch, srcloc.value, 1)
             self.mc.PINSRQ_xri(resloc.value, scratch, 1)
diff --git a/rpython/jit/backend/x86/rx86.py b/rpython/jit/backend/x86/rx86.py
--- a/rpython/jit/backend/x86/rx86.py
+++ b/rpython/jit/backend/x86/rx86.py
@@ -750,15 +750,15 @@
 
 
     # following require SSE4_1
-    PEXTRQ_rxi = xmminsn('\x66', rex_w, '\x0F\x3A\x16', register(2,8), register(1), '\xC0', immediate(3, 'b'))
-    PEXTRD_rxi = xmminsn('\x66', rex_nw, '\x0F\x3A\x16', register(2,8), register(1), '\xC0', immediate(3, 'b'))
-    PEXTRW_rxi = xmminsn('\x66', rex_nw, '\x0F\xC4', register(2,8), register(1), '\xC0', immediate(3, 'b'))
-    PEXTRB_rxi = xmminsn('\x66', rex_nw, '\x0F\x3A\x14', register(2,8), register(1), '\xC0', immediate(3, 'b'))
-    EXTRACTPS_rxi = xmminsn('\x66', rex_nw, '\x0F\x3A\x17', register(2,8), register(1), '\xC0', immediate(3, 'b'))
+    PEXTRQ_rxi = xmminsn('\x66', rex_w, '\x0F\x3A\x16', register(1), register(2,8), '\xC0', immediate(3, 'b'))
+    PEXTRD_rxi = xmminsn('\x66', rex_nw, '\x0F\x3A\x16', register(1), register(2,8), '\xC0', immediate(3, 'b'))
+    PEXTRW_rxi = xmminsn('\x66', rex_nw, '\x0F\xC5', register(1,8), register(2), '\xC0', immediate(3, 'b'))
+    PEXTRB_rxi = xmminsn('\x66', rex_nw, '\x0F\x3A\x14', register(1), register(2,8), '\xC0', immediate(3, 'b'))
+    EXTRACTPS_rxi = xmminsn('\x66', rex_nw, '\x0F\x3A\x17', register(1), register(2,8), '\xC0', immediate(3, 'b'))
     
-    PINSRQ_xri = xmminsn('\x66', rex_w, '\x0F\x3A\x22', register(1,8), register(2), '\xC0', immediate(3, 'b'))
-    PINSRD_xri = xmminsn('\x66', rex_nw, '\x0F\x3A\x22', register(1,8), register(2), '\xC0', immediate(3, 'b'))
-    PINSRW_xri = xmminsn('\x66', rex_nw, '\x0F\xC5', register(1,8), register(2), '\xC0', immediate(3, 'b'))
+    PINSRQ_xri = xmminsn('\x66', rex_w, '\x0F\x3A\x22', register(1,8), register(2,8), '\xC0', immediate(3, 'b'))
+    PINSRD_xri = xmminsn('\x66', rex_nw, '\x0F\x3A\x22', register(1,8), register(2,8), '\xC0', immediate(3, 'b'))
+    PINSRW_xri = xmminsn('\x66', rex_nw, '\x0F\xC4', register(1,8), register(2,8), '\xC0', immediate(3, 'b'))
     PINSRB_xri = xmminsn('\x66', rex_nw, '\x0F\x3A\x20', register(1,8), register(2), '\xC0', immediate(3, 'b'))
     INSERTPS_xxi = xmminsn('\x66', rex_nw, '\x0F\x3A\x21', register(1,8), register(2), '\xC0', immediate(3, 'b'))
 
diff --git a/rpython/jit/backend/x86/test/test_rx86.py b/rpython/jit/backend/x86/test/test_rx86.py
--- a/rpython/jit/backend/x86/test/test_rx86.py
+++ b/rpython/jit/backend/x86/test/test_rx86.py
@@ -14,6 +14,9 @@
     def getvalue(self):
         return ''.join(self.buffer)
 
+    def clear(self):
+        self.buffer = []
+
     def force_frame_size(self, frame_size):
         pass
 
@@ -242,3 +245,34 @@
         assert len(cls.MULTIBYTE_NOPs) == 16
         for i in range(16):
             assert len(cls.MULTIBYTE_NOPs[i]) == i
+
+def test_pextr():
+    s = CodeBuilder64()
+    s.PEXTRW_rxi(R.r11, R.xmm0,0)
+    assert s.getvalue() == '\x66\x44\x0f\xc5\xd8\x00'
+    s.clear()
+    s.PEXTRW_rxi(R.edi, R.xmm15, 15)
+    assert s.getvalue() == '\x66\x41\x0f\xc5\xff\x0f'
+    s.clear()
+    s.PEXTRD_rxi(R.eax, R.xmm11, 2)
+    assert s.getvalue() == '\x66\x44\x0f\x3a\x16\xd8\x02'
+    s.clear()
+    s.PEXTRD_rxi(R.r11, R.xmm5, 2)
+    assert s.getvalue() == '\x66\x41\x0f\x3a\x16\xeb\x02'
+    s.clear()
+    s.PEXTRQ_rxi(R.ebp, R.xmm0, 7)
+    assert s.getvalue() == '\x66\x48\x0f\x3a\x16\xc5\x07'
+    # BYTE
+    s.clear()
+    s.PEXTRB_rxi(R.eax, R.xmm13, 24)
+    assert s.getvalue() == '\x66\x44\x0f\x3a\x14\xe8\x18'
+    s.clear()
+    s.PEXTRB_rxi(R.r15, R.xmm5, 33)
+    assert s.getvalue() == '\x66\x41\x0f\x3a\x14\xef\x21'
+    # EXTR SINGLE FLOAT
+    s.clear()
+    s.EXTRACTPS_rxi(R.eax, R.xmm15, 2)
+    assert s.getvalue() == '\x66\x44\x0f\x3a\x17\xf8\x02'
+    s.clear()
+    s.EXTRACTPS_rxi(R.r11, R.xmm0, 1)
+    assert s.getvalue() == '\x66\x41\x0f\x3a\x17\xc3\x01'
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_schedule.py b/rpython/jit/metainterp/optimizeopt/test/test_schedule.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_schedule.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_schedule.py
@@ -145,7 +145,7 @@
         v10[f64|2] = vec_box(2)
         v20[f64|2] = vec_float_pack(v10[f64|2], f0, 0, 1)
         v30[f64|2] = vec_float_pack(v20[f64|2], f1, 1, 1)
-        v40[f64|2] = vec_float_expand(f5) | only expaned once
+        v40[f64|2] = vec_float_expand(f5) # only expaned once
         #
         v50[f64|2] = vec_float_add(v30[f64|2], v40[f64|2])
         v60[f64|2] = vec_float_add(v50[f64|2], v40[f64|2])

From noreply at buildbot.pypy.org  Fri Jun  5 19:57:38 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Fri,  5 Jun 2015 19:57:38 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: bytes(obj) should call obj.__index__()
 instead of obj.__int__().
Message-ID: <20150605175738.314E61C11BD@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob <me at manueljacob.de>
Branch: py3k
Changeset: r77922:b84f64795e2b
Date: 2015-06-05 19:38 +0200
http://bitbucket.org/pypy/pypy/changeset/b84f64795e2b/

Log:	bytes(obj) should call obj.__index__() instead of obj.__int__().

	Fixes #1957.

diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -701,9 +701,10 @@
             raise OperationError(space.w_TypeError, space.wrap(
                     "encoding or errors without string argument"))
         return []
-    # Is it an int?
+    # Is it an integer?
+    # Note that we're calling space.getindex_w() instead of space.int_w().
     try:
-        count = space.int_w(w_source)
+        count = space.getindex_w(w_source, space.w_OverflowError)
     except OperationError, e:
         if not e.match(space, space.w_TypeError):
             raise
diff --git a/pypy/objspace/std/test/test_bytesobject.py b/pypy/objspace/std/test/test_bytesobject.py
--- a/pypy/objspace/std/test/test_bytesobject.py
+++ b/pypy/objspace/std/test/test_bytesobject.py
@@ -732,6 +732,18 @@
                 return [3, 4]
         raises(TypeError, bytes, Z())
 
+    def test_fromobject___index__(self):
+        class WithIndex:
+            def __index__(self):
+                return 3
+        assert bytes(WithIndex()) == b'\x00\x00\x00'
+
+    def test_fromobject___int__(self):
+        class WithInt:
+            def __int__(self):
+                return 3
+        raises(TypeError, bytes, WithInt())
+
     def test_getnewargs(self):
         assert  b"foo".__getnewargs__() == (b"foo",)
 

From noreply at buildbot.pypy.org  Fri Jun  5 20:32:06 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Fri,  5 Jun 2015 20:32:06 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Back out changeset b84f64795e2b because I
 referred the wrong issue in the commit message.
Message-ID: <20150605183206.500D71C1203@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob <me at manueljacob.de>
Branch: py3k
Changeset: r77923:5ddb9db14d38
Date: 2015-06-05 20:29 +0200
http://bitbucket.org/pypy/pypy/changeset/5ddb9db14d38/

Log:	Back out changeset b84f64795e2b because I referred the wrong issue
	in the commit message.

diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -701,10 +701,9 @@
             raise OperationError(space.w_TypeError, space.wrap(
                     "encoding or errors without string argument"))
         return []
-    # Is it an integer?
-    # Note that we're calling space.getindex_w() instead of space.int_w().
+    # Is it an int?
     try:
-        count = space.getindex_w(w_source, space.w_OverflowError)
+        count = space.int_w(w_source)
     except OperationError, e:
         if not e.match(space, space.w_TypeError):
             raise
diff --git a/pypy/objspace/std/test/test_bytesobject.py b/pypy/objspace/std/test/test_bytesobject.py
--- a/pypy/objspace/std/test/test_bytesobject.py
+++ b/pypy/objspace/std/test/test_bytesobject.py
@@ -732,18 +732,6 @@
                 return [3, 4]
         raises(TypeError, bytes, Z())
 
-    def test_fromobject___index__(self):
-        class WithIndex:
-            def __index__(self):
-                return 3
-        assert bytes(WithIndex()) == b'\x00\x00\x00'
-
-    def test_fromobject___int__(self):
-        class WithInt:
-            def __int__(self):
-                return 3
-        raises(TypeError, bytes, WithInt())
-
     def test_getnewargs(self):
         assert  b"foo".__getnewargs__() == (b"foo",)
 

From noreply at buildbot.pypy.org  Fri Jun  5 20:32:07 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Fri,  5 Jun 2015 20:32:07 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: bytes(obj) should call obj.__index__()
 instead of obj.__int__().
Message-ID: <20150605183207.77E741C1203@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob <me at manueljacob.de>
Branch: py3k
Changeset: r77924:6caeb8d4c376
Date: 2015-06-05 20:32 +0200
http://bitbucket.org/pypy/pypy/changeset/6caeb8d4c376/

Log:	bytes(obj) should call obj.__index__() instead of obj.__int__().

	Fixes #1964

diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -701,9 +701,10 @@
             raise OperationError(space.w_TypeError, space.wrap(
                     "encoding or errors without string argument"))
         return []
-    # Is it an int?
+    # Is it an integer?
+    # Note that we're calling space.getindex_w() instead of space.int_w().
     try:
-        count = space.int_w(w_source)
+        count = space.getindex_w(w_source, space.w_OverflowError)
     except OperationError, e:
         if not e.match(space, space.w_TypeError):
             raise
diff --git a/pypy/objspace/std/test/test_bytesobject.py b/pypy/objspace/std/test/test_bytesobject.py
--- a/pypy/objspace/std/test/test_bytesobject.py
+++ b/pypy/objspace/std/test/test_bytesobject.py
@@ -732,6 +732,18 @@
                 return [3, 4]
         raises(TypeError, bytes, Z())
 
+    def test_fromobject___index__(self):
+        class WithIndex:
+            def __index__(self):
+                return 3
+        assert bytes(WithIndex()) == b'\x00\x00\x00'
+
+    def test_fromobject___int__(self):
+        class WithInt:
+            def __int__(self):
+                return 3
+        raises(TypeError, bytes, WithInt())
+
     def test_getnewargs(self):
         assert  b"foo".__getnewargs__() == (b"foo",)
 

From noreply at buildbot.pypy.org  Fri Jun  5 21:20:43 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Fri,  5 Jun 2015 21:20:43 +0200 (CEST)
Subject: [pypy-commit] pypy disable-unroll-for-short-loops: try to attack
 the problem differently, a bit unclear how to test the beast
Message-ID: <20150605192043.6230A1C048F@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: disable-unroll-for-short-loops
Changeset: r77925:655b7c16bafd
Date: 2015-06-05 21:20 +0200
http://bitbucket.org/pypy/pypy/changeset/655b7c16bafd/

Log:	try to attack the problem differently, a bit unclear how to test the
	beast

diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -104,34 +104,6 @@
 # ____________________________________________________________
 
 
-def disable_unrolling_if_loop_too_long(loop, metainterp):
-    threshold = metainterp.jitdriver_sd.warmstate.disable_unrolling_threshold
-    if len(loop.operations) < threshold:
-        return False
-    # now we need to patch label, like in simplify.py
-    last_op = loop.operations[-1]
-    descr = last_op.getdescr()
-    assert isinstance(descr, TargetToken)
-    descr = descr.targeting_jitcell_token
-    assert isinstance (descr, JitCellToken)
-    last_op = last_op.copy_and_change(rop.JUMP)
-    if not descr.target_tokens:
-        # fish the first label
-        for op in loop.operations:
-            if op.getopnum() == rop.LABEL:
-                target_token = op.getdescr()
-                break
-        else:
-            assert False, "can't find a label"
-        assert isinstance(target_token, TargetToken)
-        assert target_token.targeting_jitcell_token is descr
-        last_op.setdescr(target_token)
-    else:
-        assert len(descr.target_tokens) == 1
-        last_op.setdescr(descr.target_tokens[0])
-    loop.operations[-1] = last_op
-    return True
-
 def compile_loop(metainterp, greenkey, start,
                  inputargs, jumpargs,
                  full_preamble_needed=True,
@@ -177,7 +149,7 @@
     if part.quasi_immutable_deps:
         loop.quasi_immutable_deps.update(part.quasi_immutable_deps)
     if part.operations[-1].getopnum() == rop.LABEL:
-        if not disable_unrolling_if_loop_too_long(part, metainterp):
+        if start_state is not None:
             inliner = Inliner(inputargs, jumpargs)
             part.quasi_immutable_deps = None
             part.operations = [part.operations[-1]] + \
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -154,6 +154,19 @@
 
         loop.operations = self.optimizer.get_newoperations()
         if export_state:
+            jd_sd = self.optimizer.jitdriver_sd
+            threshold = jd_sd.warmstate.disable_unrolling_threshold
+            if 1 or len(loop.operations) > threshold:
+                if loop.operations[0].getopnum() == rop.LABEL:
+                    # abandoning unrolling, too long
+                    new_descr = stop_label.getdescr()
+                    if loop.operations[0].getopnum() == rop.LABEL:
+                        new_descr = loop.operations[0].getdescr()
+                    stop_label = stop_label.copy_and_change(rop.JUMP,
+                                        descr=new_descr)
+                    self.optimizer.send_extra_operation(stop_label)
+                    loop.operations = self.optimizer.get_newoperations()
+                    return None
             final_state = self.export_state(stop_label)
         else:
             final_state = None

From noreply at buildbot.pypy.org  Sat Jun  6 18:55:53 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Sat,  6 Jun 2015 18:55:53 +0200 (CEST)
Subject: [pypy-commit] pypy default: Move implementation of
 FlexibleType.to_str() to W_FlexibleBox
Message-ID: <20150606165553.09C4D1C0354@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r77926:aa1b46439de0
Date: 2015-06-06 17:39 +0100
http://bitbucket.org/pypy/pypy/changeset/aa1b46439de0/

Log:	Move implementation of FlexibleType.to_str() to W_FlexibleBox

diff --git a/pypy/module/micronumpy/boxes.py b/pypy/module/micronumpy/boxes.py
--- a/pypy/module/micronumpy/boxes.py
+++ b/pypy/module/micronumpy/boxes.py
@@ -10,6 +10,7 @@
 from rpython.rlib.rarithmetic import LONG_BIT
 from rpython.rlib.rstring import StringBuilder
 from rpython.rlib.objectmodel import specialize
+from rpython.rlib import jit
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.tool.sourcetools import func_with_new_name
 from pypy.module.micronumpy import constants as NPY
@@ -536,8 +537,20 @@
     def get_dtype(self, space):
         return self.dtype
 
+    @jit.unroll_safe
     def raw_str(self):
-        return self.arr.dtype.itemtype.to_str(self)
+        builder = StringBuilder()
+        i = self.ofs
+        end = i + self.dtype.elsize
+        with self.arr as storage:
+            while i < end:
+                assert isinstance(storage[i], str)
+                if storage[i] == '\x00':
+                    break
+                builder.append(storage[i])
+                i += 1
+            return builder.build()
+
 
 class W_VoidBox(W_FlexibleBox):
     def descr_getitem(self, space, w_item):
diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py
--- a/pypy/module/micronumpy/ndarray.py
+++ b/pypy/module/micronumpy/ndarray.py
@@ -279,7 +279,7 @@
                 s.append(separator)
                 s.append(' ')
             if self.is_scalar() and dtype.is_str():
-                s.append(dtype.itemtype.to_str(i.getitem(state)))
+                s.append(i.getitem(state).raw_str())
             else:
                 s.append(dtype.itemtype.str_format(i.getitem(state), add_quotes=True))
             state = i.next(state)
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -2051,20 +2051,8 @@
     def get_element_size(self):
         return rffi.sizeof(self.T)
 
-    @jit.unroll_safe
     def to_str(self, item):
-        builder = StringBuilder()
-        assert isinstance(item, boxes.W_FlexibleBox)
-        i = item.ofs
-        end = i + item.dtype.elsize
-        with item.arr as storage:
-            while i < end:
-                assert isinstance(storage[i], str)
-                if storage[i] == '\x00':
-                    break
-                builder.append(storage[i])
-                i += 1
-            return builder.build()
+        return item.raw_str()
 
 def str_unary_op(func):
     specialize.argtype(1)(func)

From noreply at buildbot.pypy.org  Sat Jun  6 20:29:25 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Sat,  6 Jun 2015 20:29:25 +0200 (CEST)
Subject: [pypy-commit] pypy default: Create W_Dtype.runpack_str() and
	simplify its callers
Message-ID: <20150606182925.433E81C033F@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r77927:34dee90d9ae5
Date: 2015-06-06 19:29 +0100
http://bitbucket.org/pypy/pypy/changeset/34dee90d9ae5/

Log:	Create W_Dtype.runpack_str() and simplify its callers

diff --git a/pypy/module/micronumpy/boxes.py b/pypy/module/micronumpy/boxes.py
--- a/pypy/module/micronumpy/boxes.py
+++ b/pypy/module/micronumpy/boxes.py
@@ -369,13 +369,11 @@
             if dtype.elsize != self.get_dtype(space).elsize:
                 raise OperationError(space.w_ValueError, space.wrap(
                     "new type not compatible with array."))
-        if dtype.is_str_or_unicode():
-            return dtype.coerce(space, space.wrap(self.raw_str()))
-        elif dtype.is_record():
+        if dtype.is_record():
             raise OperationError(space.w_NotImplementedError, space.wrap(
                 "viewing scalar as record not implemented"))
         else:
-            return dtype.itemtype.runpack_str(space, self.raw_str())
+            return dtype.runpack_str(space, self.raw_str())
 
     def descr_self(self, space):
         return self
diff --git a/pypy/module/micronumpy/descriptor.py b/pypy/module/micronumpy/descriptor.py
--- a/pypy/module/micronumpy/descriptor.py
+++ b/pypy/module/micronumpy/descriptor.py
@@ -397,6 +397,11 @@
             return space.wrap(0)
         return space.wrap(len(self.fields))
 
+    def runpack_str(self, space, s):
+        if self.is_str_or_unicode():
+            return self.coerce(space, space.wrap(s))
+        return self.itemtype.runpack_str(space, s)
+
     def descr_reduce(self, space):
         w_class = space.type(self)
         builder_args = space.newtuple([
diff --git a/pypy/module/micronumpy/loop.py b/pypy/module/micronumpy/loop.py
--- a/pypy/module/micronumpy/loop.py
+++ b/pypy/module/micronumpy/loop.py
@@ -566,10 +566,7 @@
     while not ai.done(state):
         fromstring_driver.jit_merge_point(dtype=dtype, itemsize=itemsize)
         sub = s[i*itemsize:i*itemsize + itemsize]
-        if dtype.is_str_or_unicode():
-            val = dtype.coerce(space, space.wrap(sub))
-        else:
-            val = dtype.itemtype.runpack_str(space, sub)
+        val = dtype.runpack_str(space, sub)
         ai.setitem(state, val)
         state = ai.next(state)
         i += 1
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -1149,13 +1149,13 @@
         return ''.join(['(', real_str, op, imag_str, ')'])
 
     def runpack_str(self, space, s):
-        comp = self.ComponentBoxType._get_dtype(space).itemtype
+        comp = self.ComponentBoxType._get_dtype(space)
         l = len(s) // 2
         real = comp.runpack_str(space, s[:l])
         imag = comp.runpack_str(space, s[l:])
         if not self.native:
-            real = comp.byteswap(real)
-            imag = comp.byteswap(imag)
+            real = comp.itemtype.byteswap(real)
+            imag = comp.itemtype.byteswap(imag)
         return self.composite(real, imag)
 
     @staticmethod

From noreply at buildbot.pypy.org  Sat Jun  6 20:46:56 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Sat,  6 Jun 2015 20:46:56 +0200 (CEST)
Subject: [pypy-commit] pypy default: test, implement kwargs to astype
Message-ID: <20150606184656.E22D11C033F@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: 
Changeset: r77928:b49bb002e037
Date: 2015-06-05 18:16 +0300
http://bitbucket.org/pypy/pypy/changeset/b49bb002e037/

Log:	test, implement kwargs to astype

diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py
--- a/pypy/module/micronumpy/concrete.py
+++ b/pypy/module/micronumpy/concrete.py
@@ -17,7 +17,6 @@
 from rpython.rtyper.annlowlevel import cast_gcref_to_instance
 from pypy.interpreter.baseobjspace import W_Root
 
-
 class BaseConcreteArray(object):
     _immutable_fields_ = ['dtype?', 'storage', 'start', 'size', 'shape[*]',
                           'strides[*]', 'backstrides[*]', 'order', 'gcstruct',
@@ -334,7 +333,7 @@
     def get_buffer(self, space, readonly):
         return ArrayBuffer(self, readonly)
 
-    def astype(self, space, dtype):
+    def astype(self, space, dtype, order):
         # copy the general pattern of the strides
         # but make the array storage contiguous in memory
         shape = self.get_shape()
@@ -350,7 +349,7 @@
         else:
             t_strides = []
             backstrides = []
-        impl = ConcreteArray(shape, dtype, self.order, t_strides, backstrides)
+        impl = ConcreteArray(shape, dtype, order, t_strides, backstrides)
         loop.setslice(space, impl.get_shape(), impl, self)
         return impl
 
diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py
--- a/pypy/module/micronumpy/ndarray.py
+++ b/pypy/module/micronumpy/ndarray.py
@@ -20,6 +20,7 @@
 from pypy.module.micronumpy.flagsobj import W_FlagsObject
 from pypy.module.micronumpy.strides import get_shape_from_iterable, \
     shape_agreement, shape_agreement_multiple, is_c_contiguous, is_f_contiguous
+from pypy.module.micronumpy.casting import can_cast_array
 
 
 def _match_dot_shapes(space, left, right):
@@ -43,6 +44,15 @@
         raise oefmt(space.w_ValueError, "objects are not aligned")
     return out_shape, right_critical_dim
 
+def get_order(proto_order, order):
+    if order == 'C':
+        return 'C'
+    elif order == 'F':
+        return 'F'
+    elif order == 'K':
+        return proto_order
+    elif order == 'A':
+        return proto_order
 
 class __extend__(W_NDimArray):
     @jit.unroll_safe
@@ -592,10 +602,11 @@
         if self.is_scalar():
             return space.wrap(0)
         dtype = self.get_dtype().descr_newbyteorder(space, NPY.NATIVE)
-        contig = self.implementation.astype(space, dtype)
+        contig = self.implementation.astype(space, dtype, self.get_order())
         return contig.argsort(space, w_axis)
 
-    def descr_astype(self, space, w_dtype):
+    @unwrap_spec(order=str, casting=str, subok=bool, copy=bool)
+    def descr_astype(self, space, w_dtype, order='K', casting='unsafe', subok=True, copy=True):
         cur_dtype = self.get_dtype()
         new_dtype = space.interp_w(descriptor.W_Dtype, space.call_function(
             space.gettypefor(descriptor.W_Dtype), w_dtype))
@@ -607,9 +618,24 @@
             if cur_dtype.num == NPY.STRING:
                 new_dtype = descriptor.variable_dtype(
                     space, 'S' + str(cur_dtype.elsize))
+        if not can_cast_array(space, self, new_dtype, casting):
+            raise oefmt(space.w_TypeError, "Cannot cast array from %s to %s"
+                        "according to the rule %s", 
+                        space.str_w(self.get_dtype().descr_repr(space)),
+                        space.str_w(new_dtype.descr_repr(space)), casting)
+        order  = get_order(self.get_order(), order)
+        if (not copy and new_dtype == self.get_dtype() and order == self.get_order()
+                and (subok or type(self) is W_NDimArray)):
+            return self
         impl = self.implementation
-        new_impl = impl.astype(space, new_dtype)
-        return wrap_impl(space, space.type(self), self, new_impl)
+        new_impl = impl.astype(space, new_dtype, order)
+        if new_impl is None:
+            return self
+        if subok:
+            w_type = space.type(self)
+        else:
+            w_type = None
+        return wrap_impl(space, w_type, self, new_impl)
 
     def descr_get_base(self, space):
         impl = self.implementation
diff --git a/pypy/module/micronumpy/nditer.py b/pypy/module/micronumpy/nditer.py
--- a/pypy/module/micronumpy/nditer.py
+++ b/pypy/module/micronumpy/nditer.py
@@ -470,7 +470,7 @@
                                     "Iterator operand required copying or "
                                     "buffering for operand %d", i)
                     impl = self.seq[i].implementation
-                    new_impl = impl.astype(space, selfd)
+                    new_impl = impl.astype(space, selfd, self.order)
                     self.seq[i] = W_NDimArray(new_impl)
         else:
             #copy them from seq
diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py
--- a/pypy/module/micronumpy/test/test_ndarray.py
+++ b/pypy/module/micronumpy/test/test_ndarray.py
@@ -2241,6 +2241,15 @@
         assert c.shape == b.shape
         assert c.strides == (8,)
 
+        exc = raises(TypeError, a.astype, 'i8', casting='safe')
+        assert exc.value.message.startswith(
+                "Cannot cast array from dtype('complex128') to dtype('int64')")
+        a = arange(6, dtype='f4').reshape(2, 3)
+        b = a.astype('f4', copy=False)
+        assert a is b
+        b = a.astype('f4', order='C', copy=False)
+        assert a is b
+
     def test_base(self):
         from numpy import array
         assert array(1).base is None

From noreply at buildbot.pypy.org  Sat Jun  6 20:46:58 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Sat,  6 Jun 2015 20:46:58 +0200 (CEST)
Subject: [pypy-commit] pypy default: sanity check
Message-ID: <20150606184658.111BE1C033F@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: 
Changeset: r77929:057398c13698
Date: 2015-06-06 20:37 +0300
http://bitbucket.org/pypy/pypy/changeset/057398c13698/

Log:	sanity check

diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py
--- a/pypy/module/micronumpy/concrete.py
+++ b/pypy/module/micronumpy/concrete.py
@@ -345,6 +345,10 @@
                 if s < mins:
                     mins = s
             t_strides = [s * t_elsize / mins for s in strides]
+            if order not in ('C', 'F'):
+                raise oefmt(space.w_ValueError, "Unknown order %s in astype", order)
+            if order != self.order:
+                t_strides = tstrides[::-1]
             backstrides = calc_backstrides(t_strides, shape)
         else:
             t_strides = []
@@ -399,6 +403,8 @@
         make_sure_not_resized(backstrides)
         self.shape = shape
         self.size = support.product(shape) * dtype.elsize
+        if order not in ('C', 'F'):
+            raise oefmt(space.w_ValueError, "Unknown order %s in astype", order)
         self.order = order
         self.dtype = dtype
         self.strides = strides

From noreply at buildbot.pypy.org  Sat Jun  6 20:46:59 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Sat,  6 Jun 2015 20:46:59 +0200 (CEST)
Subject: [pypy-commit] pypy default: refactor ndarray.flags in cpyext
Message-ID: <20150606184659.2EB701C033F@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: 
Changeset: r77930:021adba9ebbe
Date: 2015-06-06 21:44 +0300
http://bitbucket.org/pypy/pypy/changeset/021adba9ebbe/

Log:	refactor ndarray.flags in cpyext

diff --git a/pypy/module/cpyext/ndarrayobject.py b/pypy/module/cpyext/ndarrayobject.py
--- a/pypy/module/cpyext/ndarrayobject.py
+++ b/pypy/module/cpyext/ndarrayobject.py
@@ -65,15 +65,7 @@
 @cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL)
 def _PyArray_FLAGS(space, w_array):
     assert isinstance(w_array, W_NDimArray)
-    flags = NPY_BEHAVED_NS
-    if isinstance(w_array.implementation, ConcreteArray):
-        flags |= NPY_OWNDATA
-    if len(w_array.get_shape()) < 2:
-        flags |= NPY_CONTIGUOUS
-    elif w_array.implementation.order == 'C':
-        flags |= NPY_C_CONTIGUOUS
-    else:
-        flags |= NPY_F_CONTIGUOUS
+    flags = NPY_BEHAVED_NS | w_array.get_flags()
     return flags
 
 @cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL)

From noreply at buildbot.pypy.org  Sat Jun  6 20:55:33 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat,  6 Jun 2015 20:55:33 +0200 (CEST)
Subject: [pypy-commit] cffi default: Fix the logic to handle ffi.gc(x) being
 called several times with equal
Message-ID: <20150606185533.64E151C033F@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2163:ce83dce59b83
Date: 2015-06-06 20:56 +0200
http://bitbucket.org/cffi/cffi/changeset/ce83dce59b83/

Log:	Fix the logic to handle ffi.gc(x) being called several times with
	equal values of x. (PyPy 2.6.0's RPython version is already
	correct, which I guess is why the problem was noticed.)

diff --git a/c/cgc.c b/c/cgc.c
--- a/c/cgc.c
+++ b/c/cgc.c
@@ -2,79 +2,121 @@
 /* translated to C from cffi/gc_weakref.py */
 
 
-static PyObject *const_name_pop;
+static PyObject *gc_wref_remove(PyObject *ffi_wref_tup, PyObject *key)
+{
+    FFIObject *ffi;
+    PyObject *indexobj, *destructor, *cdata, *freelist, *result;
+    Py_ssize_t index;
 
-static PyObject *gc_wref_remove(PyObject *ffi_wref_data, PyObject *arg)
-{
-    PyObject *destructor, *cdata, *x;
-    PyObject *res = PyObject_CallMethodObjArgs(ffi_wref_data,
-                                               const_name_pop, arg, NULL);
-    if (res == NULL)
-        return NULL;
+    /* here, tup is a 4-tuple (ffi, destructor, cdata, index) */
+    if (!PyTuple_Check(ffi_wref_tup))
+        goto oops;    /* should never occur */
 
-    assert(PyTuple_Check(res));
-    destructor = PyTuple_GET_ITEM(res, 0);
-    cdata = PyTuple_GET_ITEM(res, 1);
-    x = PyObject_CallFunctionObjArgs(destructor, cdata, NULL);
-    Py_DECREF(res);
-    if (x == NULL)
-        return NULL;
-    Py_DECREF(x);
+    ffi = (FFIObject *)PyTuple_GET_ITEM(ffi_wref_tup, 0);
+    destructor = PyTuple_GET_ITEM(ffi_wref_tup, 1);
+    cdata = PyTuple_GET_ITEM(ffi_wref_tup, 2);
+    indexobj = PyTuple_GET_ITEM(ffi_wref_tup, 3);
 
-    Py_INCREF(Py_None);
-    return Py_None;
+    index = PyInt_AsSsize_t(indexobj);
+    if (index < 0)
+        goto oops;    /* should never occur */
+
+    /* assert gc_wrefs[index] is key */
+    if (PyList_GET_ITEM(ffi->gc_wrefs, index) != key)
+        goto oops;    /* should never occur */
+
+    /* gc_wrefs[index] = freelist */
+    /* transfer ownership of 'freelist' to 'gc_wrefs[index]' */
+    freelist = ffi->gc_wrefs_freelist;
+    PyList_SET_ITEM(ffi->gc_wrefs, index, freelist);
+
+    /* freelist = index */
+    ffi->gc_wrefs_freelist = indexobj;
+    Py_INCREF(indexobj);
+
+    /* destructor(cdata) */
+    result = PyObject_CallFunctionObjArgs(destructor, cdata, NULL);
+
+    Py_DECREF(key);    /* free the reference that was in 'gc_wrefs[index]' */
+    return result;
+
+ oops:
+    PyErr_SetString(PyExc_SystemError, "cgc: internal inconsistency");
+    /* random leaks may follow */
+    return NULL;
 }
 
 static PyMethodDef remove_callback = {
     "gc_wref_remove", (PyCFunction)gc_wref_remove, METH_O
 };
 
-static PyObject *gc_weakrefs_build(FFIObject *ffi, CDataObject *cd,
+static PyObject *gc_weakrefs_build(FFIObject *ffi, CDataObject *cdata,
                                    PyObject *destructor)
 {
-    PyObject *new_cdata, *ref = NULL, *tup = NULL;
+    PyObject *new_cdata, *ref = NULL, *tup = NULL, *remove_fn = NULL;
+    Py_ssize_t index;
+    PyObject *datalist;
 
     if (ffi->gc_wrefs == NULL) {
         /* initialize */
-        PyObject *data;
-
-        if (const_name_pop == NULL) {
-            const_name_pop = PyText_InternFromString("pop");
-            if (const_name_pop == NULL)
-                return NULL;
-        }
-        data = PyDict_New();
-        if (data == NULL)
+        datalist = PyList_New(0);
+        if (datalist == NULL)
             return NULL;
-        ffi->gc_wrefs = PyCFunction_New(&remove_callback, data);
-        Py_DECREF(data);
-        if (ffi->gc_wrefs == NULL)
-            return NULL;
+        ffi->gc_wrefs = datalist;
+        assert(ffi->gc_wrefs_freelist == NULL);
+        ffi->gc_wrefs_freelist = Py_None;
+        Py_INCREF(Py_None);
     }
 
-    new_cdata = do_cast(cd->c_type, (PyObject *)cd);
+    /* new_cdata = self.ffi.cast(typeof(cdata), cdata) */
+    new_cdata = do_cast(cdata->c_type, (PyObject *)cdata);
     if (new_cdata == NULL)
         goto error;
 
-    ref = PyWeakref_NewRef(new_cdata, ffi->gc_wrefs);
+    /* if freelist is None: */
+    datalist = ffi->gc_wrefs;
+    if (ffi->gc_wrefs_freelist == Py_None) {
+        /* index = len(gc_wrefs) */
+        index = PyList_GET_SIZE(datalist);
+        /* gc_wrefs.append(None) */
+        if (PyList_Append(datalist, Py_None) < 0)
+            goto error;
+        tup = Py_BuildValue("OOOn", ffi, destructor, cdata, index);
+    }
+    else {
+        /* index = freelist */
+        index = PyInt_AsSsize_t(ffi->gc_wrefs_freelist);
+        if (index < 0)
+            goto error;   /* should not occur */
+        tup = PyTuple_Pack(4, ffi, destructor, cdata, ffi->gc_wrefs_freelist);
+    }
+    if (tup == NULL)
+        goto error;
+
+    remove_fn = PyCFunction_New(&remove_callback, tup);
+    if (remove_fn == NULL)
+        goto error;
+
+    ref = PyWeakref_NewRef(new_cdata, remove_fn);
     if (ref == NULL)
         goto error;
 
-    tup = PyTuple_Pack(2, destructor, cd);
-    if (tup == NULL)
-        goto error;
+    /* freelist = gc_wrefs[index] (which is None if we just did append(None)) */
+    /* transfer ownership of 'datalist[index]' into gc_wrefs_freelist */
+    Py_DECREF(ffi->gc_wrefs_freelist);
+    ffi->gc_wrefs_freelist = PyList_GET_ITEM(datalist, index);
+    /* gc_wrefs[index] = ref */
+    /* transfer ownership of 'ref' into 'datalist[index]' */
+    PyList_SET_ITEM(datalist, index, ref);
+    Py_DECREF(remove_fn);
+    Py_DECREF(tup);
 
-    /* the 'self' of the function 'gc_wrefs' is actually the data dict */
-    if (PyDict_SetItem(PyCFunction_GET_SELF(ffi->gc_wrefs), ref, tup) < 0)
-        goto error;
-
-    Py_DECREF(tup);
-    Py_DECREF(ref);
     return new_cdata;
 
  error:
     Py_XDECREF(new_cdata);
     Py_XDECREF(ref);
     Py_XDECREF(tup);
+    Py_XDECREF(remove_fn);
     return NULL;
 }
diff --git a/c/ffi_obj.c b/c/ffi_obj.c
--- a/c/ffi_obj.c
+++ b/c/ffi_obj.c
@@ -23,7 +23,7 @@
 
 struct FFIObject_s {
     PyObject_HEAD
-    PyObject *gc_wrefs;
+    PyObject *gc_wrefs, *gc_wrefs_freelist;
     struct _cffi_parse_info_s info;
     char ctx_is_static, ctx_is_nonempty;
     builder_c_t types_builder;
@@ -51,6 +51,7 @@
         return NULL;
     }
     ffi->gc_wrefs = NULL;
+    ffi->gc_wrefs_freelist = NULL;
     ffi->info.ctx = &ffi->types_builder.ctx;
     ffi->info.output = internal_output;
     ffi->info.output_size = FFI_COMPLEXITY_OUTPUT;
@@ -63,6 +64,7 @@
 {
     PyObject_GC_UnTrack(ffi);
     Py_XDECREF(ffi->gc_wrefs);
+    Py_XDECREF(ffi->gc_wrefs_freelist);
 
     free_builder_c(&ffi->types_builder, ffi->ctx_is_static);
 
diff --git a/cffi/gc_weakref.py b/cffi/gc_weakref.py
--- a/cffi/gc_weakref.py
+++ b/cffi/gc_weakref.py
@@ -2,18 +2,27 @@
 
 
 class GcWeakrefs(object):
-    # code copied and adapted from WeakKeyDictionary.
-
     def __init__(self, ffi):
         self.ffi = ffi
-        self.data = data = {}
-        def remove(k):
-            destructor, cdata = data.pop(k)
-            destructor(cdata)
-        self.remove = remove
+        self.data = []
+        self.freelist = None
 
     def build(self, cdata, destructor):
         # make a new cdata of the same type as the original one
         new_cdata = self.ffi.cast(self.ffi._backend.typeof(cdata), cdata)
-        self.data[ref(new_cdata, self.remove)] = destructor, cdata
+        #
+        def remove(key):
+            assert self.data[index] is key
+            self.data[index] = self.freelist
+            self.freelist = index
+            destructor(cdata)
+        #
+        key = ref(new_cdata, remove)
+        index = self.freelist
+        if index is None:
+            index = len(self.data)
+            self.data.append(key)
+        else:
+            self.freelist = self.data[index]
+            self.data[index] = key
         return new_cdata
diff --git a/testing/cffi0/backend_tests.py b/testing/cffi0/backend_tests.py
--- a/testing/cffi0/backend_tests.py
+++ b/testing/cffi0/backend_tests.py
@@ -1457,6 +1457,63 @@
         import gc; gc.collect(); gc.collect(); gc.collect()
         assert seen == [1]
 
+    def test_gc_2(self):
+        ffi = FFI(backend=self.Backend())
+        p = ffi.new("int *", 123)
+        seen = []
+        q1 = ffi.gc(p, lambda p: seen.append(1))
+        q2 = ffi.gc(q1, lambda p: seen.append(2))
+        import gc; gc.collect()
+        assert seen == []
+        del q1, q2
+        import gc; gc.collect(); gc.collect(); gc.collect(); gc.collect()
+        assert seen == [2, 1]
+
+    def test_gc_3(self):
+        ffi = FFI(backend=self.Backend())
+        p = ffi.new("int *", 123)
+        r = ffi.new("int *", 123)
+        seen = []
+        seen_r = []
+        q1 = ffi.gc(p, lambda p: seen.append(1))
+        s1 = ffi.gc(r, lambda r: seen_r.append(4))
+        q2 = ffi.gc(q1, lambda p: seen.append(2))
+        s2 = ffi.gc(s1, lambda r: seen_r.append(5))
+        q3 = ffi.gc(q2, lambda p: seen.append(3))
+        import gc; gc.collect()
+        assert seen == []
+        assert seen_r == []
+        del q1, q2, q3, s2, s1
+        import gc; gc.collect(); gc.collect(); gc.collect(); gc.collect()
+        assert seen == [3, 2, 1]
+        assert seen_r == [5, 4]
+
+    def test_gc_4(self):
+        ffi = FFI(backend=self.Backend())
+        p = ffi.new("int *", 123)
+        seen = []
+        q1 = ffi.gc(p, lambda p: seen.append(1))
+        q2 = ffi.gc(q1, lambda p: seen.append(2))
+        q3 = ffi.gc(q2, lambda p: seen.append(3))
+        import gc; gc.collect()
+        assert seen == []
+        del q1, q3     # q2 remains, and has a hard ref to q1
+        import gc; gc.collect(); gc.collect(); gc.collect()
+        assert seen == [3]
+
+    def test_gc_finite_list(self):
+        ffi = FFI(backend=self.Backend())
+        p = ffi.new("int *", 123)
+        keepalive = []
+        for i in range(10):
+            keepalive.append(ffi.gc(p, lambda p: None))
+            assert len(ffi.gc_weakrefs.data) == i + 1  #should be a private attr
+        del keepalive[:]
+        import gc; gc.collect(); gc.collect()
+        for i in range(10):
+            keepalive.append(ffi.gc(p, lambda p: None))
+        assert len(ffi.gc_weakrefs.data) == 10
+
     def test_CData_CType(self):
         ffi = FFI(backend=self.Backend())
         assert isinstance(ffi.cast("int", 0), ffi.CData)
diff --git a/testing/cffi1/test_new_ffi_1.py b/testing/cffi1/test_new_ffi_1.py
--- a/testing/cffi1/test_new_ffi_1.py
+++ b/testing/cffi1/test_new_ffi_1.py
@@ -1409,6 +1409,47 @@
         import gc; gc.collect(); gc.collect(); gc.collect()
         assert seen == [1]
 
+    def test_gc_2(self):
+        p = ffi.new("int *", 123)
+        seen = []
+        q1 = ffi.gc(p, lambda p: seen.append(1))
+        q2 = ffi.gc(q1, lambda p: seen.append(2))
+        import gc; gc.collect()
+        assert seen == []
+        del q1, q2
+        import gc; gc.collect(); gc.collect(); gc.collect(); gc.collect()
+        assert seen == [2, 1]
+
+    def test_gc_3(self):
+        p = ffi.new("int *", 123)
+        r = ffi.new("int *", 123)
+        seen = []
+        seen_r = []
+        q1 = ffi.gc(p, lambda p: seen.append(1))
+        s1 = ffi.gc(r, lambda r: seen_r.append(4))
+        q2 = ffi.gc(q1, lambda p: seen.append(2))
+        s2 = ffi.gc(s1, lambda r: seen_r.append(5))
+        q3 = ffi.gc(q2, lambda p: seen.append(3))
+        import gc; gc.collect()
+        assert seen == []
+        assert seen_r == []
+        del q1, q2, q3, s2, s1
+        import gc; gc.collect(); gc.collect(); gc.collect(); gc.collect()
+        assert seen == [3, 2, 1]
+        assert seen_r == [5, 4]
+
+    def test_gc_4(self):
+        p = ffi.new("int *", 123)
+        seen = []
+        q1 = ffi.gc(p, lambda p: seen.append(1))
+        q2 = ffi.gc(q1, lambda p: seen.append(2))
+        q3 = ffi.gc(q2, lambda p: seen.append(3))
+        import gc; gc.collect()
+        assert seen == []
+        del q1, q3     # q2 remains, and has a hard ref to q1
+        import gc; gc.collect(); gc.collect(); gc.collect()
+        assert seen == [3]
+
     def test_CData_CType(self):
         assert isinstance(ffi.cast("int", 0), ffi.CData)
         assert isinstance(ffi.new("int *"), ffi.CData)

From noreply at buildbot.pypy.org  Sat Jun  6 21:00:28 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat,  6 Jun 2015 21:00:28 +0200 (CEST)
Subject: [pypy-commit] cffi default: whatsnew: ffi.gc fix
Message-ID: <20150606190028.1D66C1C033F@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2164:0a6cf359cc4a
Date: 2015-06-06 21:01 +0200
http://bitbucket.org/cffi/cffi/changeset/0a6cf359cc4a/

Log:	whatsnew: ffi.gc fix

diff --git a/doc/source/whatsnew.rst b/doc/source/whatsnew.rst
--- a/doc/source/whatsnew.rst
+++ b/doc/source/whatsnew.rst
@@ -13,6 +13,11 @@
 * Out-of-line ABI mode: documented a restriction__ of ``ffi.dlopen()``
   when compared to the in-line mode.
 
+* ``ffi.gc()``: when called several times with equal pointers, it was
+  accidentally registering only the last destructor, or even none at
+  all depending on details.  (It was correctly registering all of them
+  only in PyPy, and only with the out-of-line FFIs.)
+
 .. __: cdef.html#dlopen-note
 
 

From noreply at buildbot.pypy.org  Sat Jun  6 21:02:37 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat,  6 Jun 2015 21:02:37 +0200 (CEST)
Subject: [pypy-commit] pypy default: import cffi/0a6cf359cc4a
Message-ID: <20150606190237.B80E01C033F@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r77931:a24d38a16e74
Date: 2015-06-06 21:02 +0200
http://bitbucket.org/pypy/pypy/changeset/a24d38a16e74/

Log:	import cffi/0a6cf359cc4a

diff --git a/lib_pypy/cffi/gc_weakref.py b/lib_pypy/cffi/gc_weakref.py
--- a/lib_pypy/cffi/gc_weakref.py
+++ b/lib_pypy/cffi/gc_weakref.py
@@ -2,18 +2,27 @@
 
 
 class GcWeakrefs(object):
-    # code copied and adapted from WeakKeyDictionary.
-
     def __init__(self, ffi):
         self.ffi = ffi
-        self.data = data = {}
-        def remove(k):
-            destructor, cdata = data.pop(k)
-            destructor(cdata)
-        self.remove = remove
+        self.data = []
+        self.freelist = None
 
     def build(self, cdata, destructor):
         # make a new cdata of the same type as the original one
         new_cdata = self.ffi.cast(self.ffi._backend.typeof(cdata), cdata)
-        self.data[ref(new_cdata, self.remove)] = destructor, cdata
+        #
+        def remove(key):
+            assert self.data[index] is key
+            self.data[index] = self.freelist
+            self.freelist = index
+            destructor(cdata)
+        #
+        key = ref(new_cdata, remove)
+        index = self.freelist
+        if index is None:
+            index = len(self.data)
+            self.data.append(key)
+        else:
+            self.freelist = self.data[index]
+            self.data[index] = key
         return new_cdata
diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py
--- a/lib_pypy/cffi/recompiler.py
+++ b/lib_pypy/cffi/recompiler.py
@@ -775,7 +775,8 @@
             try:
                 if ftype.is_integer_type() or fbitsize >= 0:
                     # accept all integers, but complain on float or double
-                    prnt('  (void)((p->%s) << 1);' % fname)
+                    prnt("  (void)((p->%s) << 1);  /* check that '%s.%s' is "
+                         "an integer */" % (fname, cname, fname))
                     continue
                 # only accept exactly the type declared, except that '[]'
                 # is interpreted as a '*' and so will match any array length.
@@ -949,7 +950,7 @@
             prnt('{')
             prnt('  int n = (%s) <= 0;' % (name,))
             prnt('  *o = (unsigned long long)((%s) << 0);'
-                 '  /* check that we get an integer */' % (name,))
+                 '  /* check that %s is an integer */' % (name, name))
             if check_value is not None:
                 if check_value > 0:
                     check_value = '%dU' % (check_value,)
@@ -1088,8 +1089,9 @@
         self.cffi_types[index] = CffiOp(OP_PRIMITIVE, prim_index)
 
     def _emit_bytecode_UnknownIntegerType(self, tp, index):
-        s = '_cffi_prim_int(sizeof(%s), (((%s)-1) << 0) <= 0)' % (
-            tp.name, tp.name)
+        s = ('_cffi_prim_int(sizeof(%s), (\n'
+             '           ((%s)-1) << 0 /* check that %s is an integer type */\n'
+             '         ) <= 0)' % (tp.name, tp.name, tp.name))
         self.cffi_types[index] = CffiOp(OP_PRIMITIVE, s)
 
     def _emit_bytecode_RawFunctionType(self, tp, index):
diff --git a/lib_pypy/cffi/setuptools_ext.py b/lib_pypy/cffi/setuptools_ext.py
--- a/lib_pypy/cffi/setuptools_ext.py
+++ b/lib_pypy/cffi/setuptools_ext.py
@@ -18,7 +18,9 @@
     # __init__.py files may already try to import the file that
     # we are generating.
     with open(filename) as f:
-        code = compile(f.read(), filename, 'exec')
+        src = f.read()
+    src += '\n'      # Python 2.6 compatibility
+    code = compile(src, filename, 'exec')
     exec(code, glob, glob)
 
 
diff --git a/lib_pypy/cffi/vengine_gen.py b/lib_pypy/cffi/vengine_gen.py
--- a/lib_pypy/cffi/vengine_gen.py
+++ b/lib_pypy/cffi/vengine_gen.py
@@ -402,12 +402,16 @@
         else:
             assert tp is not None
             assert check_value is None
-            prnt(tp.get_c_name(' %s(void)' % funcname, name),)
-            prnt('{')
             if category == 'var':
                 ampersand = '&'
             else:
                 ampersand = ''
+            extra = ''
+            if category == 'const' and isinstance(tp, model.StructOrUnion):
+                extra = 'const *'
+                ampersand = '&'
+            prnt(tp.get_c_name(' %s%s(void)' % (extra, funcname), name))
+            prnt('{')
             prnt('  return (%s%s);' % (ampersand, name))
             prnt('}')
         prnt()
@@ -436,9 +440,14 @@
                 value += (1 << (8*self.ffi.sizeof(BLongLong)))
         else:
             assert check_value is None
-            BFunc = self.ffi._typeof_locked(tp.get_c_name('(*)(void)', name))[0]
+            fntypeextra = '(*)(void)'
+            if isinstance(tp, model.StructOrUnion):
+                fntypeextra = '*' + fntypeextra
+            BFunc = self.ffi._typeof_locked(tp.get_c_name(fntypeextra, name))[0]
             function = module.load_function(BFunc, funcname)
             value = function()
+            if isinstance(tp, model.StructOrUnion):
+                value = value[0]
         return value
 
     def _loaded_gen_constant(self, tp, name, module, library):
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
@@ -1458,6 +1458,63 @@
         import gc; gc.collect(); gc.collect(); gc.collect()
         assert seen == [1]
 
+    def test_gc_2(self):
+        ffi = FFI(backend=self.Backend())
+        p = ffi.new("int *", 123)
+        seen = []
+        q1 = ffi.gc(p, lambda p: seen.append(1))
+        q2 = ffi.gc(q1, lambda p: seen.append(2))
+        import gc; gc.collect()
+        assert seen == []
+        del q1, q2
+        import gc; gc.collect(); gc.collect(); gc.collect(); gc.collect()
+        assert seen == [2, 1]
+
+    def test_gc_3(self):
+        ffi = FFI(backend=self.Backend())
+        p = ffi.new("int *", 123)
+        r = ffi.new("int *", 123)
+        seen = []
+        seen_r = []
+        q1 = ffi.gc(p, lambda p: seen.append(1))
+        s1 = ffi.gc(r, lambda r: seen_r.append(4))
+        q2 = ffi.gc(q1, lambda p: seen.append(2))
+        s2 = ffi.gc(s1, lambda r: seen_r.append(5))
+        q3 = ffi.gc(q2, lambda p: seen.append(3))
+        import gc; gc.collect()
+        assert seen == []
+        assert seen_r == []
+        del q1, q2, q3, s2, s1
+        import gc; gc.collect(); gc.collect(); gc.collect(); gc.collect()
+        assert seen == [3, 2, 1]
+        assert seen_r == [5, 4]
+
+    def test_gc_4(self):
+        ffi = FFI(backend=self.Backend())
+        p = ffi.new("int *", 123)
+        seen = []
+        q1 = ffi.gc(p, lambda p: seen.append(1))
+        q2 = ffi.gc(q1, lambda p: seen.append(2))
+        q3 = ffi.gc(q2, lambda p: seen.append(3))
+        import gc; gc.collect()
+        assert seen == []
+        del q1, q3     # q2 remains, and has a hard ref to q1
+        import gc; gc.collect(); gc.collect(); gc.collect()
+        assert seen == [3]
+
+    def test_gc_finite_list(self):
+        ffi = FFI(backend=self.Backend())
+        p = ffi.new("int *", 123)
+        keepalive = []
+        for i in range(10):
+            keepalive.append(ffi.gc(p, lambda p: None))
+            assert len(ffi.gc_weakrefs.data) == i + 1  #should be a private attr
+        del keepalive[:]
+        import gc; gc.collect(); gc.collect()
+        for i in range(10):
+            keepalive.append(ffi.gc(p, lambda p: None))
+        assert len(ffi.gc_weakrefs.data) == 10
+
     def test_CData_CType(self):
         ffi = FFI(backend=self.Backend())
         assert isinstance(ffi.cast("int", 0), ffi.CData)
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py
@@ -2228,3 +2228,11 @@
     ffi.cdef("static const int FOO = 123;")
     e = py.test.raises(VerificationError, ffi.verify, "#define FOO 124")
     assert str(e.value).endswith("FOO has the real value 124, not 123")
+
+def test_const_struct_global():
+    ffi = FFI()
+    ffi.cdef("typedef struct { int x; ...; } T; const T myglob;")
+    lib = ffi.verify("typedef struct { double y; int x; } T;"
+                     "const T myglob = { 0.1, 42 };")
+    assert ffi.typeof(lib.myglob) == ffi.typeof("T")
+    assert lib.myglob.x == 42
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_ffi_obj.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_ffi_obj.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_ffi_obj.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_ffi_obj.py
@@ -1,5 +1,5 @@
 # Generated by pypy/tool/import_cffi.py
-import py
+import py, sys
 import _cffi_backend as _cffi1_backend
 
 
@@ -66,6 +66,7 @@
     ffi = _cffi1_backend.FFI()
     p = ffi.new("char[]", init=b"foobar\x00baz")
     assert ffi.string(p) == b"foobar"
+    assert ffi.string(cdata=p, maxlen=3) == b"foo"
 
 def test_ffi_errno():
     # xxx not really checking errno, just checking that we can read/write it
@@ -158,11 +159,18 @@
     assert str(e.value) == ("undefined struct/union name\n"
                             "struct never_heard_of_s\n"
                             "       ^")
+    e = py.test.raises(ffi.error, ffi.cast, "\t\n\x01\x1f~\x7f\x80\xff", 0)
+    assert str(e.value) == ("identifier expected\n"
+                            "  ??~???\n"
+                            "  ^")
+    e = py.test.raises(ffi.error, ffi.cast, "X" * 600, 0)
+    assert str(e.value) == ("undefined type name")
 
 def test_ffi_buffer():
     ffi = _cffi1_backend.FFI()
     a = ffi.new("signed char[]", [5, 6, 7])
     assert ffi.buffer(a)[:] == b'\x05\x06\x07'
+    assert ffi.buffer(cdata=a, size=2)[:] == b'\x05\x06'
 
 def test_ffi_from_buffer():
     import array
@@ -179,3 +187,11 @@
     ffi = _cffi1_backend.FFI()
     assert isinstance(ffi.cast("int", 42), CData)
     assert isinstance(ffi.typeof("int"), CType)
+
+def test_ffi_getwinerror():
+    if sys.platform != "win32":
+        py.test.skip("for windows")
+    ffi = _cffi1_backend.FFI()
+    n = (1 << 29) + 42
+    code, message = ffi.getwinerror(code=n)
+    assert code == n
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py
@@ -33,7 +33,9 @@
         struct ab { int a, b; };
         struct abc { int a, b, c; };
 
-        enum foq { A0, B0, CC0, D0 };
+        /* don't use A0, B0, CC0, D0 because termios.h might be included
+           and it has its own #defines for these names */
+        enum foq { cffiA0, cffiB0, cffiCC0, cffiD0 };
         enum bar { A1, B1=-2, CC1, D1, E1 };
         enum baz { A2=0x1000, B2=0x2000 };
         enum foo2 { A3, B3, C3, D3 };
@@ -879,9 +881,9 @@
 
     def test_enum(self):
         # enum foq { A0, B0, CC0, D0 };
-        assert ffi.string(ffi.cast("enum foq", 0)) == "A0"
-        assert ffi.string(ffi.cast("enum foq", 2)) == "CC0"
-        assert ffi.string(ffi.cast("enum foq", 3)) == "D0"
+        assert ffi.string(ffi.cast("enum foq", 0)) == "cffiA0"
+        assert ffi.string(ffi.cast("enum foq", 2)) == "cffiCC0"
+        assert ffi.string(ffi.cast("enum foq", 3)) == "cffiD0"
         assert ffi.string(ffi.cast("enum foq", 4)) == "4"
         # enum bar { A1, B1=-2, CC1, D1, E1 };
         assert ffi.string(ffi.cast("enum bar", 0)) == "A1"
@@ -1408,6 +1410,47 @@
         import gc; gc.collect(); gc.collect(); gc.collect()
         assert seen == [1]
 
+    def test_gc_2(self):
+        p = ffi.new("int *", 123)
+        seen = []
+        q1 = ffi.gc(p, lambda p: seen.append(1))
+        q2 = ffi.gc(q1, lambda p: seen.append(2))
+        import gc; gc.collect()
+        assert seen == []
+        del q1, q2
+        import gc; gc.collect(); gc.collect(); gc.collect(); gc.collect()
+        assert seen == [2, 1]
+
+    def test_gc_3(self):
+        p = ffi.new("int *", 123)
+        r = ffi.new("int *", 123)
+        seen = []
+        seen_r = []
+        q1 = ffi.gc(p, lambda p: seen.append(1))
+        s1 = ffi.gc(r, lambda r: seen_r.append(4))
+        q2 = ffi.gc(q1, lambda p: seen.append(2))
+        s2 = ffi.gc(s1, lambda r: seen_r.append(5))
+        q3 = ffi.gc(q2, lambda p: seen.append(3))
+        import gc; gc.collect()
+        assert seen == []
+        assert seen_r == []
+        del q1, q2, q3, s2, s1
+        import gc; gc.collect(); gc.collect(); gc.collect(); gc.collect()
+        assert seen == [3, 2, 1]
+        assert seen_r == [5, 4]
+
+    def test_gc_4(self):
+        p = ffi.new("int *", 123)
+        seen = []
+        q1 = ffi.gc(p, lambda p: seen.append(1))
+        q2 = ffi.gc(q1, lambda p: seen.append(2))
+        q3 = ffi.gc(q2, lambda p: seen.append(3))
+        import gc; gc.collect()
+        assert seen == []
+        del q1, q3     # q2 remains, and has a hard ref to q1
+        import gc; gc.collect(); gc.collect(); gc.collect()
+        assert seen == [3]
+
     def test_CData_CType(self):
         assert isinstance(ffi.cast("int", 0), ffi.CData)
         assert isinstance(ffi.new("int *"), ffi.CData)
@@ -1534,8 +1577,8 @@
         assert p.a == -52525
         #
         p = ffi.cast("enum foq", 2)
-        assert ffi.string(p) == "CC0"
-        assert ffi2.sizeof("char[CC0]") == 2
+        assert ffi.string(p) == "cffiCC0"
+        assert ffi2.sizeof("char[cffiCC0]") == 2
         #
         p = ffi.new("anon_foo_t *", [-52526])
         assert p.a == -52526
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_re_python.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_re_python.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_re_python.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_re_python.py
@@ -8,6 +8,7 @@
 
 def setup_module(mod):
     SRC = """
+    #include <string.h>
     #define FOOBAR (-42)
     static const int FOOBAZ = -43;
     #define BIGPOS 420000000000L
@@ -54,6 +55,7 @@
     struct foo_s;
     typedef struct bar_s { int x; signed char a[]; } bar_t;
     enum foo_e { AA, BB, CC };
+    int strlen(const char *);
     """)
     ffi.set_source('re_python_pysrc', None)
     ffi.emit_python_code(str(tmpdir.join('re_python_pysrc.py')))
@@ -82,10 +84,20 @@
 def test_function_with_varargs():
     import _cffi_backend
     from re_python_pysrc import ffi
-    lib = ffi.dlopen(extmod)
+    lib = ffi.dlopen(extmod, 0)
     assert lib.add43(45, ffi.cast("int", -5)) == 45
     assert type(lib.add43) is _cffi_backend.FFI.CData
 
+def test_dlopen_none():
+    import _cffi_backend
+    from re_python_pysrc import ffi
+    name = None
+    if sys.platform == 'win32':
+        import ctypes.util
+        name = ctypes.util.find_msvcrt()
+    lib = ffi.dlopen(name)
+    assert lib.strlen(b"hello") == 5
+
 def test_dlclose():
     import _cffi_backend
     from re_python_pysrc import ffi
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
@@ -993,3 +993,13 @@
     ffi.typeof('function_t*')
     lib.function(ffi.NULL)
     # assert did not crash
+
+def test_alignment_of_longlong():
+    ffi = FFI()
+    x1 = ffi.alignof('unsigned long long')
+    assert x1 in [4, 8]
+    ffi.cdef("struct foo_s { unsigned long long x; };")
+    lib = verify(ffi, 'test_alignment_of_longlong',
+                 "struct foo_s { unsigned long long x; };")
+    assert ffi.alignof('unsigned long long') == x1
+    assert ffi.alignof('struct foo_s') == x1
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py
@@ -2118,25 +2118,19 @@
     try:
         ffi1 = FFI()
         ffi1.cdef("int foo_verify_dlopen_flags;")
-
-        sys.setdlopenflags(ffi1.RTLD_GLOBAL | ffi1.RTLD_LAZY)
+        sys.setdlopenflags(ffi1.RTLD_GLOBAL | ffi1.RTLD_NOW)
         lib1 = ffi1.verify("int foo_verify_dlopen_flags;")
-        lib2 = get_second_lib()
-
-        lib1.foo_verify_dlopen_flags = 42
-        assert lib2.foo_verify_dlopen_flags == 42
-        lib2.foo_verify_dlopen_flags += 1
-        assert lib1.foo_verify_dlopen_flags == 43
     finally:
         sys.setdlopenflags(old)
 
-def get_second_lib():
-    # Hack, using modulename makes the test fail
     ffi2 = FFI()
-    ffi2.cdef("int foo_verify_dlopen_flags;")
-    lib2 = ffi2.verify("int foo_verify_dlopen_flags;",
-                       flags=ffi2.RTLD_GLOBAL | ffi2.RTLD_LAZY)
-    return lib2
+    ffi2.cdef("int *getptr(void);")
+    lib2 = ffi2.verify("""
+        extern int foo_verify_dlopen_flags;
+        static int *getptr(void) { return &foo_verify_dlopen_flags; }
+    """)
+    p = lib2.getptr()
+    assert ffi1.addressof(lib1, 'foo_verify_dlopen_flags') == p
 
 def test_consider_not_implemented_function_type():
     ffi = FFI()
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_zdist.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_zdist.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_zdist.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_zdist.py
@@ -30,13 +30,17 @@
         if hasattr(self, 'saved_cwd'):
             os.chdir(self.saved_cwd)
 
-    def run(self, args):
+    def run(self, args, cwd=None):
         env = os.environ.copy()
-        newpath = self.rootdir
-        if 'PYTHONPATH' in env:
-            newpath += os.pathsep + env['PYTHONPATH']
-        env['PYTHONPATH'] = newpath
-        subprocess.check_call([self.executable] + args, env=env)
+        # a horrible hack to prevent distutils from finding ~/.pydistutils.cfg
+        # (there is the --no-user-cfg option, but not in Python 2.6...)
+        env['HOME'] = '/this/path/does/not/exist'
+        if cwd is None:
+            newpath = self.rootdir
+            if 'PYTHONPATH' in env:
+                newpath += os.pathsep + env['PYTHONPATH']
+            env['PYTHONPATH'] = newpath
+        subprocess.check_call([self.executable] + args, cwd=cwd, env=env)
 
     def _prepare_setuptools(self):
         if hasattr(TestDist, '_setuptools_ready'):
@@ -45,8 +49,7 @@
             import setuptools
         except ImportError:
             py.test.skip("setuptools not found")
-        subprocess.check_call([self.executable, 'setup.py', 'egg_info'],
-                              cwd=self.rootdir)
+        self.run(['setup.py', 'egg_info'], cwd=self.rootdir)
         TestDist._setuptools_ready = True
 
     def check_produced_files(self, content, curdir=None):

From noreply at buildbot.pypy.org  Sun Jun  7 03:38:28 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Sun,  7 Jun 2015 03:38:28 +0200 (CEST)
Subject: [pypy-commit] pypy default: Fix translation and allow order='K'
Message-ID: <20150607013828.115CC1C0EF3@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r77932:75e2c1d303eb
Date: 2015-06-07 02:36 +0100
http://bitbucket.org/pypy/pypy/changeset/75e2c1d303eb/

Log:	Fix translation and allow order='K'

diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py
--- a/pypy/module/micronumpy/concrete.py
+++ b/pypy/module/micronumpy/concrete.py
@@ -345,10 +345,12 @@
                 if s < mins:
                     mins = s
             t_strides = [s * t_elsize / mins for s in strides]
-            if order not in ('C', 'F'):
+            if order == 'K':
+                pass
+            elif order not in ('C', 'F'):
                 raise oefmt(space.w_ValueError, "Unknown order %s in astype", order)
-            if order != self.order:
-                t_strides = tstrides[::-1]
+            elif order != self.order:
+                t_strides.reverse()
             backstrides = calc_backstrides(t_strides, shape)
         else:
             t_strides = []
@@ -379,7 +381,7 @@
         gc._trace_callback(callback, arg, storage)
         storage += step
         i += 1
-    
+
 lambda_customtrace = lambda: customtrace
 
 def _setup():
@@ -403,8 +405,6 @@
         make_sure_not_resized(backstrides)
         self.shape = shape
         self.size = support.product(shape) * dtype.elsize
-        if order not in ('C', 'F'):
-            raise oefmt(space.w_ValueError, "Unknown order %s in astype", order)
         self.order = order
         self.dtype = dtype
         self.strides = strides
@@ -445,7 +445,7 @@
         gcstruct = V_OBJECTSTORE
         flags = NPY.ARRAY_ALIGNED | NPY.ARRAY_WRITEABLE
         if storage == lltype.nullptr(RAW_STORAGE):
-            length = support.product(shape) 
+            length = support.product(shape)
             if dtype.num == NPY.OBJECT:
                 storage = dtype.itemtype.malloc(length * dtype.elsize, zero=True)
                 gcstruct = _create_objectstore(storage, length, dtype.elsize)
@@ -507,7 +507,7 @@
         ConcreteArray.__init__(self, shape, dtype, order, strides, backstrides,
                     storage, zero)
         self.flags &= ~ NPY.ARRAY_WRITEABLE
-        
+
     def descr_setitem(self, space, orig_array, w_index, w_value):
         raise OperationError(space.w_ValueError, space.wrap(
             "assignment destination is read-only"))

From noreply at buildbot.pypy.org  Sun Jun  7 06:08:07 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Sun,  7 Jun 2015 06:08:07 +0200 (CEST)
Subject: [pypy-commit] pypy default: Create W_Dtype.store()
Message-ID: <20150607040807.BE96B1C0F05@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r77933:dd0100ab58a2
Date: 2015-06-07 03:32 +0100
http://bitbucket.org/pypy/pypy/changeset/dd0100ab58a2/

Log:	Create W_Dtype.store()

diff --git a/pypy/module/micronumpy/boxes.py b/pypy/module/micronumpy/boxes.py
--- a/pypy/module/micronumpy/boxes.py
+++ b/pypy/module/micronumpy/boxes.py
@@ -593,7 +593,7 @@
             raise oefmt(space.w_IndexError, "222only integers, slices (`:`), "
                 "ellipsis (`...`), numpy.newaxis (`None`) and integer or "
                 "boolean arrays are valid indices")
-        dtype.itemtype.store(self.arr, self.ofs, ofs,
+        dtype.store(self.arr, self.ofs, ofs,
                              dtype.coerce(space, w_value))
 
     def convert_to(self, space, dtype):
diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py
--- a/pypy/module/micronumpy/concrete.py
+++ b/pypy/module/micronumpy/concrete.py
@@ -49,7 +49,7 @@
         return self.dtype.itemtype.read_bool(self, index, 0)
 
     def setitem(self, index, value):
-        self.dtype.itemtype.store(self, index, 0, value)
+        self.dtype.store(self, index, 0, value)
 
     @jit.unroll_safe
     def setslice(self, space, arr):
diff --git a/pypy/module/micronumpy/descriptor.py b/pypy/module/micronumpy/descriptor.py
--- a/pypy/module/micronumpy/descriptor.py
+++ b/pypy/module/micronumpy/descriptor.py
@@ -402,6 +402,9 @@
             return self.coerce(space, space.wrap(s))
         return self.itemtype.runpack_str(space, s)
 
+    def store(self, arr, i, offset, value):
+        return self.itemtype.store(arr, i, offset, value)
+
     def descr_reduce(self, space):
         w_class = space.type(self)
         builder_args = space.newtuple([
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -2264,8 +2264,8 @@
         itemtype = subdtype.itemtype
         if len(shape) <= 1:
             for i in range(len(items_w)):
-                w_box = itemtype.coerce(space, subdtype, items_w[i])
-                itemtype.store(arr, 0, ofs, w_box)
+                w_box = subdtype.coerce(space, items_w[i])
+                subdtype.store(arr, 0, ofs, w_box)
                 ofs += itemtype.get_element_size()
         else:
             for w_item in items_w:
@@ -2381,12 +2381,11 @@
         arr = VoidBoxStorage(dtype.elsize, dtype)
         for i in range(len(dtype.fields)):
             ofs, subdtype = dtype.fields[dtype.names[i]]
-            itemtype = subdtype.itemtype
             try:
-                w_box = itemtype.coerce(space, subdtype, items_w[i])
+                w_box = subdtype.coerce(space, items_w[i])
             except IndexError:
-                w_box = itemtype.coerce(space, subdtype, None)
-            itemtype.store(arr, 0, ofs, w_box)
+                w_box = subdtype.coerce(space, None)
+            subdtype.store(arr, 0, ofs, w_box)
         return boxes.W_VoidBox(arr, 0, dtype)
 
     def runpack_str(self, space, s):

From noreply at buildbot.pypy.org  Sun Jun  7 06:08:08 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Sun,  7 Jun 2015 06:08:08 +0200 (CEST)
Subject: [pypy-commit] pypy default: Create W_Dtype.read()
Message-ID: <20150607040808.F04FE1C0F05@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r77934:397c54d1f22a
Date: 2015-06-07 04:19 +0100
http://bitbucket.org/pypy/pypy/changeset/397c54d1f22a/

Log:	Create W_Dtype.read()

diff --git a/pypy/module/micronumpy/boxes.py b/pypy/module/micronumpy/boxes.py
--- a/pypy/module/micronumpy/boxes.py
+++ b/pypy/module/micronumpy/boxes.py
@@ -573,7 +573,7 @@
         if isinstance(dtype.itemtype, VoidType):
             read_val = dtype.itemtype.readarray(self.arr, self.ofs, ofs, dtype)
         else:
-            read_val = dtype.itemtype.read(self.arr, self.ofs, ofs, dtype)
+            read_val = dtype.read(self.arr, self.ofs, ofs)
         if isinstance (read_val, W_StringBox):
             # StringType returns a str
             return space.wrap(dtype.itemtype.to_str(read_val))
diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py
--- a/pypy/module/micronumpy/concrete.py
+++ b/pypy/module/micronumpy/concrete.py
@@ -43,7 +43,7 @@
         return backstrides
 
     def getitem(self, index):
-        return self.dtype.itemtype.read(self, index, 0)
+        return self.dtype.read(self, index, 0)
 
     def getitem_bool(self, index):
         return self.dtype.itemtype.read_bool(self, index, 0)
diff --git a/pypy/module/micronumpy/descriptor.py b/pypy/module/micronumpy/descriptor.py
--- a/pypy/module/micronumpy/descriptor.py
+++ b/pypy/module/micronumpy/descriptor.py
@@ -405,6 +405,9 @@
     def store(self, arr, i, offset, value):
         return self.itemtype.store(arr, i, offset, value)
 
+    def read(self, arr, i, offset):
+        return self.itemtype.read(arr, i, offset, self)
+
     def descr_reduce(self, space):
         w_class = space.type(self)
         builder_args = space.newtuple([
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -2322,10 +2322,11 @@
         ret_unwrapped = []
         for name in dt.names:
             ofs, dtype = dt.fields[name]
+            # XXX: code duplication with W_VoidBox.descr_getitem()
             if isinstance(dtype.itemtype, VoidType):
                 read_val = dtype.itemtype.readarray(item.arr, ofs, 0, dtype)
             else:
-                read_val = dtype.itemtype.read(item.arr, ofs, 0, dtype)
+                read_val = dtype.read(item.arr, ofs, 0)
             if isinstance (read_val, boxes.W_StringBox):
                 # StringType returns a str
                 read_val = space.wrap(dtype.itemtype.to_str(read_val))
@@ -2419,9 +2420,8 @@
         dtype = box.dtype
         for name in dtype.names:
             ofs, subdtype = dtype.fields[name]
-            itemtype = subdtype.itemtype
-            subbox = itemtype.read(box.arr, box.ofs, ofs, subdtype)
-            items.append(itemtype.to_builtin_type(space, subbox))
+            subbox = subdtype.read(box.arr, box.ofs, ofs)
+            items.append(subdtype.itemtype.to_builtin_type(space, subbox))
         return space.newtuple(items)
 
     @jit.unroll_safe
@@ -2431,12 +2431,12 @@
         first = True
         for name in box.dtype.names:
             ofs, subdtype = box.dtype.fields[name]
-            tp = subdtype.itemtype
             if first:
                 first = False
             else:
                 pieces.append(", ")
-            val = tp.read(box.arr, box.ofs, ofs, subdtype)
+            val = subdtype.read(box.arr, box.ofs, ofs)
+            tp = subdtype.itemtype
             pieces.append(tp.str_format(val, add_quotes=add_quotes))
         pieces.append(")")
         return "".join(pieces)

From noreply at buildbot.pypy.org  Sun Jun  7 06:08:10 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Sun,  7 Jun 2015 06:08:10 +0200 (CEST)
Subject: [pypy-commit] pypy default: make dtype argument to itemtype.read()
	mandatory
Message-ID: <20150607040810.211341C0F05@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r77935:5f4b0bd8073e
Date: 2015-06-07 04:32 +0100
http://bitbucket.org/pypy/pypy/changeset/5f4b0bd8073e/

Log:	make dtype argument to itemtype.read() mandatory

diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -210,7 +210,7 @@
             value = byteswap(value)
         raw_storage_setitem_unaligned(storage, i + offset, value)
 
-    def read(self, arr, i, offset, dtype=None):
+    def read(self, arr, i, offset, dtype):
         with arr as storage:
             return self.box(self._read(storage, i, offset))
 
@@ -1227,7 +1227,7 @@
             imag = byteswap(imag)
         return real, imag
 
-    def read(self, arr, i, offset, dtype=None):
+    def read(self, arr, i, offset, dtype):
         with arr as storage:
             real, imag = self._read(storage, i, offset)
             return self.box_complex(real, imag)
@@ -1795,7 +1795,7 @@
         self._write(arr.storage, i, offset, self.unbox(box),
                     arr.gcstruct)
 
-    def read(self, arr, i, offset, dtype=None):
+    def read(self, arr, i, offset, dtype):
         return self.box(self._read(arr.storage, i, offset))
 
     def byteswap(self, w_v):
@@ -2106,9 +2106,7 @@
             for k in range(size):
                 storage[k + offset + i] = box_storage[k + box.ofs]
 
-    def read(self, arr, i, offset, dtype=None):
-        if dtype is None:
-            dtype = arr.dtype
+    def read(self, arr, i, offset, dtype):
         return boxes.W_StringBox(arr, i + offset, dtype)
 
     def str_format(self, item, add_quotes=True):
@@ -2197,7 +2195,7 @@
         assert isinstance(box, boxes.W_UnicodeBox)
         raise oefmt(self.space.w_NotImplementedError, "unicode type not completed")
 
-    def read(self, arr, i, offset, dtype=None):
+    def read(self, arr, i, offset, dtype):
         raise oefmt(self.space.w_NotImplementedError, "unicode type not completed")
 
     def str_format(self, item, add_quotes=True):
@@ -2301,9 +2299,7 @@
                                     dtype.subdtype)
         return W_NDimArray(implementation)
 
-    def read(self, arr, i, offset, dtype=None):
-        if dtype is None:
-            dtype = arr.dtype
+    def read(self, arr, i, offset, dtype):
         return boxes.W_VoidBox(arr, i + offset, dtype)
 
     @jit.unroll_safe
@@ -2345,9 +2341,7 @@
     kind = NPY.VOIDLTR
     char = NPY.VOIDLTR
 
-    def read(self, arr, i, offset, dtype=None):
-        if dtype is None:
-            dtype = arr.dtype
+    def read(self, arr, i, offset, dtype):
         return boxes.W_VoidBox(arr, i + offset, dtype)
 
     @jit.unroll_safe

From noreply at buildbot.pypy.org  Sun Jun  7 06:08:11 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Sun,  7 Jun 2015 06:08:11 +0200 (CEST)
Subject: [pypy-commit] pypy default: Create W_Dtype.read_bool()
Message-ID: <20150607040811.3CBBA1C0F05@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r77936:bcb81b2d92aa
Date: 2015-06-07 04:53 +0100
http://bitbucket.org/pypy/pypy/changeset/bcb81b2d92aa/

Log:	Create W_Dtype.read_bool()

diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py
--- a/pypy/module/micronumpy/concrete.py
+++ b/pypy/module/micronumpy/concrete.py
@@ -46,7 +46,7 @@
         return self.dtype.read(self, index, 0)
 
     def getitem_bool(self, index):
-        return self.dtype.itemtype.read_bool(self, index, 0)
+        return self.dtype.read_bool(self, index, 0)
 
     def setitem(self, index, value):
         self.dtype.store(self, index, 0, value)
diff --git a/pypy/module/micronumpy/descriptor.py b/pypy/module/micronumpy/descriptor.py
--- a/pypy/module/micronumpy/descriptor.py
+++ b/pypy/module/micronumpy/descriptor.py
@@ -408,6 +408,9 @@
     def read(self, arr, i, offset):
         return self.itemtype.read(arr, i, offset, self)
 
+    def read_bool(self, arr, i, offset):
+        return self.itemtype.read_bool(arr, i, offset, self)
+
     def descr_reduce(self, space):
         w_class = space.type(self)
         builder_args = space.newtuple([

From noreply at buildbot.pypy.org  Sun Jun  7 06:08:12 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Sun,  7 Jun 2015 06:08:12 +0200 (CEST)
Subject: [pypy-commit] pypy default: Remove 'native' attribute from itemtypes
Message-ID: <20150607040812.5CFDC1C0F05@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r77937:36a1899115e0
Date: 2015-06-07 05:08 +0100
http://bitbucket.org/pypy/pypy/changeset/36a1899115e0/

Log:	Remove 'native' attribute from itemtypes

diff --git a/pypy/module/micronumpy/boxes.py b/pypy/module/micronumpy/boxes.py
--- a/pypy/module/micronumpy/boxes.py
+++ b/pypy/module/micronumpy/boxes.py
@@ -67,7 +67,8 @@
         assert isinstance(multiarray, MixedModule)
         scalar = multiarray.get("scalar")
 
-        ret = space.newtuple([scalar, space.newtuple([space.wrap(self._get_dtype(space)), space.wrap(self.raw_str())])])
+        ret = space.newtuple([scalar, space.newtuple(
+            [space.wrap(self._get_dtype(space)), space.wrap(self.raw_str())])])
         return ret
 
 
diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py
--- a/pypy/module/micronumpy/concrete.py
+++ b/pypy/module/micronumpy/concrete.py
@@ -414,8 +414,9 @@
         self.gcstruct = V_OBJECTSTORE
 
     def fill(self, space, box):
-        self.dtype.itemtype.fill(self.storage, self.dtype.elsize,
-                                 box, 0, self.size, 0, self.gcstruct)
+        self.dtype.itemtype.fill(
+            self.storage, self.dtype.elsize, self.dtype.is_native(),
+            box, 0, self.size, 0, self.gcstruct)
 
     def set_shape(self, space, orig_array, new_shape):
         strides, backstrides = calc_strides(new_shape, self.dtype,
diff --git a/pypy/module/micronumpy/descriptor.py b/pypy/module/micronumpy/descriptor.py
--- a/pypy/module/micronumpy/descriptor.py
+++ b/pypy/module/micronumpy/descriptor.py
@@ -400,10 +400,10 @@
     def runpack_str(self, space, s):
         if self.is_str_or_unicode():
             return self.coerce(space, space.wrap(s))
-        return self.itemtype.runpack_str(space, s)
+        return self.itemtype.runpack_str(space, s, self.is_native())
 
     def store(self, arr, i, offset, value):
-        return self.itemtype.store(arr, i, offset, value)
+        return self.itemtype.store(arr, i, offset, value, self.is_native())
 
     def read(self, arr, i, offset):
         return self.itemtype.read(arr, i, offset, self)
@@ -506,11 +506,10 @@
                 endian = NPY.OPPBYTE if self.is_native() else NPY.NATBYTE
             elif newendian != NPY.IGNORE:
                 endian = newendian
-        itemtype = self.itemtype.__class__(space, endian in (NPY.NATIVE, NPY.NATBYTE))
         fields = self.fields
         if fields is None:
             fields = {}
-        return W_Dtype(itemtype,
+        return W_Dtype(self.itemtype,
                        self.w_box_type, byteorder=endian, elsize=self.elsize,
                        names=self.names, fields=fields,
                        shape=self.shape, subdtype=self.subdtype)
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -133,12 +133,11 @@
     return dispatcher
 
 class BaseType(object):
-    _immutable_fields_ = ['native', 'space']
+    _immutable_fields_ = ['space']
     strlen = 0  # chars needed to print any possible value of the type
 
-    def __init__(self, space, native=True):
+    def __init__(self, space):
         assert isinstance(space, ObjSpace)
-        self.native = native
         self.space = space
 
     def __repr__(self):
@@ -199,37 +198,38 @@
     def default_fromstring(self, space):
         raise NotImplementedError
 
-    def _read(self, storage, i, offset):
+    def _read(self, storage, i, offset, native):
         res = raw_storage_getitem_unaligned(self.T, storage, i + offset)
-        if not self.native:
+        if not native:
             res = byteswap(res)
         return res
 
-    def _write(self, storage, i, offset, value):
-        if not self.native:
+    def _write(self, storage, i, offset, value, native):
+        if not native:
             value = byteswap(value)
         raw_storage_setitem_unaligned(storage, i + offset, value)
 
     def read(self, arr, i, offset, dtype):
         with arr as storage:
-            return self.box(self._read(storage, i, offset))
-
-    def read_bool(self, arr, i, offset):
+            return self.box(self._read(storage, i, offset, dtype.is_native()))
+
+    def read_bool(self, arr, i, offset, dtype):
         with arr as storage:
-            return bool(self.for_computation(self._read(storage, i, offset)))
-
-    def store(self, arr, i, offset, box):
+            return bool(self.for_computation(
+                self._read(storage, i, offset, dtype.is_native())))
+
+    def store(self, arr, i, offset, box, native):
         with arr as storage:
-            self._write(storage, i, offset, self.unbox(box))
-
-    def fill(self, storage, width, box, start, stop, offset, gcstruct):
+            self._write(storage, i, offset, self.unbox(box), native)
+
+    def fill(self, storage, width, native, box, start, stop, offset, gcstruct):
         value = self.unbox(box)
         for i in xrange(start, stop, width):
-            self._write(storage, i, offset, value)
-
-    def runpack_str(self, space, s):
+            self._write(storage, i, offset, value, native)
+
+    def runpack_str(self, space, s, native):
         v = rffi.cast(self.T, runpack(self.format_code, s))
-        if not self.native:
+        if not native:
             v = byteswap(v)
         return self.box(v)
 
@@ -1058,10 +1058,10 @@
     def box(self, value):
         return self.BoxType(rffi.cast(rffi.DOUBLE, value))
 
-    def runpack_str(self, space, s):
+    def runpack_str(self, space, s, native):
         assert len(s) == 2
         fval = self.box(unpack_float(s, native_is_bigendian))
-        if not self.native:
+        if not native:
             fval = self.byteswap(fval)
         return fval
 
@@ -1074,19 +1074,19 @@
         swapped = byteswap(rffi.cast(self._STORAGE_T, hbits))
         return self.box(float_unpack(r_ulonglong(swapped), 2))
 
-    def _read(self, storage, i, offset):
+    def _read(self, storage, i, offset, native):
         hbits = raw_storage_getitem_unaligned(self._STORAGE_T, storage, i + offset)
-        if not self.native:
+        if not native:
             hbits = byteswap(hbits)
         return float_unpack(r_ulonglong(hbits), 2)
 
-    def _write(self, storage, i, offset, value):
+    def _write(self, storage, i, offset, value, native):
         try:
             hbits = float_pack(value, 2)
         except OverflowError:
             hbits = float_pack(rfloat.INFINITY, 2)
         hbits = rffi.cast(self._STORAGE_T, hbits)
-        if not self.native:
+        if not native:
             hbits = byteswap(hbits)
         raw_storage_setitem_unaligned(storage, i + offset, hbits)
 
@@ -1148,12 +1148,12 @@
         op = '+' if imag >= 0 or rfloat.isnan(imag) else ''
         return ''.join(['(', real_str, op, imag_str, ')'])
 
-    def runpack_str(self, space, s):
+    def runpack_str(self, space, s, native):
         comp = self.ComponentBoxType._get_dtype(space)
         l = len(s) // 2
         real = comp.runpack_str(space, s[:l])
         imag = comp.runpack_str(space, s[l:])
-        if not self.native:
+        if not native:
             real = comp.itemtype.byteswap(real)
             imag = comp.itemtype.byteswap(imag)
         return self.composite(real, imag)
@@ -1174,9 +1174,10 @@
         real, imag = self.for_computation(self.unbox(v))
         return bool(real) or bool(imag)
 
-    def read_bool(self, arr, i, offset):
+    def read_bool(self, arr, i, offset, dtype):
         with arr as storage:
-            v = self.for_computation(self._read(storage, i, offset))
+            v = self.for_computation(
+                self._read(storage, i, offset, dtype.is_native()))
             return bool(v[0]) or bool(v[1])
 
     def get_element_size(self):
@@ -1219,35 +1220,35 @@
         assert isinstance(box, self.BoxType)
         return box.real, box.imag
 
-    def _read(self, storage, i, offset):
+    def _read(self, storage, i, offset, native):
         real = raw_storage_getitem_unaligned(self.T, storage, i + offset)
         imag = raw_storage_getitem_unaligned(self.T, storage, i + offset + rffi.sizeof(self.T))
-        if not self.native:
+        if not native:
             real = byteswap(real)
             imag = byteswap(imag)
         return real, imag
 
     def read(self, arr, i, offset, dtype):
         with arr as storage:
-            real, imag = self._read(storage, i, offset)
+            real, imag = self._read(storage, i, offset, dtype.is_native())
             return self.box_complex(real, imag)
 
-    def _write(self, storage, i, offset, value):
+    def _write(self, storage, i, offset, value, native):
         real, imag = value
-        if not self.native:
+        if not native:
             real = byteswap(real)
             imag = byteswap(imag)
         raw_storage_setitem_unaligned(storage, i + offset, real)
         raw_storage_setitem_unaligned(storage, i + offset + rffi.sizeof(self.T), imag)
 
-    def store(self, arr, i, offset, box):
+    def store(self, arr, i, offset, box, native):
         with arr as storage:
-            self._write(storage, i, offset, self.unbox(box))
-
-    def fill(self, storage, width, box, start, stop, offset, gcstruct):
+            self._write(storage, i, offset, self.unbox(box), native)
+
+    def fill(self, storage, width, native, box, start, stop, offset, gcstruct):
         value = self.unbox(box)
         for i in xrange(start, stop, width):
-            self._write(storage, i, offset, value)
+            self._write(storage, i, offset, value, native)
 
     @complex_binary_op
     def add(self, v1, v2):
@@ -1745,10 +1746,10 @@
         char = NPY.LONGDOUBLELTR
         BoxType = boxes.W_FloatLongBox
 
-        def runpack_str(self, space, s):
+        def runpack_str(self, space, s, native):
             assert len(s) == boxes.long_double_size
             fval = self.box(unpack_float80(s, native_is_bigendian))
-            if not self.native:
+            if not native:
                 fval = self.byteswap(fval)
             return fval
 
@@ -1788,7 +1789,7 @@
         # return the item itself
         return self.unbox(self.box(w_item))
 
-    def store(self, arr, i, offset, box):
+    def store(self, arr, i, offset, box, native):
         if arr.gcstruct is V_OBJECTSTORE:
             raise oefmt(self.space.w_NotImplementedError,
                 "cannot store object in array with no gc hook")
@@ -1814,7 +1815,7 @@
         raw_storage_setitem_unaligned(storage, i + offset, value)
 
     @jit.dont_look_inside
-    def _read(self, storage, i, offset):
+    def _read(self, storage, i, offset, native=True):
         res = raw_storage_getitem_unaligned(self.T, storage, i + offset)
         if we_are_translated():
             gcref = rffi.cast(llmemory.GCREF, res)
@@ -1823,7 +1824,7 @@
             w_obj = _all_objs_for_tests[res]
         return w_obj
 
-    def fill(self, storage, width, box, start, stop, offset, gcstruct):
+    def fill(self, storage, width, native, box, start, stop, offset, gcstruct):
         value = self.unbox(box)
         for i in xrange(start, stop, width):
             self._write(storage, i, offset, value, gcstruct)
@@ -1866,7 +1867,7 @@
     def str_format(self, box, add_quotes=True):
         return self.space.str_w(self.space.repr(self.unbox(box)))
 
-    def runpack_str(self, space, s):
+    def runpack_str(self, space, s, native):
         raise oefmt(space.w_NotImplementedError,
                     "fromstring not implemented for object type")
 
@@ -2093,7 +2094,7 @@
                 storage[j] = '\x00'
             return boxes.W_StringBox(arr,  0, arr.dtype)
 
-    def store(self, arr, i, offset, box):
+    def store(self, arr, i, offset, box, native):
         assert isinstance(box, boxes.W_StringBox)
         size = min(arr.dtype.elsize - offset, box.arr.size - box.ofs)
         with arr as storage:
@@ -2171,7 +2172,7 @@
     def bool(self, v):
         return bool(self.to_str(v))
 
-    def fill(self, storage, width, box, start, stop, offset, gcstruct):
+    def fill(self, storage, width, native, box, start, stop, offset, gcstruct):
         for i in xrange(start, stop, width):
             self._store(storage, i, offset, box, width)
 
@@ -2191,7 +2192,7 @@
         raise OperationError(space.w_NotImplementedError, space.wrap(
             "coerce (probably from set_item) not implemented for unicode type"))
 
-    def store(self, arr, i, offset, box):
+    def store(self, arr, i, offset, box, native):
         assert isinstance(box, boxes.W_UnicodeBox)
         raise oefmt(self.space.w_NotImplementedError, "unicode type not completed")
 
@@ -2238,7 +2239,7 @@
     def bool(self, v):
         raise oefmt(self.space.w_NotImplementedError, "unicode type not completed")
 
-    def fill(self, storage, width, box, start, stop, offset, gcstruct):
+    def fill(self, storage, width, native, box, start, stop, offset, gcstruct):
         raise oefmt(self.space.w_NotImplementedError, "unicode type not completed")
 
 
@@ -2280,13 +2281,13 @@
         return boxes.W_VoidBox(arr, 0, dtype)
 
     @jit.unroll_safe
-    def store(self, arr, i, ofs, box):
+    def store(self, arr, i, offset, box, native):
         assert i == 0
         assert isinstance(box, boxes.W_VoidBox)
         assert box.dtype is box.arr.dtype
         with arr as arr_storage, box.arr as box_storage:
             for k in range(box.arr.dtype.elsize):
-                arr_storage[k + ofs] = box_storage[k + box.ofs]
+                arr_storage[k + offset] = box_storage[k + box.ofs]
 
     def readarray(self, arr, i, offset, dtype=None):
         from pypy.module.micronumpy.base import W_NDimArray
@@ -2383,14 +2384,14 @@
             subdtype.store(arr, 0, ofs, w_box)
         return boxes.W_VoidBox(arr, 0, dtype)
 
-    def runpack_str(self, space, s):
+    def runpack_str(self, space, s, native):
         raise oefmt(space.w_NotImplementedError,
                     "fromstring not implemented for record types")
 
-    def store(self, arr, i, ofs, box):
+    def store(self, arr, i, offset, box, native):
         assert isinstance(box, boxes.W_VoidBox)
         with arr as storage:
-            self._store(storage, i, ofs, box, box.dtype.elsize)
+            self._store(storage, i, offset, box, box.dtype.elsize)
 
     @jit.unroll_safe
     def _store(self, storage, i, ofs, box, size):
@@ -2398,7 +2399,7 @@
             for k in range(size):
                 storage[k + i + ofs] = box_storage[k + box.ofs]
 
-    def fill(self, storage, width, box, start, stop, offset, gcstruct):
+    def fill(self, storage, width, native, box, start, stop, offset, gcstruct):
         assert isinstance(box, boxes.W_VoidBox)
         assert width == box.dtype.elsize
         for i in xrange(start, stop, width):

From noreply at buildbot.pypy.org  Sun Jun  7 11:51:43 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sun,  7 Jun 2015 11:51:43 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Remove 'associated_segment_num' and
	keep only
Message-ID: <20150607095143.927631C026B@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r1792:939e4d807eb3
Date: 2015-06-07 11:52 +0200
http://bitbucket.org/pypy/stmgc/changeset/939e4d807eb3/

Log:	Remove 'associated_segment_num' and keep only
	'last_associated_segment_num'.

diff --git a/c8/stm/core.h b/c8/stm/core.h
--- a/c8/stm/core.h
+++ b/c8/stm/core.h
@@ -170,6 +170,10 @@
     TS_INEVITABLE,
 };
 
+#define in_transaction(tl)                                              \
+    (get_segment((tl)->last_associated_segment_num)->running_thread == (tl))
+
+
 /* Commit Log things */
 struct stm_undo_s {
   union {
diff --git a/c8/stm/extra.c b/c8/stm/extra.c
--- a/c8/stm/extra.c
+++ b/c8/stm/extra.c
@@ -8,7 +8,7 @@
 {
     dprintf(("register_callbacks: tl=%p key=%p callback=%p index=%ld\n",
              tl, key, callback, index));
-    if (tl->associated_segment_num == -1) {
+    if (!in_transaction(tl)) {
         /* check that the provided thread-local is really running a
            transaction, and do nothing otherwise. */
         dprintf(("  NOT IN TRANSACTION\n"));
diff --git a/c8/stm/forksupport.c b/c8/stm/forksupport.c
--- a/c8/stm/forksupport.c
+++ b/c8/stm/forksupport.c
@@ -41,6 +41,7 @@
     bool was_in_transaction = _stm_in_transaction(this_tl);
     if (!was_in_transaction)
         stm_start_transaction(this_tl);
+    assert(in_transaction(this_tl));
 
     stm_become_inevitable(this_tl, "fork");
     /* Note that the line above can still fail and abort, which should
@@ -83,7 +84,8 @@
     struct stm_priv_segment_info_s *pr = get_priv_segment(i);
     stm_thread_local_t *tl = pr->pub.running_thread;
     dprintf(("forksupport_child: abort in seg%ld\n", i));
-    assert(tl->associated_segment_num == i);
+    assert(tl->last_associated_segment_num == i);
+    assert(in_transaction(tl));
     assert(pr->transaction_state != TS_INEVITABLE);
     set_gs_register(get_segment_base(i));
     assert(STM_SEGMENT->segment_num == i);
@@ -150,7 +152,7 @@
 
     /* Restore a few things: the new pthread_self(), and the %gs
        register */
-    int segnum = fork_this_tl->associated_segment_num;
+    int segnum = fork_this_tl->last_associated_segment_num;
     assert(1 <= segnum && segnum < NB_SEGMENTS);
     *_get_cpth(fork_this_tl) = pthread_self();
     set_gs_register(get_segment_base(segnum));
diff --git a/c8/stm/setup.c b/c8/stm/setup.c
--- a/c8/stm/setup.c
+++ b/c8/stm/setup.c
@@ -244,7 +244,6 @@
     /* assign numbers consecutively, but that's for tests; we could also
        assign the same number to all of them and they would get their own
        numbers automatically. */
-    tl->associated_segment_num = -1;
     tl->last_associated_segment_num = num + 1;
     tl->thread_local_counter = ++thread_local_counters;
     *_get_cpth(tl) = pthread_self();
diff --git a/c8/stm/sync.c b/c8/stm/sync.c
--- a/c8/stm/sync.c
+++ b/c8/stm/sync.c
@@ -176,8 +176,10 @@
     sync_ctl.in_use1[num+1] = 1;
     assert(STM_SEGMENT->segment_num == num+1);
     assert(STM_SEGMENT->running_thread == NULL);
-    tl->associated_segment_num = tl->last_associated_segment_num;
+    assert(tl->last_associated_segment_num == STM_SEGMENT->segment_num);
+    assert(!in_transaction(tl));
     STM_SEGMENT->running_thread = tl;
+    assert(in_transaction(tl));
     return true;
 }
 
@@ -188,9 +190,10 @@
     cond_signal(C_SEGMENT_FREE);
 
     assert(STM_SEGMENT->running_thread == tl);
-    assert(tl->associated_segment_num == tl->last_associated_segment_num);
-    tl->associated_segment_num = -1;
+    assert(tl->last_associated_segment_num == STM_SEGMENT->segment_num);
+    assert(in_transaction(tl));
     STM_SEGMENT->running_thread = NULL;
+    assert(!in_transaction(tl));
 
     assert(sync_ctl.in_use1[tl->last_associated_segment_num] == 1);
     sync_ctl.in_use1[tl->last_associated_segment_num] = 0;
@@ -204,22 +207,15 @@
 
 bool _stm_in_transaction(stm_thread_local_t *tl)
 {
-    if (tl->associated_segment_num == -1) {
-        return false;
-    }
-    else {
-        int num = tl->associated_segment_num;
-        OPT_ASSERT(1 <= num && num < NB_SEGMENTS);
-        OPT_ASSERT(num == tl->last_associated_segment_num);
-        OPT_ASSERT(get_segment(num)->running_thread == tl);
-        return true;
-    }
+    int num = tl->last_associated_segment_num;
+    OPT_ASSERT(1 <= num && num < NB_SEGMENTS);
+    return in_transaction(tl);
 }
 
 void _stm_test_switch(stm_thread_local_t *tl)
 {
     assert(_stm_in_transaction(tl));
-    set_gs_register(get_segment_base(tl->associated_segment_num));
+    set_gs_register(get_segment_base(tl->last_associated_segment_num));
     assert(STM_SEGMENT->running_thread == tl);
     exec_local_finalizers();
 }
diff --git a/c8/stmgc.h b/c8/stmgc.h
--- a/c8/stmgc.h
+++ b/c8/stmgc.h
@@ -69,8 +69,7 @@
        (this field is not modified on a successful commit) */
     long last_abort__bytes_in_nursery;
     /* the next fields are handled internally by the library */
-    int associated_segment_num;
-    int last_associated_segment_num;
+    int last_associated_segment_num;   /* always a valid seg num */
     int thread_local_counter;
     struct stm_thread_local_s *prev, *next;
     void *creating_pthread[2];
diff --git a/c8/test/support.py b/c8/test/support.py
--- a/c8/test/support.py
+++ b/c8/test/support.py
@@ -29,7 +29,6 @@
     char *mem_clear_on_abort;
     size_t mem_bytes_to_clear_on_abort;
     long last_abort__bytes_in_nursery;
-    int associated_segment_num;
     int last_associated_segment_num;
     struct stm_thread_local_s *prev, *next;
     void *creating_pthread[2];
@@ -798,8 +797,8 @@
         seen = set()
         for tl1 in self.tls:
             if lib._stm_in_transaction(tl1):
-                assert tl1.associated_segment_num not in seen
-                seen.add(tl1.associated_segment_num)
+                assert tl1.last_associated_segment_num not in seen
+                seen.add(tl1.last_associated_segment_num)
 
     def commit_transaction(self):
         tl = self.tls[self.current_thread]
diff --git a/c8/test/test_finalizer.py b/c8/test/test_finalizer.py
--- a/c8/test/test_finalizer.py
+++ b/c8/test/test_finalizer.py
@@ -13,9 +13,10 @@
             segnum = lib.current_segment_num()
             tlnum = '?'
             for n, tl in enumerate(self.tls):
-                if tl.associated_segment_num == segnum:
-                    tlnum = n
-                    break
+                if lib._stm_in_transaction(tl):
+                    if tl.last_associated_segment_num == segnum:
+                        tlnum = n
+                        break
             self.light_finalizers_called.append((obj, tlnum))
         self.light_finalizers_called = []
         lib.stmcb_light_finalizer = light_finalizer

From noreply at buildbot.pypy.org  Sun Jun  7 14:34:12 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sun,  7 Jun 2015 14:34:12 +0200 (CEST)
Subject: [pypy-commit] cffi release-1.1: hg merge default
Message-ID: <20150607123412.258091C02FD@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: release-1.1
Changeset: r2165:8636109593c6
Date: 2015-06-07 14:33 +0200
http://bitbucket.org/cffi/cffi/changeset/8636109593c6/

Log:	hg merge default

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -5158,12 +5158,14 @@
     return PyText_FromStringAndSize(s, namelen + replacelen);
 }
 
-static PyObject *b_string(PyObject *self, PyObject *args)
+static PyObject *b_string(PyObject *self, PyObject *args, PyObject *kwds)
 {
     CDataObject *cd;
     Py_ssize_t maxlen = -1;
-    if (!PyArg_ParseTuple(args, "O!|n:string",
-                          &CData_Type, &cd, &maxlen))
+    static char *keywords[] = {"cdata", "maxlen", NULL};
+
+    if (!PyArg_ParseTupleAndKeywords(args, kwds, "O!|n:string", keywords,
+                                     &CData_Type, &cd, &maxlen))
         return NULL;
 
     if (cd->c_type->ct_itemdescr != NULL &&
@@ -5246,12 +5248,14 @@
     return NULL;
 }
 
-static PyObject *b_buffer(PyObject *self, PyObject *args)
+static PyObject *b_buffer(PyObject *self, PyObject *args, PyObject *kwds)
 {
     CDataObject *cd;
     Py_ssize_t size = -1;
-    if (!PyArg_ParseTuple(args, "O!|n:buffer",
-                          &CData_Type, &cd, &size))
+    static char *keywords[] = {"cdata", "size", NULL};
+
+    if (!PyArg_ParseTupleAndKeywords(args, kwds, "O!|n:buffer", keywords,
+                                     &CData_Type, &cd, &size))
         return NULL;
 
     if (cd->c_type->ct_flags & CT_POINTER) {
@@ -5351,6 +5355,12 @@
         return NULL;
     }
     x = (PyObject *)(raw + 42);
+    if (Py_REFCNT(x) <= 0) {
+        Py_FatalError("ffi.from_handle() detected that the address passed "
+                      "points to garbage. If it is really the result of "
+                      "ffi.new_handle(), then the Python object has already "
+                      "been garbage collected");
+    }
     Py_INCREF(x);
     return x;
 }
@@ -5790,15 +5800,15 @@
     {"typeoffsetof", b_typeoffsetof, METH_VARARGS},
     {"rawaddressof", b_rawaddressof, METH_VARARGS},
     {"getcname", b_getcname, METH_VARARGS},
-    {"string", b_string, METH_VARARGS},
-    {"buffer", b_buffer, METH_VARARGS},
+    {"string", (PyCFunction)b_string, METH_VARARGS | METH_KEYWORDS},
+    {"buffer", (PyCFunction)b_buffer, METH_VARARGS | METH_KEYWORDS},
     {"get_errno", b_get_errno, METH_NOARGS},
     {"set_errno", b_set_errno, METH_O},
     {"newp_handle", b_newp_handle, METH_VARARGS},
     {"from_handle", b_from_handle, METH_O},
     {"from_buffer", b_from_buffer, METH_VARARGS},
 #ifdef MS_WIN32
-    {"getwinerror", b_getwinerror, METH_VARARGS},
+    {"getwinerror", (PyCFunction)b_getwinerror, METH_VARARGS | METH_KEYWORDS},
 #endif
     {"_get_types", b__get_types, METH_NOARGS},
     {"_testfunc", b__testfunc, METH_VARARGS},
diff --git a/c/cgc.c b/c/cgc.c
--- a/c/cgc.c
+++ b/c/cgc.c
@@ -2,79 +2,121 @@
 /* translated to C from cffi/gc_weakref.py */
 
 
-static PyObject *const_name_pop;
+static PyObject *gc_wref_remove(PyObject *ffi_wref_tup, PyObject *key)
+{
+    FFIObject *ffi;
+    PyObject *indexobj, *destructor, *cdata, *freelist, *result;
+    Py_ssize_t index;
 
-static PyObject *gc_wref_remove(PyObject *ffi_wref_data, PyObject *arg)
-{
-    PyObject *destructor, *cdata, *x;
-    PyObject *res = PyObject_CallMethodObjArgs(ffi_wref_data,
-                                               const_name_pop, arg, NULL);
-    if (res == NULL)
-        return NULL;
+    /* here, tup is a 4-tuple (ffi, destructor, cdata, index) */
+    if (!PyTuple_Check(ffi_wref_tup))
+        goto oops;    /* should never occur */
 
-    assert(PyTuple_Check(res));
-    destructor = PyTuple_GET_ITEM(res, 0);
-    cdata = PyTuple_GET_ITEM(res, 1);
-    x = PyObject_CallFunctionObjArgs(destructor, cdata, NULL);
-    Py_DECREF(res);
-    if (x == NULL)
-        return NULL;
-    Py_DECREF(x);
+    ffi = (FFIObject *)PyTuple_GET_ITEM(ffi_wref_tup, 0);
+    destructor = PyTuple_GET_ITEM(ffi_wref_tup, 1);
+    cdata = PyTuple_GET_ITEM(ffi_wref_tup, 2);
+    indexobj = PyTuple_GET_ITEM(ffi_wref_tup, 3);
 
-    Py_INCREF(Py_None);
-    return Py_None;
+    index = PyInt_AsSsize_t(indexobj);
+    if (index < 0)
+        goto oops;    /* should never occur */
+
+    /* assert gc_wrefs[index] is key */
+    if (PyList_GET_ITEM(ffi->gc_wrefs, index) != key)
+        goto oops;    /* should never occur */
+
+    /* gc_wrefs[index] = freelist */
+    /* transfer ownership of 'freelist' to 'gc_wrefs[index]' */
+    freelist = ffi->gc_wrefs_freelist;
+    PyList_SET_ITEM(ffi->gc_wrefs, index, freelist);
+
+    /* freelist = index */
+    ffi->gc_wrefs_freelist = indexobj;
+    Py_INCREF(indexobj);
+
+    /* destructor(cdata) */
+    result = PyObject_CallFunctionObjArgs(destructor, cdata, NULL);
+
+    Py_DECREF(key);    /* free the reference that was in 'gc_wrefs[index]' */
+    return result;
+
+ oops:
+    PyErr_SetString(PyExc_SystemError, "cgc: internal inconsistency");
+    /* random leaks may follow */
+    return NULL;
 }
 
 static PyMethodDef remove_callback = {
     "gc_wref_remove", (PyCFunction)gc_wref_remove, METH_O
 };
 
-static PyObject *gc_weakrefs_build(FFIObject *ffi, CDataObject *cd,
+static PyObject *gc_weakrefs_build(FFIObject *ffi, CDataObject *cdata,
                                    PyObject *destructor)
 {
-    PyObject *new_cdata, *ref = NULL, *tup = NULL;
+    PyObject *new_cdata, *ref = NULL, *tup = NULL, *remove_fn = NULL;
+    Py_ssize_t index;
+    PyObject *datalist;
 
     if (ffi->gc_wrefs == NULL) {
         /* initialize */
-        PyObject *data;
-
-        if (const_name_pop == NULL) {
-            const_name_pop = PyText_InternFromString("pop");
-            if (const_name_pop == NULL)
-                return NULL;
-        }
-        data = PyDict_New();
-        if (data == NULL)
+        datalist = PyList_New(0);
+        if (datalist == NULL)
             return NULL;
-        ffi->gc_wrefs = PyCFunction_New(&remove_callback, data);
-        Py_DECREF(data);
-        if (ffi->gc_wrefs == NULL)
-            return NULL;
+        ffi->gc_wrefs = datalist;
+        assert(ffi->gc_wrefs_freelist == NULL);
+        ffi->gc_wrefs_freelist = Py_None;
+        Py_INCREF(Py_None);
     }
 
-    new_cdata = do_cast(cd->c_type, (PyObject *)cd);
+    /* new_cdata = self.ffi.cast(typeof(cdata), cdata) */
+    new_cdata = do_cast(cdata->c_type, (PyObject *)cdata);
     if (new_cdata == NULL)
         goto error;
 
-    ref = PyWeakref_NewRef(new_cdata, ffi->gc_wrefs);
+    /* if freelist is None: */
+    datalist = ffi->gc_wrefs;
+    if (ffi->gc_wrefs_freelist == Py_None) {
+        /* index = len(gc_wrefs) */
+        index = PyList_GET_SIZE(datalist);
+        /* gc_wrefs.append(None) */
+        if (PyList_Append(datalist, Py_None) < 0)
+            goto error;
+        tup = Py_BuildValue("OOOn", ffi, destructor, cdata, index);
+    }
+    else {
+        /* index = freelist */
+        index = PyInt_AsSsize_t(ffi->gc_wrefs_freelist);
+        if (index < 0)
+            goto error;   /* should not occur */
+        tup = PyTuple_Pack(4, ffi, destructor, cdata, ffi->gc_wrefs_freelist);
+    }
+    if (tup == NULL)
+        goto error;
+
+    remove_fn = PyCFunction_New(&remove_callback, tup);
+    if (remove_fn == NULL)
+        goto error;
+
+    ref = PyWeakref_NewRef(new_cdata, remove_fn);
     if (ref == NULL)
         goto error;
 
-    tup = PyTuple_Pack(2, destructor, cd);
-    if (tup == NULL)
-        goto error;
+    /* freelist = gc_wrefs[index] (which is None if we just did append(None)) */
+    /* transfer ownership of 'datalist[index]' into gc_wrefs_freelist */
+    Py_DECREF(ffi->gc_wrefs_freelist);
+    ffi->gc_wrefs_freelist = PyList_GET_ITEM(datalist, index);
+    /* gc_wrefs[index] = ref */
+    /* transfer ownership of 'ref' into 'datalist[index]' */
+    PyList_SET_ITEM(datalist, index, ref);
+    Py_DECREF(remove_fn);
+    Py_DECREF(tup);
 
-    /* the 'self' of the function 'gc_wrefs' is actually the data dict */
-    if (PyDict_SetItem(PyCFunction_GET_SELF(ffi->gc_wrefs), ref, tup) < 0)
-        goto error;
-
-    Py_DECREF(tup);
-    Py_DECREF(ref);
     return new_cdata;
 
  error:
     Py_XDECREF(new_cdata);
     Py_XDECREF(ref);
     Py_XDECREF(tup);
+    Py_XDECREF(remove_fn);
     return NULL;
 }
diff --git a/c/ffi_obj.c b/c/ffi_obj.c
--- a/c/ffi_obj.c
+++ b/c/ffi_obj.c
@@ -23,7 +23,7 @@
 
 struct FFIObject_s {
     PyObject_HEAD
-    PyObject *gc_wrefs;
+    PyObject *gc_wrefs, *gc_wrefs_freelist;
     struct _cffi_parse_info_s info;
     char ctx_is_static, ctx_is_nonempty;
     builder_c_t types_builder;
@@ -51,6 +51,7 @@
         return NULL;
     }
     ffi->gc_wrefs = NULL;
+    ffi->gc_wrefs_freelist = NULL;
     ffi->info.ctx = &ffi->types_builder.ctx;
     ffi->info.output = internal_output;
     ffi->info.output_size = FFI_COMPLEXITY_OUTPUT;
@@ -63,6 +64,7 @@
 {
     PyObject_GC_UnTrack(ffi);
     Py_XDECREF(ffi->gc_wrefs);
+    Py_XDECREF(ffi->gc_wrefs_freelist);
 
     free_builder_c(&ffi->types_builder, ffi->ctx_is_static);
 
@@ -140,6 +142,38 @@
 #define ACCEPT_ALL      (ACCEPT_STRING | ACCEPT_CTYPE | ACCEPT_CDATA)
 #define CONSIDER_FN_AS_FNPTR  8
 
+static CTypeDescrObject *_ffi_bad_type(FFIObject *ffi, char *input_text)
+{
+    size_t length = strlen(input_text);
+    char *extra;
+
+    if (length > 500) {
+        extra = "";
+    }
+    else {
+        char *p;
+        size_t i, num_spaces = ffi->info.error_location;
+        extra = alloca(length + num_spaces + 4);
+        p = extra;
+        *p++ = '\n';
+        for (i = 0; i < length; i++) {
+            if (' ' <= input_text[i] && input_text[i] < 0x7f)
+                *p++ = input_text[i];
+            else if (input_text[i] == '\t' || input_text[i] == '\n')
+                *p++ = ' ';
+            else
+                *p++ = '?';
+        }
+        *p++ = '\n';
+        memset(p, ' ', num_spaces);
+        p += num_spaces;
+        *p++ = '^';
+        *p++ = 0;
+    }
+    PyErr_Format(FFIError, "%s%s", ffi->info.error_message, extra);
+    return NULL;
+}
+
 static CTypeDescrObject *_ffi_type(FFIObject *ffi, PyObject *arg,
                                    int accept)
 {
@@ -153,15 +187,9 @@
         if (x == NULL) {
             char *input_text = PyText_AS_UTF8(arg);
             int err, index = parse_c_type(&ffi->info, input_text);
-            if (index < 0) {
-                size_t num_spaces = ffi->info.error_location;
-                char *spaces = alloca(num_spaces + 1);
-                memset(spaces, ' ', num_spaces);
-                spaces[num_spaces] = '\0';
-                PyErr_Format(FFIError, "%s\n%s\n%s^", ffi->info.error_message,
-                             input_text, spaces);
-                return NULL;
-            }
+            if (index < 0)
+                return _ffi_bad_type(ffi, input_text);
+
             x = realize_c_type_or_func(&ffi->types_builder,
                                        ffi->info.output, index);
             if (x == NULL)
@@ -774,7 +802,7 @@
 static PyMethodDef ffi_methods[] = {
  {"addressof",  (PyCFunction)ffi_addressof,  METH_VARARGS, ffi_addressof_doc},
  {"alignof",    (PyCFunction)ffi_alignof,    METH_O,       ffi_alignof_doc},
- {"buffer",     (PyCFunction)ffi_buffer,     METH_VARARGS, ffi_buffer_doc},
+ {"buffer",     (PyCFunction)ffi_buffer,     METH_VKW,     ffi_buffer_doc},
  {"callback",   (PyCFunction)ffi_callback,   METH_VKW,     ffi_callback_doc},
  {"cast",       (PyCFunction)ffi_cast,       METH_VARARGS, ffi_cast_doc},
  {"dlclose",    (PyCFunction)ffi_dlclose,    METH_VARARGS, ffi_dlclose_doc},
@@ -784,14 +812,14 @@
  {"gc",         (PyCFunction)ffi_gc,         METH_VKW,     ffi_gc_doc},
  {"getctype",   (PyCFunction)ffi_getctype,   METH_VKW,     ffi_getctype_doc},
 #ifdef MS_WIN32
- {"getwinerror",(PyCFunction)ffi_getwinerror,METH_VARARGS, ffi_getwinerror_doc},
+ {"getwinerror",(PyCFunction)ffi_getwinerror,METH_VKW,     ffi_getwinerror_doc},
 #endif
  {"integer_const",(PyCFunction)ffi_int_const,METH_VKW,     ffi_int_const_doc},
  {"new",        (PyCFunction)ffi_new,        METH_VKW,     ffi_new_doc},
  {"new_handle", (PyCFunction)ffi_new_handle, METH_O,       ffi_new_handle_doc},
  {"offsetof",   (PyCFunction)ffi_offsetof,   METH_VARARGS, ffi_offsetof_doc},
  {"sizeof",     (PyCFunction)ffi_sizeof,     METH_O,       ffi_sizeof_doc},
- {"string",     (PyCFunction)ffi_string,     METH_VARARGS, ffi_string_doc},
+ {"string",     (PyCFunction)ffi_string,     METH_VKW,     ffi_string_doc},
  {"typeof",     (PyCFunction)ffi_typeof,     METH_O,       ffi_typeof_doc},
  {NULL}
 };
diff --git a/c/misc_win32.h b/c/misc_win32.h
--- a/c/misc_win32.h
+++ b/c/misc_win32.h
@@ -82,14 +82,15 @@
 }
 
 #if PY_MAJOR_VERSION >= 3
-static PyObject *b_getwinerror(PyObject *self, PyObject *args)
+static PyObject *b_getwinerror(PyObject *self, PyObject *args, PyObject *kwds)
 {
     int err = -1;
     int len;
     WCHAR *s_buf = NULL; /* Free via LocalFree */
     PyObject *v, *message;
+    static char *keywords[] = {"code", NULL};
 
-    if (!PyArg_ParseTuple(args, "|i", &err))
+    if (!PyArg_ParseTupleAndKeywords(args, kwds, "|i", keywords, &err))
         return NULL;
 
     if (err == -1) {
@@ -129,7 +130,7 @@
     return v;
 }
 #else
-static PyObject *b_getwinerror(PyObject *self, PyObject *args)
+static PyObject *b_getwinerror(PyObject *self, PyObject *args, PyObject *kwds)
 {
     int err = -1;
     int len;
@@ -137,8 +138,9 @@
     char *s_buf = NULL; /* Free via LocalFree */
     char s_small_buf[28]; /* Room for "Windows Error 0xFFFFFFFF" */
     PyObject *v;
+    static char *keywords[] = {"code", NULL};
 
-    if (!PyArg_ParseTuple(args, "|i", &err))
+    if (!PyArg_ParseTupleAndKeywords(args, kwds, "|i", keywords, &err))
         return NULL;
 
     if (err == -1) {
diff --git a/cffi/gc_weakref.py b/cffi/gc_weakref.py
--- a/cffi/gc_weakref.py
+++ b/cffi/gc_weakref.py
@@ -2,18 +2,27 @@
 
 
 class GcWeakrefs(object):
-    # code copied and adapted from WeakKeyDictionary.
-
     def __init__(self, ffi):
         self.ffi = ffi
-        self.data = data = {}
-        def remove(k):
-            destructor, cdata = data.pop(k)
-            destructor(cdata)
-        self.remove = remove
+        self.data = []
+        self.freelist = None
 
     def build(self, cdata, destructor):
         # make a new cdata of the same type as the original one
         new_cdata = self.ffi.cast(self.ffi._backend.typeof(cdata), cdata)
-        self.data[ref(new_cdata, self.remove)] = destructor, cdata
+        #
+        def remove(key):
+            assert self.data[index] is key
+            self.data[index] = self.freelist
+            self.freelist = index
+            destructor(cdata)
+        #
+        key = ref(new_cdata, remove)
+        index = self.freelist
+        if index is None:
+            index = len(self.data)
+            self.data.append(key)
+        else:
+            self.freelist = self.data[index]
+            self.data[index] = key
         return new_cdata
diff --git a/cffi/recompiler.py b/cffi/recompiler.py
--- a/cffi/recompiler.py
+++ b/cffi/recompiler.py
@@ -775,7 +775,8 @@
             try:
                 if ftype.is_integer_type() or fbitsize >= 0:
                     # accept all integers, but complain on float or double
-                    prnt('  (void)((p->%s) << 1);' % fname)
+                    prnt("  (void)((p->%s) << 1);  /* check that '%s.%s' is "
+                         "an integer */" % (fname, cname, fname))
                     continue
                 # only accept exactly the type declared, except that '[]'
                 # is interpreted as a '*' and so will match any array length.
@@ -949,7 +950,7 @@
             prnt('{')
             prnt('  int n = (%s) <= 0;' % (name,))
             prnt('  *o = (unsigned long long)((%s) << 0);'
-                 '  /* check that we get an integer */' % (name,))
+                 '  /* check that %s is an integer */' % (name, name))
             if check_value is not None:
                 if check_value > 0:
                     check_value = '%dU' % (check_value,)
@@ -1088,8 +1089,9 @@
         self.cffi_types[index] = CffiOp(OP_PRIMITIVE, prim_index)
 
     def _emit_bytecode_UnknownIntegerType(self, tp, index):
-        s = '_cffi_prim_int(sizeof(%s), (((%s)-1) << 0) <= 0)' % (
-            tp.name, tp.name)
+        s = ('_cffi_prim_int(sizeof(%s), (\n'
+             '           ((%s)-1) << 0 /* check that %s is an integer type */\n'
+             '         ) <= 0)' % (tp.name, tp.name, tp.name))
         self.cffi_types[index] = CffiOp(OP_PRIMITIVE, s)
 
     def _emit_bytecode_RawFunctionType(self, tp, index):
diff --git a/cffi/setuptools_ext.py b/cffi/setuptools_ext.py
--- a/cffi/setuptools_ext.py
+++ b/cffi/setuptools_ext.py
@@ -18,7 +18,9 @@
     # __init__.py files may already try to import the file that
     # we are generating.
     with open(filename) as f:
-        code = compile(f.read(), filename, 'exec')
+        src = f.read()
+    src += '\n'      # Python 2.6 compatibility
+    code = compile(src, filename, 'exec')
     exec(code, glob, glob)
 
 
diff --git a/cffi/vengine_gen.py b/cffi/vengine_gen.py
--- a/cffi/vengine_gen.py
+++ b/cffi/vengine_gen.py
@@ -402,12 +402,16 @@
         else:
             assert tp is not None
             assert check_value is None
-            prnt(tp.get_c_name(' %s(void)' % funcname, name),)
-            prnt('{')
             if category == 'var':
                 ampersand = '&'
             else:
                 ampersand = ''
+            extra = ''
+            if category == 'const' and isinstance(tp, model.StructOrUnion):
+                extra = 'const *'
+                ampersand = '&'
+            prnt(tp.get_c_name(' %s%s(void)' % (extra, funcname), name))
+            prnt('{')
             prnt('  return (%s%s);' % (ampersand, name))
             prnt('}')
         prnt()
@@ -436,9 +440,14 @@
                 value += (1 << (8*self.ffi.sizeof(BLongLong)))
         else:
             assert check_value is None
-            BFunc = self.ffi._typeof_locked(tp.get_c_name('(*)(void)', name))[0]
+            fntypeextra = '(*)(void)'
+            if isinstance(tp, model.StructOrUnion):
+                fntypeextra = '*' + fntypeextra
+            BFunc = self.ffi._typeof_locked(tp.get_c_name(fntypeextra, name))[0]
             function = module.load_function(BFunc, funcname)
             value = function()
+            if isinstance(tp, model.StructOrUnion):
+                value = value[0]
         return value
 
     def _loaded_gen_constant(self, tp, name, module, library):
diff --git a/doc/source/cdef.rst b/doc/source/cdef.rst
--- a/doc/source/cdef.rst
+++ b/doc/source/cdef.rst
@@ -278,6 +278,18 @@
 needed.  (Alternatively, the out-of-line FFIs have a method
 ``ffi.dlclose(lib)``.)
 
+.. _dlopen-note:
+
+Note: the old version of ``ffi.dlopen()`` from the in-line ABI mode
+tries to use ``ctypes.util.find_library()`` if it cannot directly find
+the library.  The newer out-of-line ``ffi.dlopen()`` no longer does it
+automatically; it simply passes the argument it receives to the
+underlying ``dlopen()`` or ``LoadLibrary()`` function.  If needed, it
+is up to you to use ``ctypes.util.find_library()`` or any other way to
+look for the library's filename.  This also means that
+``ffi.dlopen(None)`` no longer work on Windows; try instead
+``ffi.dlopen(ctypes.util.find_library('c'))``.
+
 
 ffi.set_source(): preparing out-of-line modules
 -----------------------------------------------
@@ -375,7 +387,7 @@
 
 *  *New in version 1.1:* integer types: the syntax "``typedef
    int... foo_t;``" declares the type ``foo_t`` as an integer type
-   whose exact size and signness is not specified.  The compiler will
+   whose exact size and signedness is not specified.  The compiler will
    figure it out.  (Note that this requires ``set_source()``; it does
    not work with ``verify()``.)  The ``int...`` can be replaced with
    ``long...`` or ``unsigned long long...`` or any other primitive
diff --git a/doc/source/overview.rst b/doc/source/overview.rst
--- a/doc/source/overview.rst
+++ b/doc/source/overview.rst
@@ -82,9 +82,21 @@
 
     from _simple_example import ffi
 
-    lib = ffi.dlopen(None)         # or path to a library
+    lib = ffi.dlopen(None)      # Unix: open the standard C library
+    #import ctypes.util         # or, try this on Windows:
+    #lib = ffi.dlopen(ctypes.util.find_library("c"))
+
     lib.printf(b"hi there, number %d\n", ffi.cast("int", 2))
 
+Note that this ``ffi.dlopen()``, unlike the one from in-line mode,
+does not invoke any additional magic to locate the library: it must be
+a path name (with or without a directory), as required by the C
+``dlopen()`` or ``LoadLibrary()`` functions.  This means that
+``ffi.dlopen("libfoo.so")`` is ok, but ``ffi.dlopen("foo")`` is not.
+In the latter case, you could replace it with
+``ffi.dlopen(ctypes.util.find_library("foo"))``.  Also, None is only
+recognized on Unix to open the standard C library.
+
 For distribution purposes, remember that there is a new
 ``_simple_example.py`` file generated.  You can either include it
 statically within your project's source files, or, with Setuptools,
@@ -202,6 +214,13 @@
 .. _struct: http://docs.python.org/library/struct.html
 .. _array: http://docs.python.org/library/array.html
 
+This example also admits an out-of-line equivalent.  It is similar to
+`Out-of-line example (ABI level, out-of-line)`_ above, but without any
+call to ``ffi.dlopen()``.  In the main program, you write ``from
+_simple_example import ffi`` and then the same content as the in-line
+example above starting from the line ``image = ffi.new("pixel_t[]",
+800*600)``.
+
 
 .. _performance:
 
diff --git a/doc/source/whatsnew.rst b/doc/source/whatsnew.rst
--- a/doc/source/whatsnew.rst
+++ b/doc/source/whatsnew.rst
@@ -3,11 +3,29 @@
 ======================
 
 
+1.1.1
+=====
+
+* Out-of-line mode: ``ffi.string()``, ``ffi.buffer()`` and
+  ``ffi.getwinerror()`` didn't accept their arguments as keyword
+  arguments, unlike their in-line mode equivalent.  (It worked in PyPy.)
+
+* Out-of-line ABI mode: documented a restriction__ of ``ffi.dlopen()``
+  when compared to the in-line mode.
+
+* ``ffi.gc()``: when called several times with equal pointers, it was
+  accidentally registering only the last destructor, or even none at
+  all depending on details.  (It was correctly registering all of them
+  only in PyPy, and only with the out-of-line FFIs.)
+
+.. __: cdef.html#dlopen-note
+
+
 1.1.0
 =====
 
 * Out-of-line API mode: we can now declare integer types with
-  ``typedef int... foo_t;``.  The exact size and signness of ``foo_t``
+  ``typedef int... foo_t;``.  The exact size and signedness of ``foo_t``
   is figured out by the compiler.
 
 * Out-of-line API mode: we can now declare multidimensional arrays
diff --git a/testing/cffi0/backend_tests.py b/testing/cffi0/backend_tests.py
--- a/testing/cffi0/backend_tests.py
+++ b/testing/cffi0/backend_tests.py
@@ -1457,6 +1457,63 @@
         import gc; gc.collect(); gc.collect(); gc.collect()
         assert seen == [1]
 
+    def test_gc_2(self):
+        ffi = FFI(backend=self.Backend())
+        p = ffi.new("int *", 123)
+        seen = []
+        q1 = ffi.gc(p, lambda p: seen.append(1))
+        q2 = ffi.gc(q1, lambda p: seen.append(2))
+        import gc; gc.collect()
+        assert seen == []
+        del q1, q2
+        import gc; gc.collect(); gc.collect(); gc.collect(); gc.collect()
+        assert seen == [2, 1]
+
+    def test_gc_3(self):
+        ffi = FFI(backend=self.Backend())
+        p = ffi.new("int *", 123)
+        r = ffi.new("int *", 123)
+        seen = []
+        seen_r = []
+        q1 = ffi.gc(p, lambda p: seen.append(1))
+        s1 = ffi.gc(r, lambda r: seen_r.append(4))
+        q2 = ffi.gc(q1, lambda p: seen.append(2))
+        s2 = ffi.gc(s1, lambda r: seen_r.append(5))
+        q3 = ffi.gc(q2, lambda p: seen.append(3))
+        import gc; gc.collect()
+        assert seen == []
+        assert seen_r == []
+        del q1, q2, q3, s2, s1
+        import gc; gc.collect(); gc.collect(); gc.collect(); gc.collect()
+        assert seen == [3, 2, 1]
+        assert seen_r == [5, 4]
+
+    def test_gc_4(self):
+        ffi = FFI(backend=self.Backend())
+        p = ffi.new("int *", 123)
+        seen = []
+        q1 = ffi.gc(p, lambda p: seen.append(1))
+        q2 = ffi.gc(q1, lambda p: seen.append(2))
+        q3 = ffi.gc(q2, lambda p: seen.append(3))
+        import gc; gc.collect()
+        assert seen == []
+        del q1, q3     # q2 remains, and has a hard ref to q1
+        import gc; gc.collect(); gc.collect(); gc.collect()
+        assert seen == [3]
+
+    def test_gc_finite_list(self):
+        ffi = FFI(backend=self.Backend())
+        p = ffi.new("int *", 123)
+        keepalive = []
+        for i in range(10):
+            keepalive.append(ffi.gc(p, lambda p: None))
+            assert len(ffi.gc_weakrefs.data) == i + 1  #should be a private attr
+        del keepalive[:]
+        import gc; gc.collect(); gc.collect()
+        for i in range(10):
+            keepalive.append(ffi.gc(p, lambda p: None))
+        assert len(ffi.gc_weakrefs.data) == 10
+
     def test_CData_CType(self):
         ffi = FFI(backend=self.Backend())
         assert isinstance(ffi.cast("int", 0), ffi.CData)
diff --git a/testing/cffi0/test_verify.py b/testing/cffi0/test_verify.py
--- a/testing/cffi0/test_verify.py
+++ b/testing/cffi0/test_verify.py
@@ -2227,3 +2227,11 @@
     ffi.cdef("static const int FOO = 123;")
     e = py.test.raises(VerificationError, ffi.verify, "#define FOO 124")
     assert str(e.value).endswith("FOO has the real value 124, not 123")
+
+def test_const_struct_global():
+    ffi = FFI()
+    ffi.cdef("typedef struct { int x; ...; } T; const T myglob;")
+    lib = ffi.verify("typedef struct { double y; int x; } T;"
+                     "const T myglob = { 0.1, 42 };")
+    assert ffi.typeof(lib.myglob) == ffi.typeof("T")
+    assert lib.myglob.x == 42
diff --git a/testing/cffi1/test_ffi_obj.py b/testing/cffi1/test_ffi_obj.py
--- a/testing/cffi1/test_ffi_obj.py
+++ b/testing/cffi1/test_ffi_obj.py
@@ -1,4 +1,4 @@
-import py
+import py, sys
 import _cffi_backend as _cffi1_backend
 
 
@@ -65,6 +65,7 @@
     ffi = _cffi1_backend.FFI()
     p = ffi.new("char[]", init=b"foobar\x00baz")
     assert ffi.string(p) == b"foobar"
+    assert ffi.string(cdata=p, maxlen=3) == b"foo"
 
 def test_ffi_errno():
     # xxx not really checking errno, just checking that we can read/write it
@@ -157,11 +158,18 @@
     assert str(e.value) == ("undefined struct/union name\n"
                             "struct never_heard_of_s\n"
                             "       ^")
+    e = py.test.raises(ffi.error, ffi.cast, "\t\n\x01\x1f~\x7f\x80\xff", 0)
+    assert str(e.value) == ("identifier expected\n"
+                            "  ??~???\n"
+                            "  ^")
+    e = py.test.raises(ffi.error, ffi.cast, "X" * 600, 0)
+    assert str(e.value) == ("undefined type name")
 
 def test_ffi_buffer():
     ffi = _cffi1_backend.FFI()
     a = ffi.new("signed char[]", [5, 6, 7])
     assert ffi.buffer(a)[:] == b'\x05\x06\x07'
+    assert ffi.buffer(cdata=a, size=2)[:] == b'\x05\x06'
 
 def test_ffi_from_buffer():
     import array
@@ -178,3 +186,11 @@
     ffi = _cffi1_backend.FFI()
     assert isinstance(ffi.cast("int", 42), CData)
     assert isinstance(ffi.typeof("int"), CType)
+
+def test_ffi_getwinerror():
+    if sys.platform != "win32":
+        py.test.skip("for windows")
+    ffi = _cffi1_backend.FFI()
+    n = (1 << 29) + 42
+    code, message = ffi.getwinerror(code=n)
+    assert code == n
diff --git a/testing/cffi1/test_new_ffi_1.py b/testing/cffi1/test_new_ffi_1.py
--- a/testing/cffi1/test_new_ffi_1.py
+++ b/testing/cffi1/test_new_ffi_1.py
@@ -32,7 +32,9 @@
         struct ab { int a, b; };
         struct abc { int a, b, c; };
 
-        enum foq { A0, B0, CC0, D0 };
+        /* don't use A0, B0, CC0, D0 because termios.h might be included
+           and it has its own #defines for these names */
+        enum foq { cffiA0, cffiB0, cffiCC0, cffiD0 };
         enum bar { A1, B1=-2, CC1, D1, E1 };
         enum baz { A2=0x1000, B2=0x2000 };
         enum foo2 { A3, B3, C3, D3 };
@@ -878,9 +880,9 @@
 
     def test_enum(self):
         # enum foq { A0, B0, CC0, D0 };
-        assert ffi.string(ffi.cast("enum foq", 0)) == "A0"
-        assert ffi.string(ffi.cast("enum foq", 2)) == "CC0"
-        assert ffi.string(ffi.cast("enum foq", 3)) == "D0"
+        assert ffi.string(ffi.cast("enum foq", 0)) == "cffiA0"
+        assert ffi.string(ffi.cast("enum foq", 2)) == "cffiCC0"
+        assert ffi.string(ffi.cast("enum foq", 3)) == "cffiD0"
         assert ffi.string(ffi.cast("enum foq", 4)) == "4"
         # enum bar { A1, B1=-2, CC1, D1, E1 };
         assert ffi.string(ffi.cast("enum bar", 0)) == "A1"
@@ -1407,6 +1409,47 @@
         import gc; gc.collect(); gc.collect(); gc.collect()
         assert seen == [1]
 
+    def test_gc_2(self):
+        p = ffi.new("int *", 123)
+        seen = []
+        q1 = ffi.gc(p, lambda p: seen.append(1))
+        q2 = ffi.gc(q1, lambda p: seen.append(2))
+        import gc; gc.collect()
+        assert seen == []
+        del q1, q2
+        import gc; gc.collect(); gc.collect(); gc.collect(); gc.collect()
+        assert seen == [2, 1]
+
+    def test_gc_3(self):
+        p = ffi.new("int *", 123)
+        r = ffi.new("int *", 123)
+        seen = []
+        seen_r = []
+        q1 = ffi.gc(p, lambda p: seen.append(1))
+        s1 = ffi.gc(r, lambda r: seen_r.append(4))
+        q2 = ffi.gc(q1, lambda p: seen.append(2))
+        s2 = ffi.gc(s1, lambda r: seen_r.append(5))
+        q3 = ffi.gc(q2, lambda p: seen.append(3))
+        import gc; gc.collect()
+        assert seen == []
+        assert seen_r == []
+        del q1, q2, q3, s2, s1
+        import gc; gc.collect(); gc.collect(); gc.collect(); gc.collect()
+        assert seen == [3, 2, 1]
+        assert seen_r == [5, 4]
+
+    def test_gc_4(self):
+        p = ffi.new("int *", 123)
+        seen = []
+        q1 = ffi.gc(p, lambda p: seen.append(1))
+        q2 = ffi.gc(q1, lambda p: seen.append(2))
+        q3 = ffi.gc(q2, lambda p: seen.append(3))
+        import gc; gc.collect()
+        assert seen == []
+        del q1, q3     # q2 remains, and has a hard ref to q1
+        import gc; gc.collect(); gc.collect(); gc.collect()
+        assert seen == [3]
+
     def test_CData_CType(self):
         assert isinstance(ffi.cast("int", 0), ffi.CData)
         assert isinstance(ffi.new("int *"), ffi.CData)
@@ -1533,8 +1576,8 @@
         assert p.a == -52525
         #
         p = ffi.cast("enum foq", 2)
-        assert ffi.string(p) == "CC0"
-        assert ffi2.sizeof("char[CC0]") == 2
+        assert ffi.string(p) == "cffiCC0"
+        assert ffi2.sizeof("char[cffiCC0]") == 2
         #
         p = ffi.new("anon_foo_t *", [-52526])
         assert p.a == -52526
diff --git a/testing/cffi1/test_re_python.py b/testing/cffi1/test_re_python.py
--- a/testing/cffi1/test_re_python.py
+++ b/testing/cffi1/test_re_python.py
@@ -7,6 +7,7 @@
 
 def setup_module(mod):
     SRC = """
+    #include <string.h>
     #define FOOBAR (-42)
     static const int FOOBAZ = -43;
     #define BIGPOS 420000000000L
@@ -53,6 +54,7 @@
     struct foo_s;
     typedef struct bar_s { int x; signed char a[]; } bar_t;
     enum foo_e { AA, BB, CC };
+    int strlen(const char *);
     """)
     ffi.set_source('re_python_pysrc', None)
     ffi.emit_python_code(str(tmpdir.join('re_python_pysrc.py')))
@@ -81,10 +83,20 @@
 def test_function_with_varargs():
     import _cffi_backend
     from re_python_pysrc import ffi
-    lib = ffi.dlopen(extmod)
+    lib = ffi.dlopen(extmod, 0)
     assert lib.add43(45, ffi.cast("int", -5)) == 45
     assert type(lib.add43) is _cffi_backend.FFI.CData
 
+def test_dlopen_none():
+    import _cffi_backend
+    from re_python_pysrc import ffi
+    name = None
+    if sys.platform == 'win32':
+        import ctypes.util
+        name = ctypes.util.find_msvcrt()
+    lib = ffi.dlopen(name)
+    assert lib.strlen(b"hello") == 5
+
 def test_dlclose():
     import _cffi_backend
     from re_python_pysrc import ffi
diff --git a/testing/cffi1/test_recompiler.py b/testing/cffi1/test_recompiler.py
--- a/testing/cffi1/test_recompiler.py
+++ b/testing/cffi1/test_recompiler.py
@@ -992,3 +992,13 @@
     ffi.typeof('function_t*')
     lib.function(ffi.NULL)
     # assert did not crash
+
+def test_alignment_of_longlong():
+    ffi = FFI()
+    x1 = ffi.alignof('unsigned long long')
+    assert x1 in [4, 8]
+    ffi.cdef("struct foo_s { unsigned long long x; };")
+    lib = verify(ffi, 'test_alignment_of_longlong',
+                 "struct foo_s { unsigned long long x; };")
+    assert ffi.alignof('unsigned long long') == x1
+    assert ffi.alignof('struct foo_s') == x1
diff --git a/testing/cffi1/test_verify1.py b/testing/cffi1/test_verify1.py
--- a/testing/cffi1/test_verify1.py
+++ b/testing/cffi1/test_verify1.py
@@ -2117,25 +2117,19 @@
     try:
         ffi1 = FFI()
         ffi1.cdef("int foo_verify_dlopen_flags;")
-
-        sys.setdlopenflags(ffi1.RTLD_GLOBAL | ffi1.RTLD_LAZY)
+        sys.setdlopenflags(ffi1.RTLD_GLOBAL | ffi1.RTLD_NOW)
         lib1 = ffi1.verify("int foo_verify_dlopen_flags;")
-        lib2 = get_second_lib()
-
-        lib1.foo_verify_dlopen_flags = 42
-        assert lib2.foo_verify_dlopen_flags == 42
-        lib2.foo_verify_dlopen_flags += 1
-        assert lib1.foo_verify_dlopen_flags == 43
     finally:
         sys.setdlopenflags(old)
 
-def get_second_lib():
-    # Hack, using modulename makes the test fail
     ffi2 = FFI()
-    ffi2.cdef("int foo_verify_dlopen_flags;")
-    lib2 = ffi2.verify("int foo_verify_dlopen_flags;",
-                       flags=ffi2.RTLD_GLOBAL | ffi2.RTLD_LAZY)
-    return lib2
+    ffi2.cdef("int *getptr(void);")
+    lib2 = ffi2.verify("""
+        extern int foo_verify_dlopen_flags;
+        static int *getptr(void) { return &foo_verify_dlopen_flags; }
+    """)
+    p = lib2.getptr()
+    assert ffi1.addressof(lib1, 'foo_verify_dlopen_flags') == p
 
 def test_consider_not_implemented_function_type():
     ffi = FFI()
diff --git a/testing/cffi1/test_zdist.py b/testing/cffi1/test_zdist.py
--- a/testing/cffi1/test_zdist.py
+++ b/testing/cffi1/test_zdist.py
@@ -29,13 +29,17 @@
         if hasattr(self, 'saved_cwd'):
             os.chdir(self.saved_cwd)
 
-    def run(self, args):
+    def run(self, args, cwd=None):
         env = os.environ.copy()
-        newpath = self.rootdir
-        if 'PYTHONPATH' in env:
-            newpath += os.pathsep + env['PYTHONPATH']
-        env['PYTHONPATH'] = newpath
-        subprocess.check_call([self.executable] + args, env=env)
+        # a horrible hack to prevent distutils from finding ~/.pydistutils.cfg
+        # (there is the --no-user-cfg option, but not in Python 2.6...)
+        env['HOME'] = '/this/path/does/not/exist'
+        if cwd is None:
+            newpath = self.rootdir
+            if 'PYTHONPATH' in env:
+                newpath += os.pathsep + env['PYTHONPATH']
+            env['PYTHONPATH'] = newpath
+        subprocess.check_call([self.executable] + args, cwd=cwd, env=env)
 
     def _prepare_setuptools(self):
         if hasattr(TestDist, '_setuptools_ready'):
@@ -44,8 +48,7 @@
             import setuptools
         except ImportError:
             py.test.skip("setuptools not found")
-        subprocess.check_call([self.executable, 'setup.py', 'egg_info'],
-                              cwd=self.rootdir)
+        self.run(['setup.py', 'egg_info'], cwd=self.rootdir)
         TestDist._setuptools_ready = True
 
     def check_produced_files(self, content, curdir=None):

From noreply at buildbot.pypy.org  Sun Jun  7 14:34:13 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sun,  7 Jun 2015 14:34:13 +0200 (CEST)
Subject: [pypy-commit] cffi release-1.1: Bump version number to 1.1.1
Message-ID: <20150607123413.4C62D1C02FD@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: release-1.1
Changeset: r2166:e912e8961f9e
Date: 2015-06-07 14:34 +0200
http://bitbucket.org/cffi/cffi/changeset/e912e8961f9e/

Log:	Bump version number to 1.1.1

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -6063,7 +6063,7 @@
     if (v == NULL || PyModule_AddObject(m, "_C_API", v) < 0)
         INITERROR;
 
-    v = PyText_FromString("1.1.0");
+    v = PyText_FromString("1.1.1");
     if (v == NULL || PyModule_AddObject(m, "__version__", v) < 0)
         INITERROR;
 
diff --git a/c/test_c.py b/c/test_c.py
--- a/c/test_c.py
+++ b/c/test_c.py
@@ -3346,4 +3346,4 @@
 
 def test_version():
     # this test is here mostly for PyPy
-    assert __version__ == "1.1.0"
+    assert __version__ == "1.1.1"
diff --git a/cffi/__init__.py b/cffi/__init__.py
--- a/cffi/__init__.py
+++ b/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI, CDefError, FFIError
 from .ffiplatform import VerificationError, VerificationMissing
 
-__version__ = "1.1.0"
-__version_info__ = (1, 1, 0)
+__version__ = "1.1.1"
+__version_info__ = (1, 1, 1)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/doc/source/conf.py b/doc/source/conf.py
--- a/doc/source/conf.py
+++ b/doc/source/conf.py
@@ -47,7 +47,7 @@
 # The short X.Y version.
 version = '1.1'
 # The full version, including alpha/beta/rc tags.
-release = '1.1.0'
+release = '1.1.1'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
diff --git a/doc/source/installation.rst b/doc/source/installation.rst
--- a/doc/source/installation.rst
+++ b/doc/source/installation.rst
@@ -51,13 +51,13 @@
 
 Download and Installation:
 
-* http://pypi.python.org/packages/source/c/cffi/cffi-1.1.0.tar.gz
+* http://pypi.python.org/packages/source/c/cffi/cffi-1.1.1.tar.gz
 
    - Or grab the most current version by following the instructions below.
 
-   - MD5: b58d43a708e757f63a905c6a0d9ecf7a
+   - MD5: ...
 
-   - SHA: 7c36b783156eaf985b35a56c43f3eecac37e262c
+   - SHA: ...
 
 * Or get it from the `Bitbucket page`_:
   ``hg clone https://bitbucket.org/cffi/cffi``
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -144,7 +144,7 @@
 
 `Mailing list <https://groups.google.com/forum/#!forum/python-cffi>`_
 """,
-        version='1.1.0',
+        version='1.1.1',
         packages=['cffi'] if cpython else [],
         package_data={'cffi': ['_cffi_include.h', 'parse_c_type.h']}
                      if cpython else {},

From noreply at buildbot.pypy.org  Sun Jun  7 15:12:55 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sun,  7 Jun 2015 15:12:55 +0200 (CEST)
Subject: [pypy-commit] cffi release-1.1: Some details
Message-ID: <20150607131255.2505D1C02FD@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: release-1.1
Changeset: r2167:ad072efcccfd
Date: 2015-06-07 15:13 +0200
http://bitbucket.org/cffi/cffi/changeset/ad072efcccfd/

Log:	Some details

diff --git a/c/lib_obj.c b/c/lib_obj.c
--- a/c/lib_obj.c
+++ b/c/lib_obj.c
@@ -300,7 +300,7 @@
     case _CFFI_OP_GLOBAL_VAR:
     {
         /* global variable of the exact type specified here */
-        size_t g_size = (size_t)g->size_or_direct_fn;
+        Py_ssize_t g_size = (Py_ssize_t)g->size_or_direct_fn;
         ct = realize_c_type(types_builder, types_builder->ctx.types,
                             _CFFI_GETARG(g->type_op));
         if (ct == NULL)
diff --git a/c/realize_c_type.c b/c/realize_c_type.c
--- a/c/realize_c_type.c
+++ b/c/realize_c_type.c
@@ -60,7 +60,7 @@
 static void free_builder_c(builder_c_t *builder, int ctx_is_static)
 {
     if (!ctx_is_static) {
-        int i;
+        size_t i;
         const void *mem[] = {builder->ctx.types,
                              builder->ctx.globals,
                              builder->ctx.struct_unions,
diff --git a/testing/cffi0/test_zintegration.py b/testing/cffi0/test_zintegration.py
--- a/testing/cffi0/test_zintegration.py
+++ b/testing/cffi0/test_zintegration.py
@@ -1,5 +1,4 @@
 import py, os, sys, shutil
-import imp
 import subprocess
 from testing.udir import udir
 
@@ -15,28 +14,12 @@
     except OSError as e:
         py.test.skip("Cannot execute virtualenv: %s" % (e,))
 
-    try:
-        deepcopy = os.symlink
-    except:
-        import shutil, errno
-        def deepcopy(src, dst):
-            try:
-                shutil.copytree(src, dst)
-            except OSError as e:
-                if e.errno in (errno.ENOTDIR, errno.EINVAL):
-                    shutil.copy(src, dst)
-                else:
-                    print('got errno')
-                    print(e.errno)
-                    print('not')
-                    print(errno.ENOTDIR)
-                    raise
-
     site_packages = None
     for dirpath, dirnames, filenames in os.walk(str(tmpdir)):
         if os.path.basename(dirpath) == 'site-packages':
             site_packages = dirpath
             break
+    paths = ""
     if site_packages:
         try:
             from cffi import _pycparser
@@ -49,15 +32,22 @@
                 pass
             else:
                 modules += ('ply',)   # needed for older versions of pycparser
+        paths = []
         for module in modules:
-            target = imp.find_module(module)[1]
-            deepcopy(target, os.path.join(site_packages,
-                                            os.path.basename(target)))
-    return tmpdir
+            target = __import__(module, None, None, [])
+            src = os.path.abspath(target.__file__)
+            for end in ['__init__.pyc', '__init__.pyo', '__init__.py']:
+                if src.lower().endswith(end):
+                    src = src[:-len(end)-1]
+                    break
+            paths.append(os.path.dirname(src))
+        paths = os.pathsep.join(paths)
+    return tmpdir, paths
 
 SNIPPET_DIR = py.path.local(__file__).join('..', 'snippets')
 
-def really_run_setup_and_program(dirname, venv_dir, python_snippet):
+def really_run_setup_and_program(dirname, venv_dir_and_paths, python_snippet):
+    venv_dir, paths = venv_dir_and_paths
     def remove(dir):
         dir = str(SNIPPET_DIR.join(dirname, dir))
         shutil.rmtree(dir, ignore_errors=True)
@@ -75,9 +65,11 @@
         else:
             bindir = 'bin'
         vp = str(venv_dir.join(bindir).join('python'))
-        subprocess.check_call((vp, 'setup.py', 'clean'))
-        subprocess.check_call((vp, 'setup.py', 'install'))
-        subprocess.check_call((vp, str(python_f)))
+        env = os.environ.copy()
+        env['PYTHONPATH'] = paths
+        subprocess.check_call((vp, 'setup.py', 'clean'), env=env)
+        subprocess.check_call((vp, 'setup.py', 'install'), env=env)
+        subprocess.check_call((vp, str(python_f)), env=env)
     finally:
         os.chdir(olddir)
 
diff --git a/testing/cffi1/test_ffi_obj.py b/testing/cffi1/test_ffi_obj.py
--- a/testing/cffi1/test_ffi_obj.py
+++ b/testing/cffi1/test_ffi_obj.py
@@ -159,9 +159,10 @@
                             "struct never_heard_of_s\n"
                             "       ^")
     e = py.test.raises(ffi.error, ffi.cast, "\t\n\x01\x1f~\x7f\x80\xff", 0)
+    marks = "?" if sys.version_info < (3,) else "??"
     assert str(e.value) == ("identifier expected\n"
-                            "  ??~???\n"
-                            "  ^")
+                            "  ??~?%s%s\n"
+                            "  ^" % (marks, marks))
     e = py.test.raises(ffi.error, ffi.cast, "X" * 600, 0)
     assert str(e.value) == ("undefined type name")
 

From noreply at buildbot.pypy.org  Sun Jun  7 15:22:12 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sun,  7 Jun 2015 15:22:12 +0200 (CEST)
Subject: [pypy-commit] cffi release-1.1: md5/sha1
Message-ID: <20150607132212.3A90A1C0EE0@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: release-1.1
Changeset: r2168:8daca9ce750e
Date: 2015-06-07 15:22 +0200
http://bitbucket.org/cffi/cffi/changeset/8daca9ce750e/

Log:	md5/sha1

diff --git a/doc/source/installation.rst b/doc/source/installation.rst
--- a/doc/source/installation.rst
+++ b/doc/source/installation.rst
@@ -55,9 +55,9 @@
 
    - Or grab the most current version by following the instructions below.
 
-   - MD5: ...
+   - MD5: f397363bfbf99048accb0498ffc3e72b
 
-   - SHA: ...
+   - SHA: 8c4f4d1078d05c796c12fc6d8f8cea25aaff0148
 
 * Or get it from the `Bitbucket page`_:
   ``hg clone https://bitbucket.org/cffi/cffi``

From noreply at buildbot.pypy.org  Sun Jun  7 15:22:13 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sun,  7 Jun 2015 15:22:13 +0200 (CEST)
Subject: [pypy-commit] cffi default: hg merge release-1.1
Message-ID: <20150607132213.532341C0EE0@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2169:5c10cf136437
Date: 2015-06-07 15:22 +0200
http://bitbucket.org/cffi/cffi/changeset/5c10cf136437/

Log:	hg merge release-1.1

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -6063,7 +6063,7 @@
     if (v == NULL || PyModule_AddObject(m, "_C_API", v) < 0)
         INITERROR;
 
-    v = PyText_FromString("1.1.0");
+    v = PyText_FromString("1.1.1");
     if (v == NULL || PyModule_AddObject(m, "__version__", v) < 0)
         INITERROR;
 
diff --git a/c/lib_obj.c b/c/lib_obj.c
--- a/c/lib_obj.c
+++ b/c/lib_obj.c
@@ -300,7 +300,7 @@
     case _CFFI_OP_GLOBAL_VAR:
     {
         /* global variable of the exact type specified here */
-        size_t g_size = (size_t)g->size_or_direct_fn;
+        Py_ssize_t g_size = (Py_ssize_t)g->size_or_direct_fn;
         ct = realize_c_type(types_builder, types_builder->ctx.types,
                             _CFFI_GETARG(g->type_op));
         if (ct == NULL)
diff --git a/c/realize_c_type.c b/c/realize_c_type.c
--- a/c/realize_c_type.c
+++ b/c/realize_c_type.c
@@ -60,7 +60,7 @@
 static void free_builder_c(builder_c_t *builder, int ctx_is_static)
 {
     if (!ctx_is_static) {
-        int i;
+        size_t i;
         const void *mem[] = {builder->ctx.types,
                              builder->ctx.globals,
                              builder->ctx.struct_unions,
diff --git a/c/test_c.py b/c/test_c.py
--- a/c/test_c.py
+++ b/c/test_c.py
@@ -3346,4 +3346,4 @@
 
 def test_version():
     # this test is here mostly for PyPy
-    assert __version__ == "1.1.0"
+    assert __version__ == "1.1.1"
diff --git a/cffi/__init__.py b/cffi/__init__.py
--- a/cffi/__init__.py
+++ b/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI, CDefError, FFIError
 from .ffiplatform import VerificationError, VerificationMissing
 
-__version__ = "1.1.0"
-__version_info__ = (1, 1, 0)
+__version__ = "1.1.1"
+__version_info__ = (1, 1, 1)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/doc/source/conf.py b/doc/source/conf.py
--- a/doc/source/conf.py
+++ b/doc/source/conf.py
@@ -47,7 +47,7 @@
 # The short X.Y version.
 version = '1.1'
 # The full version, including alpha/beta/rc tags.
-release = '1.1.0'
+release = '1.1.1'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
diff --git a/doc/source/installation.rst b/doc/source/installation.rst
--- a/doc/source/installation.rst
+++ b/doc/source/installation.rst
@@ -51,13 +51,13 @@
 
 Download and Installation:
 
-* http://pypi.python.org/packages/source/c/cffi/cffi-1.1.0.tar.gz
+* http://pypi.python.org/packages/source/c/cffi/cffi-1.1.1.tar.gz
 
    - Or grab the most current version by following the instructions below.
 
-   - MD5: b58d43a708e757f63a905c6a0d9ecf7a
+   - MD5: f397363bfbf99048accb0498ffc3e72b
 
-   - SHA: 7c36b783156eaf985b35a56c43f3eecac37e262c
+   - SHA: 8c4f4d1078d05c796c12fc6d8f8cea25aaff0148
 
 * Or get it from the `Bitbucket page`_:
   ``hg clone https://bitbucket.org/cffi/cffi``
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -144,7 +144,7 @@
 
 `Mailing list <https://groups.google.com/forum/#!forum/python-cffi>`_
 """,
-        version='1.1.0',
+        version='1.1.1',
         packages=['cffi'] if cpython else [],
         package_data={'cffi': ['_cffi_include.h', 'parse_c_type.h']}
                      if cpython else {},
diff --git a/testing/cffi0/test_zintegration.py b/testing/cffi0/test_zintegration.py
--- a/testing/cffi0/test_zintegration.py
+++ b/testing/cffi0/test_zintegration.py
@@ -1,5 +1,4 @@
 import py, os, sys, shutil
-import imp
 import subprocess
 from testing.udir import udir
 
@@ -15,28 +14,12 @@
     except OSError as e:
         py.test.skip("Cannot execute virtualenv: %s" % (e,))
 
-    try:
-        deepcopy = os.symlink
-    except:
-        import shutil, errno
-        def deepcopy(src, dst):
-            try:
-                shutil.copytree(src, dst)
-            except OSError as e:
-                if e.errno in (errno.ENOTDIR, errno.EINVAL):
-                    shutil.copy(src, dst)
-                else:
-                    print('got errno')
-                    print(e.errno)
-                    print('not')
-                    print(errno.ENOTDIR)
-                    raise
-
     site_packages = None
     for dirpath, dirnames, filenames in os.walk(str(tmpdir)):
         if os.path.basename(dirpath) == 'site-packages':
             site_packages = dirpath
             break
+    paths = ""
     if site_packages:
         try:
             from cffi import _pycparser
@@ -49,15 +32,22 @@
                 pass
             else:
                 modules += ('ply',)   # needed for older versions of pycparser
+        paths = []
         for module in modules:
-            target = imp.find_module(module)[1]
-            deepcopy(target, os.path.join(site_packages,
-                                            os.path.basename(target)))
-    return tmpdir
+            target = __import__(module, None, None, [])
+            src = os.path.abspath(target.__file__)
+            for end in ['__init__.pyc', '__init__.pyo', '__init__.py']:
+                if src.lower().endswith(end):
+                    src = src[:-len(end)-1]
+                    break
+            paths.append(os.path.dirname(src))
+        paths = os.pathsep.join(paths)
+    return tmpdir, paths
 
 SNIPPET_DIR = py.path.local(__file__).join('..', 'snippets')
 
-def really_run_setup_and_program(dirname, venv_dir, python_snippet):
+def really_run_setup_and_program(dirname, venv_dir_and_paths, python_snippet):
+    venv_dir, paths = venv_dir_and_paths
     def remove(dir):
         dir = str(SNIPPET_DIR.join(dirname, dir))
         shutil.rmtree(dir, ignore_errors=True)
@@ -75,9 +65,11 @@
         else:
             bindir = 'bin'
         vp = str(venv_dir.join(bindir).join('python'))
-        subprocess.check_call((vp, 'setup.py', 'clean'))
-        subprocess.check_call((vp, 'setup.py', 'install'))
-        subprocess.check_call((vp, str(python_f)))
+        env = os.environ.copy()
+        env['PYTHONPATH'] = paths
+        subprocess.check_call((vp, 'setup.py', 'clean'), env=env)
+        subprocess.check_call((vp, 'setup.py', 'install'), env=env)
+        subprocess.check_call((vp, str(python_f)), env=env)
     finally:
         os.chdir(olddir)
 
diff --git a/testing/cffi1/test_ffi_obj.py b/testing/cffi1/test_ffi_obj.py
--- a/testing/cffi1/test_ffi_obj.py
+++ b/testing/cffi1/test_ffi_obj.py
@@ -159,9 +159,10 @@
                             "struct never_heard_of_s\n"
                             "       ^")
     e = py.test.raises(ffi.error, ffi.cast, "\t\n\x01\x1f~\x7f\x80\xff", 0)
+    marks = "?" if sys.version_info < (3,) else "??"
     assert str(e.value) == ("identifier expected\n"
-                            "  ??~???\n"
-                            "  ^")
+                            "  ??~?%s%s\n"
+                            "  ^" % (marks, marks))
     e = py.test.raises(ffi.error, ffi.cast, "X" * 600, 0)
     assert str(e.value) == ("undefined type name")
 

From noreply at buildbot.pypy.org  Sun Jun  7 15:59:50 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sun,  7 Jun 2015 15:59:50 +0200 (CEST)
Subject: [pypy-commit] cffi default: Support "[][...]", "[5][...]", etc.
Message-ID: <20150607135950.6D1111C1239@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2170:792c0cbe632d
Date: 2015-06-07 16:00 +0200
http://bitbucket.org/cffi/cffi/changeset/792c0cbe632d/

Log:	Support "[][...]", "[5][...]", etc.

diff --git a/cffi/cparser.py b/cffi/cparser.py
--- a/cffi/cparser.py
+++ b/cffi/cparser.py
@@ -337,7 +337,7 @@
                 length = self._parse_constant(
                     typenode.dim, partial_length_ok=partial_length_ok)
             tp = self._get_type(typenode.type,
-                                partial_length_ok=(length == '...'))
+                                partial_length_ok=partial_length_ok)
             return model.ArrayType(tp, length)
         #
         if isinstance(typenode, pycparser.c_ast.PtrDecl):
diff --git a/cffi/recompiler.py b/cffi/recompiler.py
--- a/cffi/recompiler.py
+++ b/cffi/recompiler.py
@@ -749,10 +749,12 @@
     # named structs or unions
 
     def _field_type(self, tp_struct, field_name, tp_field):
-        if isinstance(tp_field, model.ArrayType) and tp_field.length == '...':
-            ptr_struct_name = tp_struct.get_c_name('*')
-            actual_length = '_cffi_array_len(((%s)0)->%s)' % (
-                ptr_struct_name, field_name)
+        if isinstance(tp_field, model.ArrayType):
+            actual_length = tp_field.length
+            if actual_length == '...':
+                ptr_struct_name = tp_struct.get_c_name('*')
+                actual_length = '_cffi_array_len(((%s)0)->%s)' % (
+                    ptr_struct_name, field_name)
             tp_item = self._field_type(tp_struct, '%s[0]' % field_name,
                                        tp_field.item)
             tp_field = model.ArrayType(tp_item, actual_length)
@@ -1055,8 +1057,10 @@
     # global variables
 
     def _global_type(self, tp, global_name):
-        if isinstance(tp, model.ArrayType) and tp.length == '...':
-            actual_length = '_cffi_array_len(%s)' % (global_name,)
+        if isinstance(tp, model.ArrayType):
+            actual_length = tp.length
+            if actual_length == '...':
+                actual_length = '_cffi_array_len(%s)' % (global_name,)
             tp_item = self._global_type(tp.item, '%s[0]' % global_name)
             tp = model.ArrayType(tp_item, actual_length)
         return tp
diff --git a/doc/source/whatsnew.rst b/doc/source/whatsnew.rst
--- a/doc/source/whatsnew.rst
+++ b/doc/source/whatsnew.rst
@@ -3,6 +3,19 @@
 ======================
 
 
+1.1.2
+=====
+
+* Out-of-line mode: ``int a[][...];`` can be used to declare a structure
+  field or global variable which is simultanously of total length
+  unknown to the C compiler (the ``[]`` part), but each element is an
+  array of N integers, where the value of N *is* known to the C compiler
+  (the ``int [...]`` part around).  Similarly, ``int a[5][...];`` is
+  supported (but probably less useful).  Remember that in the order of
+  the C syntax, it means an array of 5 things, each of which is an array
+  of N integers---and ask the C compiler for the value of N.
+
+
 1.1.1
 =====
 
diff --git a/testing/cffi1/test_recompiler.py b/testing/cffi1/test_recompiler.py
--- a/testing/cffi1/test_recompiler.py
+++ b/testing/cffi1/test_recompiler.py
@@ -924,6 +924,18 @@
     assert ffi.typeof(s.a) == ffi.typeof("int[5][8]")
     assert ffi.typeof(s.a[0]) == ffi.typeof("int[8]")
 
+def test_struct_array_guess_length_3():
+    ffi = FFI()
+    ffi.cdef("struct foo_s { int a[][...]; };")
+    lib = verify(ffi, 'test_struct_array_guess_length_3',
+                 "struct foo_s { int x; int a[5][7]; int y; };")
+    assert ffi.sizeof('struct foo_s') == 37 * ffi.sizeof('int')
+    s = ffi.new("struct foo_s *")
+    assert ffi.typeof(s.a) == ffi.typeof("int(*)[7]")
+    assert s.a[4][6] == 0
+    py.test.raises(IndexError, 's.a[4][7]')
+    assert ffi.typeof(s.a[0]) == ffi.typeof("int[7]")
+
 def test_global_var_array_2():
     ffi = FFI()
     ffi.cdef("int a[...][...];")
@@ -935,6 +947,27 @@
     assert ffi.typeof(lib.a) == ffi.typeof("int[10][8]")
     assert ffi.typeof(lib.a[0]) == ffi.typeof("int[8]")
 
+def test_global_var_array_3():
+    ffi = FFI()
+    ffi.cdef("int a[][...];")
+    lib = verify(ffi, 'test_global_var_array_3', 'int a[10][8];')
+    lib.a[9][7] = 123456
+    assert lib.a[9][7] == 123456
+    py.test.raises(IndexError, 'lib.a[0][8]')
+    assert ffi.typeof(lib.a) == ffi.typeof("int(*)[8]")
+    assert ffi.typeof(lib.a[0]) == ffi.typeof("int[8]")
+
+def test_global_var_array_4():
+    ffi = FFI()
+    ffi.cdef("int a[10][...];")
+    lib = verify(ffi, 'test_global_var_array_4', 'int a[10][8];')
+    lib.a[9][7] = 123456
+    assert lib.a[9][7] == 123456
+    py.test.raises(IndexError, 'lib.a[0][8]')
+    py.test.raises(IndexError, 'lib.a[10][8]')
+    assert ffi.typeof(lib.a) == ffi.typeof("int[10][8]")
+    assert ffi.typeof(lib.a[0]) == ffi.typeof("int[8]")
+
 def test_some_integer_type():
     ffi = FFI()
     ffi.cdef("""

From noreply at buildbot.pypy.org  Sun Jun  7 17:16:59 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Sun,  7 Jun 2015 17:16:59 +0200 (CEST)
Subject: [pypy-commit] pypy default: allow only C, F in concrete type order
Message-ID: <20150607151659.03E1D1C02FD@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: 
Changeset: r77938:ccf856368ecd
Date: 2015-06-07 16:10 +0300
http://bitbucket.org/pypy/pypy/changeset/ccf856368ecd/

Log:	allow only C, F in concrete type order

diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py
--- a/pypy/module/micronumpy/concrete.py
+++ b/pypy/module/micronumpy/concrete.py
@@ -338,23 +338,21 @@
         # but make the array storage contiguous in memory
         shape = self.get_shape()
         strides = self.get_strides()
-        if len(strides) > 0:
+        if order not in ('C', 'F'):
+            raise oefmt(space.w_ValueError, "Unknown order %s in astype", order)
+        if len(strides) == 0:
+            t_strides = []
+            backstrides = []
+        elif order != self.order:
+            t_strides, backstrides = calc_strides(shape, dtype, order)
+        else:
             mins = strides[0]
             t_elsize = dtype.elsize
             for s in strides:
                 if s < mins:
                     mins = s
             t_strides = [s * t_elsize / mins for s in strides]
-            if order == 'K':
-                pass
-            elif order not in ('C', 'F'):
-                raise oefmt(space.w_ValueError, "Unknown order %s in astype", order)
-            elif order != self.order:
-                t_strides.reverse()
             backstrides = calc_backstrides(t_strides, shape)
-        else:
-            t_strides = []
-            backstrides = []
         impl = ConcreteArray(shape, dtype, order, t_strides, backstrides)
         loop.setslice(space, impl.get_shape(), impl, self)
         return impl
diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py
--- a/pypy/module/micronumpy/ndarray.py
+++ b/pypy/module/micronumpy/ndarray.py
@@ -44,16 +44,6 @@
         raise oefmt(space.w_ValueError, "objects are not aligned")
     return out_shape, right_critical_dim
 
-def get_order(proto_order, order):
-    if order == 'C':
-        return 'C'
-    elif order == 'F':
-        return 'F'
-    elif order == 'K':
-        return proto_order
-    elif order == 'A':
-        return proto_order
-
 class __extend__(W_NDimArray):
     @jit.unroll_safe
     def descr_get_shape(self, space):
@@ -620,10 +610,10 @@
                     space, 'S' + str(cur_dtype.elsize))
         if not can_cast_array(space, self, new_dtype, casting):
             raise oefmt(space.w_TypeError, "Cannot cast array from %s to %s"
-                        "according to the rule %s", 
+                        "according to the rule %s",
                         space.str_w(self.get_dtype().descr_repr(space)),
                         space.str_w(new_dtype.descr_repr(space)), casting)
-        order  = get_order(self.get_order(), order)
+        order  = support.get_order_as_CF(self.get_order(), order)
         if (not copy and new_dtype == self.get_dtype() and order == self.get_order()
                 and (subok or type(self) is W_NDimArray)):
             return self
diff --git a/pypy/module/micronumpy/nditer.py b/pypy/module/micronumpy/nditer.py
--- a/pypy/module/micronumpy/nditer.py
+++ b/pypy/module/micronumpy/nditer.py
@@ -460,17 +460,18 @@
         # handle w_op_dtypes part 2: copy where needed if possible
         if len(self.dtypes) > 0:
             for i in range(len(self.seq)):
-                selfd = self.dtypes[i]
+                self_d = self.dtypes[i]
                 seq_d = self.seq[i].get_dtype()
-                if not selfd:
+                if not self_d:
                     self.dtypes[i] = seq_d
-                elif selfd != seq_d:
+                elif self_d != seq_d:
                     if not 'r' in self.op_flags[i].tmp_copy:
                         raise oefmt(space.w_TypeError,
                                     "Iterator operand required copying or "
                                     "buffering for operand %d", i)
                     impl = self.seq[i].implementation
-                    new_impl = impl.astype(space, selfd, self.order)
+                    order = support.get_order_as_CF(impl.order, self.order)
+                    new_impl = impl.astype(space, self_d, order)
                     self.seq[i] = W_NDimArray(new_impl)
         else:
             #copy them from seq
diff --git a/pypy/module/micronumpy/support.py b/pypy/module/micronumpy/support.py
--- a/pypy/module/micronumpy/support.py
+++ b/pypy/module/micronumpy/support.py
@@ -161,3 +161,14 @@
     w_priority_r = space.findattr(w_rhs, space.wrap('__array_priority__')) or w_zero
     # XXX what is better, unwrapping values or space.gt?
     return space.is_true(space.gt(w_priority_r, w_priority_l))
+
+def get_order_as_CF(proto_order, req_order):
+    if req_order == 'C':
+        return 'C'
+    elif req_order == 'F':
+        return 'F'
+    elif req_order == 'K':
+        return proto_order
+    elif req_order == 'A':
+        return proto_order
+

From noreply at buildbot.pypy.org  Sun Jun  7 17:17:00 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Sun,  7 Jun 2015 17:17:00 +0200 (CEST)
Subject: [pypy-commit] pypy default: test, fix converting object dtype to str
Message-ID: <20150607151700.6CB571C02FD@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: 
Changeset: r77939:e6db7b9cfa59
Date: 2015-06-07 18:16 +0300
http://bitbucket.org/pypy/pypy/changeset/e6db7b9cfa59/

Log:	test, fix converting object dtype to str

diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py
--- a/pypy/module/micronumpy/ndarray.py
+++ b/pypy/module/micronumpy/ndarray.py
@@ -604,10 +604,14 @@
             raise oefmt(space.w_NotImplementedError,
                         "astype(%s) not implemented yet",
                         new_dtype.get_name())
-        if new_dtype.num == NPY.STRING and new_dtype.elsize == 0:
-            if cur_dtype.num == NPY.STRING:
-                new_dtype = descriptor.variable_dtype(
-                    space, 'S' + str(cur_dtype.elsize))
+        if new_dtype.is_str() and new_dtype.elsize == 0:
+            elsize = 0
+            itype = cur_dtype.itemtype
+            for i in range(self.get_size()):
+                elsize = max(elsize, len(itype.str_format(self.implementation.getitem(i), add_quotes=False)))
+            new_dtype = descriptor.variable_dtype(
+                    space, 'S' + str(elsize))
+
         if not can_cast_array(space, self, new_dtype, casting):
             raise oefmt(space.w_TypeError, "Cannot cast array from %s to %s"
                         "according to the rule %s",
diff --git a/pypy/module/micronumpy/test/test_object_arrays.py b/pypy/module/micronumpy/test/test_object_arrays.py
--- a/pypy/module/micronumpy/test/test_object_arrays.py
+++ b/pypy/module/micronumpy/test/test_object_arrays.py
@@ -83,8 +83,8 @@
     def test_complex_op(self):
         import numpy as np
         import sys
-        a = np.array(['abc', 'def'], dtype=object) 
-        b = np.array([1, 2, 3], dtype=object) 
+        a = np.array(['abc', 'def'], dtype=object)
+        b = np.array([1, 2, 3], dtype=object)
         c = np.array([complex(1, 1), complex(1, -1)], dtype=object)
         for arg in (a,b,c):
             assert (arg == np.real(arg)).all()
@@ -164,3 +164,11 @@
         a = np.array([(1, 'object')], dt)
         # Wrong way - should complain about writing buffer to object dtype
         raises(ValueError, np.array, [1, 'object'], dt)
+
+    def test_astype(self):
+        import numpy as np
+        a = np.array([b'a' * 100], dtype='O')
+        assert 'a' * 100 in str(a)
+        b = a.astype('S')
+        assert 'a' * 100 in str(b)
+

From noreply at buildbot.pypy.org  Sun Jun  7 19:16:54 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sun,  7 Jun 2015 19:16:54 +0200 (CEST)
Subject: [pypy-commit] cffi default: Rephrase
Message-ID: <20150607171654.62C231C02FD@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2171:075c4ebb1e4f
Date: 2015-06-07 19:17 +0200
http://bitbucket.org/cffi/cffi/changeset/075c4ebb1e4f/

Log:	Rephrase

diff --git a/doc/source/whatsnew.rst b/doc/source/whatsnew.rst
--- a/doc/source/whatsnew.rst
+++ b/doc/source/whatsnew.rst
@@ -7,13 +7,12 @@
 =====
 
 * Out-of-line mode: ``int a[][...];`` can be used to declare a structure
-  field or global variable which is simultanously of total length
-  unknown to the C compiler (the ``[]`` part), but each element is an
-  array of N integers, where the value of N *is* known to the C compiler
-  (the ``int [...]`` part around).  Similarly, ``int a[5][...];`` is
-  supported (but probably less useful).  Remember that in the order of
-  the C syntax, it means an array of 5 things, each of which is an array
-  of N integers---and ask the C compiler for the value of N.
+  field or global variable which is, simultaneously, of total length
+  unknown to the C compiler (the ``a[]`` part) and each element is
+  itself an array of N integers, where the value of N *is* known to the
+  C compiler (the ``int`` and ``[...]`` parts around it).  Similarly,
+  ``int a[5][...];`` is supported (but probably less useful: remember
+  that in C it means ``int (a[5])[...];``).
 
 
 1.1.1

From noreply at buildbot.pypy.org  Sun Jun  7 19:36:23 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sun,  7 Jun 2015 19:36:23 +0200 (CEST)
Subject: [pypy-commit] cffi default: Move this to the future cffi 1.2,
 and explain it more in cdef.rst.
Message-ID: <20150607173623.6AA0E1C02FD@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2172:4ca0daf6ac3b
Date: 2015-06-07 19:37 +0200
http://bitbucket.org/cffi/cffi/changeset/4ca0daf6ac3b/

Log:	Move this to the future cffi 1.2, and explain it more in cdef.rst.

diff --git a/doc/source/cdef.rst b/doc/source/cdef.rst
--- a/doc/source/cdef.rst
+++ b/doc/source/cdef.rst
@@ -415,6 +415,15 @@
    no attempt is made to complete it.  *New in version 1.1:* support
    for multidimensional arrays: "``int n[...][...];``".
 
+   *New in version 1.2:* "``int m[][...];``", i.e. ``...`` can be used
+   in the innermost dimensions without being also used in the outermost
+   dimension.  In the example given, the length of the ``m`` array is
+   assumed not to be known to the C compiler, but the length of every
+   item (like the sub-array ``m[0]``) is always known the C compiler.
+   In other words, only the outermost dimension can be specified as
+   ``[]``, both in C and in CFFI, but any dimension can be given as
+   ``[...]`` in CFFI.
+
 *  enums: if you don't know the exact order (or values) of the declared
    constants, then use this syntax: "``enum foo { A, B, C, ... };``"
    (with a trailing "``...``").  The C compiler will be used to figure
diff --git a/doc/source/whatsnew.rst b/doc/source/whatsnew.rst
--- a/doc/source/whatsnew.rst
+++ b/doc/source/whatsnew.rst
@@ -3,7 +3,7 @@
 ======================
 
 
-1.1.2
+1.2.0
 =====
 
 * Out-of-line mode: ``int a[][...];`` can be used to declare a structure

From noreply at buildbot.pypy.org  Sun Jun  7 20:58:11 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sun,  7 Jun 2015 20:58:11 +0200 (CEST)
Subject: [pypy-commit] pypy default: forgot to check this in
Message-ID: <20150607185811.2B1301C122D@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r77940:6ce28e4823d0
Date: 2015-06-07 20:57 +0200
http://bitbucket.org/pypy/pypy/changeset/6ce28e4823d0/

Log:	forgot to check this in

diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: cffi
-Version: 1.1.0
+Version: 1.1.1
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI, CDefError, FFIError
 from .ffiplatform import VerificationError, VerificationMissing
 
-__version__ = "1.1.0"
-__version_info__ = (1, 1, 0)
+__version__ = "1.1.1"
+__version_info__ = (1, 1, 1)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -2,7 +2,7 @@
 from pypy.interpreter.mixedmodule import MixedModule
 from rpython.rlib import rdynload
 
-VERSION = "1.1.0"
+VERSION = "1.1.1"
 
 
 class Module(MixedModule):
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -3335,4 +3335,4 @@
 
 def test_version():
     # this test is here mostly for PyPy
-    assert __version__ == "1.1.0"
+    assert __version__ == "1.1.1"

From noreply at buildbot.pypy.org  Sun Jun  7 20:58:12 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sun,  7 Jun 2015 20:58:12 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge heads
Message-ID: <20150607185812.5E9141C122D@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r77941:8c54d426c810
Date: 2015-06-07 20:58 +0200
http://bitbucket.org/pypy/pypy/changeset/8c54d426c810/

Log:	merge heads

diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: cffi
-Version: 1.1.0
+Version: 1.1.1
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI, CDefError, FFIError
 from .ffiplatform import VerificationError, VerificationMissing
 
-__version__ = "1.1.0"
-__version_info__ = (1, 1, 0)
+__version__ = "1.1.1"
+__version_info__ = (1, 1, 1)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -2,7 +2,7 @@
 from pypy.interpreter.mixedmodule import MixedModule
 from rpython.rlib import rdynload
 
-VERSION = "1.1.0"
+VERSION = "1.1.1"
 
 
 class Module(MixedModule):
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -3335,4 +3335,4 @@
 
 def test_version():
     # this test is here mostly for PyPy
-    assert __version__ == "1.1.0"
+    assert __version__ == "1.1.1"

From noreply at buildbot.pypy.org  Sun Jun  7 21:06:21 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sun,  7 Jun 2015 21:06:21 +0200 (CEST)
Subject: [pypy-commit] pypy default: Issue #2060: some attempt at getting
 the no-sse2 version working
Message-ID: <20150607190621.D5D0D1C1239@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r77942:0d4c2de3d999
Date: 2015-06-07 21:04 +0200
http://bitbucket.org/pypy/pypy/changeset/0d4c2de3d999/

Log:	Issue #2060: some attempt at getting the no-sse2 version working

diff --git a/rpython/jit/backend/llsupport/assembler.py b/rpython/jit/backend/llsupport/assembler.py
--- a/rpython/jit/backend/llsupport/assembler.py
+++ b/rpython/jit/backend/llsupport/assembler.py
@@ -109,10 +109,13 @@
                 kind='unicode')
         else:
             self.malloc_slowpath_unicode = None
-        self.cond_call_slowpath = [self._build_cond_call_slowpath(False, False),
-                                   self._build_cond_call_slowpath(False, True),
-                                   self._build_cond_call_slowpath(True, False),
-                                   self._build_cond_call_slowpath(True, True)]
+        lst = [0, 0, 0, 0]
+        lst[0] = self._build_cond_call_slowpath(False, False)
+        lst[1] = self._build_cond_call_slowpath(False, True)
+        if self.cpu.supports_floats:
+            lst[2] = self._build_cond_call_slowpath(True, False)
+            lst[3] = self._build_cond_call_slowpath(True, True)
+        self.cond_call_slowpath = lst
 
         self._build_stack_check_slowpath()
         self._build_release_gil(gc_ll_descr.gcrootmap)
diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -382,7 +382,8 @@
             # we have one word to align
             mc.SUB_ri(esp.value, 7 * WORD) # align and reserve some space
             mc.MOV_sr(WORD, eax.value) # save for later
-            mc.MOVSD_sx(2 * WORD, xmm0.value)   # 32-bit: also 3 * WORD
+            if self.cpu.supports_floats:
+                mc.MOVSD_sx(2 * WORD, xmm0.value)   # 32-bit: also 3 * WORD
             if IS_X86_32:
                 mc.MOV_sr(4 * WORD, edx.value)
                 mc.MOV_sr(0, ebp.value)
@@ -423,7 +424,8 @@
         else:
             if IS_X86_32:
                 mc.MOV_rs(edx.value, 4 * WORD)
-            mc.MOVSD_xs(xmm0.value, 2 * WORD)
+            if self.cpu.supports_floats:
+                mc.MOVSD_xs(xmm0.value, 2 * WORD)
             mc.MOV_rs(eax.value, WORD) # restore
             self._restore_exception(mc, exc0, exc1)
             mc.MOV(exc0, RawEspLoc(WORD * 5, REF))

From noreply at buildbot.pypy.org  Sun Jun  7 22:00:09 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sun,  7 Jun 2015 22:00:09 +0200 (CEST)
Subject: [pypy-commit] pypy default: Test fix: this test used to randomly
 close file descriptor 1!
Message-ID: <20150607200009.228461C080A@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r77943:d7d00adb03ae
Date: 2015-06-07 22:00 +0200
http://bitbucket.org/pypy/pypy/changeset/d7d00adb03ae/

Log:	Test fix: this test used to randomly close file descriptor 1!

diff --git a/pypy/module/_multiprocessing/test/test_connection.py b/pypy/module/_multiprocessing/test/test_connection.py
--- a/pypy/module/_multiprocessing/test/test_connection.py
+++ b/pypy/module/_multiprocessing/test/test_connection.py
@@ -189,9 +189,11 @@
         assert data2 == '\x00\x00\x00\x04defg'
 
     def test_repr(self):
-        import _multiprocessing
-        c = _multiprocessing.Connection(1)
-        assert repr(c) == '<read-write Connection, handle 1>'
+        import _multiprocessing, os
+        fd = os.dup(1)     # closed by Connection.__del__
+        c = _multiprocessing.Connection(fd)
+        assert repr(c) == '<read-write Connection, handle %d>' % fd
         if hasattr(_multiprocessing, 'PipeConnection'):
-            c = _multiprocessing.PipeConnection(1)
-            assert repr(c) == '<read-write PipeConnection, handle 1>'
+            fd = os.dup(1)     # closed by PipeConnection.__del__
+            c = _multiprocessing.PipeConnection(fd)
+            assert repr(c) == '<read-write PipeConnection, handle %d>' % fd

From noreply at buildbot.pypy.org  Sun Jun  7 22:00:10 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sun,  7 Jun 2015 22:00:10 +0200 (CEST)
Subject: [pypy-commit] pypy default: A bit more care about __del__ on
 half-initialized instances
Message-ID: <20150607200010.400781C080A@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r77944:4156be89b516
Date: 2015-06-07 22:00 +0200
http://bitbucket.org/pypy/pypy/changeset/4156be89b516/

Log:	A bit more care about __del__ on half-initialized instances

diff --git a/pypy/module/_multiprocessing/interp_connection.py b/pypy/module/_multiprocessing/interp_connection.py
--- a/pypy/module/_multiprocessing/interp_connection.py
+++ b/pypy/module/_multiprocessing/interp_connection.py
@@ -28,6 +28,7 @@
 
 class W_BaseConnection(W_Root):
     BUFFER_SIZE = 1024
+    buffer = lltype.nullptr(rffi.CCHARP.TO)
 
     def __init__(self, flags):
         self.flags = flags
@@ -35,7 +36,8 @@
                                     flavor='raw')
 
     def __del__(self):
-        lltype.free(self.buffer, flavor='raw')
+        if self.buffer:
+            lltype.free(self.buffer, flavor='raw')
         try:
             self.do_close()
         except OSError:
@@ -204,6 +206,7 @@
 
 class W_FileConnection(W_BaseConnection):
     INVALID_HANDLE_VALUE = -1
+    fd = INVALID_HANDLE_VALUE
 
     if sys.platform == 'win32':
         def WRITE(self, data):

From noreply at buildbot.pypy.org  Sun Jun  7 22:08:51 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sun,  7 Jun 2015 22:08:51 +0200 (CEST)
Subject: [pypy-commit] cffi default: Comment out that line from conf.py. It
 gives a warning with the newer
Message-ID: <20150607200851.EDE2D1C1239@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2173:eb4e88e0507e
Date: 2015-06-07 22:09 +0200
http://bitbucket.org/cffi/cffi/changeset/eb4e88e0507e/

Log:	Comment out that line from conf.py. It gives a warning with the
	newer sphinx 1.3...

diff --git a/doc/source/conf.py b/doc/source/conf.py
--- a/doc/source/conf.py
+++ b/doc/source/conf.py
@@ -91,7 +91,7 @@
 
 # The theme to use for HTML and HTML Help pages.  Major themes that come with
 # Sphinx are currently 'default' and 'sphinxdoc'.
-html_theme = 'default'
+#html_theme = 'default'
 
 # Theme options are theme-specific and customize the look and feel of a theme
 # further.  For a list of options available for each theme, see the

From noreply at buildbot.pypy.org  Sun Jun  7 22:29:26 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sun,  7 Jun 2015 22:29:26 +0200 (CEST)
Subject: [pypy-commit] pypy default: From this last-level helper,
 don't propagate further OSErrors
Message-ID: <20150607202926.3724C1C1239@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r77945:76611312113e
Date: 2015-06-07 22:29 +0200
http://bitbucket.org/pypy/pypy/changeset/76611312113e/

Log:	From this last-level helper, don't propagate further OSErrors

diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -21,7 +21,10 @@
     this_dir = os.path.dirname(sys.argv[0])
 
 def debug(msg):
-    os.write(2, "debug: " + msg + '\n')
+    try:
+        os.write(2, "debug: " + msg + '\n')
+    except OSError:
+        pass     # bah, no working stderr :-(
 
 # __________  Entry point  __________
 

From noreply at buildbot.pypy.org  Mon Jun  8 08:39:18 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Mon,  8 Jun 2015 08:39:18 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: tested insert x86 opcodes
Message-ID: <20150608063918.1EBA21C0460@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r77946:815c8b4a7c5b
Date: 2015-06-08 08:39 +0200
http://bitbucket.org/pypy/pypy/changeset/815c8b4a7c5b/

Log:	tested insert x86 opcodes

diff --git a/rpython/jit/backend/x86/rx86.py b/rpython/jit/backend/x86/rx86.py
--- a/rpython/jit/backend/x86/rx86.py
+++ b/rpython/jit/backend/x86/rx86.py
@@ -756,9 +756,9 @@
     PEXTRB_rxi = xmminsn('\x66', rex_nw, '\x0F\x3A\x14', register(1), register(2,8), '\xC0', immediate(3, 'b'))
     EXTRACTPS_rxi = xmminsn('\x66', rex_nw, '\x0F\x3A\x17', register(1), register(2,8), '\xC0', immediate(3, 'b'))
     
-    PINSRQ_xri = xmminsn('\x66', rex_w, '\x0F\x3A\x22', register(1,8), register(2,8), '\xC0', immediate(3, 'b'))
-    PINSRD_xri = xmminsn('\x66', rex_nw, '\x0F\x3A\x22', register(1,8), register(2,8), '\xC0', immediate(3, 'b'))
-    PINSRW_xri = xmminsn('\x66', rex_nw, '\x0F\xC4', register(1,8), register(2,8), '\xC0', immediate(3, 'b'))
+    PINSRQ_xri = xmminsn('\x66', rex_w, '\x0F\x3A\x22', register(1,8), register(2), '\xC0', immediate(3, 'b'))
+    PINSRD_xri = xmminsn('\x66', rex_nw, '\x0F\x3A\x22', register(1,8), register(2), '\xC0', immediate(3, 'b'))
+    PINSRW_xri = xmminsn('\x66', rex_nw, '\x0F\xC4', register(1,8), register(2), '\xC0', immediate(3, 'b'))
     PINSRB_xri = xmminsn('\x66', rex_nw, '\x0F\x3A\x20', register(1,8), register(2), '\xC0', immediate(3, 'b'))
     INSERTPS_xxi = xmminsn('\x66', rex_nw, '\x0F\x3A\x21', register(1,8), register(2), '\xC0', immediate(3, 'b'))
 
diff --git a/rpython/jit/backend/x86/test/test_rx86.py b/rpython/jit/backend/x86/test/test_rx86.py
--- a/rpython/jit/backend/x86/test/test_rx86.py
+++ b/rpython/jit/backend/x86/test/test_rx86.py
@@ -276,3 +276,46 @@
     s.clear()
     s.EXTRACTPS_rxi(R.r11, R.xmm0, 1)
     assert s.getvalue() == '\x66\x41\x0f\x3a\x17\xc3\x01'
+    s.clear()
+    s.EXTRACTPS_rxi(R.eax, R.xmm0, 1)
+    assert s.getvalue() == '\x66\x0f\x3a\x17\xc0\x01'
+    s.clear()
+    s.EXTRACTPS_rxi(R.r15, R.xmm15, 4)
+    assert s.getvalue() == '\x66\x45\x0f\x3a\x17\xff\x04'
+
+def test_pinsr():
+    s = CodeBuilder64()
+    s.PINSRW_xri(R.xmm0, R.r11,0)
+    assert s.getvalue() == '\x66\x41\x0f\xc4\xc3\x00'
+    s.clear()
+    s.PINSRW_xri(R.xmm15, R.edi, 15)
+    assert s.getvalue() == '\x66\x44\x0f\xc4\xff\x0f'
+    s.clear()
+    s.PINSRD_xri(R.xmm11, R.eax, 2)
+    assert s.getvalue() == '\x66\x44\x0f\x3a\x22\xd8\x02'
+    s.clear()
+    s.PINSRD_xri(R.xmm5, R.r11, 2)
+    assert s.getvalue() == '\x66\x41\x0f\x3a\x22\xeb\x02'
+    s.clear()
+    s.PINSRQ_xri(R.xmm0, R.ebp, 7)
+    assert s.getvalue() == '\x66\x48\x0f\x3a\x22\xc5\x07'
+    # BYTE
+    s.clear()
+    s.PINSRB_xri(R.xmm13, R.eax, 24)
+    assert s.getvalue() == '\x66\x44\x0f\x3a\x20\xe8\x18'
+    s.clear()
+    s.PINSRB_xri(R.xmm5, R.r15, 33)
+    assert s.getvalue() == '\x66\x41\x0f\x3a\x20\xef\x21'
+    # EXTR SINGLE FLOAT
+    s.clear()
+    s.INSERTPS_xxi(R.xmm15, R.xmm0, 2)
+    assert s.getvalue() == '\x66\x44\x0f\x3a\x21\xf8\x02'
+    s.clear()
+    s.INSERTPS_xxi(R.xmm0, R.xmm11, 1)
+    assert s.getvalue() == '\x66\x41\x0f\x3a\x21\xc3\x01'
+    s.clear()
+    s.INSERTPS_xxi(R.xmm0, R.xmm0, 1)
+    assert s.getvalue() == '\x66\x0f\x3a\x21\xc0\x01'
+    s.clear()
+    s.INSERTPS_xxi(R.xmm15, R.xmm15, 4)
+    assert s.getvalue() == '\x66\x45\x0f\x3a\x21\xff\x04'

From noreply at buildbot.pypy.org  Mon Jun  8 10:24:50 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Mon,  8 Jun 2015 10:24:50 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: a failing test
Message-ID: <20150608082450.471821C0460@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77947:8cc581d569ad
Date: 2015-06-08 10:21 +0200
http://bitbucket.org/pypy/pypy/changeset/8cc581d569ad/

Log:	a failing test

diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -5584,5 +5584,25 @@
         """
         self.optimize_loop(ops, ops)
 
+    def test_random_call_forcing_strgetitem(self):
+        ops = """
+        [p3, i15]
+        i13 = strgetitem(p3, i15)
+        p0 = newstr(1)
+        p2 = new_with_vtable(descr=nodesize)
+        setfield_gc(p2, p0, descr=otherdescr)
+        strsetitem(p0, 0, i13)
+        i2 = strgetitem(p0, 0)
+        i3 = call_pure_i(1, i2, descr=nonwritedescr)
+        finish(i3)
+        """
+        expected = """
+        [p3, i15]
+        i13 = strgetitem(p3, i15)
+        i3 = call_i(1, i13, descr=nonwritedescr)
+        finish(i3)
+        """
+        self.optimize_loop(ops, expected)
+
 class TestLLtype(BaseTestOptimizeBasic, LLtypeMixin):
     pass

From noreply at buildbot.pypy.org  Mon Jun  8 10:24:51 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Mon,  8 Jun 2015 10:24:51 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: tiny fix
Message-ID: <20150608082451.8273A1C0460@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77948:ad4e2cf34b10
Date: 2015-06-08 10:24 +0200
http://bitbucket.org/pypy/pypy/changeset/ad4e2cf34b10/

Log:	tiny fix

diff --git a/rpython/jit/metainterp/optimizeopt/vstring.py b/rpython/jit/metainterp/optimizeopt/vstring.py
--- a/rpython/jit/metainterp/optimizeopt/vstring.py
+++ b/rpython/jit/metainterp/optimizeopt/vstring.py
@@ -571,6 +571,8 @@
             if vindex.is_constant():
                 result = sinfo.getitem(vindex.getint())
                 if result is not None:
+                    if op is not None:
+                        self.make_equal_to(op, result)
                     return result
         #
         vindex = self.getintbound(index)

From noreply at buildbot.pypy.org  Mon Jun  8 10:28:38 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Mon,  8 Jun 2015 10:28:38 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: removed manual test since it is tested
	in auto test now
Message-ID: <20150608082838.B4C1B1C0460@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r77949:1fc0d9cd2612
Date: 2015-06-08 10:28 +0200
http://bitbucket.org/pypy/pypy/changeset/1fc0d9cd2612/

Log:	removed manual test since it is tested in auto test now fixed some
	other tests in the x86 backend

diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -232,14 +232,14 @@
         c = astype(|1|, int16)
         c[0] = 16i
         b = a + c
-        d = b -> 7:9
+        d = b -> 7:15
         sum(d)
         """
     def test_int16_expand(self):
         result = self.run("int16_expand")
-        i = 2
+        i = 8
         assert int(result) == i*16 + sum(range(7,7+i))
-        self.check_vectorized(2, 2)
+        self.check_vectorized(3, 2) # TODO sum at the end
 
     def define_int8_expand():
         return """
@@ -253,7 +253,7 @@
     def test_int8_expand(self):
         result = self.run("int8_expand")
         assert int(result) == 16*8 + sum(range(0,17))
-        self.check_vectorized(2, 2)
+        self.check_vectorized(3, 2)
 
     def define_int32_add_const():
         return """
diff --git a/rpython/jit/backend/tool/viewcode.py b/rpython/jit/backend/tool/viewcode.py
--- a/rpython/jit/backend/tool/viewcode.py
+++ b/rpython/jit/backend/tool/viewcode.py
@@ -57,6 +57,7 @@
         'x86_32': 'i386',
         'x86_64': 'i386:x86-64',
         'x86-64': 'i386:x86-64',
+        'x86-64-sse4': 'i386:x86-64',
         'i386': 'i386',
         'arm': 'arm',
         'arm_32': 'arm',
diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -2615,24 +2615,24 @@
             return # already the right size
         if size == 4 and tosize == 2:
             scratch = X86_64_SCRATCH_REG
-            self.mc.PSHUFLW_xxi(resloc.value, srcloc.value, 0b11111000)
-            self.mc.PEXTRW_rxi(scratch.value, srcloc.value, 4)
-            self.mc.PINSRW_xri(resloc.value, scratch.value, 2)
-            self.mc.PEXTRW_rxi(scratch.value, srcloc.value, 6)
-            self.mc.PINSRW_xri(resloc.value, scratch.value, 3)
+            self.mc.PSHUFLW_xxi8(resloc.value, srcloc.value, 0b11111000)
+            self.mc.PEXTRW_rxi8(scratch.value, srcloc.value, 4)
+            self.mc.PINSRW_xri8(resloc.value, scratch.value, 2)
+            self.mc.PEXTRW_rxi8(scratch.value, srcloc.value, 6)
+            self.mc.PINSRW_xri8(resloc.value, scratch.value, 3)
         elif size == 4 and tosize == 8:
             scratch = X86_64_SCRATCH_REG.value
-            self.mc.PEXTRD_rxi(scratch, srcloc.value, 1)
-            self.mc.PINSRQ_xri(resloc.value, scratch, 1)
-            self.mc.PEXTRD_rxi(scratch, srcloc.value, 0)
-            self.mc.PINSRQ_xri(resloc.value, scratch, 0)
+            self.mc.PEXTRD_rxi8(scratch, srcloc.value, 1)
+            self.mc.PINSRQ_xri8(resloc.value, scratch, 1)
+            self.mc.PEXTRD_rxi8(scratch, srcloc.value, 0)
+            self.mc.PINSRQ_xri8(resloc.value, scratch, 0)
         elif size == 8 and tosize == 4:
             # is there a better sequence to move them?
             scratch = X86_64_SCRATCH_REG.value
-            self.mc.PEXTRQ_rxi(scratch, srcloc.value, 0)
-            self.mc.PINSRD_xri(resloc.value, scratch, 0)
-            self.mc.PEXTRQ_rxi(scratch, srcloc.value, 1)
-            self.mc.PINSRD_xri(resloc.value, scratch, 1)
+            self.mc.PEXTRQ_rxi8(scratch, srcloc.value, 0)
+            self.mc.PINSRD_xri8(resloc.value, scratch, 0)
+            self.mc.PEXTRQ_rxi8(scratch, srcloc.value, 1)
+            self.mc.PINSRD_xri8(resloc.value, scratch, 1)
         else:
             raise NotImplementedError("sign ext missing: " + str(size) + " -> " + str(tosize))
 
@@ -2653,19 +2653,19 @@
         assert not srcloc.is_xmm
         size = sizeloc.value
         if size == 1:
-            self.mc.PINSRB_xri(resloc.value, srcloc.value, 0)
+            self.mc.PINSRB_xri8(resloc.value, srcloc.value, 0)
             self.mc.PSHUFB(resloc, heap(self.expand_byte_mask_addr))
         elif size == 2:
-            self.mc.PINSRW_xri(resloc.value, srcloc.value, 0)
-            self.mc.PINSRW_xri(resloc.value, srcloc.value, 4)
-            self.mc.PSHUFLW_xxi(resloc.value, resloc.value, 0)
-            self.mc.PSHUFHW_xxi(resloc.value, resloc.value, 0)
+            self.mc.PINSRW_xri8(resloc.value, srcloc.value, 0)
+            self.mc.PINSRW_xri8(resloc.value, srcloc.value, 4)
+            self.mc.PSHUFLW_xxi8(resloc.value, resloc.value, 0)
+            self.mc.PSHUFHW_xxi8(resloc.value, resloc.value, 0)
         elif size == 4:
-            self.mc.PINSRD_xri(resloc.value, srcloc.value, 0)
-            self.mc.PSHUFD_xxi(resloc.value, resloc.value, 0)
+            self.mc.PINSRD_xri8(resloc.value, srcloc.value, 0)
+            self.mc.PSHUFD_xxi8(resloc.value, resloc.value, 0)
         elif size == 8:
-            self.mc.PINSRQ_xri(resloc.value, srcloc.value, 0)
-            self.mc.PINSRQ_xri(resloc.value, srcloc.value, 1)
+            self.mc.PINSRQ_xri8(resloc.value, srcloc.value, 0)
+            self.mc.PINSRQ_xri8(resloc.value, srcloc.value, 1)
         else:
             raise NotImplementedError("missing size %d for int expand" % (size,))
 
@@ -2676,34 +2676,36 @@
         srcidx = srcidxloc.value
         residx = residxloc.value
         count = countloc.value
+        # for small data type conversion this can be quite costy
+        # j = pack(i,4,4)
         si = srcidx
         ri = residx
         k = count
         while k > 0:
             if size == 8:
                 if resultloc.is_xmm:
-                    self.mc.PEXTRQ_rxi(X86_64_SCRATCH_REG.value, sourceloc.value, si)
-                    self.mc.PINSRQ_xri(resultloc.value, X86_64_SCRATCH_REG.value, ri)
+                    self.mc.PEXTRQ_rxi8(X86_64_SCRATCH_REG.value, sourceloc.value, si)
+                    self.mc.PINSRQ_xri8(resultloc.value, X86_64_SCRATCH_REG.value, ri)
                 else:
-                    self.mc.PEXTRQ_rxi(resultloc.value, sourceloc.value, si)
+                    self.mc.PEXTRQ_rxi8(resultloc.value, sourceloc.value, si)
             elif size == 4:
                 if resultloc.is_xmm:
-                    self.mc.PEXTRD_rxi(X86_64_SCRATCH_REG.value, sourceloc.value, si)
-                    self.mc.PINSRD_xri(resultloc.value, X86_64_SCRATCH_REG.value, ri)
+                    self.mc.PEXTRD_rxi8(X86_64_SCRATCH_REG.value, sourceloc.value, si)
+                    self.mc.PINSRD_xri8(resultloc.value, X86_64_SCRATCH_REG.value, ri)
                 else:
-                    self.mc.PEXTRD_rxi(resultloc.value, sourceloc.value, si)
+                    self.mc.PEXTRD_rxi8(resultloc.value, sourceloc.value, si)
             elif size == 2:
                 if resultloc.is_xmm:
-                    self.mc.PEXTRW_rxi(X86_64_SCRATCH_REG.value, sourceloc.value, si)
-                    self.mc.PINSRW_xri(resultloc.value, X86_64_SCRATCH_REG.value, ri)
+                    self.mc.PEXTRW_rxi8(X86_64_SCRATCH_REG.value, sourceloc.value, si)
+                    self.mc.PINSRW_xri8(resultloc.value, X86_64_SCRATCH_REG.value, ri)
                 else:
-                    self.mc.PEXTRW_rxi(resultloc.value, sourceloc.value, si)
+                    self.mc.PEXTRW_rxi8(resultloc.value, sourceloc.value, si)
             elif size == 1:
                 if resultloc.is_xmm:
-                    self.mc.PEXTRB_rxi(X86_64_SCRATCH_REG.value, sourceloc.value, si)
-                    self.mc.PINSRB_xri(resultloc.value, X86_64_SCRATCH_REG.value, ri)
+                    self.mc.PEXTRB_rxi8(X86_64_SCRATCH_REG.value, sourceloc.value, si)
+                    self.mc.PINSRB_xri8(resultloc.value, X86_64_SCRATCH_REG.value, ri)
                 else:
-                    self.mc.PEXTRB_rxi(resultloc.value, sourceloc.value, si)
+                    self.mc.PEXTRB_rxi8(resultloc.value, sourceloc.value, si)
             si += 1
             ri += 1
             k -= 1
@@ -2732,9 +2734,9 @@
                         self.mov(X86_64_XMM_SCRATCH_REG, srcloc)
                         src = X86_64_XMM_SCRATCH_REG.value
                     select = ((si & 0x3) << 6)|((ri & 0x3) << 4)
-                    self.mc.INSERTPS_xxi(resloc.value, src, select)
+                    self.mc.INSERTPS_xxi8(resloc.value, src, select)
                 else:
-                    self.mc.PEXTRD_rxi(resloc.value, srcloc.value, si)
+                    self.mc.PEXTRD_rxi8(resloc.value, srcloc.value, si)
                 si += 1
                 ri += 1
                 k -= 1
@@ -2755,12 +2757,12 @@
                         # r = (s[1], r[1])
                         if resloc != srcloc:
                             self.mc.UNPCKHPD(resloc, srcloc)
-                        self.mc.SHUFPD_xxi(resloc.value, resloc.value, 1)
+                        self.mc.SHUFPD_xxi8(resloc.value, resloc.value, 1)
                     else:
                         assert residx == 1
                         # r = (r[0], s[1])
                         if resloc != srcloc:
-                            self.mc.SHUFPD_xxi(resloc.value, resloc.value, 1)
+                            self.mc.SHUFPD_xxi8(resloc.value, resloc.value, 1)
                             self.mc.UNPCKHPD(resloc, srcloc)
                         # if they are equal nothing is to be done
 
diff --git a/rpython/jit/backend/x86/rx86.py b/rpython/jit/backend/x86/rx86.py
--- a/rpython/jit/backend/x86/rx86.py
+++ b/rpython/jit/backend/x86/rx86.py
@@ -728,10 +728,9 @@
     MOVD32_xb = xmminsn('\x66', rex_nw, '\x0F\x6E', register(1, 8), stack_bp(2))
     MOVD32_xs = xmminsn('\x66', rex_nw, '\x0F\x6E', register(1, 8), stack_sp(2))
 
-    PSRAD_xi = xmminsn('\x66', rex_nw, '\x0F\x72', register(1), '\xE0', immediate(2, 'b'))
-
     MOVSS_xx = xmminsn('\xF3', rex_nw, '\x0F\x10', register(1,8), register(2), '\xC0')
 
+    PSRAD_xi = xmminsn('\x66', rex_nw, '\x0F\x72', register(1), '\xE0', immediate(2, 'b'))
     PSRLDQ_xi = xmminsn('\x66', rex_nw, '\x0F\x73', register(1), 
                         orbyte(0x3 << 3), '\xC0', immediate(2, 'b'))
     UNPCKLPD_xx = xmminsn('\x66', rex_nw, '\x0F\x14', register(1, 8), register(2), '\xC0')
diff --git a/rpython/jit/backend/x86/test/test_rx86.py b/rpython/jit/backend/x86/test/test_rx86.py
--- a/rpython/jit/backend/x86/test/test_rx86.py
+++ b/rpython/jit/backend/x86/test/test_rx86.py
@@ -245,77 +245,3 @@
         assert len(cls.MULTIBYTE_NOPs) == 16
         for i in range(16):
             assert len(cls.MULTIBYTE_NOPs[i]) == i
-
-def test_pextr():
-    s = CodeBuilder64()
-    s.PEXTRW_rxi(R.r11, R.xmm0,0)
-    assert s.getvalue() == '\x66\x44\x0f\xc5\xd8\x00'
-    s.clear()
-    s.PEXTRW_rxi(R.edi, R.xmm15, 15)
-    assert s.getvalue() == '\x66\x41\x0f\xc5\xff\x0f'
-    s.clear()
-    s.PEXTRD_rxi(R.eax, R.xmm11, 2)
-    assert s.getvalue() == '\x66\x44\x0f\x3a\x16\xd8\x02'
-    s.clear()
-    s.PEXTRD_rxi(R.r11, R.xmm5, 2)
-    assert s.getvalue() == '\x66\x41\x0f\x3a\x16\xeb\x02'
-    s.clear()
-    s.PEXTRQ_rxi(R.ebp, R.xmm0, 7)
-    assert s.getvalue() == '\x66\x48\x0f\x3a\x16\xc5\x07'
-    # BYTE
-    s.clear()
-    s.PEXTRB_rxi(R.eax, R.xmm13, 24)
-    assert s.getvalue() == '\x66\x44\x0f\x3a\x14\xe8\x18'
-    s.clear()
-    s.PEXTRB_rxi(R.r15, R.xmm5, 33)
-    assert s.getvalue() == '\x66\x41\x0f\x3a\x14\xef\x21'
-    # EXTR SINGLE FLOAT
-    s.clear()
-    s.EXTRACTPS_rxi(R.eax, R.xmm15, 2)
-    assert s.getvalue() == '\x66\x44\x0f\x3a\x17\xf8\x02'
-    s.clear()
-    s.EXTRACTPS_rxi(R.r11, R.xmm0, 1)
-    assert s.getvalue() == '\x66\x41\x0f\x3a\x17\xc3\x01'
-    s.clear()
-    s.EXTRACTPS_rxi(R.eax, R.xmm0, 1)
-    assert s.getvalue() == '\x66\x0f\x3a\x17\xc0\x01'
-    s.clear()
-    s.EXTRACTPS_rxi(R.r15, R.xmm15, 4)
-    assert s.getvalue() == '\x66\x45\x0f\x3a\x17\xff\x04'
-
-def test_pinsr():
-    s = CodeBuilder64()
-    s.PINSRW_xri(R.xmm0, R.r11,0)
-    assert s.getvalue() == '\x66\x41\x0f\xc4\xc3\x00'
-    s.clear()
-    s.PINSRW_xri(R.xmm15, R.edi, 15)
-    assert s.getvalue() == '\x66\x44\x0f\xc4\xff\x0f'
-    s.clear()
-    s.PINSRD_xri(R.xmm11, R.eax, 2)
-    assert s.getvalue() == '\x66\x44\x0f\x3a\x22\xd8\x02'
-    s.clear()
-    s.PINSRD_xri(R.xmm5, R.r11, 2)
-    assert s.getvalue() == '\x66\x41\x0f\x3a\x22\xeb\x02'
-    s.clear()
-    s.PINSRQ_xri(R.xmm0, R.ebp, 7)
-    assert s.getvalue() == '\x66\x48\x0f\x3a\x22\xc5\x07'
-    # BYTE
-    s.clear()
-    s.PINSRB_xri(R.xmm13, R.eax, 24)
-    assert s.getvalue() == '\x66\x44\x0f\x3a\x20\xe8\x18'
-    s.clear()
-    s.PINSRB_xri(R.xmm5, R.r15, 33)
-    assert s.getvalue() == '\x66\x41\x0f\x3a\x20\xef\x21'
-    # EXTR SINGLE FLOAT
-    s.clear()
-    s.INSERTPS_xxi(R.xmm15, R.xmm0, 2)
-    assert s.getvalue() == '\x66\x44\x0f\x3a\x21\xf8\x02'
-    s.clear()
-    s.INSERTPS_xxi(R.xmm0, R.xmm11, 1)
-    assert s.getvalue() == '\x66\x41\x0f\x3a\x21\xc3\x01'
-    s.clear()
-    s.INSERTPS_xxi(R.xmm0, R.xmm0, 1)
-    assert s.getvalue() == '\x66\x0f\x3a\x21\xc0\x01'
-    s.clear()
-    s.INSERTPS_xxi(R.xmm15, R.xmm15, 4)
-    assert s.getvalue() == '\x66\x45\x0f\x3a\x21\xff\x04'
diff --git a/rpython/jit/backend/x86/test/test_rx86_32_auto_encoding.py b/rpython/jit/backend/x86/test/test_rx86_32_auto_encoding.py
--- a/rpython/jit/backend/x86/test/test_rx86_32_auto_encoding.py
+++ b/rpython/jit/backend/x86/test/test_rx86_32_auto_encoding.py
@@ -196,6 +196,8 @@
                 instrname = 'MOVD'
             if argmodes == 'xb':
                 py.test.skip('"as" uses an undocumented alternate encoding??')
+            if argmodes == 'xx' and self.WORD != 8:
+                instrname = 'MOVQ'
         #
         for args in args_lists:
             suffix = ""
@@ -328,6 +330,15 @@
                 (instrname == 'MULTIBYTE')
         )
 
+    def should_skip_instruction_bit32(self, instrname, argmodes):
+        if self.WORD != 8:
+            return (
+                # the test suite uses 64 bit registers instead of 32 bit...
+                (instrname == 'PEXTRQ') or
+                (instrname == 'PINSRQ')
+            )
+
+        return False
 
 
     def complete_test(self, methname):
@@ -336,7 +347,8 @@
         else:
             instrname, argmodes = methname, ''
 
-        if self.should_skip_instruction(instrname, argmodes):
+        if self.should_skip_instruction(instrname, argmodes) or \
+           self.should_skip_instruction_bit32(instrname, argmodes):
             print "Skipping %s" % methname
             return
 
@@ -370,6 +382,19 @@
         else:
             instr_suffix = None
 
+        if instrname.find('EXTR') != -1 or \
+           instrname.find('INSR') != -1 or \
+           instrname.find('INSERT') != -1 or \
+           instrname.find('EXTRACT') != -1 or \
+           instrname.find('SRLDQ') != -1 or \
+           instrname.find('SHUF') != -1:
+            realargmodes = []
+            for mode in argmodes:
+                if mode == 'i':
+                    mode = 'i8'
+                realargmodes.append(mode)
+            argmodes = realargmodes
+
         print "Testing %s with argmodes=%r" % (instrname, argmodes)
         self.methname = methname
         self.is_xmm_insn = getattr(getattr(self.X86_CodeBuilder,
diff --git a/rpython/jit/backend/x86/test/test_rx86_64_auto_encoding.py b/rpython/jit/backend/x86/test/test_rx86_64_auto_encoding.py
--- a/rpython/jit/backend/x86/test/test_rx86_64_auto_encoding.py
+++ b/rpython/jit/backend/x86/test/test_rx86_64_auto_encoding.py
@@ -24,7 +24,10 @@
         return (
                 super(TestRx86_64, self).should_skip_instruction(instrname, argmodes) or
                 # Not testing FSTP on 64-bit for now
-                (instrname == 'FSTP')
+                (instrname == 'FSTP') or
+                # the test suite uses 64 bit registers instead of 32 bit...
+                (instrname == 'PEXTRD') or
+                (instrname == 'PINSRD')
         )
 
     def array_tests(self):

From noreply at buildbot.pypy.org  Mon Jun  8 10:58:29 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Mon,  8 Jun 2015 10:58:29 +0200 (CEST)
Subject: [pypy-commit] benchmarks single-run: another, slightly different,
	benchmark
Message-ID: <20150608085829.33DF01C0460@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: single-run
Changeset: r327:bc2c68b23fcd
Date: 2015-06-08 10:59 +0200
http://bitbucket.org/pypy/benchmarks/changeset/bc2c68b23fcd/

Log:	another, slightly different, benchmark

diff --git a/lib/pypy/include/pypy_decl.h b/lib/pypy/include/pypy_decl.h
--- a/lib/pypy/include/pypy_decl.h
+++ b/lib/pypy/include/pypy_decl.h
@@ -508,87 +508,87 @@
 PyAPI_FUNC(Signed) _Py_HashPointer(void *arg0);
 PyAPI_FUNC(PyObject *) _Py_InitPyPyModule(const char *arg0, PyMethodDef *arg1, const char *arg2, PyObject *arg3, int arg4);
 PyAPI_FUNC(void) _Py_NewReference(PyObject *arg0);
-PyAPI_DATA(PyTypeObject) PyStaticMethod_Type;
-PyAPI_DATA(PyObject*) PyExc_EnvironmentError;
-PyAPI_DATA(PyTypeObject) PySlice_Type;
-PyAPI_DATA(PyObject*) PyExc_IOError;
-PyAPI_DATA(PyObject*) PyExc_RuntimeError;
+PyAPI_DATA(PyObject) _Py_NoneStruct;
+PyAPI_DATA(PyObject) _Py_TrueStruct;
+PyAPI_DATA(PyObject) _Py_ZeroStruct;
+PyAPI_DATA(PyObject) _Py_NotImplementedStruct;
+PyAPI_DATA(PyObject) _Py_EllipsisObject;
+PyAPI_DATA(PyDateTime_CAPI*) PyDateTimeAPI;
+PyAPI_DATA(PyObject*) PyExc_ArithmeticError;
+PyAPI_DATA(PyObject*) PyExc_AssertionError;
 PyAPI_DATA(PyObject*) PyExc_AttributeError;
-PyAPI_DATA(PyObject*) PyExc_NameError;
-PyAPI_DATA(PyObject*) PyExc_MemoryError;
-PyAPI_DATA(PyObject*) PyExc_SystemExit;
-PyAPI_DATA(PyTypeObject) PyModule_Type;
-PyAPI_DATA(PyTypeObject) PyBaseObject_Type;
-PyAPI_DATA(PyObject*) PyExc_FloatingPointError;
-PyAPI_DATA(PyObject*) PyExc_UnicodeDecodeError;
-PyAPI_DATA(PyObject*) PyExc_Exception;
-PyAPI_DATA(PyObject*) PyExc_TypeError;
-PyAPI_DATA(PyObject*) PyExc_SystemError;
-PyAPI_DATA(PyObject*) PyExc_ReferenceError;
-PyAPI_DATA(PyTypeObject) PyNotImplemented_Type;
-PyAPI_DATA(PyTypeObject) PySet_Type;
-PyAPI_DATA(PyObject*) PyExc_TabError;
-PyAPI_DATA(PyTypeObject) PyDict_Type;
-PyAPI_DATA(PyTypeObject) PyByteArray_Type;
-PyAPI_DATA(PyTypeObject) PyNone_Type;
-PyAPI_DATA(PyTypeObject) PyLong_Type;
-PyAPI_DATA(PyTypeObject) PyWrapperDescr_Type;
-PyAPI_DATA(PyObject*) PyExc_PendingDeprecationWarning;
-PyAPI_DATA(PyObject*) PyExc_OverflowError;
 PyAPI_DATA(PyObject*) PyExc_BaseException;
-PyAPI_DATA(PyObject*) PyExc_StandardError;
-PyAPI_DATA(PyObject*) PyExc_Warning;
-PyAPI_DATA(PyTypeObject) PyTuple_Type;
-PyAPI_DATA(PyObject*) PyExc_UnicodeError;
-PyAPI_DATA(PyTypeObject) PyProperty_Type;
-PyAPI_DATA(PyObject*) PyExc_IndexError;
-PyAPI_DATA(PyTypeObject) PyCell_Type;
-PyAPI_DATA(PyObject*) PyExc_FutureWarning;
-PyAPI_DATA(PyObject) _Py_ZeroStruct;
-PyAPI_DATA(PyObject*) PyExc_UnboundLocalError;
-PyAPI_DATA(PyObject) _Py_NotImplementedStruct;
-PyAPI_DATA(PyTypeObject) PyList_Type;
-PyAPI_DATA(PyTypeObject) PyComplex_Type;
-PyAPI_DATA(PyTypeObject) PyFrozenSet_Type;
-PyAPI_DATA(PyTypeObject) PyUnicode_Type;
-PyAPI_DATA(PyTypeObject) PyCFunction_Type;
+PyAPI_DATA(PyObject*) PyExc_BufferError;
 PyAPI_DATA(PyObject*) PyExc_BytesWarning;
 PyAPI_DATA(PyObject*) PyExc_DeprecationWarning;
+PyAPI_DATA(PyObject*) PyExc_EOFError;
+PyAPI_DATA(PyObject*) PyExc_EnvironmentError;
+PyAPI_DATA(PyObject*) PyExc_Exception;
+PyAPI_DATA(PyObject*) PyExc_FloatingPointError;
+PyAPI_DATA(PyObject*) PyExc_FutureWarning;
+PyAPI_DATA(PyObject*) PyExc_GeneratorExit;
+PyAPI_DATA(PyObject*) PyExc_IOError;
+PyAPI_DATA(PyObject*) PyExc_ImportError;
+PyAPI_DATA(PyObject*) PyExc_ImportWarning;
+PyAPI_DATA(PyObject*) PyExc_IndentationError;
+PyAPI_DATA(PyObject*) PyExc_IndexError;
+PyAPI_DATA(PyObject*) PyExc_KeyError;
+PyAPI_DATA(PyObject*) PyExc_KeyboardInterrupt;
+PyAPI_DATA(PyObject*) PyExc_LookupError;
+PyAPI_DATA(PyObject*) PyExc_MemoryError;
+PyAPI_DATA(PyObject*) PyExc_NameError;
+PyAPI_DATA(PyObject*) PyExc_NotImplementedError;
+PyAPI_DATA(PyObject*) PyExc_OSError;
+PyAPI_DATA(PyObject*) PyExc_OverflowError;
+PyAPI_DATA(PyObject*) PyExc_PendingDeprecationWarning;
+PyAPI_DATA(PyObject*) PyExc_ReferenceError;
+PyAPI_DATA(PyObject*) PyExc_RuntimeError;
+PyAPI_DATA(PyObject*) PyExc_RuntimeWarning;
+PyAPI_DATA(PyObject*) PyExc_StandardError;
+PyAPI_DATA(PyObject*) PyExc_StopIteration;
 PyAPI_DATA(PyObject*) PyExc_SyntaxError;
+PyAPI_DATA(PyObject*) PyExc_SyntaxWarning;
+PyAPI_DATA(PyObject*) PyExc_SystemExit;
+PyAPI_DATA(PyObject*) PyExc_SystemError;
+PyAPI_DATA(PyObject*) PyExc_TabError;
+PyAPI_DATA(PyObject*) PyExc_TypeError;
+PyAPI_DATA(PyObject*) PyExc_UnboundLocalError;
+PyAPI_DATA(PyObject*) PyExc_UnicodeDecodeError;
+PyAPI_DATA(PyObject*) PyExc_UnicodeEncodeError;
+PyAPI_DATA(PyObject*) PyExc_UnicodeError;
+PyAPI_DATA(PyObject*) PyExc_UnicodeTranslateError;
 PyAPI_DATA(PyObject*) PyExc_UnicodeWarning;
+PyAPI_DATA(PyObject*) PyExc_UserWarning;
+PyAPI_DATA(PyObject*) PyExc_ValueError;
+PyAPI_DATA(PyObject*) PyExc_Warning;
 PyAPI_DATA(PyObject*) PyExc_ZeroDivisionError;
+PyAPI_DATA(PyTypeObject) PyType_Type;
+PyAPI_DATA(PyTypeObject) PyString_Type;
+PyAPI_DATA(PyTypeObject) PyUnicode_Type;
+PyAPI_DATA(PyTypeObject) PyBaseString_Type;
+PyAPI_DATA(PyTypeObject) PyDict_Type;
+PyAPI_DATA(PyTypeObject) PyTuple_Type;
+PyAPI_DATA(PyTypeObject) PyList_Type;
+PyAPI_DATA(PyTypeObject) PySet_Type;
+PyAPI_DATA(PyTypeObject) PyFrozenSet_Type;
+PyAPI_DATA(PyTypeObject) PyInt_Type;
+PyAPI_DATA(PyTypeObject) PyBool_Type;
 PyAPI_DATA(PyTypeObject) PyFloat_Type;
-PyAPI_DATA(PyObject*) PyExc_RuntimeWarning;
-PyAPI_DATA(PyObject) _Py_NoneStruct;
-PyAPI_DATA(PyObject*) PyExc_IndentationError;
-PyAPI_DATA(PyObject*) PyExc_AssertionError;
-PyAPI_DATA(PyObject*) PyExc_GeneratorExit;
-PyAPI_DATA(PyObject*) PyExc_ImportWarning;
-PyAPI_DATA(PyObject*) PyExc_UnicodeEncodeError;
-PyAPI_DATA(PyTypeObject) PyInt_Type;
-PyAPI_DATA(PyTypeObject) PyString_Type;
-PyAPI_DATA(PyTypeObject) PyBool_Type;
-PyAPI_DATA(PyObject*) PyExc_OSError;
-PyAPI_DATA(PyObject*) PyExc_KeyError;
-PyAPI_DATA(PyObject*) PyExc_SyntaxWarning;
-PyAPI_DATA(PyTypeObject) PyBaseString_Type;
-PyAPI_DATA(PyObject*) PyExc_StopIteration;
-PyAPI_DATA(PyObject*) PyExc_NotImplementedError;
-PyAPI_DATA(PyObject*) PyExc_ImportError;
-PyAPI_DATA(PyDateTime_CAPI*) PyDateTimeAPI;
-PyAPI_DATA(PyObject*) PyExc_UserWarning;
-PyAPI_DATA(PyObject) _Py_TrueStruct;
-PyAPI_DATA(PyObject*) PyExc_ArithmeticError;
+PyAPI_DATA(PyTypeObject) PyLong_Type;
+PyAPI_DATA(PyTypeObject) PyComplex_Type;
+PyAPI_DATA(PyTypeObject) PyByteArray_Type;
+PyAPI_DATA(PyTypeObject) PyMemoryView_Type;
+PyAPI_DATA(PyTypeObject) PyBaseObject_Type;
+PyAPI_DATA(PyTypeObject) PyNone_Type;
+PyAPI_DATA(PyTypeObject) PyNotImplemented_Type;
+PyAPI_DATA(PyTypeObject) PyCell_Type;
+PyAPI_DATA(PyTypeObject) PyModule_Type;
+PyAPI_DATA(PyTypeObject) PyProperty_Type;
+PyAPI_DATA(PyTypeObject) PySlice_Type;
 PyAPI_DATA(PyTypeObject) PyClass_Type;
-PyAPI_DATA(PyTypeObject) PyType_Type;
-PyAPI_DATA(PyTypeObject) PyMemoryView_Type;
-PyAPI_DATA(PyObject*) PyExc_UnicodeTranslateError;
-PyAPI_DATA(PyObject*) PyExc_LookupError;
-PyAPI_DATA(PyObject*) PyExc_EOFError;
-PyAPI_DATA(PyObject*) PyExc_BufferError;
-PyAPI_DATA(PyObject*) PyExc_ValueError;
-PyAPI_DATA(PyObject) _Py_EllipsisObject;
-PyAPI_DATA(PyObject*) PyExc_KeyboardInterrupt;
+PyAPI_DATA(PyTypeObject) PyStaticMethod_Type;
+PyAPI_DATA(PyTypeObject) PyCFunction_Type;
+PyAPI_DATA(PyTypeObject) PyWrapperDescr_Type;
 #undef Signed    /* xxx temporary fix */
 
 #undef Unsigned  /* xxx temporary fix */
diff --git a/warmup/function_call.py b/warmup/function_call.py
--- a/warmup/function_call.py
+++ b/warmup/function_call.py
@@ -35,5 +35,6 @@
     f(i)
 """
     l.append(time.time() - t0)
+    #l.append(0)
 
 print l
diff --git a/warmup/function_call2.py b/warmup/function_call2.py
new file mode 100644
--- /dev/null
+++ b/warmup/function_call2.py
@@ -0,0 +1,34 @@
+
+import time
+l = []
+
+for i in range(200):
+    if i % 10 == 0:
+        print i
+    t0 = time.time()
+    exec """
+
+def k(a, b, c):
+    pass
+
+def g(a, b, c):
+    k(a, b + 1, c + 2)
+    k(a, b + 1, c + 2)
+    k(a, b + 1, c + 2)
+    k(a, b + 1, c + 2)
+    k(a, b + 1, c + 2)
+
+def f(i):
+    g(i, i + 1, i + 2)
+    g(i, i + 1, i + 2)
+    g(i, i + 1, i + 2)
+    g(i, i + 1, i + 2)
+    g(i, i + 1, i + 2)
+    g(i, i + 1, i + 2)
+for i in range(2000):
+    f(i)
+"""
+    l.append(time.time() - t0)
+    #l.append(0)
+
+print l

From noreply at buildbot.pypy.org  Mon Jun  8 10:59:52 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Mon,  8 Jun 2015 10:59:52 +0200 (CEST)
Subject: [pypy-commit] benchmarks default: add another, slightly different,
	benchmark
Message-ID: <20150608085952.A5FCD1C0460@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r328:7674214f995f
Date: 2015-06-08 11:00 +0200
http://bitbucket.org/pypy/benchmarks/changeset/7674214f995f/

Log:	add another, slightly different, benchmark

diff --git a/warmup/function_call2.py b/warmup/function_call2.py
new file mode 100644
--- /dev/null
+++ b/warmup/function_call2.py
@@ -0,0 +1,34 @@
+
+import time
+l = []
+
+for i in range(200):
+    if i % 10 == 0:
+        print i
+    t0 = time.time()
+    exec """
+
+def k(a, b, c):
+    pass
+
+def g(a, b, c):
+    k(a, b + 1, c + 2)
+    k(a, b + 1, c + 2)
+    k(a, b + 1, c + 2)
+    k(a, b + 1, c + 2)
+    k(a, b + 1, c + 2)
+
+def f(i):
+    g(i, i + 1, i + 2)
+    g(i, i + 1, i + 2)
+    g(i, i + 1, i + 2)
+    g(i, i + 1, i + 2)
+    g(i, i + 1, i + 2)
+    g(i, i + 1, i + 2)
+for i in range(2000):
+    f(i)
+"""
+    l.append(time.time() - t0)
+    #l.append(0)
+
+print l

From noreply at buildbot.pypy.org  Mon Jun  8 12:09:19 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Mon,  8 Jun 2015 12:09:19 +0200 (CEST)
Subject: [pypy-commit] pypy disable-unroll-for-short-loops: "tweak" the
	parameter
Message-ID: <20150608100919.C2E021C0460@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: disable-unroll-for-short-loops
Changeset: r77950:82aafa07b4e3
Date: 2015-06-08 12:09 +0200
http://bitbucket.org/pypy/pypy/changeset/82aafa07b4e3/

Log:	"tweak" the parameter

diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py
--- a/rpython/rlib/jit.py
+++ b/rpython/rlib/jit.py
@@ -565,7 +565,7 @@
               'retrace_limit': 5,
               'max_retrace_guards': 15,
               'max_unroll_loops': 0,
-              'disable_unrolling': 1000,
+              'disable_unrolling': 100,
               'enable_opts': 'all',
               'max_unroll_recursion': 7,
               }

From noreply at buildbot.pypy.org  Mon Jun  8 13:34:49 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Mon,  8 Jun 2015 13:34:49 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: try to disable pending setfields
Message-ID: <20150608113449.1FB8E1C0478@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77951:ee93fcd055df
Date: 2015-06-08 13:34 +0200
http://bitbucket.org/pypy/pypy/changeset/ee93fcd055df/

Log:	try to disable pending setfields

diff --git a/rpython/jit/metainterp/optimizeopt/heap.py b/rpython/jit/metainterp/optimizeopt/heap.py
--- a/rpython/jit/metainterp/optimizeopt/heap.py
+++ b/rpython/jit/metainterp/optimizeopt/heap.py
@@ -422,6 +422,8 @@
                 cf.force_lazy_setfield(self, None)
 
     def force_lazy_setfields_and_arrayitems_for_guard(self):
+        self.force_all_lazy_setfields_and_arrayitems()
+        return []
         pendingfields = []
         for descr, cf in self.cached_fields.iteritems():
             op = cf._lazy_setfield

From noreply at buildbot.pypy.org  Mon Jun  8 14:09:02 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Mon,  8 Jun 2015 14:09:02 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: revert the previous checkin does not
 seem to be the problem
Message-ID: <20150608120902.A854D1C0460@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77952:34c82a309912
Date: 2015-06-08 14:09 +0200
http://bitbucket.org/pypy/pypy/changeset/34c82a309912/

Log:	revert the previous checkin does not seem to be the problem

diff --git a/rpython/jit/metainterp/optimizeopt/heap.py b/rpython/jit/metainterp/optimizeopt/heap.py
--- a/rpython/jit/metainterp/optimizeopt/heap.py
+++ b/rpython/jit/metainterp/optimizeopt/heap.py
@@ -422,8 +422,6 @@
                 cf.force_lazy_setfield(self, None)
 
     def force_lazy_setfields_and_arrayitems_for_guard(self):
-        self.force_all_lazy_setfields_and_arrayitems()
-        return []
         pendingfields = []
         for descr, cf in self.cached_fields.iteritems():
             op = cf._lazy_setfield

From noreply at buildbot.pypy.org  Mon Jun  8 14:15:43 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Mon,  8 Jun 2015 14:15:43 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: reverted the 8 immediate suffix (solved
 differently for tests)
Message-ID: <20150608121543.C488D1C0460@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r77953:6e0e98c3d70a
Date: 2015-06-08 10:50 +0200
http://bitbucket.org/pypy/pypy/changeset/6e0e98c3d70a/

Log:	reverted the 8 immediate suffix (solved differently for tests)

diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -243,17 +243,18 @@
 
     def define_int8_expand():
         return """
-        a = astype(|30|, int16)
-        c = astype(|1|, int16)
+        a = astype(|30|, int8)
+        c = astype(|1|, int8)
         c[0] = 8i
         b = a + c
         d = b -> 0:17
         sum(d)
         """
     def test_int8_expand(self):
+        py.test.skip("TODO implement assembler")
         result = self.run("int8_expand")
-        assert int(result) == 16*8 + sum(range(0,17))
-        self.check_vectorized(3, 2)
+        assert int(result) == 8*8 + sum(range(0,17))
+        self.check_vectorized(3, 2) # TODO sum at the end
 
     def define_int32_add_const():
         return """
diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -2615,24 +2615,24 @@
             return # already the right size
         if size == 4 and tosize == 2:
             scratch = X86_64_SCRATCH_REG
-            self.mc.PSHUFLW_xxi8(resloc.value, srcloc.value, 0b11111000)
-            self.mc.PEXTRW_rxi8(scratch.value, srcloc.value, 4)
-            self.mc.PINSRW_xri8(resloc.value, scratch.value, 2)
-            self.mc.PEXTRW_rxi8(scratch.value, srcloc.value, 6)
-            self.mc.PINSRW_xri8(resloc.value, scratch.value, 3)
+            self.mc.PSHUFLW_xxi(resloc.value, srcloc.value, 0b11111000)
+            self.mc.PEXTRW_rxi(scratch.value, srcloc.value, 4)
+            self.mc.PINSRW_xri(resloc.value, scratch.value, 2)
+            self.mc.PEXTRW_rxi(scratch.value, srcloc.value, 6)
+            self.mc.PINSRW_xri(resloc.value, scratch.value, 3)
         elif size == 4 and tosize == 8:
             scratch = X86_64_SCRATCH_REG.value
-            self.mc.PEXTRD_rxi8(scratch, srcloc.value, 1)
-            self.mc.PINSRQ_xri8(resloc.value, scratch, 1)
-            self.mc.PEXTRD_rxi8(scratch, srcloc.value, 0)
-            self.mc.PINSRQ_xri8(resloc.value, scratch, 0)
+            self.mc.PEXTRD_rxi(scratch, srcloc.value, 1)
+            self.mc.PINSRQ_xri(resloc.value, scratch, 1)
+            self.mc.PEXTRD_rxi(scratch, srcloc.value, 0)
+            self.mc.PINSRQ_xri(resloc.value, scratch, 0)
         elif size == 8 and tosize == 4:
             # is there a better sequence to move them?
             scratch = X86_64_SCRATCH_REG.value
-            self.mc.PEXTRQ_rxi8(scratch, srcloc.value, 0)
-            self.mc.PINSRD_xri8(resloc.value, scratch, 0)
-            self.mc.PEXTRQ_rxi8(scratch, srcloc.value, 1)
-            self.mc.PINSRD_xri8(resloc.value, scratch, 1)
+            self.mc.PEXTRQ_rxi(scratch, srcloc.value, 0)
+            self.mc.PINSRD_xri(resloc.value, scratch, 0)
+            self.mc.PEXTRQ_rxi(scratch, srcloc.value, 1)
+            self.mc.PINSRD_xri(resloc.value, scratch, 1)
         else:
             raise NotImplementedError("sign ext missing: " + str(size) + " -> " + str(tosize))
 
@@ -2653,19 +2653,19 @@
         assert not srcloc.is_xmm
         size = sizeloc.value
         if size == 1:
-            self.mc.PINSRB_xri8(resloc.value, srcloc.value, 0)
+            self.mc.PINSRB_xri(resloc.value, srcloc.value, 0)
             self.mc.PSHUFB(resloc, heap(self.expand_byte_mask_addr))
         elif size == 2:
-            self.mc.PINSRW_xri8(resloc.value, srcloc.value, 0)
-            self.mc.PINSRW_xri8(resloc.value, srcloc.value, 4)
-            self.mc.PSHUFLW_xxi8(resloc.value, resloc.value, 0)
-            self.mc.PSHUFHW_xxi8(resloc.value, resloc.value, 0)
+            self.mc.PINSRW_xri(resloc.value, srcloc.value, 0)
+            self.mc.PINSRW_xri(resloc.value, srcloc.value, 4)
+            self.mc.PSHUFLW_xxi(resloc.value, resloc.value, 0)
+            self.mc.PSHUFHW_xxi(resloc.value, resloc.value, 0)
         elif size == 4:
-            self.mc.PINSRD_xri8(resloc.value, srcloc.value, 0)
-            self.mc.PSHUFD_xxi8(resloc.value, resloc.value, 0)
+            self.mc.PINSRD_xri(resloc.value, srcloc.value, 0)
+            self.mc.PSHUFD_xxi(resloc.value, resloc.value, 0)
         elif size == 8:
-            self.mc.PINSRQ_xri8(resloc.value, srcloc.value, 0)
-            self.mc.PINSRQ_xri8(resloc.value, srcloc.value, 1)
+            self.mc.PINSRQ_xri(resloc.value, srcloc.value, 0)
+            self.mc.PINSRQ_xri(resloc.value, srcloc.value, 1)
         else:
             raise NotImplementedError("missing size %d for int expand" % (size,))
 
@@ -2684,28 +2684,28 @@
         while k > 0:
             if size == 8:
                 if resultloc.is_xmm:
-                    self.mc.PEXTRQ_rxi8(X86_64_SCRATCH_REG.value, sourceloc.value, si)
-                    self.mc.PINSRQ_xri8(resultloc.value, X86_64_SCRATCH_REG.value, ri)
+                    self.mc.PEXTRQ_rxi(X86_64_SCRATCH_REG.value, sourceloc.value, si)
+                    self.mc.PINSRQ_xri(resultloc.value, X86_64_SCRATCH_REG.value, ri)
                 else:
-                    self.mc.PEXTRQ_rxi8(resultloc.value, sourceloc.value, si)
+                    self.mc.PEXTRQ_rxi(resultloc.value, sourceloc.value, si)
             elif size == 4:
                 if resultloc.is_xmm:
-                    self.mc.PEXTRD_rxi8(X86_64_SCRATCH_REG.value, sourceloc.value, si)
-                    self.mc.PINSRD_xri8(resultloc.value, X86_64_SCRATCH_REG.value, ri)
+                    self.mc.PEXTRD_rxi(X86_64_SCRATCH_REG.value, sourceloc.value, si)
+                    self.mc.PINSRD_xri(resultloc.value, X86_64_SCRATCH_REG.value, ri)
                 else:
-                    self.mc.PEXTRD_rxi8(resultloc.value, sourceloc.value, si)
+                    self.mc.PEXTRD_rxi(resultloc.value, sourceloc.value, si)
             elif size == 2:
                 if resultloc.is_xmm:
-                    self.mc.PEXTRW_rxi8(X86_64_SCRATCH_REG.value, sourceloc.value, si)
-                    self.mc.PINSRW_xri8(resultloc.value, X86_64_SCRATCH_REG.value, ri)
+                    self.mc.PEXTRW_rxi(X86_64_SCRATCH_REG.value, sourceloc.value, si)
+                    self.mc.PINSRW_xri(resultloc.value, X86_64_SCRATCH_REG.value, ri)
                 else:
-                    self.mc.PEXTRW_rxi8(resultloc.value, sourceloc.value, si)
+                    self.mc.PEXTRW_rxi(resultloc.value, sourceloc.value, si)
             elif size == 1:
                 if resultloc.is_xmm:
-                    self.mc.PEXTRB_rxi8(X86_64_SCRATCH_REG.value, sourceloc.value, si)
-                    self.mc.PINSRB_xri8(resultloc.value, X86_64_SCRATCH_REG.value, ri)
+                    self.mc.PEXTRB_rxi(X86_64_SCRATCH_REG.value, sourceloc.value, si)
+                    self.mc.PINSRB_xri(resultloc.value, X86_64_SCRATCH_REG.value, ri)
                 else:
-                    self.mc.PEXTRB_rxi8(resultloc.value, sourceloc.value, si)
+                    self.mc.PEXTRB_rxi(resultloc.value, sourceloc.value, si)
             si += 1
             ri += 1
             k -= 1
@@ -2734,9 +2734,9 @@
                         self.mov(X86_64_XMM_SCRATCH_REG, srcloc)
                         src = X86_64_XMM_SCRATCH_REG.value
                     select = ((si & 0x3) << 6)|((ri & 0x3) << 4)
-                    self.mc.INSERTPS_xxi8(resloc.value, src, select)
+                    self.mc.INSERTPS_xxi(resloc.value, src, select)
                 else:
-                    self.mc.PEXTRD_rxi8(resloc.value, srcloc.value, si)
+                    self.mc.PEXTRD_rxi(resloc.value, srcloc.value, si)
                 si += 1
                 ri += 1
                 k -= 1
@@ -2757,12 +2757,12 @@
                         # r = (s[1], r[1])
                         if resloc != srcloc:
                             self.mc.UNPCKHPD(resloc, srcloc)
-                        self.mc.SHUFPD_xxi8(resloc.value, resloc.value, 1)
+                        self.mc.SHUFPD_xxi(resloc.value, resloc.value, 1)
                     else:
                         assert residx == 1
                         # r = (r[0], s[1])
                         if resloc != srcloc:
-                            self.mc.SHUFPD_xxi8(resloc.value, resloc.value, 1)
+                            self.mc.SHUFPD_xxi(resloc.value, resloc.value, 1)
                             self.mc.UNPCKHPD(resloc, srcloc)
                         # if they are equal nothing is to be done
 
diff --git a/rpython/jit/backend/x86/test/test_rx86_32_auto_encoding.py b/rpython/jit/backend/x86/test/test_rx86_32_auto_encoding.py
--- a/rpython/jit/backend/x86/test/test_rx86_32_auto_encoding.py
+++ b/rpython/jit/backend/x86/test/test_rx86_32_auto_encoding.py
@@ -332,6 +332,7 @@
 
     def should_skip_instruction_bit32(self, instrname, argmodes):
         if self.WORD != 8:
+            # those are tested in the 64 bit test case
             return (
                 # the test suite uses 64 bit registers instead of 32 bit...
                 (instrname == 'PEXTRQ') or
diff --git a/rpython/jit/backend/x86/test/test_rx86_64_auto_encoding.py b/rpython/jit/backend/x86/test/test_rx86_64_auto_encoding.py
--- a/rpython/jit/backend/x86/test/test_rx86_64_auto_encoding.py
+++ b/rpython/jit/backend/x86/test/test_rx86_64_auto_encoding.py
@@ -26,6 +26,7 @@
                 # Not testing FSTP on 64-bit for now
                 (instrname == 'FSTP') or
                 # the test suite uses 64 bit registers instead of 32 bit...
+                # it is tested in the 32 bit test!
                 (instrname == 'PEXTRD') or
                 (instrname == 'PINSRD')
         )

From noreply at buildbot.pypy.org  Mon Jun  8 14:15:45 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Mon,  8 Jun 2015 14:15:45 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: started to find reduce/accumulation
 functions that are vectorizable (e.g. sum)
Message-ID: <20150608121545.171951C0460@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r77954:90b97695ef67
Date: 2015-06-08 14:15 +0200
http://bitbucket.org/pypy/pypy/changeset/90b97695ef67/

Log:	started to find reduce/accumulation functions that are vectorizable
	(e.g. sum)

diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -330,8 +330,7 @@
     def test_sum(self):
         result = self.run("sum")
         assert result == sum(range(30))
-        # TODO impl reduce
-        self.check_vectorized(1, 0)
+        self.check_vectorized(1, 1)
 
     def define_cumsum():
         return """
@@ -343,6 +342,7 @@
     def test_cumsum(self):
         result = self.run("cumsum")
         assert result == 15
+        # not vectorizable, has one back edge
         self.check_vectorized(1, 0)
 
     def define_axissum():
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py b/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
@@ -1041,6 +1041,33 @@
         vopt = self.vectorize(self.parse_loop(ops),1)
         self.assert_equal(vopt.loop, self.parse_loop(opt, add_label=False))
 
+    def test_accumulate_basic(self):
+        trace = """
+        [p0, i0, f0]
+        guard_early_exit() [p0, i0, f0]
+        f1 = raw_load(p0, i0, descr=floatarraydescr)
+        f2 = float_add(f0, f1)
+        i1 = int_add(i0, 8)
+        i2 = int_lt(i1, 100)
+        guard_false(i2) [p0, i0, f2]
+        jump(p0, i1, f2)
+        """
+        trace_opt = """
+        [p0, i0, v2[f64|2]]
+        guard_early_exit() [p0, i0, v2[f64|2]]
+        i1 = int_add(i0, 16)
+        i2 = int_lt(i1, 100)
+        guard_false(i2) [p0, i0, v[f64|2]]
+        i10 = int_add(i0, 16)
+        i20 = int_lt(i10, 100)
+        v1[f64|2] = vec_raw_load(p0, i0, 2, descr=floatarraydescr)
+        v3[f64|2] = vec_float_hadd(v2[f64|2], v1[f64|2])
+        jump(p0, i1, v3[f64|2])
+        """
+        opt = self.vectorize(self.parse_loop(trace))
+        self.debug_print_operations(opt.loop)
+
+
     def test_element_f45_in_guard_failargs(self):
         ops = """
         [p36, i28, p9, i37, p14, f34, p12, p38, f35, p39, i40, i41, p42, i43, i44, i21, i4, i0, i18]
@@ -1325,23 +1352,5 @@
         opt = self.vectorize(self.parse_loop(trace))
         self.debug_print_operations(opt.loop)
 
-    def test_reduction_basic(self):
-        trace = """
-        [p5, i6, p2, i7, p1, p8, i9, i10, f11, i12, i13, i14]
-        guard_early_exit() [p2, p1, p5, f11, i9, i6, i10, i7, p8]
-        f15 = raw_load(i12, i10, descr=floatarraydescr)
-        guard_not_invalidated() [p2, p1, f15, p5, f11, i9, i6, i10, i7, p8]
-        f16 = float_add(f11, f15)
-        raw_store(i13, i7, f16, descr=floatarraydescr)
-        i18 = int_add(i7, 8)
-        i20 = int_add(i9, 1)
-        i22 = int_add(i10, 8)
-        i23 = int_ge(i20, i14)
-        guard_false(i23) [p2, p1, i20, i18, f16, i22, p5, None, None, i6, None, None, p8]
-        jump(p5, i6, p2, i18, p1, p8, i20, i22, f16, i12, i13, i14)
-        """
-        opt = self.vectorize(self.parse_loop(trace))
-        self.debug_print_operations(opt.loop)
-
 class TestLLtype(BaseTestVectorize, LLtypeMixin):
     pass
diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/vectorize.py
@@ -136,10 +136,7 @@
         self._newoperations.append(op)
 
     def unroll_loop_iterations(self, loop, unroll_count):
-        """ Unroll the loop X times. unroll_count is an integral how
-        often to further unroll the loop.
-        """
-
+        """ Unroll the loop X times. unroll_count + 1 = unroll_factor """
         op_count = len(loop.operations)
 
         label_op = loop.operations[0].clone()
@@ -293,8 +290,8 @@
                 # that point forward:
                 if node_a.is_before(node_b):
                     if memref_a.is_adjacent_to(memref_b):
-                        if self.packset.can_be_packed(node_a, node_b, None):
-                            pair = Pair(node_a,node_b)
+                        pair = self.packset.can_be_packed(node_a, node_b, None)
+                        if pair:
                             self.packset.packs.append(pair)
 
     def extend_packset(self):
@@ -315,8 +312,9 @@
                 rnode = rdep.to
                 isomorph = isomorphic(lnode.getoperation(), rnode.getoperation())
                 if isomorph and lnode.is_before(rnode):
-                    if self.packset.can_be_packed(lnode, rnode, pack):
-                        self.packset.add_pair(lnode, rnode)
+                    pair = self.packset.can_be_packed(lnode, rnode, pack)
+                    if pair:
+                        self.packset.packs.append(pair)
 
     def follow_def_uses(self, pack):
         assert isinstance(pack, Pair)
@@ -1322,17 +1320,50 @@
         p = Pair(l,r)
         self.packs.append(p)
 
+    def accumulates(self, lnode, rnode, origin_pack):
+        # lnode and rnode are isomorphic and dependent
+        lop = lnode.getoperation()
+        opnum = lop.getopnum()
+        rop = rnode.getoperation()
+
+        if opnum in (rop.FLOAT_ADD, rop.INT_ADD):
+            assert lop.numargs() == 2 and lop.result is not None
+            accum, accum_pos = self.getaccumulator_variable(lop, rop, origin_pack)
+            if not accum:
+                return False
+            loaded_pos = (accum_pos + 1) % 2
+            # the dependency exists only because of the result of lnode
+            for dep in lnode.provides():
+                if dep.to is rnode:
+                    if not dep.because_of(accum):
+                        # not quite ... this is not handlable
+                        return False
+            # this can be handled by accumulation
+            return True
+
+        return False
+
+    def getaccumulator_variable(self, lop, rop, origin_pack):
+        args = rop.getarglist()
+        for arg, i in enumerate(args):
+            if arg is lop.result:
+                return arg, i
+
+        return None, -1
+
     def can_be_packed(self, lnode, rnode, origin_pack):
         if isomorphic(lnode.getoperation(), rnode.getoperation()):
-            if lnode.independent(rnode):
+            independent = lnode.independent(rnode)
+            if independent or self.accumulates(lnode, rnode, origin_pack):
                 for pack in self.packs:
                     if pack.left == lnode or \
                        pack.right == rnode:
-                        return False
+                        return None
                 if origin_pack is None:
-                    return True
-                return self.profitable_pack(lnode, rnode, origin_pack)
-        return False
+                    return Pair(lnode, rnode)
+                if self.profitable_pack(lnode, rnode, origin_pack)
+                    return Pair(lnode, rnode)
+        return None
 
     def profitable_pack(self, lnode, rnode, origin_pack):
         lpacknode = origin_pack.left

From noreply at buildbot.pypy.org  Mon Jun  8 15:03:33 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Mon,  8 Jun 2015 15:03:33 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: added accumulation pair
Message-ID: <20150608130333.6B00E1C1239@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r77955:a3b3e446b864
Date: 2015-06-08 14:38 +0200
http://bitbucket.org/pypy/pypy/changeset/a3b3e446b864/

Log:	added accumulation pair

diff --git a/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py b/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
@@ -1061,7 +1061,7 @@
         i10 = int_add(i0, 16)
         i20 = int_lt(i10, 100)
         v1[f64|2] = vec_raw_load(p0, i0, 2, descr=floatarraydescr)
-        v3[f64|2] = vec_float_hadd(v2[f64|2], v1[f64|2])
+        v3[f64|2] = vec_float_add(v2[f64|2], v1[f64|2])
         jump(p0, i1, v3[f64|2])
         """
         opt = self.vectorize(self.parse_loop(trace))
diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/vectorize.py
@@ -1320,7 +1320,7 @@
         p = Pair(l,r)
         self.packs.append(p)
 
-    def accumulates(self, lnode, rnode, origin_pack):
+    def accumulates_pair(self, lnode, rnode, origin_pack):
         # lnode and rnode are isomorphic and dependent
         lop = lnode.getoperation()
         opnum = lop.getopnum()
@@ -1330,18 +1330,17 @@
             assert lop.numargs() == 2 and lop.result is not None
             accum, accum_pos = self.getaccumulator_variable(lop, rop, origin_pack)
             if not accum:
-                return False
-            loaded_pos = (accum_pos + 1) % 2
+                return None
             # the dependency exists only because of the result of lnode
             for dep in lnode.provides():
                 if dep.to is rnode:
                     if not dep.because_of(accum):
                         # not quite ... this is not handlable
-                        return False
+                        return None
             # this can be handled by accumulation
-            return True
+            return AccumPair(lnode, rnode, accum, accum_pos)
 
-        return False
+        return None
 
     def getaccumulator_variable(self, lop, rop, origin_pack):
         args = rop.getarglist()
@@ -1353,18 +1352,23 @@
 
     def can_be_packed(self, lnode, rnode, origin_pack):
         if isomorphic(lnode.getoperation(), rnode.getoperation()):
-            independent = lnode.independent(rnode)
-            if independent or self.accumulates(lnode, rnode, origin_pack):
-                for pack in self.packs:
-                    if pack.left == lnode or \
-                       pack.right == rnode:
-                        return None
+            if lnode.independent(rnode):
+                if self.contains_pair(lnode, rnode):
+                    return None
                 if origin_pack is None:
                     return Pair(lnode, rnode)
                 if self.profitable_pack(lnode, rnode, origin_pack)
                     return Pair(lnode, rnode)
+            else:
+                return self.accumulates_pair(lnode, rnode, origin_pack):
         return None
 
+    def contains_pair(self, lnode, rnode):
+        for pack in self.packs:
+            if pack.left is lnode or pack.right is rnode:
+                return True
+        return False
+
     def profitable_pack(self, lnode, rnode, origin_pack):
         lpacknode = origin_pack.left
         if self.prohibit_packing(lpacknode.getoperation(), lnode.getoperation()):
@@ -1424,6 +1428,8 @@
         for i,node in enumerate(self.operations):
             node.pack = self
             node.pack_position = i
+        self.accum_variable = None
+        self.accum_position = -1
 
     def opcount(self):
         return len(self.operations)
@@ -1441,11 +1447,15 @@
         assert isinstance(other, Pack)
         rightmost = self.operations[-1]
         leftmost = other.operations[0]
-        return rightmost == leftmost
+        both_same_type = self.is_accumulating() == other.is_accumulating()
+        return rightmost == leftmost and both_same_type
 
     def __repr__(self):
         return "Pack(%r)" % self.operations
 
+    def is_accumulating(self):
+        return accum_position != -1
+
 class Pair(Pack):
     """ A special Pack object with only two statements. """
     def __init__(self, left, right):
@@ -1457,5 +1467,15 @@
 
     def __eq__(self, other):
         if isinstance(other, Pair):
-            return self.left == other.left and \
-                   self.right == other.right
+            return self.left is other.left and \
+                   self.right is other.right
+
+class AccumPair(Pair):
+    def __init__(self, left, right, accum_var, accum_pos):
+        assert isinstance(left, Node)
+        assert isinstance(right, Node)
+        Pair.__init__(self, left, right)
+        self.left = left
+        self.right = right
+        self.accum_variable = accum_var
+        self.accum_position = accum_pos

From noreply at buildbot.pypy.org  Mon Jun  8 15:03:34 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Mon,  8 Jun 2015 15:03:34 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: finding reduceables works,
 scheduling needs to be adapted next
Message-ID: <20150608130334.A3C4A1C1239@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r77956:ac80f41576c2
Date: 2015-06-08 15:03 +0200
http://bitbucket.org/pypy/pypy/changeset/ac80f41576c2/

Log:	finding reduceables works, scheduling needs to be adapted next

diff --git a/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py b/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
@@ -1065,6 +1065,8 @@
         jump(p0, i1, v3[f64|2])
         """
         opt = self.vectorize(self.parse_loop(trace))
+        assert len(opt.packset.accum_vars) == 1
+        assert opt.loop.inputargs[2] in opt.packset.accum_vars
         self.debug_print_operations(opt.loop)
 
 
diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/vectorize.py
@@ -292,7 +292,7 @@
                     if memref_a.is_adjacent_to(memref_b):
                         pair = self.packset.can_be_packed(node_a, node_b, None)
                         if pair:
-                            self.packset.packs.append(pair)
+                            self.packset.add_pack(pair)
 
     def extend_packset(self):
         pack_count = self.packset.pack_count()
@@ -314,7 +314,7 @@
                 if isomorph and lnode.is_before(rnode):
                     pair = self.packset.can_be_packed(lnode, rnode, pack)
                     if pair:
-                        self.packset.packs.append(pair)
+                        self.packset.add_pack(pair)
 
     def follow_def_uses(self, pack):
         assert isinstance(pack, Pair)
@@ -324,8 +324,9 @@
                 rnode = rdep.to
                 isomorph = isomorphic(lnode.getoperation(), rnode.getoperation())
                 if isomorph and lnode.is_before(rnode):
-                    if self.packset.can_be_packed(lnode, rnode, pack):
-                        self.packset.add_pair(lnode, rnode)
+                    pair = self.packset.can_be_packed(lnode, rnode, pack)
+                    if pair:
+                        self.packset.add_pack(pair)
 
     def combine_packset(self):
         if len(self.packset.packs) == 0:
@@ -1312,23 +1313,27 @@
         self.operations = operations
         self.unroll_count = unroll_count
         self.smallest_type_bytes = smallest_type_bytes
+        self.accum_vars = {}
 
     def pack_count(self):
         return len(self.packs)
 
-    def add_pair(self, l, r):
-        p = Pair(l,r)
-        self.packs.append(p)
+    def add_pack(self, pack):
+        if pack.is_accumulating():
+            # remember the variable and the position in this map
+            self.accum_vars[pack.accum_variable] = pack.accum_variable
+        self.packs.append(pack)
 
     def accumulates_pair(self, lnode, rnode, origin_pack):
         # lnode and rnode are isomorphic and dependent
+        assert isinstance(origin_pack, Pair)
         lop = lnode.getoperation()
         opnum = lop.getopnum()
-        rop = rnode.getoperation()
 
         if opnum in (rop.FLOAT_ADD, rop.INT_ADD):
+            roper = rnode.getoperation()
             assert lop.numargs() == 2 and lop.result is not None
-            accum, accum_pos = self.getaccumulator_variable(lop, rop, origin_pack)
+            accum, accum_pos = self.getaccumulator_variable(lop, roper, origin_pack)
             if not accum:
                 return None
             # the dependency exists only because of the result of lnode
@@ -1337,6 +1342,17 @@
                     if not dep.because_of(accum):
                         # not quite ... this is not handlable
                         return None
+
+            # in either of the two cases the arguments are mixed,
+            # which is not handled currently
+            var_pos = (accum_pos + 1) % 2
+            plop = origin_pack.left.getoperation()
+            if lop.getarg(var_pos) is not plop.result:
+                return None
+            prop = origin_pack.right.getoperation()
+            if roper.getarg(var_pos) is not prop.result:
+                return None
+
             # this can be handled by accumulation
             return AccumPair(lnode, rnode, accum, accum_pos)
 
@@ -1344,10 +1360,11 @@
 
     def getaccumulator_variable(self, lop, rop, origin_pack):
         args = rop.getarglist()
-        for arg, i in enumerate(args):
+        for i, arg in enumerate(args):
+            print arg, "is", lop.result
             if arg is lop.result:
                 return arg, i
-
+        #
         return None, -1
 
     def can_be_packed(self, lnode, rnode, origin_pack):
@@ -1357,10 +1374,12 @@
                     return None
                 if origin_pack is None:
                     return Pair(lnode, rnode)
-                if self.profitable_pack(lnode, rnode, origin_pack)
+                if self.profitable_pack(lnode, rnode, origin_pack):
                     return Pair(lnode, rnode)
             else:
-                return self.accumulates_pair(lnode, rnode, origin_pack):
+                if self.contains_pair(lnode, rnode):
+                    return None
+                return self.accumulates_pair(lnode, rnode, origin_pack)
         return None
 
     def contains_pair(self, lnode, rnode):
@@ -1399,6 +1418,8 @@
         for op in pack_j.operations[1:]:
             operations.append(op)
         self.packs[i] = pack = Pack(operations)
+        pack.accum_variable = pack_i.accum_variable
+        pack.accum_position = pack_i.accum_position
 
         # instead of deleting an item in the center of pack array,
         # the last element is assigned to position j and
@@ -1411,13 +1432,6 @@
             del self.packs[last_pos]
         return last_pos
 
-    def pack_for_operation(self, node):
-        for pack in self.packs:
-            for node2 in pack.operations:
-                if node == node2:
-                    return pack
-        return None
-
 class Pack(object):
     """ A pack is a set of n statements that are:
         * isomorphic
@@ -1447,14 +1461,15 @@
         assert isinstance(other, Pack)
         rightmost = self.operations[-1]
         leftmost = other.operations[0]
-        both_same_type = self.is_accumulating() == other.is_accumulating()
-        return rightmost == leftmost and both_same_type
+        return rightmost == leftmost and \
+               self.accum_variable == other.accum_variable and \
+               self.accum_position == other.accum_position
 
     def __repr__(self):
         return "Pack(%r)" % self.operations
 
     def is_accumulating(self):
-        return accum_position != -1
+        return self.accum_variable is not None
 
 class Pair(Pack):
     """ A special Pack object with only two statements. """

From noreply at buildbot.pypy.org  Mon Jun  8 15:43:41 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Mon,  8 Jun 2015 15:43:41 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: removed allocation of comparison objects
 to track guard comparisons. This is handled by the guard strengthing
 operation
Message-ID: <20150608134341.BAF5D1C0478@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r77957:3bc0e1fb8db8
Date: 2015-06-08 15:12 +0200
http://bitbucket.org/pypy/pypy/changeset/3bc0e1fb8db8/

Log:	removed allocation of comparison objects to track guard comparisons.
	This is handled by the guard strengthing operation

diff --git a/rpython/jit/metainterp/optimizeopt/dependency.py b/rpython/jit/metainterp/optimizeopt/dependency.py
--- a/rpython/jit/metainterp/optimizeopt/dependency.py
+++ b/rpython/jit/metainterp/optimizeopt/dependency.py
@@ -810,7 +810,7 @@
 
 
 class IntegralForwardModification(object):
-    """ Calculates integral modifications on an integer box. """
+    """ Calculates integral modifications on integer boxes. """
     def __init__(self, memory_refs, index_vars, comparison_vars, invariant_vars):
         self.index_vars = index_vars
         self.comparison_vars = comparison_vars
@@ -828,21 +828,6 @@
             var = self.index_vars[arg] = IndexVar(arg)
         return var
 
-    bool_func_source = """
-    def operation_{name}(self, op, node):
-        box_a0 = op.getarg(0)
-        box_a1 = op.getarg(1)
-        left = self.index_vars.get(box_a0, None)
-        right = self.index_vars.get(box_a1, None)
-        box_r = op.result
-        self.comparison_vars[box_r] = CompareOperation(op.getopnum(), left, right)
-    """
-    for name in ['INT_LT', 'INT_LE', 'INT_EQ', 'INT_NE', 'INT_NE',
-                 'INT_GT', 'INT_GE', 'UINT_LT', 'UINT_LE', 'UINT_GT',
-                 'UINT_GE']:
-        exec py.code.Source(bool_func_source.format(name=name)).compile()
-    del bool_func_source
-
     additive_func_source = """
     def operation_{name}(self, op, node):
         box_r = op.result
@@ -923,25 +908,6 @@
 IntegralForwardModification.inspect_operation = integral_dispatch_opt
 del integral_dispatch_opt
 
-class CompareOperation(object):
-    def __init__(self, opnum, lindex_var, rindex_var):
-        self.opnum = opnum
-        self.lindex_var = lindex_var
-        self.rindex_var = rindex_var
-
-    def getindex_vars(self):
-        if self.lindex_var and self.rindex_var:
-            return (self.lindex_var, self.rindex_var)
-        elif self.lindex_var:
-            return (self.lindex_var, None)
-        elif self.rindex_var:
-            return (self.rindex_var, None)
-        else:
-            return (None, None)
-
-    def adapt_operation(self, op):
-        pass
-
 class IndexVar(AbstractValue):
     """ IndexVar is an AbstractValue only to ensure that a box can be assigned
         to the same variable as an index var.

From noreply at buildbot.pypy.org  Mon Jun  8 15:43:43 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Mon,  8 Jun 2015 15:43:43 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: moved out guard strengthening (on arith
 level) and scheduling from vectorize.py and schedule.py
Message-ID: <20150608134343.1DEA61C0478@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r77958:cec809035d0c
Date: 2015-06-08 15:25 +0200
http://bitbucket.org/pypy/pypy/changeset/cec809035d0c/

Log:	moved out guard strengthening (on arith level) and scheduling from
	vectorize.py and schedule.py

diff --git a/rpython/jit/metainterp/optimizeopt/dependency.py b/rpython/jit/metainterp/optimizeopt/dependency.py
--- a/rpython/jit/metainterp/optimizeopt/dependency.py
+++ b/rpython/jit/metainterp/optimizeopt/dependency.py
@@ -736,79 +736,6 @@
             return dot
         raise NotImplementedError("dot only for debug purpose")
 
-class SchedulerData(object):
-    pass
-class Scheduler(object):
-    def __init__(self, graph, sched_data):
-        assert isinstance(sched_data, SchedulerData)
-        self.graph = graph
-        self.schedulable_nodes = self.graph.schedulable_nodes
-        self.sched_data = sched_data
-
-    def has_more(self):
-        return len(self.schedulable_nodes) > 0
-
-    def next(self, position):
-        i = self._next(self.schedulable_nodes)
-        if i >= 0:
-            candidate = self.schedulable_nodes[i]
-            del self.schedulable_nodes[i]
-            return self.schedule(candidate, position)
-
-        raise RuntimeError("schedule failed cannot continue")
-
-    def _next(self, candidate_list):
-        i = len(candidate_list)-1
-        while i >= 0:
-            candidate = candidate_list[i]
-            if candidate.emitted:
-                del candidate_list[i]
-                i -= 1
-                continue
-            if self.schedulable(candidate):
-                return i
-            i -= 1
-        return -1
-
-    def schedulable(self, candidate):
-        if candidate.pack:
-            for node in candidate.pack.operations:
-                if node.depends_count() > 0:
-                    return False
-        return candidate.depends_count() == 0
-
-    def schedule(self, candidate, position):
-        if candidate.pack:
-            pack = candidate.pack
-            vops = self.sched_data.as_vector_operation(pack)
-            for node in pack.operations:
-                self.scheduled(node, position)
-            return vops
-        else:
-            self.scheduled(candidate, position)
-            return [candidate.getoperation()]
-
-    def scheduled(self, node, position):
-        node.position = position
-        for dep in node.provides()[:]: # COPY
-            to = dep.to
-            node.remove_edge_to(to)
-            if not to.emitted and to.depends_count() == 0:
-                # sorts them by priority
-                nodes = self.schedulable_nodes
-                i = len(nodes)-1
-                while i >= 0:
-                    itnode = nodes[i]
-                    if itnode.priority < to.priority:
-                        nodes.insert(i+1, to)
-                        break
-                    i -= 1
-                else:
-                    nodes.insert(0, to)
-        node.clear_dependencies()
-        node.emitted = True
-
-
 class IntegralForwardModification(object):
     """ Calculates integral modifications on integer boxes. """
     def __init__(self, memory_refs, index_vars, comparison_vars, invariant_vars):
diff --git a/rpython/jit/metainterp/optimizeopt/guard.py b/rpython/jit/metainterp/optimizeopt/guard.py
new file mode 100644
--- /dev/null
+++ b/rpython/jit/metainterp/optimizeopt/guard.py
@@ -0,0 +1,224 @@
+"""
+NOTE this strengthing optimization is only used in the vecopt.
+It needs also the information about integral modifications
+gathered with IntegralForwardModification
+"""
+
+class Guard(object):
+    """ An object wrapper around a guard. Helps to determine
+        if one guard implies another
+    """
+    def __init__(self, index, op, cmp_op, lhs, lhs_arg, rhs, rhs_arg):
+        self.index = index
+        self.op = op
+        self.cmp_op = cmp_op
+        self.lhs = lhs
+        self.rhs = rhs
+        self.lhs_arg = lhs_arg
+        self.rhs_arg = rhs_arg
+        self.implied = False
+        self.stronger = False
+
+    def implies(self, guard, opt):
+        if self.op.getopnum() != guard.op.getopnum():
+            return False
+
+        my_key = opt._get_key(self.cmp_op)
+        ot_key = opt._get_key(guard.cmp_op)
+
+        if my_key[1] == ot_key[1]:
+            # same operation
+            lc = self.compare(self.lhs, guard.lhs)
+            rc = self.compare(self.rhs, guard.rhs)
+            opnum = self.get_compare_opnum()
+            if opnum == -1:
+                return False
+            # x < y  = -1,-2,...
+            # x == y = 0
+            # x > y  = 1,2,...
+            if opnum == rop.INT_LT:
+                return (lc > 0 and rc >= 0) or (lc == 0 and rc >= 0)
+            if opnum == rop.INT_LE:
+                return (lc >= 0 and rc >= 0) or (lc == 0 and rc >= 0)
+            if opnum == rop.INT_GT:
+                return (lc < 0 and rc >= 0) or (lc == 0 and rc > 0)
+            if opnum == rop.INT_GE:
+                return (lc <= 0 and rc >= 0) or (lc == 0 and rc >= 0)
+        return False
+
+    def get_compare_opnum(self):
+        opnum = self.op.getopnum()
+        if opnum == rop.GUARD_TRUE:
+            return self.cmp_op.getopnum()
+        else:
+            return self.cmp_op.boolinverse
+
+    def inhert_attributes(self, other):
+        myop = self.op
+        otherop = other.op
+        assert isinstance(otherop, GuardResOp)
+        assert isinstance(myop, GuardResOp)
+        self.stronger = True
+        self.index = other.index
+
+        descr = myop.getdescr()
+        descr.copy_all_attributes_from(other.op.getdescr())
+        myop.rd_frame_info_list = otherop.rd_frame_info_list
+        myop.rd_snapshot = otherop.rd_snapshot
+        myop.setfailargs(otherop.getfailargs())
+
+    def compare(self, key1, key2):
+        if isinstance(key1, Box):
+            assert isinstance(key2, Box)
+            assert key1 is key2 # key of hash enforces this
+            return 0
+        #
+        if isinstance(key1, ConstInt):
+            assert isinstance(key2, ConstInt)
+            v1 = key1.value
+            v2 = key2.value
+            if v1 == v2:
+                return 0
+            elif v1 < v2:
+                return -1
+            else:
+                return 1
+        #
+        if isinstance(key1, IndexVar):
+            assert isinstance(key2, IndexVar)
+            return key1.compare(key2)
+        #
+        raise AssertionError("cannot compare: " + str(key1) + " <=> " + str(key2))
+
+    def emit_varops(self, opt, var, old_arg):
+        if isinstance(var, IndexVar):
+            box = var.emit_operations(opt)
+            opt.renamer.start_renaming(old_arg, box)
+            return box
+        else:
+            return var
+
+    def emit_operations(self, opt):
+        lhs, opnum, rhs = opt._get_key(self.cmp_op)
+        # create trace instructions for the index
+        box_lhs = self.emit_varops(opt, self.lhs, self.lhs_arg)
+        box_rhs = self.emit_varops(opt, self.rhs, self.rhs_arg)
+        box_result = self.cmp_op.result.clonebox()
+        opt.emit_operation(ResOperation(opnum, [box_lhs, box_rhs], box_result))
+        # guard
+        guard = self.op.clone()
+        guard.setarg(0, box_result)
+        opt.emit_operation(guard)
+
+class GuardStrengthenOpt(object):
+    def __init__(self, index_vars):
+        self.index_vars = index_vars
+        self._newoperations = []
+        self._same_as = {}
+
+    def find_compare_guard_bool(self, boolarg, operations, index):
+        i = index - 1
+        # most likely hit in the first iteration
+        while i > 0:
+            op = operations[i]
+            if op.result and op.result == boolarg:
+                return op
+            i -= 1
+
+        raise AssertionError("guard_true/false first arg not defined")
+
+    def _get_key(self, cmp_op):
+        if cmp_op and rop.INT_LT <= cmp_op.getopnum() <= rop.INT_GE:
+            lhs_arg = cmp_op.getarg(0)
+            rhs_arg = cmp_op.getarg(1)
+            lhs_index_var = self.index_vars.get(lhs_arg, None)
+            rhs_index_var = self.index_vars.get(rhs_arg, None)
+
+            cmp_opnum = cmp_op.getopnum()
+            # get the key, this identifies the guarded operation
+            if lhs_index_var and rhs_index_var:
+                key = (lhs_index_var.getvariable(), cmp_opnum, rhs_index_var.getvariable())
+            elif lhs_index_var:
+                key = (lhs_index_var.getvariable(), cmp_opnum, rhs_arg)
+            elif rhs_index_var:
+                key = (lhs_arg, cmp_opnum, rhs_index_var)
+            else:
+                key = (lhs_arg, cmp_opnum, rhs_arg)
+            return key
+        return (None, 0, None)
+
+    def get_key(self, guard_bool, operations, i):
+        cmp_op = self.find_compare_guard_bool(guard_bool.getarg(0), operations, i)
+        return self._get_key(cmp_op)
+
+    def propagate_all_forward(self, loop):
+        """ strengthens the guards that protect an integral value """
+        strongest_guards = {}
+        guards = {}
+        # the guards are ordered. guards[i] is before guards[j] iff i < j
+        operations = loop.operations
+        last_guard = None
+        for i,op in enumerate(operations):
+            op = operations[i]
+            if op.is_guard() and op.getopnum() in (rop.GUARD_TRUE, rop.GUARD_FALSE):
+                cmp_op = self.find_compare_guard_bool(op.getarg(0), operations, i)
+                key = self._get_key(cmp_op)
+                if key[0] is not None:
+                    lhs_arg = cmp_op.getarg(0)
+                    lhs = self.index_vars.get(lhs_arg, lhs_arg)
+                    rhs_arg = cmp_op.getarg(1)
+                    rhs = self.index_vars.get(rhs_arg, rhs_arg)
+                    other = strongest_guards.get(key, None)
+                    if not other:
+                        guard = Guard(i, op, cmp_op,
+                                      lhs, lhs_arg,
+                                      rhs, rhs_arg)
+                        strongest_guards[key] = guard
+                        # nothing known, at this position emit the guard
+                        guards[i] = guard
+                    else: # implicit index(strongest) < index(current)
+                        guard = Guard(i, op, cmp_op,
+                                      lhs, lhs_arg, rhs, rhs_arg)
+                        if guard.implies(other, self):
+                            guard.inhert_attributes(other)
+
+                            strongest_guards[key] = guard
+                            guards[other.index] = guard
+                            # do not mark as emit
+                            continue
+                        elif other.implies(guard, self):
+                            guard.implied = True
+                        # mark as emit
+                        guards[i] = guard
+                else:
+                    # emit non guard_true/false guards
+                    guards[i] = Guard(i, op, None, None, None, None, None)
+
+        strongest_guards = None
+        #
+        self.renamer = Renamer()
+        last_op_idx = len(operations)-1
+        for i,op in enumerate(operations):
+            op = operations[i]
+            if op.is_guard() and op.getopnum() in (rop.GUARD_TRUE, rop.GUARD_FALSE):
+                guard = guards.get(i, None)
+                if not guard or guard.implied:
+                    # this guard is implied or marked as not emitted (= None)
+                    continue
+                if guard.stronger:
+                    guard.emit_operations(self)
+                    continue
+            if op.result:
+                index_var = self.index_vars.get(op.result, None)
+                if index_var:
+                    if not index_var.is_identity():
+                        index_var.emit_operations(self, op.result)
+                        continue
+            self.emit_operation(op)
+
+        loop.operations = self._newoperations[:]
+
+    def emit_operation(self, op):
+        self.renamer.rename(op)
+        self._newoperations.append(op)
+
diff --git a/rpython/jit/metainterp/optimizeopt/schedule.py b/rpython/jit/metainterp/optimizeopt/schedule.py
new file mode 100644
--- /dev/null
+++ b/rpython/jit/metainterp/optimizeopt/schedule.py
@@ -0,0 +1,266 @@
+
+class SchedulerData(object):
+    pass
+class Scheduler(object):
+    def __init__(self, graph, sched_data):
+        assert isinstance(sched_data, SchedulerData)
+        self.graph = graph
+        self.schedulable_nodes = self.graph.schedulable_nodes
+        self.sched_data = sched_data
+
+    def has_more(self):
+        return len(self.schedulable_nodes) > 0
+
+    def next(self, position):
+        i = self._next(self.schedulable_nodes)
+        if i >= 0:
+            candidate = self.schedulable_nodes[i]
+            del self.schedulable_nodes[i]
+            return self.schedule(candidate, position)
+
+        raise AssertionError("schedule failed cannot continue. possible reason: cycle")
+
+    def _next(self, candidate_list):
+        i = len(candidate_list)-1
+        while i >= 0:
+            candidate = candidate_list[i]
+            if candidate.emitted:
+                del candidate_list[i]
+                i -= 1
+                continue
+            if self.schedulable(candidate):
+                return i
+            i -= 1
+        return -1
+
+    def schedulable(self, candidate):
+        if candidate.pack:
+            for node in candidate.pack.operations:
+                if node.depends_count() > 0:
+                    return False
+        return candidate.depends_count() == 0
+
+    def schedule(self, candidate, position):
+        if candidate.pack:
+            pack = candidate.pack
+            vops = self.sched_data.as_vector_operation(pack)
+            for node in pack.operations:
+                self.scheduled(node, position)
+            return vops
+        else:
+            self.scheduled(candidate, position)
+            return [candidate.getoperation()]
+
+    def scheduled(self, node, position):
+        node.position = position
+        for dep in node.provides()[:]: # COPY
+            to = dep.to
+            node.remove_edge_to(to)
+            if not to.emitted and to.depends_count() == 0:
+                # sorts them by priority
+                nodes = self.schedulable_nodes
+                i = len(nodes)-1
+                while i >= 0:
+                    itnode = nodes[i]
+                    if itnode.priority < to.priority:
+                        nodes.insert(i+1, to)
+                        break
+                    i -= 1
+                else:
+                    nodes.insert(0, to)
+        node.clear_dependencies()
+        node.emitted = True
+
+PT_FLOAT_2 = PackType(FLOAT, 4, False, 2)
+PT_DOUBLE_2 = PackType(FLOAT, 8, False, 2)
+PT_FLOAT_GENERIC = PackType(INT, -1, True)
+PT_INT64 = PackType(INT, 8, True)
+PT_INT32_2 = PackType(INT, 4, True, 2)
+PT_INT_GENERIC = PackType(INT, -1, True)
+PT_GENERIC = PackType(PackType.UNKNOWN_TYPE, -1, False)
+
+INT_RES = PT_INT_GENERIC
+FLOAT_RES = PT_FLOAT_GENERIC
+
+class OpToVectorOpConv(OpToVectorOp):
+    def __init__(self, intype, outtype):
+        self.from_size = intype.getsize()
+        self.to_size = outtype.getsize()
+        OpToVectorOp.__init__(self, (intype, ), outtype)
+
+    def determine_input_type(self, op):
+        return self.arg_ptypes[0]
+
+    def determine_output_type(self, op):
+        return self.result_ptype
+
+    def split_pack(self, pack):
+        if self.from_size > self.to_size:
+            # cast down
+            return OpToVectorOp.split_pack(self, pack)
+        op0 = pack.operations[0].getoperation()
+        _, vbox = self.sched_data.getvector_of_box(op0.getarg(0))
+        vec_reg_size = self.sched_data.vec_reg_size
+        if vbox.getcount() * self.to_size > vec_reg_size:
+            return vec_reg_size // self.to_size
+        return len(pack.operations)
+
+    def new_result_vector_box(self):
+        type = self.output_type.gettype()
+        size = self.to_size
+        count = self.output_type.getcount()
+        vec_reg_size = self.sched_data.vec_reg_size
+        if count * size > vec_reg_size:
+            count = vec_reg_size // size
+        signed = self.output_type.signed
+        return BoxVector(type, count, size, signed)
+
+class SignExtToVectorOp(OpToVectorOp):
+    def __init__(self, intype, outtype):
+        OpToVectorOp.__init__(self, intype, outtype)
+        self.size = -1
+
+    def split_pack(self, pack):
+        op0 = pack.operations[0].getoperation()
+        sizearg = op0.getarg(1)
+        assert isinstance(sizearg, ConstInt)
+        self.size = sizearg.value
+        if self.input_type.getsize() > self.size:
+            # cast down
+            return OpToVectorOp.split_pack(self, pack)
+        _, vbox = self.sched_data.getvector_of_box(op0.getarg(0))
+        vec_reg_size = self.sched_data.vec_reg_size
+        if vbox.getcount() * self.size > vec_reg_size:
+            return vec_reg_size // self.size
+        return vbox.getcount()
+
+    def new_result_vector_box(self):
+        type = self.output_type.gettype()
+        count = self.input_type.getcount()
+        vec_reg_size = self.sched_data.vec_reg_size
+        if count * self.size > vec_reg_size:
+            count = vec_reg_size // self.size
+        signed = self.input_type.signed
+        return BoxVector(type, count, self.size, signed)
+
+class LoadToVectorLoad(OpToVectorOp):
+    def __init__(self):
+        OpToVectorOp.__init__(self, (), PT_GENERIC)
+
+    def determine_input_type(self, op):
+        return None
+
+    def determine_output_type(self, op):
+        return PackType.by_descr(op.getdescr(), self.sched_data.vec_reg_size)
+
+    def before_argument_transform(self, args):
+        args.append(ConstInt(len(self.pack.operations)))
+
+    def getsplitsize(self):
+        return self.output_type.getsize()
+
+    def new_result_vector_box(self):
+        type = self.output_type.gettype()
+        size = self.output_type.getsize()
+        count = len(self.pack.operations)
+        signed = self.output_type.signed
+        return BoxVector(type, count, size, signed)
+
+class StoreToVectorStore(OpToVectorOp):
+    def __init__(self):
+        OpToVectorOp.__init__(self, (None, None, PT_GENERIC), None)
+        self.has_descr = True
+
+    def determine_input_type(self, op):
+        return PackType.by_descr(op.getdescr(), self.sched_data.vec_reg_size)
+
+    def determine_output_type(self, op):
+        return None
+
+INT_OP_TO_VOP = OpToVectorOp((PT_INT_GENERIC, PT_INT_GENERIC), INT_RES)
+FLOAT_OP_TO_VOP = OpToVectorOp((PT_FLOAT_GENERIC, PT_FLOAT_GENERIC), FLOAT_RES)
+FLOAT_SINGLE_ARG_OP_TO_VOP = OpToVectorOp((PT_FLOAT_GENERIC,), FLOAT_RES)
+LOAD_TRANS = LoadToVectorLoad()
+STORE_TRANS = StoreToVectorStore()
+
+# note that the following definition is x86 machine
+# specific.
+ROP_ARG_RES_VECTOR = {
+    rop.VEC_INT_ADD:     INT_OP_TO_VOP,
+    rop.VEC_INT_SUB:     INT_OP_TO_VOP,
+    rop.VEC_INT_MUL:     INT_OP_TO_VOP,
+    rop.VEC_INT_AND:     INT_OP_TO_VOP,
+    rop.VEC_INT_OR:      INT_OP_TO_VOP,
+    rop.VEC_INT_XOR:     INT_OP_TO_VOP,
+
+    rop.VEC_INT_SIGNEXT: SignExtToVectorOp((PT_INT_GENERIC,), INT_RES),
+
+    rop.VEC_FLOAT_ADD:   FLOAT_OP_TO_VOP,
+    rop.VEC_FLOAT_SUB:   FLOAT_OP_TO_VOP,
+    rop.VEC_FLOAT_MUL:   FLOAT_OP_TO_VOP,
+    rop.VEC_FLOAT_TRUEDIV:   FLOAT_OP_TO_VOP,
+    rop.VEC_FLOAT_ABS:   FLOAT_SINGLE_ARG_OP_TO_VOP,
+    rop.VEC_FLOAT_NEG:   FLOAT_SINGLE_ARG_OP_TO_VOP,
+    rop.VEC_FLOAT_EQ:    OpToVectorOp((PT_FLOAT_GENERIC,PT_FLOAT_GENERIC), INT_RES),
+
+    rop.VEC_RAW_LOAD:         LOAD_TRANS,
+    rop.VEC_GETARRAYITEM_RAW: LOAD_TRANS,
+    rop.VEC_RAW_STORE:        STORE_TRANS,
+    rop.VEC_SETARRAYITEM_RAW: STORE_TRANS,
+
+    rop.VEC_CAST_FLOAT_TO_SINGLEFLOAT: OpToVectorOpConv(PT_DOUBLE_2, PT_FLOAT_2),
+    rop.VEC_CAST_SINGLEFLOAT_TO_FLOAT: OpToVectorOpConv(PT_FLOAT_2, PT_DOUBLE_2),
+    rop.VEC_CAST_FLOAT_TO_INT: OpToVectorOpConv(PT_DOUBLE_2, PT_INT32_2),
+    rop.VEC_CAST_INT_TO_FLOAT: OpToVectorOpConv(PT_INT32_2, PT_DOUBLE_2),
+}
+
+class VecScheduleData(SchedulerData):
+    def __init__(self, vec_reg_size):
+        self.box_to_vbox = {}
+        self.vec_reg_size = vec_reg_size
+        self.invariant_oplist = []
+        self.invariant_vector_vars = []
+        self.expanded_map = {}
+
+    def as_vector_operation(self, pack):
+        op_count = len(pack.operations)
+        assert op_count > 1
+        self.pack = pack
+        # properties that hold for the pack are:
+        # + isomorphism (see func above)
+        # + tight packed (no room between vector elems)
+
+        op0 = pack.operations[0].getoperation()
+        tovector = ROP_ARG_RES_VECTOR.get(op0.vector, None)
+        if tovector is None:
+            raise NotImplementedError("missing vecop for '%s'" % (op0.getopname(),))
+        oplist = []
+        tovector.as_vector_operation(pack, self, oplist)
+        return oplist
+
+    def getvector_of_box(self, arg):
+        return self.box_to_vbox.get(arg, (-1, None))
+
+    def setvector_of_box(self, box, off, vector):
+        self.box_to_vbox[box] = (off, vector)
+
+    def prepend_invariant_operations(self, oplist):
+        if len(self.invariant_oplist) > 0:
+            label = oplist[0]
+            assert label.getopnum() == rop.LABEL
+            jump = oplist[-1]
+            assert jump.getopnum() == rop.JUMP
+
+            label_args = label.getarglist()
+            jump_args = jump.getarglist()
+            for var in self.invariant_vector_vars:
+                label_args.append(var)
+                jump_args.append(var)
+
+            oplist[0] = label.copy_and_change(label.getopnum(), label_args, None, label.getdescr())
+            oplist[-1] = jump.copy_and_change(jump.getopnum(), jump_args, None, jump.getdescr())
+
+            return self.invariant_oplist + oplist
+
+        return oplist
+
diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/vectorize.py
@@ -9,7 +9,9 @@
 from rpython.jit.metainterp.optimizeopt.optimizer import Optimizer, Optimization
 from rpython.jit.metainterp.optimizeopt.util import make_dispatcher_method
 from rpython.jit.metainterp.optimizeopt.dependency import (DependencyGraph, 
-        MemoryRef, Scheduler, SchedulerData, Node, IndexVar)
+        MemoryRef, Node, IndexVar)
+from rpython.jit.metainterp.optimizeopt.schedule import VecScheduleData, Scheduler
+from rpython.jit.metainterp.optimizeopt.guard import GuardStrengthenOpt
 from rpython.jit.metainterp.resoperation import (rop, ResOperation, GuardResOp)
 from rpython.rlib.objectmodel import we_are_translated
 from rpython.rlib.debug import debug_print, debug_start, debug_stop
@@ -509,224 +511,6 @@
         rec_snap = self.rename_rd_snapshot(snapshot.prev, clone)
         return Snapshot(rec_snap, boxes)
 
-class Guard(object):
-    """ An object wrapper around a guard. Helps to determine
-        if one guard implies another
-    """
-    def __init__(self, index, op, cmp_op, lhs, lhs_arg, rhs, rhs_arg):
-        self.index = index
-        self.op = op
-        self.cmp_op = cmp_op
-        self.lhs = lhs
-        self.rhs = rhs
-        self.lhs_arg = lhs_arg
-        self.rhs_arg = rhs_arg
-        self.implied = False
-        self.stronger = False
-
-    def implies(self, guard, opt):
-        if self.op.getopnum() != guard.op.getopnum():
-            return False
-
-        my_key = opt._get_key(self.cmp_op)
-        ot_key = opt._get_key(guard.cmp_op)
-
-        if my_key[1] == ot_key[1]:
-            # same operation
-            lc = self.compare(self.lhs, guard.lhs)
-            rc = self.compare(self.rhs, guard.rhs)
-            opnum = self.get_compare_opnum()
-            if opnum == -1:
-                return False
-            # x < y  = -1,-2,...
-            # x == y = 0
-            # x > y  = 1,2,...
-            if opnum == rop.INT_LT:
-                return (lc > 0 and rc >= 0) or (lc == 0 and rc >= 0)
-            if opnum == rop.INT_LE:
-                return (lc >= 0 and rc >= 0) or (lc == 0 and rc >= 0)
-            if opnum == rop.INT_GT:
-                return (lc < 0 and rc >= 0) or (lc == 0 and rc > 0)
-            if opnum == rop.INT_GE:
-                return (lc <= 0 and rc >= 0) or (lc == 0 and rc >= 0)
-        return False
-
-    def get_compare_opnum(self):
-        opnum = self.op.getopnum()
-        if opnum == rop.GUARD_TRUE:
-            return self.cmp_op.getopnum()
-        else:
-            return self.cmp_op.boolinverse
-
-    def inhert_attributes(self, other):
-        myop = self.op
-        otherop = other.op
-        assert isinstance(otherop, GuardResOp)
-        assert isinstance(myop, GuardResOp)
-        self.stronger = True
-        self.index = other.index
-
-        descr = myop.getdescr()
-        descr.copy_all_attributes_from(other.op.getdescr())
-        myop.rd_frame_info_list = otherop.rd_frame_info_list
-        myop.rd_snapshot = otherop.rd_snapshot
-        myop.setfailargs(otherop.getfailargs())
-
-    def compare(self, key1, key2):
-        if isinstance(key1, Box):
-            assert isinstance(key2, Box)
-            assert key1 is key2 # key of hash enforces this
-            return 0
-        #
-        if isinstance(key1, ConstInt):
-            assert isinstance(key2, ConstInt)
-            v1 = key1.value
-            v2 = key2.value
-            if v1 == v2:
-                return 0
-            elif v1 < v2:
-                return -1
-            else:
-                return 1
-        #
-        if isinstance(key1, IndexVar):
-            assert isinstance(key2, IndexVar)
-            return key1.compare(key2)
-        #
-        raise AssertionError("cannot compare: " + str(key1) + " <=> " + str(key2))
-
-    def emit_varops(self, opt, var, old_arg):
-        if isinstance(var, IndexVar):
-            box = var.emit_operations(opt)
-            opt.renamer.start_renaming(old_arg, box)
-            return box
-        else:
-            return var
-
-    def emit_operations(self, opt):
-        lhs, opnum, rhs = opt._get_key(self.cmp_op)
-        # create trace instructions for the index
-        box_lhs = self.emit_varops(opt, self.lhs, self.lhs_arg)
-        box_rhs = self.emit_varops(opt, self.rhs, self.rhs_arg)
-        box_result = self.cmp_op.result.clonebox()
-        opt.emit_operation(ResOperation(opnum, [box_lhs, box_rhs], box_result))
-        # guard
-        guard = self.op.clone()
-        guard.setarg(0, box_result)
-        opt.emit_operation(guard)
-
-class GuardStrengthenOpt(object):
-    def __init__(self, index_vars):
-        self.index_vars = index_vars
-        self._newoperations = []
-        self._same_as = {}
-
-    def find_compare_guard_bool(self, boolarg, operations, index):
-        i = index - 1
-        # most likely hit in the first iteration
-        while i > 0:
-            op = operations[i]
-            if op.result and op.result == boolarg:
-                return op
-            i -= 1
-
-        raise AssertionError("guard_true/false first arg not defined")
-
-    def _get_key(self, cmp_op):
-        if cmp_op and rop.INT_LT <= cmp_op.getopnum() <= rop.INT_GE:
-            lhs_arg = cmp_op.getarg(0)
-            rhs_arg = cmp_op.getarg(1)
-            lhs_index_var = self.index_vars.get(lhs_arg, None)
-            rhs_index_var = self.index_vars.get(rhs_arg, None)
-
-            cmp_opnum = cmp_op.getopnum()
-            # get the key, this identifies the guarded operation
-            if lhs_index_var and rhs_index_var:
-                key = (lhs_index_var.getvariable(), cmp_opnum, rhs_index_var.getvariable())
-            elif lhs_index_var:
-                key = (lhs_index_var.getvariable(), cmp_opnum, rhs_arg)
-            elif rhs_index_var:
-                key = (lhs_arg, cmp_opnum, rhs_index_var)
-            else:
-                key = (lhs_arg, cmp_opnum, rhs_arg)
-            return key
-        return (None, 0, None)
-
-    def get_key(self, guard_bool, operations, i):
-        cmp_op = self.find_compare_guard_bool(guard_bool.getarg(0), operations, i)
-        return self._get_key(cmp_op)
-
-    def propagate_all_forward(self, loop):
-        """ strengthens the guards that protect an integral value """
-        strongest_guards = {}
-        guards = {}
-        # the guards are ordered. guards[i] is before guards[j] iff i < j
-        operations = loop.operations
-        last_guard = None
-        for i,op in enumerate(operations):
-            op = operations[i]
-            if op.is_guard() and op.getopnum() in (rop.GUARD_TRUE, rop.GUARD_FALSE):
-                cmp_op = self.find_compare_guard_bool(op.getarg(0), operations, i)
-                key = self._get_key(cmp_op)
-                if key[0] is not None:
-                    lhs_arg = cmp_op.getarg(0)
-                    lhs = self.index_vars.get(lhs_arg, lhs_arg)
-                    rhs_arg = cmp_op.getarg(1)
-                    rhs = self.index_vars.get(rhs_arg, rhs_arg)
-                    other = strongest_guards.get(key, None)
-                    if not other:
-                        guard = Guard(i, op, cmp_op,
-                                      lhs, lhs_arg,
-                                      rhs, rhs_arg)
-                        strongest_guards[key] = guard
-                        # nothing known, at this position emit the guard
-                        guards[i] = guard
-                    else: # implicit index(strongest) < index(current)
-                        guard = Guard(i, op, cmp_op,
-                                      lhs, lhs_arg, rhs, rhs_arg)
-                        if guard.implies(other, self):
-                            guard.inhert_attributes(other)
-
-                            strongest_guards[key] = guard
-                            guards[other.index] = guard
-                            # do not mark as emit
-                            continue
-                        elif other.implies(guard, self):
-                            guard.implied = True
-                        # mark as emit
-                        guards[i] = guard
-                else:
-                    # emit non guard_true/false guards
-                    guards[i] = Guard(i, op, None, None, None, None, None)
-
-        strongest_guards = None
-        #
-        self.renamer = Renamer()
-        last_op_idx = len(operations)-1
-        for i,op in enumerate(operations):
-            op = operations[i]
-            if op.is_guard() and op.getopnum() in (rop.GUARD_TRUE, rop.GUARD_FALSE):
-                guard = guards.get(i, None)
-                if not guard or guard.implied:
-                    # this guard is implied or marked as not emitted (= None)
-                    continue
-                if guard.stronger:
-                    guard.emit_operations(self)
-                    continue
-            if op.result:
-                index_var = self.index_vars.get(op.result, None)
-                if index_var:
-                    if not index_var.is_identity():
-                        index_var.emit_operations(self, op.result)
-                        continue
-            self.emit_operation(op)
-
-        loop.operations = self._newoperations[:]
-
-    def emit_operation(self, op):
-        self.renamer.rename(op)
-        self._newoperations.append(op)
-
 class CostModel(object):
     def __init__(self, threshold):
         self.threshold = threshold
@@ -1103,201 +887,6 @@
         invariant_vars.append(vbox)
         return vbox
 
-class OpToVectorOpConv(OpToVectorOp):
-    def __init__(self, intype, outtype):
-        self.from_size = intype.getsize()
-        self.to_size = outtype.getsize()
-        OpToVectorOp.__init__(self, (intype, ), outtype)
-
-    def determine_input_type(self, op):
-        return self.arg_ptypes[0]
-
-    def determine_output_type(self, op):
-        return self.result_ptype
-
-    def split_pack(self, pack):
-        if self.from_size > self.to_size:
-            # cast down
-            return OpToVectorOp.split_pack(self, pack)
-        op0 = pack.operations[0].getoperation()
-        _, vbox = self.sched_data.getvector_of_box(op0.getarg(0))
-        vec_reg_size = self.sched_data.vec_reg_size
-        if vbox.getcount() * self.to_size > vec_reg_size:
-            return vec_reg_size // self.to_size
-        return len(pack.operations)
-
-    def new_result_vector_box(self):
-        type = self.output_type.gettype()
-        size = self.to_size
-        count = self.output_type.getcount()
-        vec_reg_size = self.sched_data.vec_reg_size
-        if count * size > vec_reg_size:
-            count = vec_reg_size // size
-        signed = self.output_type.signed
-        return BoxVector(type, count, size, signed)
-
-class SignExtToVectorOp(OpToVectorOp):
-    def __init__(self, intype, outtype):
-        OpToVectorOp.__init__(self, intype, outtype)
-        self.size = -1
-
-    def split_pack(self, pack):
-        op0 = pack.operations[0].getoperation()
-        sizearg = op0.getarg(1)
-        assert isinstance(sizearg, ConstInt)
-        self.size = sizearg.value
-        if self.input_type.getsize() > self.size:
-            # cast down
-            return OpToVectorOp.split_pack(self, pack)
-        _, vbox = self.sched_data.getvector_of_box(op0.getarg(0))
-        vec_reg_size = self.sched_data.vec_reg_size
-        if vbox.getcount() * self.size > vec_reg_size:
-            return vec_reg_size // self.size
-        return vbox.getcount()
-
-    def new_result_vector_box(self):
-        type = self.output_type.gettype()
-        count = self.input_type.getcount()
-        vec_reg_size = self.sched_data.vec_reg_size
-        if count * self.size > vec_reg_size:
-            count = vec_reg_size // self.size
-        signed = self.input_type.signed
-        return BoxVector(type, count, self.size, signed)
-
-PT_GENERIC = PackType(PackType.UNKNOWN_TYPE, -1, False)
-
-class LoadToVectorLoad(OpToVectorOp):
-    def __init__(self):
-        OpToVectorOp.__init__(self, (), PT_GENERIC)
-
-    def determine_input_type(self, op):
-        return None
-
-    def determine_output_type(self, op):
-        return PackType.by_descr(op.getdescr(), self.sched_data.vec_reg_size)
-
-    def before_argument_transform(self, args):
-        args.append(ConstInt(len(self.pack.operations)))
-
-    def getsplitsize(self):
-        return self.output_type.getsize()
-
-    def new_result_vector_box(self):
-        type = self.output_type.gettype()
-        size = self.output_type.getsize()
-        count = len(self.pack.operations)
-        signed = self.output_type.signed
-        return BoxVector(type, count, size, signed)
-
-class StoreToVectorStore(OpToVectorOp):
-    def __init__(self):
-        OpToVectorOp.__init__(self, (None, None, PT_GENERIC), None)
-        self.has_descr = True
-
-    def determine_input_type(self, op):
-        return PackType.by_descr(op.getdescr(), self.sched_data.vec_reg_size)
-
-    def determine_output_type(self, op):
-        return None
-
-PT_FLOAT_2 = PackType(FLOAT, 4, False, 2)
-PT_DOUBLE_2 = PackType(FLOAT, 8, False, 2)
-PT_FLOAT_GENERIC = PackType(INT, -1, True)
-PT_INT64 = PackType(INT, 8, True)
-PT_INT32_2 = PackType(INT, 4, True, 2)
-PT_INT_GENERIC = PackType(INT, -1, True)
-PT_GENERIC = PackType(PackType.UNKNOWN_TYPE, -1, False)
-
-INT_RES = PT_INT_GENERIC
-FLOAT_RES = PT_FLOAT_GENERIC
-
-INT_OP_TO_VOP = OpToVectorOp((PT_INT_GENERIC, PT_INT_GENERIC), INT_RES)
-FLOAT_OP_TO_VOP = OpToVectorOp((PT_FLOAT_GENERIC, PT_FLOAT_GENERIC), FLOAT_RES)
-FLOAT_SINGLE_ARG_OP_TO_VOP = OpToVectorOp((PT_FLOAT_GENERIC,), FLOAT_RES)
-LOAD_TRANS = LoadToVectorLoad()
-STORE_TRANS = StoreToVectorStore()
-
-# note that the following definition is x86 machine
-# specific.
-ROP_ARG_RES_VECTOR = {
-    rop.VEC_INT_ADD:     INT_OP_TO_VOP,
-    rop.VEC_INT_SUB:     INT_OP_TO_VOP,
-    rop.VEC_INT_MUL:     INT_OP_TO_VOP,
-    rop.VEC_INT_AND:     INT_OP_TO_VOP,
-    rop.VEC_INT_OR:      INT_OP_TO_VOP,
-    rop.VEC_INT_XOR:     INT_OP_TO_VOP,
-
-    rop.VEC_INT_SIGNEXT: SignExtToVectorOp((PT_INT_GENERIC,), INT_RES),
-
-    rop.VEC_FLOAT_ADD:   FLOAT_OP_TO_VOP,
-    rop.VEC_FLOAT_SUB:   FLOAT_OP_TO_VOP,
-    rop.VEC_FLOAT_MUL:   FLOAT_OP_TO_VOP,
-    rop.VEC_FLOAT_TRUEDIV:   FLOAT_OP_TO_VOP,
-    rop.VEC_FLOAT_ABS:   FLOAT_SINGLE_ARG_OP_TO_VOP,
-    rop.VEC_FLOAT_NEG:   FLOAT_SINGLE_ARG_OP_TO_VOP,
-    rop.VEC_FLOAT_EQ:    OpToVectorOp((PT_FLOAT_GENERIC,PT_FLOAT_GENERIC), INT_RES),
-
-    rop.VEC_RAW_LOAD:         LOAD_TRANS,
-    rop.VEC_GETARRAYITEM_RAW: LOAD_TRANS,
-    rop.VEC_RAW_STORE:        STORE_TRANS,
-    rop.VEC_SETARRAYITEM_RAW: STORE_TRANS,
-
-    rop.VEC_CAST_FLOAT_TO_SINGLEFLOAT: OpToVectorOpConv(PT_DOUBLE_2, PT_FLOAT_2),
-    rop.VEC_CAST_SINGLEFLOAT_TO_FLOAT: OpToVectorOpConv(PT_FLOAT_2, PT_DOUBLE_2),
-    rop.VEC_CAST_FLOAT_TO_INT: OpToVectorOpConv(PT_DOUBLE_2, PT_INT32_2),
-    rop.VEC_CAST_INT_TO_FLOAT: OpToVectorOpConv(PT_INT32_2, PT_DOUBLE_2),
-}
-
-class VecScheduleData(SchedulerData):
-    def __init__(self, vec_reg_size):
-        self.box_to_vbox = {}
-        self.vec_reg_size = vec_reg_size
-        self.invariant_oplist = []
-        self.invariant_vector_vars = []
-        self.expanded_map = {}
-
-    def as_vector_operation(self, pack):
-        op_count = len(pack.operations)
-        assert op_count > 1
-        self.pack = pack
-        # properties that hold for the pack are:
-        # + isomorphism (see func above)
-        # + tight packed (no room between vector elems)
-
-        op0 = pack.operations[0].getoperation()
-        tovector = ROP_ARG_RES_VECTOR.get(op0.vector, None)
-        if tovector is None:
-            raise NotImplementedError("missing vecop for '%s'" % (op0.getopname(),))
-        oplist = []
-        tovector.as_vector_operation(pack, self, oplist)
-        return oplist
-
-    def getvector_of_box(self, arg):
-        return self.box_to_vbox.get(arg, (-1, None))
-
-    def setvector_of_box(self, box, off, vector):
-        self.box_to_vbox[box] = (off, vector)
-
-    def prepend_invariant_operations(self, oplist):
-        if len(self.invariant_oplist) > 0:
-            label = oplist[0]
-            assert label.getopnum() == rop.LABEL
-            jump = oplist[-1]
-            assert jump.getopnum() == rop.JUMP
-
-            label_args = label.getarglist()
-            jump_args = jump.getarglist()
-            for var in self.invariant_vector_vars:
-                label_args.append(var)
-                jump_args.append(var)
-
-            oplist[0] = label.copy_and_change(label.getopnum(), label_args, None, label.getdescr())
-            oplist[-1] = jump.copy_and_change(jump.getopnum(), jump_args, None, jump.getdescr())
-
-            return self.invariant_oplist + oplist
-
-        return oplist
-
 def isomorphic(l_op, r_op):
     """ Subject of definition """
     if l_op.getopnum() == r_op.getopnum():

From noreply at buildbot.pypy.org  Mon Jun  8 15:43:44 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Mon,  8 Jun 2015 15:43:44 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: finished refactor the structure
Message-ID: <20150608134344.43F191C0478@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r77959:fab36fa4cf6d
Date: 2015-06-08 15:39 +0200
http://bitbucket.org/pypy/pypy/changeset/fab36fa4cf6d/

Log:	finished refactor the structure

diff --git a/rpython/jit/metainterp/optimizeopt/guard.py b/rpython/jit/metainterp/optimizeopt/guard.py
--- a/rpython/jit/metainterp/optimizeopt/guard.py
+++ b/rpython/jit/metainterp/optimizeopt/guard.py
@@ -4,6 +4,13 @@
 gathered with IntegralForwardModification
 """
 
+from rpython.jit.metainterp.optimizeopt.util import Renamer
+from rpython.jit.metainterp.optimizeopt.dependency import (DependencyGraph,
+        MemoryRef, Node, IndexVar)
+from rpython.jit.metainterp.resoperation import (rop, ResOperation, GuardResOp)
+from rpython.jit.metainterp.history import (ConstInt, BoxVector, 
+        BoxFloat, BoxInt, ConstFloat, Box)
+
 class Guard(object):
     """ An object wrapper around a guard. Helps to determine
         if one guard implies another
diff --git a/rpython/jit/metainterp/optimizeopt/schedule.py b/rpython/jit/metainterp/optimizeopt/schedule.py
--- a/rpython/jit/metainterp/optimizeopt/schedule.py
+++ b/rpython/jit/metainterp/optimizeopt/schedule.py
@@ -1,3 +1,12 @@
+
+from rpython.jit.metainterp.history import (FLOAT,INT,ConstInt,BoxVector,
+        BoxFloat,BoxInt)
+from rpython.jit.metainterp.resoperation import (rop, ResOperation, GuardResOp)
+from rpython.jit.metainterp.optimizeopt.dependency import (DependencyGraph,
+        MemoryRef, Node, IndexVar)
+from rpython.jit.metainterp.optimizeopt.util import Renamer
+from rpython.rlib.objectmodel import we_are_translated
+
 
 class SchedulerData(object):
     pass
@@ -71,6 +80,60 @@
         node.clear_dependencies()
         node.emitted = True
 
+class PackType(object):
+    UNKNOWN_TYPE = '-'
+
+    def __init__(self, type, size, signed, count=-1):
+        assert type in (FLOAT, INT, PackType.UNKNOWN_TYPE)
+        self.type = type
+        self.size = size
+        self.signed = signed
+        self.count = count
+
+    def gettype(self):
+        return self.type
+
+    def getsize(self):
+        return self.size
+
+    def getsigned(self):
+        return self.signed
+
+    def get_byte_size(self):
+        return self.size
+
+    def getcount(self):
+        return self.count
+
+    @staticmethod
+    def by_descr(descr, vec_reg_size):
+        _t = INT
+        if descr.is_array_of_floats() or descr.concrete_type == FLOAT:
+            _t = FLOAT
+        size = descr.get_item_size_in_bytes()
+        pt = PackType(_t, size, descr.is_item_signed(), vec_reg_size // size)
+        return pt
+
+    def is_valid(self):
+        return self.type != PackType.UNKNOWN_TYPE and self.size > 0
+
+    def new_vector_box(self, count):
+        return BoxVector(self.type, count, self.size, self.signed)
+
+    def __repr__(self):
+        return 'PackType(%s, %d, %d, #%d)' % (self.type, self.size, self.signed, self.count)
+
+    @staticmethod
+    def of(box, count=-1):
+        assert isinstance(box, BoxVector)
+        if count == -1:
+            count = box.item_count
+        return PackType(box.item_type, box.item_size, box.item_signed, count)
+
+    def clone(self):
+        return PackType(self.type, self.size, self.signed, self.count)
+
+
 PT_FLOAT_2 = PackType(FLOAT, 4, False, 2)
 PT_DOUBLE_2 = PackType(FLOAT, 8, False, 2)
 PT_FLOAT_GENERIC = PackType(INT, -1, True)
@@ -82,6 +145,285 @@
 INT_RES = PT_INT_GENERIC
 FLOAT_RES = PT_FLOAT_GENERIC
 
+class OpToVectorOp(object):
+    def __init__(self, arg_ptypes, result_ptype):
+        self.arg_ptypes = [a for a in arg_ptypes] # do not use a tuple. rpython cannot union
+        self.result_ptype = result_ptype
+        self.preamble_ops = None
+        self.sched_data = None
+        self.pack = None
+        self.input_type = None
+        self.output_type = None
+
+    def clone_vbox_set_count(self, box, count):
+        return BoxVector(box.item_type, count, box.item_size, box.item_signed)
+
+    def is_vector_arg(self, i):
+        if i < 0 or i >= len(self.arg_ptypes):
+            return False
+        return self.arg_ptypes[i] is not None
+
+    def getsplitsize(self):
+        return self.input_type.getsize()
+
+    def determine_input_type(self, op):
+        arg = op.getarg(0)
+        _, vbox = self.sched_data.getvector_of_box(op.getarg(0))
+        if vbox:
+            return PackType.of(vbox)
+        else:
+            vec_reg_size = self.sched_data.vec_reg_size
+            if isinstance(arg, ConstInt) or isinstance(arg, BoxInt):
+                return PackType(INT, 8, True, 2)
+            elif isinstance(arg, ConstFloat) or isinstance(arg, BoxFloat):
+                return PackType(FLOAT, 8, True, 2)
+            else:
+                raise NotImplementedError("arg %s not supported" % (arg,))
+
+    def determine_output_type(self, op):
+        return self.determine_input_type(op)
+
+    def update_input_output(self, pack):
+        op0 = pack.operations[0].getoperation()
+        self.input_type = self.determine_input_type(op0)
+        self.output_type = self.determine_output_type(op0)
+
+    def as_vector_operation(self, pack, sched_data, oplist):
+        self.sched_data = sched_data
+        self.preamble_ops = oplist
+        self.update_input_output(pack)
+
+
+        off = 0
+        stride = self.split_pack(pack)
+        left = len(pack.operations)
+        assert stride > 0
+        while off < len(pack.operations):
+            if left < stride:
+                self.preamble_ops.append(pack.operations[off].getoperation())
+                off += 1
+                continue
+            ops = pack.operations[off:off+stride]
+            self.pack = Pack(ops)
+            self.transform_pack(ops, off, stride)
+            off += stride
+            left -= stride
+
+        self.pack = None
+        self.preamble_ops = None
+        self.sched_data = None
+        self.input_type = None
+        self.output_type = None
+
+    def split_pack(self, pack):
+        pack_count = len(pack.operations)
+        vec_reg_size = self.sched_data.vec_reg_size
+        bytes = pack_count * self.getsplitsize()
+        if bytes > vec_reg_size:
+            return vec_reg_size // self.getsplitsize()
+        if bytes < vec_reg_size:
+            return 1
+        return pack_count
+
+    def before_argument_transform(self, args):
+        pass
+
+    def transform_pack(self, ops, off, stride):
+        op = self.pack.operations[0].getoperation()
+        args = op.getarglist()
+        #
+        self.before_argument_transform(args)
+        #
+        for i,arg in enumerate(args):
+            if self.is_vector_arg(i):
+                args[i] = self.transform_argument(args[i], i, off)
+        #
+        result = op.result
+        result = self.transform_result(result, off)
+        #
+        vop = ResOperation(op.vector, args, result, op.getdescr())
+        self.preamble_ops.append(vop)
+
+    def transform_result(self, result, off):
+        if result is None:
+            return None
+        vbox = self.new_result_vector_box()
+        #
+        # mark the position and the vbox in the hash
+        for i, node in enumerate(self.pack.operations):
+            op = node.getoperation()
+            self.sched_data.setvector_of_box(op.result, i, vbox)
+        return vbox
+
+    def new_result_vector_box(self):
+        type = self.output_type.gettype()
+        size = self.output_type.getsize()
+        count = min(self.output_type.getcount(), len(self.pack.operations))
+        signed = self.output_type.signed
+        return BoxVector(type, count, size, signed)
+
+    def transform_argument(self, arg, argidx, off):
+        ops = self.pack.operations
+        box_pos, vbox = self.sched_data.getvector_of_box(arg)
+        if not vbox:
+            # constant/variable expand this box
+            vbox = self.expand(ops, arg, argidx)
+            box_pos = 0
+
+        # use the input as an indicator for the pack type
+        packable = self.sched_data.vec_reg_size // self.input_type.getsize()
+        packed = vbox.item_count
+        assert packed >= 0
+        assert packable >= 0
+        if packed < packable:
+            # the argument is scattered along different vector boxes
+            args = [op.getoperation().getarg(argidx) for op in ops]
+            vbox = self._pack(vbox, packed, args, packable)
+            self.update_input_output(self.pack)
+        elif packed > packable:
+            # the argument has more items than the operation is able to process!
+            vbox = self.unpack(vbox, off, packable, self.input_type)
+            self.update_input_output(self.pack)
+        #
+        if off != 0 and box_pos != 0:
+            # The original box is at a position != 0 but it
+            # is required to be at position 0. Unpack it!
+            vbox = self.unpack(vbox, off, len(ops), self.input_type)
+            self.update_input_output(self.pack)
+        # convert size i64 -> i32, i32 -> i64, ...
+        if self.input_type.getsize() > 0 and \
+           self.input_type.getsize() != vbox.getsize():
+            vbox = self.extend(vbox, self.input_type)
+        #
+        return vbox
+
+    def extend(self, vbox, newtype):
+        assert vbox.gettype() == newtype.gettype()
+        if vbox.gettype() == INT:
+            return self.extend_int(vbox, newtype)
+        else:
+            raise NotImplementedError("cannot yet extend float")
+
+    def extend_int(self, vbox, newtype):
+        vbox_cloned = newtype.new_vector_box(vbox.item_count)
+        op = ResOperation(rop.VEC_INT_SIGNEXT, 
+                          [vbox, ConstInt(newtype.getsize())],
+                          vbox_cloned)
+        self.preamble_ops.append(op)
+        return vbox_cloned
+
+    def unpack(self, vbox, index, count, arg_ptype):
+        vbox_cloned = self.clone_vbox_set_count(vbox, count)
+        opnum = rop.VEC_FLOAT_UNPACK
+        if vbox.item_type == INT:
+            opnum = rop.VEC_INT_UNPACK
+        op = ResOperation(opnum, [vbox, ConstInt(index), ConstInt(count)], vbox_cloned)
+        self.preamble_ops.append(op)
+        return vbox_cloned
+
+    def _pack(self, tgt_box, index, args, packable):
+        """ If there are two vector boxes:
+          v1 = [<empty>,<emtpy>,X,Y]
+          v2 = [A,B,<empty>,<empty>]
+          this function creates a box pack instruction to merge them to:
+          v1/2 = [A,B,X,Y]
+        """
+        opnum = rop.VEC_FLOAT_PACK
+        if tgt_box.item_type == INT:
+            opnum = rop.VEC_INT_PACK
+        arg_count = len(args)
+        i = index
+        while i < arg_count and tgt_box.item_count < packable:
+            arg = args[i]
+            pos, src_box = self.sched_data.getvector_of_box(arg)
+            if pos == -1:
+                i += 1
+                continue
+            count = tgt_box.item_count + src_box.item_count
+            new_box = self.clone_vbox_set_count(tgt_box, count)
+            op = ResOperation(opnum, [tgt_box, src_box, ConstInt(i),
+                                      ConstInt(src_box.item_count)], new_box)
+            self.preamble_ops.append(op)
+            if not we_are_translated():
+                self._check_vec_pack(op)
+            i += src_box.item_count
+
+            # overwrite the new positions, arguments now live in new_box
+            # at a new position
+            for j in range(i):
+                arg = args[j]
+                self.sched_data.setvector_of_box(arg, j, new_box)
+            tgt_box = new_box
+        _, vbox = self.sched_data.getvector_of_box(args[0])
+        return vbox
+
+    def _check_vec_pack(self, op):
+        result = op.result
+        arg0 = op.getarg(0)
+        arg1 = op.getarg(1)
+        index = op.getarg(2)
+        count = op.getarg(3)
+        assert isinstance(result, BoxVector)
+        assert isinstance(arg0, BoxVector)
+        assert isinstance(index, ConstInt)
+        assert isinstance(count, ConstInt)
+        assert arg0.item_size == result.item_size
+        if isinstance(arg1, BoxVector):
+            assert arg1.item_size == result.item_size
+        else:
+            assert count.value == 1
+        assert index.value < result.item_count
+        assert index.value + count.value <= result.item_count
+        assert result.item_count > arg0.item_count
+
+    def expand(self, nodes, arg, argidx):
+        vbox = self.input_type.new_vector_box(len(nodes))
+        box_type = arg.type
+        expanded_map = self.sched_data.expanded_map
+        invariant_ops = self.sched_data.invariant_oplist
+        invariant_vars = self.sched_data.invariant_vector_vars
+        if isinstance(arg, BoxVector):
+            box_type = arg.item_type
+
+        # note that heterogenous nodes are not yet tracked
+        already_expanded = expanded_map.get(arg, None)
+        if already_expanded:
+            return already_expanded
+
+        for i, node in enumerate(nodes):
+            op = node.getoperation()
+            if not arg.same_box(op.getarg(argidx)):
+                break
+            i += 1
+        else:
+            expand_opnum = rop.VEC_FLOAT_EXPAND
+            if box_type == INT:
+                expand_opnum = rop.VEC_INT_EXPAND
+            op = ResOperation(expand_opnum, [arg], vbox)
+            invariant_ops.append(op)
+            invariant_vars.append(vbox)
+            expanded_map[arg] = vbox
+            return vbox
+
+        op = ResOperation(rop.VEC_BOX, [ConstInt(len(nodes))], vbox)
+        invariant_ops.append(op)
+        opnum = rop.VEC_FLOAT_PACK
+        if arg.type == INT:
+            opnum = rop.VEC_INT_PACK
+        for i,node in enumerate(nodes):
+            op = node.getoperation()
+            arg = op.getarg(argidx)
+            new_box = vbox.clonebox()
+            ci = ConstInt(i)
+            c1 = ConstInt(1)
+            op = ResOperation(opnum, [vbox,arg,ci,c1], new_box)
+            vbox = new_box
+            invariant_ops.append(op)
+
+        invariant_vars.append(vbox)
+        return vbox
+
+
 class OpToVectorOpConv(OpToVectorOp):
     def __init__(self, intype, outtype):
         self.from_size = intype.getsize()
@@ -264,3 +606,65 @@
 
         return oplist
 
+class Pack(object):
+    """ A pack is a set of n statements that are:
+        * isomorphic
+        * independent
+    """
+    def __init__(self, ops):
+        self.operations = ops
+        for i,node in enumerate(self.operations):
+            node.pack = self
+            node.pack_position = i
+        self.accum_variable = None
+        self.accum_position = -1
+
+    def opcount(self):
+        return len(self.operations)
+
+    def opnum(self):
+        assert len(self.operations) > 0
+        return self.operations[0].getoperation().getopnum()
+
+    def clear(self):
+        for node in self.operations:
+            node.pack = None
+            node.pack_position = -1
+
+    def rightmost_match_leftmost(self, other):
+        assert isinstance(other, Pack)
+        rightmost = self.operations[-1]
+        leftmost = other.operations[0]
+        return rightmost == leftmost and \
+               self.accum_variable == other.accum_variable and \
+               self.accum_position == other.accum_position
+
+    def __repr__(self):
+        return "Pack(%r)" % self.operations
+
+    def is_accumulating(self):
+        return self.accum_variable is not None
+
+class Pair(Pack):
+    """ A special Pack object with only two statements. """
+    def __init__(self, left, right):
+        assert isinstance(left, Node)
+        assert isinstance(right, Node)
+        self.left = left
+        self.right = right
+        Pack.__init__(self, [left, right])
+
+    def __eq__(self, other):
+        if isinstance(other, Pair):
+            return self.left is other.left and \
+                   self.right is other.right
+
+class AccumPair(Pair):
+    def __init__(self, left, right, accum_var, accum_pos):
+        assert isinstance(left, Node)
+        assert isinstance(right, Node)
+        Pair.__init__(self, left, right)
+        self.left = left
+        self.right = right
+        self.accum_variable = accum_var
+        self.accum_position = accum_pos
diff --git a/rpython/jit/metainterp/optimizeopt/util.py b/rpython/jit/metainterp/optimizeopt/util.py
--- a/rpython/jit/metainterp/optimizeopt/util.py
+++ b/rpython/jit/metainterp/optimizeopt/util.py
@@ -4,10 +4,11 @@
 from rpython.rlib.objectmodel import r_dict, compute_identity_hash, specialize
 from rpython.rlib.rarithmetic import intmask
 from rpython.rlib.unroll import unrolling_iterable
+from rpython.rlib.debug import make_sure_not_resized
+from rpython.rlib.objectmodel import we_are_translated
 from rpython.jit.metainterp import resoperation
-from rpython.rlib.debug import make_sure_not_resized
 from rpython.jit.metainterp.resoperation import rop
-from rpython.rlib.objectmodel import we_are_translated
+from rpython.jit.metainterp.resume import Snapshot
 
 # ____________________________________________________________
 # Misc. utilities
@@ -188,3 +189,53 @@
                         assert False
     assert len(oplist1) == len(oplist2)
     return True
+
+class Renamer(object):
+    def __init__(self):
+        self.rename_map = {}
+
+    def rename_box(self, box):
+        return self.rename_map.get(box, box)
+
+    def start_renaming(self, var, tovar):
+        self.rename_map[var] = tovar
+
+    def rename(self, op):
+        for i, arg in enumerate(op.getarglist()):
+            arg = self.rename_map.get(arg, arg)
+            op.setarg(i, arg)
+
+        if op.is_guard():
+            assert isinstance(op, resoperation.GuardResOp)
+            op.rd_snapshot = self.rename_rd_snapshot(op.rd_snapshot)
+            self.rename_failargs(op)
+
+        return True
+
+    def rename_failargs(self, guard, clone=False):
+        if guard.getfailargs() is not None:
+            if clone:
+                args = guard.getfailargs()[:]
+            else:
+                args = guard.getfailargs()
+            for i,arg in enumerate(args):
+                value = self.rename_map.get(arg,arg)
+                args[i] = value
+            return args
+        return None
+
+    def rename_rd_snapshot(self, snapshot, clone=False):
+        # snapshots are nested like the MIFrames
+        if snapshot is None:
+            return None
+        if clone:
+            boxes = snapshot.boxes[:]
+        else:
+            boxes = snapshot.boxes
+        for i,box in enumerate(boxes):
+            value = self.rename_map.get(box,box)
+            boxes[i] = value
+        #
+        rec_snap = self.rename_rd_snapshot(snapshot.prev, clone)
+        return Snapshot(rec_snap, boxes)
+
diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/vectorize.py
@@ -7,10 +7,10 @@
 from rpython.jit.metainterp.history import (ConstInt, VECTOR, FLOAT, INT,
         BoxVector, BoxFloat, BoxInt, ConstFloat, TargetToken, JitCellToken, Box)
 from rpython.jit.metainterp.optimizeopt.optimizer import Optimizer, Optimization
-from rpython.jit.metainterp.optimizeopt.util import make_dispatcher_method
-from rpython.jit.metainterp.optimizeopt.dependency import (DependencyGraph, 
+from rpython.jit.metainterp.optimizeopt.util import make_dispatcher_method, Renamer
+from rpython.jit.metainterp.optimizeopt.dependency import (DependencyGraph,
         MemoryRef, Node, IndexVar)
-from rpython.jit.metainterp.optimizeopt.schedule import VecScheduleData, Scheduler
+from rpython.jit.metainterp.optimizeopt.schedule import VecScheduleData, Scheduler, Pack, Pair, AccumPair
 from rpython.jit.metainterp.optimizeopt.guard import GuardStrengthenOpt
 from rpython.jit.metainterp.resoperation import (rop, ResOperation, GuardResOp)
 from rpython.rlib.objectmodel import we_are_translated
@@ -462,55 +462,6 @@
                 guard_node.edge_to(ee_guard_node, label='pullup-last-guard')
                 guard_node.relax_guard_to(ee_guard_node)
 
-class Renamer(object):
-    def __init__(self):
-        self.rename_map = {}
-
-    def rename_box(self, box):
-        return self.rename_map.get(box, box)
-
-    def start_renaming(self, var, tovar):
-        self.rename_map[var] = tovar
-
-    def rename(self, op):
-        for i, arg in enumerate(op.getarglist()):
-            arg = self.rename_map.get(arg, arg)
-            op.setarg(i, arg)
-
-        if op.is_guard():
-            assert isinstance(op, GuardResOp)
-            op.rd_snapshot = self.rename_rd_snapshot(op.rd_snapshot)
-            self.rename_failargs(op)
-
-        return True
-
-    def rename_failargs(self, guard, clone=False):
-        if guard.getfailargs() is not None:
-            if clone:
-                args = guard.getfailargs()[:]
-            else:
-                args = guard.getfailargs()
-            for i,arg in enumerate(args):
-                value = self.rename_map.get(arg,arg)
-                args[i] = value
-            return args
-        return None
-
-    def rename_rd_snapshot(self, snapshot, clone=False):
-        # snapshots are nested like the MIFrames
-        if snapshot is None:
-            return None
-        if clone:
-            boxes = snapshot.boxes[:]
-        else:
-            boxes = snapshot.boxes
-        for i,box in enumerate(boxes):
-            value = self.rename_map.get(box,box)
-            boxes[i] = value
-        #
-        rec_snap = self.rename_rd_snapshot(snapshot.prev, clone)
-        return Snapshot(rec_snap, boxes)
-
 class CostModel(object):
     def __init__(self, threshold):
         self.threshold = threshold
@@ -556,336 +507,6 @@
         return 1
 
 
-class PackType(object):
-    UNKNOWN_TYPE = '-'
-
-    def __init__(self, type, size, signed, count=-1):
-        assert type in (FLOAT, INT, PackType.UNKNOWN_TYPE)
-        self.type = type
-        self.size = size
-        self.signed = signed
-        self.count = count
-
-    def gettype(self):
-        return self.type
-
-    def getsize(self):
-        return self.size
-
-    def getsigned(self):
-        return self.signed
-
-    def get_byte_size(self):
-        return self.size
-
-    def getcount(self):
-        return self.count
-
-    @staticmethod
-    def by_descr(descr, vec_reg_size):
-        _t = INT
-        if descr.is_array_of_floats() or descr.concrete_type == FLOAT:
-            _t = FLOAT
-        size = descr.get_item_size_in_bytes()
-        pt = PackType(_t, size, descr.is_item_signed(), vec_reg_size // size)
-        return pt
-
-    def is_valid(self):
-        return self.type != PackType.UNKNOWN_TYPE and self.size > 0
-
-    def new_vector_box(self, count):
-        return BoxVector(self.type, count, self.size, self.signed)
-
-    def __repr__(self):
-        return 'PackType(%s, %d, %d, #%d)' % (self.type, self.size, self.signed, self.count)
-
-    @staticmethod
-    def of(box, count=-1):
-        assert isinstance(box, BoxVector)
-        if count == -1:
-            count = box.item_count
-        return PackType(box.item_type, box.item_size, box.item_signed, count)
-
-    def clone(self):
-        return PackType(self.type, self.size, self.signed, self.count)
-
-class OpToVectorOp(object):
-    def __init__(self, arg_ptypes, result_ptype):
-        self.arg_ptypes = [a for a in arg_ptypes] # do not use a tuple. rpython cannot union
-        self.result_ptype = result_ptype
-        self.preamble_ops = None
-        self.sched_data = None
-        self.pack = None
-        self.input_type = None
-        self.output_type = None
-
-    def clone_vbox_set_count(self, box, count):
-        return BoxVector(box.item_type, count, box.item_size, box.item_signed)
-
-    def is_vector_arg(self, i):
-        if i < 0 or i >= len(self.arg_ptypes):
-            return False
-        return self.arg_ptypes[i] is not None
-
-    def getsplitsize(self):
-        return self.input_type.getsize()
-
-    def determine_input_type(self, op):
-        arg = op.getarg(0)
-        _, vbox = self.sched_data.getvector_of_box(op.getarg(0))
-        if vbox:
-            return PackType.of(vbox)
-        else:
-            vec_reg_size = self.sched_data.vec_reg_size
-            if isinstance(arg, ConstInt) or isinstance(arg, BoxInt):
-                return PackType(INT, 8, True, 2)
-            elif isinstance(arg, ConstFloat) or isinstance(arg, BoxFloat):
-                return PackType(FLOAT, 8, True, 2)
-            else:
-                raise NotImplementedError("arg %s not supported" % (arg,))
-
-    def determine_output_type(self, op):
-        return self.determine_input_type(op)
-
-    def update_input_output(self, pack):
-        op0 = pack.operations[0].getoperation()
-        self.input_type = self.determine_input_type(op0)
-        self.output_type = self.determine_output_type(op0)
-
-    def as_vector_operation(self, pack, sched_data, oplist):
-        self.sched_data = sched_data
-        self.preamble_ops = oplist
-        self.update_input_output(pack)
-
-
-        off = 0
-        stride = self.split_pack(pack)
-        left = len(pack.operations)
-        assert stride > 0
-        while off < len(pack.operations):
-            if left < stride:
-                self.preamble_ops.append(pack.operations[off].getoperation())
-                off += 1
-                continue
-            ops = pack.operations[off:off+stride]
-            self.pack = Pack(ops)
-            self.transform_pack(ops, off, stride)
-            off += stride
-            left -= stride
-
-        self.pack = None
-        self.preamble_ops = None
-        self.sched_data = None
-        self.input_type = None
-        self.output_type = None
-
-    def split_pack(self, pack):
-        pack_count = len(pack.operations)
-        vec_reg_size = self.sched_data.vec_reg_size
-        bytes = pack_count * self.getsplitsize()
-        if bytes > vec_reg_size:
-            return vec_reg_size // self.getsplitsize()
-        if bytes < vec_reg_size:
-            return 1
-        return pack_count
-
-    def before_argument_transform(self, args):
-        pass
-
-    def transform_pack(self, ops, off, stride):
-        op = self.pack.operations[0].getoperation()
-        args = op.getarglist()
-        #
-        self.before_argument_transform(args)
-        #
-        for i,arg in enumerate(args):
-            if self.is_vector_arg(i):
-                args[i] = self.transform_argument(args[i], i, off)
-        #
-        result = op.result
-        result = self.transform_result(result, off)
-        #
-        vop = ResOperation(op.vector, args, result, op.getdescr())
-        self.preamble_ops.append(vop)
-
-    def transform_result(self, result, off):
-        if result is None:
-            return None
-        vbox = self.new_result_vector_box()
-        #
-        # mark the position and the vbox in the hash
-        for i, node in enumerate(self.pack.operations):
-            op = node.getoperation()
-            self.sched_data.setvector_of_box(op.result, i, vbox)
-        return vbox
-
-    def new_result_vector_box(self):
-        type = self.output_type.gettype()
-        size = self.output_type.getsize()
-        count = min(self.output_type.getcount(), len(self.pack.operations))
-        signed = self.output_type.signed
-        return BoxVector(type, count, size, signed)
-
-    def transform_argument(self, arg, argidx, off):
-        ops = self.pack.operations
-        box_pos, vbox = self.sched_data.getvector_of_box(arg)
-        if not vbox:
-            # constant/variable expand this box
-            vbox = self.expand(ops, arg, argidx)
-            box_pos = 0
-
-        # use the input as an indicator for the pack type
-        packable = self.sched_data.vec_reg_size // self.input_type.getsize()
-        packed = vbox.item_count
-        assert packed >= 0
-        assert packable >= 0
-        if packed < packable:
-            # the argument is scattered along different vector boxes
-            args = [op.getoperation().getarg(argidx) for op in ops]
-            vbox = self._pack(vbox, packed, args, packable)
-            self.update_input_output(self.pack)
-        elif packed > packable:
-            # the argument has more items than the operation is able to process!
-            vbox = self.unpack(vbox, off, packable, self.input_type)
-            self.update_input_output(self.pack)
-        #
-        if off != 0 and box_pos != 0:
-            # The original box is at a position != 0 but it
-            # is required to be at position 0. Unpack it!
-            vbox = self.unpack(vbox, off, len(ops), self.input_type)
-            self.update_input_output(self.pack)
-        # convert size i64 -> i32, i32 -> i64, ...
-        if self.input_type.getsize() > 0 and \
-           self.input_type.getsize() != vbox.getsize():
-            vbox = self.extend(vbox, self.input_type)
-        #
-        return vbox
-
-    def extend(self, vbox, newtype):
-        assert vbox.gettype() == newtype.gettype()
-        if vbox.gettype() == INT:
-            return self.extend_int(vbox, newtype)
-        else:
-            raise NotImplementedError("cannot yet extend float")
-
-    def extend_int(self, vbox, newtype):
-        vbox_cloned = newtype.new_vector_box(vbox.item_count)
-        op = ResOperation(rop.VEC_INT_SIGNEXT, 
-                          [vbox, ConstInt(newtype.getsize())],
-                          vbox_cloned)
-        self.preamble_ops.append(op)
-        return vbox_cloned
-
-    def unpack(self, vbox, index, count, arg_ptype):
-        vbox_cloned = self.clone_vbox_set_count(vbox, count)
-        opnum = rop.VEC_FLOAT_UNPACK
-        if vbox.item_type == INT:
-            opnum = rop.VEC_INT_UNPACK
-        op = ResOperation(opnum, [vbox, ConstInt(index), ConstInt(count)], vbox_cloned)
-        self.preamble_ops.append(op)
-        return vbox_cloned
-
-    def _pack(self, tgt_box, index, args, packable):
-        """ If there are two vector boxes:
-          v1 = [<empty>,<emtpy>,X,Y]
-          v2 = [A,B,<empty>,<empty>]
-          this function creates a box pack instruction to merge them to:
-          v1/2 = [A,B,X,Y]
-        """
-        opnum = rop.VEC_FLOAT_PACK
-        if tgt_box.item_type == INT:
-            opnum = rop.VEC_INT_PACK
-        arg_count = len(args)
-        i = index
-        while i < arg_count and tgt_box.item_count < packable:
-            arg = args[i]
-            pos, src_box = self.sched_data.getvector_of_box(arg)
-            if pos == -1:
-                i += 1
-                continue
-            count = tgt_box.item_count + src_box.item_count
-            new_box = self.clone_vbox_set_count(tgt_box, count)
-            op = ResOperation(opnum, [tgt_box, src_box, ConstInt(i),
-                                      ConstInt(src_box.item_count)], new_box)
-            self.preamble_ops.append(op)
-            if not we_are_translated():
-                self._check_vec_pack(op)
-            i += src_box.item_count
-
-            # overwrite the new positions, arguments now live in new_box
-            # at a new position
-            for j in range(i):
-                arg = args[j]
-                self.sched_data.setvector_of_box(arg, j, new_box)
-            tgt_box = new_box
-        _, vbox = self.sched_data.getvector_of_box(args[0])
-        return vbox
-
-    def _check_vec_pack(self, op):
-        result = op.result
-        arg0 = op.getarg(0)
-        arg1 = op.getarg(1)
-        index = op.getarg(2)
-        count = op.getarg(3)
-        assert isinstance(result, BoxVector)
-        assert isinstance(arg0, BoxVector)
-        assert isinstance(index, ConstInt)
-        assert isinstance(count, ConstInt)
-        assert arg0.item_size == result.item_size
-        if isinstance(arg1, BoxVector):
-            assert arg1.item_size == result.item_size
-        else:
-            assert count.value == 1
-        assert index.value < result.item_count
-        assert index.value + count.value <= result.item_count
-        assert result.item_count > arg0.item_count
-
-    def expand(self, nodes, arg, argidx):
-        vbox = self.input_type.new_vector_box(len(nodes))
-        box_type = arg.type
-        expanded_map = self.sched_data.expanded_map
-        invariant_ops = self.sched_data.invariant_oplist
-        invariant_vars = self.sched_data.invariant_vector_vars
-        if isinstance(arg, BoxVector):
-            box_type = arg.item_type
-
-        # note that heterogenous nodes are not yet tracked
-        already_expanded = expanded_map.get(arg, None)
-        if already_expanded:
-            return already_expanded
-
-        for i, node in enumerate(nodes):
-            op = node.getoperation()
-            if not arg.same_box(op.getarg(argidx)):
-                break
-            i += 1
-        else:
-            expand_opnum = rop.VEC_FLOAT_EXPAND
-            if box_type == INT:
-                expand_opnum = rop.VEC_INT_EXPAND
-            op = ResOperation(expand_opnum, [arg], vbox)
-            invariant_ops.append(op)
-            invariant_vars.append(vbox)
-            expanded_map[arg] = vbox
-            return vbox
-
-        op = ResOperation(rop.VEC_BOX, [ConstInt(len(nodes))], vbox)
-        invariant_ops.append(op)
-        opnum = rop.VEC_FLOAT_PACK
-        if arg.type == INT:
-            opnum = rop.VEC_INT_PACK
-        for i,node in enumerate(nodes):
-            op = node.getoperation()
-            arg = op.getarg(argidx)
-            new_box = vbox.clonebox()
-            ci = ConstInt(i)
-            c1 = ConstInt(1)
-            op = ResOperation(opnum, [vbox,arg,ci,c1], new_box)
-            vbox = new_box
-            invariant_ops.append(op)
-
-        invariant_vars.append(vbox)
-        return vbox
 
 def isomorphic(l_op, r_op):
     """ Subject of definition """
@@ -1021,65 +642,3 @@
             del self.packs[last_pos]
         return last_pos
 
-class Pack(object):
-    """ A pack is a set of n statements that are:
-        * isomorphic
-        * independent
-    """
-    def __init__(self, ops):
-        self.operations = ops
-        for i,node in enumerate(self.operations):
-            node.pack = self
-            node.pack_position = i
-        self.accum_variable = None
-        self.accum_position = -1
-
-    def opcount(self):
-        return len(self.operations)
-
-    def opnum(self):
-        assert len(self.operations) > 0
-        return self.operations[0].getoperation().getopnum()
-
-    def clear(self):
-        for node in self.operations:
-            node.pack = None
-            node.pack_position = -1
-
-    def rightmost_match_leftmost(self, other):
-        assert isinstance(other, Pack)
-        rightmost = self.operations[-1]
-        leftmost = other.operations[0]
-        return rightmost == leftmost and \
-               self.accum_variable == other.accum_variable and \
-               self.accum_position == other.accum_position
-
-    def __repr__(self):
-        return "Pack(%r)" % self.operations
-
-    def is_accumulating(self):
-        return self.accum_variable is not None
-
-class Pair(Pack):
-    """ A special Pack object with only two statements. """
-    def __init__(self, left, right):
-        assert isinstance(left, Node)
-        assert isinstance(right, Node)
-        self.left = left
-        self.right = right
-        Pack.__init__(self, [left, right])
-
-    def __eq__(self, other):
-        if isinstance(other, Pair):
-            return self.left is other.left and \
-                   self.right is other.right
-
-class AccumPair(Pair):
-    def __init__(self, left, right, accum_var, accum_pos):
-        assert isinstance(left, Node)
-        assert isinstance(right, Node)
-        Pair.__init__(self, left, right)
-        self.left = left
-        self.right = right
-        self.accum_variable = accum_var
-        self.accum_position = accum_pos

From noreply at buildbot.pypy.org  Mon Jun  8 15:58:05 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Mon,  8 Jun 2015 15:58:05 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: can_be_packed did not consider the case
 if origin pack is None, some small other refactorings
Message-ID: <20150608135805.583C01C080A@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r77960:5d3c3f5b1a5a
Date: 2015-06-08 15:56 +0200
http://bitbucket.org/pypy/pypy/changeset/5d3c3f5b1a5a/

Log:	can_be_packed did not consider the case if origin pack is None, some
	small other refactorings

diff --git a/rpython/jit/metainterp/optimizeopt/schedule.py b/rpython/jit/metainterp/optimizeopt/schedule.py
--- a/rpython/jit/metainterp/optimizeopt/schedule.py
+++ b/rpython/jit/metainterp/optimizeopt/schedule.py
@@ -1,6 +1,6 @@
 
 from rpython.jit.metainterp.history import (FLOAT,INT,ConstInt,BoxVector,
-        BoxFloat,BoxInt)
+        BoxFloat,BoxInt,ConstFloat)
 from rpython.jit.metainterp.resoperation import (rop, ResOperation, GuardResOp)
 from rpython.jit.metainterp.optimizeopt.dependency import (DependencyGraph,
         MemoryRef, Node, IndexVar)
@@ -525,8 +525,7 @@
 LOAD_TRANS = LoadToVectorLoad()
 STORE_TRANS = StoreToVectorStore()
 
-# note that the following definition is x86 machine
-# specific.
+# note that the following definition is x86 arch specific
 ROP_ARG_RES_VECTOR = {
     rop.VEC_INT_ADD:     INT_OP_TO_VOP,
     rop.VEC_INT_SUB:     INT_OP_TO_VOP,
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_schedule.py b/rpython/jit/metainterp/optimizeopt/test/test_schedule.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_schedule.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_schedule.py
@@ -23,7 +23,7 @@
             'int': self.int32arraydescr,
         }
         loop = opparse("        [p0,p1,p2,p3,p4,p5,i0,i1,i2,i3,i4,i5,i6,i7,i8,i9,f0,f1,f2,f3,f4,f5,v103204[i32|4]]\n" + source + \
-                       "\n        jump(p0,p1,p2,p3,p4,p5,i0,i1,i2,i3,i4,i5,i6,i7,i8,i9,f0,f1,f2,f3,f4,f5)",
+                       "\n        jump(p0,p1,p2,p3,p4,p5,i0,i1,i2,i3,i4,i5,i6,i7,i8,i9,f0,f1,f2,f3,f4,f5,v103204[i32|4])",
                        cpu=self.cpu,
                        namespace=ns)
         if inc_label_jump:
diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/vectorize.py
@@ -589,7 +589,8 @@
             else:
                 if self.contains_pair(lnode, rnode):
                     return None
-                return self.accumulates_pair(lnode, rnode, origin_pack)
+                if origin_pack is not None:
+                    return self.accumulates_pair(lnode, rnode, origin_pack)
         return None
 
     def contains_pair(self, lnode, rnode):

From noreply at buildbot.pypy.org  Mon Jun  8 19:29:17 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Mon,  8 Jun 2015 19:29:17 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: A fix for preserving pointerness
Message-ID: <20150608172917.0F5641C1F7D@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77961:dc515403d063
Date: 2015-06-08 19:29 +0200
http://bitbucket.org/pypy/pypy/changeset/dc515403d063/

Log:	A fix for preserving pointerness

diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py
--- a/rpython/jit/backend/llsupport/rewrite.py
+++ b/rpython/jit/backend/llsupport/rewrite.py
@@ -568,7 +568,7 @@
             if self.gc_ll_descr.can_use_nursery_malloc(total_size):
                 # if the total size is still reasonable, merge it
                 self._op_malloc_nursery.setarg(0, ConstInt(total_size))
-                op = ResOperation(rop.INT_ADD,
+                op = ResOperation(rop.NURSERY_PTR_INCREMENT,
                                   [self._v_last_malloced_nursery,
                                    ConstInt(self._previous_size)])
                 self.replace_op_with(v_result, op)
diff --git a/rpython/jit/backend/llsupport/test/test_rewrite.py b/rpython/jit/backend/llsupport/test/test_rewrite.py
--- a/rpython/jit/backend/llsupport/test/test_rewrite.py
+++ b/rpython/jit/backend/llsupport/test/test_rewrite.py
@@ -332,9 +332,9 @@
             p0 = call_malloc_nursery(   \
                                %(sdescr.size + tdescr.size + sdescr.size)d)
             setfield_gc(p0, 1234, descr=tiddescr)
-            p1 = int_add(p0, %(sdescr.size)d)
+            p1 = nursery_ptr_increment(p0, %(sdescr.size)d)
             setfield_gc(p1, 5678, descr=tiddescr)
-            p2 = int_add(p1, %(tdescr.size)d)
+            p2 = nursery_ptr_increment(p1, %(tdescr.size)d)
             setfield_gc(p2, 1234, descr=tiddescr)
             zero_ptr_field(p1, %(tdescr.gc_fielddescrs[0].offset)s)
             jump()
@@ -366,7 +366,7 @@
                                 %(sdescr.size +                        \
                                   adescr.basesize + 10 * adescr.itemsize)d)
             setfield_gc(p0, 1234, descr=tiddescr)
-            p1 = int_add(p0, %(sdescr.size)d)
+            p1 = nursery_ptr_increment(p0, %(sdescr.size)d)
             setfield_gc(p1, 4321, descr=tiddescr)
             setfield_gc(p1, 10, descr=alendescr)
             jump()
@@ -398,13 +398,13 @@
             p0 = call_malloc_nursery(%(4 * (bdescr.basesize + 8))d)
             setfield_gc(p0, 8765, descr=tiddescr)
             setfield_gc(p0, 5, descr=blendescr)
-            p1 = int_add(p0, %(bdescr.basesize + 8)d)
+            p1 = nursery_ptr_increment(p0, %(bdescr.basesize + 8)d)
             setfield_gc(p1, 8765, descr=tiddescr)
             setfield_gc(p1, 5, descr=blendescr)
-            p2 = int_add(p1, %(bdescr.basesize + 8)d)
+            p2 = nursery_ptr_increment(p1, %(bdescr.basesize + 8)d)
             setfield_gc(p2, 8765, descr=tiddescr)
             setfield_gc(p2, 5, descr=blendescr)
-            p3 = int_add(p2, %(bdescr.basesize + 8)d)
+            p3 = nursery_ptr_increment(p2, %(bdescr.basesize + 8)d)
             setfield_gc(p3, 8765, descr=tiddescr)
             setfield_gc(p3, 5, descr=blendescr)
             jump()
@@ -420,7 +420,7 @@
             []
             p0 = call_malloc_nursery(%(4*WORD)d)
             setfield_gc(p0, 9000, descr=tiddescr)
-            p1 = int_add(p0, %(2*WORD)d)
+            p1 = nursery_ptr_increment(p0, %(2*WORD)d)
             setfield_gc(p1, 9000, descr=tiddescr)
             jump()
         """)
@@ -504,7 +504,7 @@
                               %(2 * (bdescr.basesize + 104))d)
             setfield_gc(p0, 8765, descr=tiddescr)
             setfield_gc(p0, 101, descr=blendescr)
-            p1 = int_add(p0, %(bdescr.basesize + 104)d)
+            p1 = nursery_ptr_increment(p0, %(bdescr.basesize + 104)d)
             setfield_gc(p1, 8765, descr=tiddescr)
             setfield_gc(p1, 102, descr=blendescr)
             p2 = call_malloc_nursery(    \
@@ -571,7 +571,7 @@
             setfield_gc(p0, %(strdescr.tid)d, descr=tiddescr)
             setfield_gc(p0, 14, descr=strlendescr)
             setfield_gc(p0, 0, descr=strhashdescr)
-            p1 = int_add(p0, %(strdescr.basesize + 16 * strdescr.itemsize)d)
+            p1 = nursery_ptr_increment(p0, %(strdescr.basesize + 16 * strdescr.itemsize)d)
             setfield_gc(p1, %(unicodedescr.tid)d, descr=tiddescr)
             setfield_gc(p1, 10, descr=unicodelendescr)
             setfield_gc(p1, 0, descr=unicodehashdescr)
@@ -731,7 +731,7 @@
             []
             p0 = call_malloc_nursery(%(tdescr.size + sdescr.size)d)
             setfield_gc(p0, 5678, descr=tiddescr)
-            p1 = int_add(p0, %(tdescr.size)d)
+            p1 = nursery_ptr_increment(p0, %(tdescr.size)d)
             setfield_gc(p1, 1234, descr=tiddescr)
             # <<<no cond_call_gc_wb here>>>
             setfield_gc(p0, p1, descr=tzdescr)
diff --git a/rpython/jit/metainterp/executor.py b/rpython/jit/metainterp/executor.py
--- a/rpython/jit/metainterp/executor.py
+++ b/rpython/jit/metainterp/executor.py
@@ -378,6 +378,7 @@
                          rop.CALL_MALLOC_NURSERY,
                          rop.CALL_MALLOC_NURSERY_VARSIZE,
                          rop.CALL_MALLOC_NURSERY_VARSIZE_FRAME,
+                         rop.NURSERY_PTR_INCREMENT,
                          rop.LABEL,
                          ):      # list of opcodes never executed by pyjitpl
                 continue
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -695,6 +695,7 @@
     'PTR_NE/2b/i',
     'INSTANCE_PTR_EQ/2b/i',
     'INSTANCE_PTR_NE/2b/i',
+    'NURSERY_PTR_INCREMENT/2/r',
     #
     'ARRAYLEN_GC/1d/i',
     'STRLEN/1/i',

From noreply at buildbot.pypy.org  Mon Jun  8 20:15:39 2015
From: noreply at buildbot.pypy.org (stefanor)
Date: Mon,  8 Jun 2015 20:15:39 +0200 (CEST)
Subject: [pypy-commit] pypy default: vmprof uses x86_64 assembler,
 so it's only available on x86_64
Message-ID: <20150608181539.826CA1C0460@cobra.cs.uni-duesseldorf.de>

Author: Stefano Rivera <stefano at rivera.za.net>
Branch: 
Changeset: r77962:5565f24726f0
Date: 2015-06-08 11:15 -0700
http://bitbucket.org/pypy/pypy/changeset/5565f24726f0/

Log:	vmprof uses x86_64 assembler, so it's only available on x86_64

	Not all 64bit linux platforms.

diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -1,3 +1,4 @@
+import os
 import sys
 
 import py
@@ -38,7 +39,7 @@
     "_csv", "cppyy", "_pypyjson"
 ])
 
-if sys.platform.startswith('linux') and sys.maxint > 2147483647:
+if sys.platform.startswith('linux') and os.uname()[4] == 'x86_64':
     working_modules.add('_vmprof')
 
 translation_modules = default_modules.copy()

From noreply at buildbot.pypy.org  Mon Jun  8 20:30:28 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Mon,  8 Jun 2015 20:30:28 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: pfff
Message-ID: <20150608183028.C09981C048F@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77963:513287d1c288
Date: 2015-06-08 20:30 +0200
http://bitbucket.org/pypy/pypy/changeset/513287d1c288/

Log:	pfff

diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -1074,6 +1074,7 @@
     genop_int_neg = _unaryop("NEG")
     genop_int_invert = _unaryop("NOT")
     genop_int_add = _binaryop_or_lea("ADD", is_add=True)
+    genop_nursery_ptr_increment = _binaryop_or_lea('ADD', is_add=True)
     genop_int_sub = _binaryop_or_lea("SUB", is_add=False)
     genop_int_mul = _binaryop("IMUL")
     genop_int_and = _binaryop("AND")
diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -467,6 +467,8 @@
         else:
             self._consider_binop_symm(op)
 
+    consider_nursery_ptr_increment = consider_int_add
+
     def consider_int_sub(self, op):
         loc = self.loc(op.getarg(0))
         y = op.getarg(1)

From noreply at buildbot.pypy.org  Mon Jun  8 20:41:50 2015
From: noreply at buildbot.pypy.org (MarkusH)
Date: Mon,  8 Jun 2015 20:41:50 +0200 (CEST)
Subject: [pypy-commit] pypy issue2062: Fixed #2062 -- Treated
 date/datetime/time/timedelta repr like on CPython
Message-ID: <20150608184150.18E1F1C080A@cobra.cs.uni-duesseldorf.de>

Author: Markus Holtermann <info at markusholtermann.eu>
Branch: issue2062
Changeset: r77964:a5b003f9b84e
Date: 2015-06-08 20:29 +0200
http://bitbucket.org/pypy/pypy/changeset/a5b003f9b84e/

Log:	Fixed #2062 -- Treated date/datetime/time/timedelta repr like on
	CPython

diff --git a/lib_pypy/datetime.py b/lib_pypy/datetime.py
--- a/lib_pypy/datetime.py
+++ b/lib_pypy/datetime.py
@@ -536,16 +536,17 @@
         return self
 
     def __repr__(self):
+        module = "datetime." if self.__class__ is timedelta else ""
         if self._microseconds:
-            return "%s(%d, %d, %d)" % ('datetime.' + self.__class__.__name__,
+            return "%s(%d, %d, %d)" % (module + self.__class__.__name__,
                                        self._days,
                                        self._seconds,
                                        self._microseconds)
         if self._seconds:
-            return "%s(%d, %d)" % ('datetime.' + self.__class__.__name__,
+            return "%s(%d, %d)" % (module + self.__class__.__name__,
                                    self._days,
                                    self._seconds)
-        return "%s(%d)" % ('datetime.' + self.__class__.__name__, self._days)
+        return "%s(%d)" % (module + self.__class__.__name__, self._days)
 
     def __str__(self):
         mm, ss = divmod(self._seconds, 60)
@@ -798,7 +799,8 @@
         >>> repr(dt)
         'datetime.datetime(2010, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)'
         """
-        return "%s(%d, %d, %d)" % ('datetime.' + self.__class__.__name__,
+        module = "datetime." if self.__class__ is date else ""
+        return "%s(%d, %d, %d)" % (module + self.__class__.__name__,
                                    self._year,
                                    self._month,
                                    self._day)
@@ -1286,7 +1288,8 @@
             s = ", %d" % self._second
         else:
             s = ""
-        s= "%s(%d, %d%s)" % ('datetime.' + self.__class__.__name__,
+        module = "datetime." if self.__class__ is time else ""
+        s= "%s(%d, %d%s)" % (module + self.__class__.__name__,
                              self._hour, self._minute, s)
         if self._tzinfo is not None:
             assert s[-1:] == ")"
@@ -1698,7 +1701,8 @@
         if L[-1] == 0:
             del L[-1]
         s = ", ".join(map(str, L))
-        s = "%s(%s)" % ('datetime.' + self.__class__.__name__, s)
+        module = "datetime." if self.__class__ is datetime else ""
+        s = "%s(%s)" % (module + self.__class__.__name__, s)
         if self._tzinfo is not None:
             assert s[-1:] == ")"
             s = s[:-1] + ", tzinfo=%r" % self._tzinfo + ")"
diff --git a/pypy/module/test_lib_pypy/test_datetime.py b/pypy/module/test_lib_pypy/test_datetime.py
--- a/pypy/module/test_lib_pypy/test_datetime.py
+++ b/pypy/module/test_lib_pypy/test_datetime.py
@@ -10,6 +10,36 @@
         expected = "datetime.datetime(1, 2, 3, 0, 0)"
         assert repr(datetime.datetime(1,2,3)) == expected
 
+    def test_repr_overridden(self):
+        class date_safe(datetime.date):
+            pass
+
+        class datetime_safe(datetime.datetime):
+            pass
+
+        class time_safe(datetime.time):
+            pass
+
+        class timedelta_safe(datetime.timedelta):
+            pass
+
+        checks = (
+            (datetime.date(2015, 6, 8), "datetime.date(2015, 6, 8)"),
+            (datetime.datetime(2015, 6, 8, 12, 34, 56), "datetime.datetime(2015, 6, 8, 12, 34, 56)"),
+            (datetime.time(12, 34, 56), "datetime.time(12, 34, 56)"),
+            (datetime.timedelta(1), "datetime.timedelta(1)"),
+            (datetime.timedelta(1, 2), "datetime.timedelta(1, 2)"),
+            (datetime.timedelta(1, 2, 3), "datetime.timedelta(1, 2, 3)"),
+            (date_safe(2015, 6, 8), "date_safe(2015, 6, 8)"),
+            (datetime_safe(2015, 6, 8, 12, 34, 56), "datetime_safe(2015, 6, 8, 12, 34, 56)"),
+            (time_safe(12, 34, 56), "time_safe(12, 34, 56)"),
+            (timedelta_safe(1), "timedelta_safe(1)"),
+            (timedelta_safe(1, 2), "timedelta_safe(1, 2)"),
+            (timedelta_safe(1, 2, 3), "timedelta_safe(1, 2, 3)"),
+        )
+        for obj, expected in checks:
+            assert repr(obj) == expected
+
     def test_attributes(self):
         for x in [datetime.date.today(),
                   datetime.time(),

From noreply at buildbot.pypy.org  Mon Jun  8 20:41:51 2015
From: noreply at buildbot.pypy.org (MarkusH)
Date: Mon,  8 Jun 2015 20:41:51 +0200 (CEST)
Subject: [pypy-commit] pypy issue2062: Cleaned up datetime __repr__ tests
Message-ID: <20150608184151.535661C080A@cobra.cs.uni-duesseldorf.de>

Author: Markus Holtermann <info at markusholtermann.eu>
Branch: issue2062
Changeset: r77965:3d0c1e1af8df
Date: 2015-06-08 20:38 +0200
http://bitbucket.org/pypy/pypy/changeset/3d0c1e1af8df/

Log:	Cleaned up datetime __repr__ tests

diff --git a/pypy/module/test_lib_pypy/test_datetime.py b/pypy/module/test_lib_pypy/test_datetime.py
--- a/pypy/module/test_lib_pypy/test_datetime.py
+++ b/pypy/module/test_lib_pypy/test_datetime.py
@@ -6,9 +6,16 @@
 
 class BaseTestDatetime:
     def test_repr(self):
-        print datetime
-        expected = "datetime.datetime(1, 2, 3, 0, 0)"
-        assert repr(datetime.datetime(1,2,3)) == expected
+        checks = (
+            (datetime.date(2015, 6, 8), "datetime.date(2015, 6, 8)"),
+            (datetime.datetime(2015, 6, 8, 12, 34, 56), "datetime.datetime(2015, 6, 8, 12, 34, 56)"),
+            (datetime.time(12, 34, 56), "datetime.time(12, 34, 56)"),
+            (datetime.timedelta(1), "datetime.timedelta(1)"),
+            (datetime.timedelta(1, 2), "datetime.timedelta(1, 2)"),
+            (datetime.timedelta(1, 2, 3), "datetime.timedelta(1, 2, 3)"),
+        )
+        for obj, expected in checks:
+            assert repr(obj) == expected
 
     def test_repr_overridden(self):
         class date_safe(datetime.date):
@@ -24,12 +31,6 @@
             pass
 
         checks = (
-            (datetime.date(2015, 6, 8), "datetime.date(2015, 6, 8)"),
-            (datetime.datetime(2015, 6, 8, 12, 34, 56), "datetime.datetime(2015, 6, 8, 12, 34, 56)"),
-            (datetime.time(12, 34, 56), "datetime.time(12, 34, 56)"),
-            (datetime.timedelta(1), "datetime.timedelta(1)"),
-            (datetime.timedelta(1, 2), "datetime.timedelta(1, 2)"),
-            (datetime.timedelta(1, 2, 3), "datetime.timedelta(1, 2, 3)"),
             (date_safe(2015, 6, 8), "date_safe(2015, 6, 8)"),
             (datetime_safe(2015, 6, 8, 12, 34, 56), "datetime_safe(2015, 6, 8, 12, 34, 56)"),
             (time_safe(12, 34, 56), "time_safe(12, 34, 56)"),

From noreply at buildbot.pypy.org  Mon Jun  8 20:41:52 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  8 Jun 2015 20:41:52 +0200 (CEST)
Subject: [pypy-commit] pypy default: Merged in MarkusH/pypy/issue2062 (pull
	request #325)
Message-ID: <20150608184152.791C11C080A@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <armin.rigo at gmail.com>
Branch: 
Changeset: r77966:8c313d798d12
Date: 2015-06-08 20:42 +0200
http://bitbucket.org/pypy/pypy/changeset/8c313d798d12/

Log:	Merged in MarkusH/pypy/issue2062 (pull request #325)

	Fixed #2062 -- Treated date/datetime/time/timedelta repr like on
	CPython

diff --git a/lib_pypy/datetime.py b/lib_pypy/datetime.py
--- a/lib_pypy/datetime.py
+++ b/lib_pypy/datetime.py
@@ -536,16 +536,17 @@
         return self
 
     def __repr__(self):
+        module = "datetime." if self.__class__ is timedelta else ""
         if self._microseconds:
-            return "%s(%d, %d, %d)" % ('datetime.' + self.__class__.__name__,
+            return "%s(%d, %d, %d)" % (module + self.__class__.__name__,
                                        self._days,
                                        self._seconds,
                                        self._microseconds)
         if self._seconds:
-            return "%s(%d, %d)" % ('datetime.' + self.__class__.__name__,
+            return "%s(%d, %d)" % (module + self.__class__.__name__,
                                    self._days,
                                    self._seconds)
-        return "%s(%d)" % ('datetime.' + self.__class__.__name__, self._days)
+        return "%s(%d)" % (module + self.__class__.__name__, self._days)
 
     def __str__(self):
         mm, ss = divmod(self._seconds, 60)
@@ -798,7 +799,8 @@
         >>> repr(dt)
         'datetime.datetime(2010, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)'
         """
-        return "%s(%d, %d, %d)" % ('datetime.' + self.__class__.__name__,
+        module = "datetime." if self.__class__ is date else ""
+        return "%s(%d, %d, %d)" % (module + self.__class__.__name__,
                                    self._year,
                                    self._month,
                                    self._day)
@@ -1286,7 +1288,8 @@
             s = ", %d" % self._second
         else:
             s = ""
-        s= "%s(%d, %d%s)" % ('datetime.' + self.__class__.__name__,
+        module = "datetime." if self.__class__ is time else ""
+        s= "%s(%d, %d%s)" % (module + self.__class__.__name__,
                              self._hour, self._minute, s)
         if self._tzinfo is not None:
             assert s[-1:] == ")"
@@ -1698,7 +1701,8 @@
         if L[-1] == 0:
             del L[-1]
         s = ", ".join(map(str, L))
-        s = "%s(%s)" % ('datetime.' + self.__class__.__name__, s)
+        module = "datetime." if self.__class__ is datetime else ""
+        s = "%s(%s)" % (module + self.__class__.__name__, s)
         if self._tzinfo is not None:
             assert s[-1:] == ")"
             s = s[:-1] + ", tzinfo=%r" % self._tzinfo + ")"
diff --git a/pypy/module/test_lib_pypy/test_datetime.py b/pypy/module/test_lib_pypy/test_datetime.py
--- a/pypy/module/test_lib_pypy/test_datetime.py
+++ b/pypy/module/test_lib_pypy/test_datetime.py
@@ -6,9 +6,40 @@
 
 class BaseTestDatetime:
     def test_repr(self):
-        print datetime
-        expected = "datetime.datetime(1, 2, 3, 0, 0)"
-        assert repr(datetime.datetime(1,2,3)) == expected
+        checks = (
+            (datetime.date(2015, 6, 8), "datetime.date(2015, 6, 8)"),
+            (datetime.datetime(2015, 6, 8, 12, 34, 56), "datetime.datetime(2015, 6, 8, 12, 34, 56)"),
+            (datetime.time(12, 34, 56), "datetime.time(12, 34, 56)"),
+            (datetime.timedelta(1), "datetime.timedelta(1)"),
+            (datetime.timedelta(1, 2), "datetime.timedelta(1, 2)"),
+            (datetime.timedelta(1, 2, 3), "datetime.timedelta(1, 2, 3)"),
+        )
+        for obj, expected in checks:
+            assert repr(obj) == expected
+
+    def test_repr_overridden(self):
+        class date_safe(datetime.date):
+            pass
+
+        class datetime_safe(datetime.datetime):
+            pass
+
+        class time_safe(datetime.time):
+            pass
+
+        class timedelta_safe(datetime.timedelta):
+            pass
+
+        checks = (
+            (date_safe(2015, 6, 8), "date_safe(2015, 6, 8)"),
+            (datetime_safe(2015, 6, 8, 12, 34, 56), "datetime_safe(2015, 6, 8, 12, 34, 56)"),
+            (time_safe(12, 34, 56), "time_safe(12, 34, 56)"),
+            (timedelta_safe(1), "timedelta_safe(1)"),
+            (timedelta_safe(1, 2), "timedelta_safe(1, 2)"),
+            (timedelta_safe(1, 2, 3), "timedelta_safe(1, 2, 3)"),
+        )
+        for obj, expected in checks:
+            assert repr(obj) == expected
 
     def test_attributes(self):
         for x in [datetime.date.today(),

From noreply at buildbot.pypy.org  Mon Jun  8 21:11:04 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Mon,  8 Jun 2015 21:11:04 +0200 (CEST)
Subject: [pypy-commit] pypy default: Ensure that W_Dtype.byteorder is a char,
 not a string, and hopefully fix performance regression
Message-ID: <20150608191104.3DD551C0478@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r77967:97360f2c7194
Date: 2015-06-08 20:09 +0100
http://bitbucket.org/pypy/pypy/changeset/97360f2c7194/

Log:	Ensure that W_Dtype.byteorder is a char, not a string, and hopefully
	fix performance regression

diff --git a/pypy/module/micronumpy/descriptor.py b/pypy/module/micronumpy/descriptor.py
--- a/pypy/module/micronumpy/descriptor.py
+++ b/pypy/module/micronumpy/descriptor.py
@@ -5,8 +5,10 @@
 from pypy.interpreter.gateway import interp2app, unwrap_spec
 from pypy.interpreter.typedef import (TypeDef, GetSetProperty,
                                       interp_attrproperty, interp_attrproperty_w)
+from rpython.annotator.model import SomeChar
 from rpython.rlib import jit
-from rpython.rlib.objectmodel import specialize, compute_hash, we_are_translated
+from rpython.rlib.objectmodel import (
+        specialize, compute_hash, we_are_translated, enforceargs)
 from rpython.rlib.rarithmetic import r_longlong, r_ulonglong
 from pypy.module.micronumpy import types, boxes, support, constants as NPY
 from .base import W_NDimArray
@@ -38,6 +40,15 @@
     out = W_NDimArray.from_shape(space, shape, dtype)
     return out
 
+def byteorder_w(space, w_str):
+    order = space.str_w(w_str)
+    if len(order) != 1:
+        raise oefmt(space.w_ValueError,
+                "endian is not 1-char string in Numpy dtype unpickling")
+    endian = order[0]
+    if endian not in (NPY.LITTLE, NPY.BIG, NPY.NATIVE, NPY.IGNORE):
+        raise oefmt(space.w_ValueError, "Invalid byteorder %s", endian)
+    return endian
 
 
 class W_Dtype(W_Root):
@@ -45,15 +56,13 @@
         "itemtype?", "w_box_type", "byteorder?", "names?", "fields?",
         "elsize?", "alignment?", "shape?", "subdtype?", "base?"]
 
-    def __init__(self, itemtype, w_box_type, byteorder=None, names=[],
+    @enforceargs(byteorder=SomeChar())
+    def __init__(self, itemtype, w_box_type, byteorder=NPY.NATIVE, names=[],
                  fields={}, elsize=None, shape=[], subdtype=None):
         self.itemtype = itemtype
         self.w_box_type = w_box_type
-        if byteorder is None:
-            if itemtype.get_element_size() == 1 or isinstance(itemtype, types.ObjectType):
-                byteorder = NPY.IGNORE
-            else:
-                byteorder = NPY.NATIVE
+        if itemtype.get_element_size() == 1 or isinstance(itemtype, types.ObjectType):
+            byteorder = NPY.IGNORE
         self.byteorder = byteorder
         self.names = names
         self.fields = fields
@@ -137,7 +146,8 @@
         return bool(self.fields)
 
     def is_native(self):
-        return self.byteorder in (NPY.NATIVE, NPY.NATBYTE)
+        # Use ord() to ensure that self.byteorder is a char and JITs properly
+        return ord(self.byteorder) in (ord(NPY.NATIVE), ord(NPY.NATBYTE))
 
     def as_signed(self, space):
         """Convert from an unsigned integer dtype to its signed partner"""
@@ -446,7 +456,7 @@
                         "can't handle version %d of numpy.dtype pickle",
                         version)
 
-        endian = space.str_w(space.getitem(w_data, space.wrap(1)))
+        endian = byteorder_w(space, space.getitem(w_data, space.wrap(1)))
         if endian == NPY.NATBYTE:
             endian = NPY.NATIVE
 

From noreply at buildbot.pypy.org  Mon Jun  8 21:40:19 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Mon,  8 Jun 2015 21:40:19 +0200 (CEST)
Subject: [pypy-commit] pypy disable-unroll-for-short-loops: close to be
	merged branch
Message-ID: <20150608194019.39A741C0460@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: disable-unroll-for-short-loops
Changeset: r77968:0ab9d5c2be13
Date: 2015-06-08 21:39 +0200
http://bitbucket.org/pypy/pypy/changeset/0ab9d5c2be13/

Log:	close to be merged branch


From noreply at buildbot.pypy.org  Mon Jun  8 21:40:20 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Mon,  8 Jun 2015 21:40:20 +0200 (CEST)
Subject: [pypy-commit] pypy default: Merge disable-unroll-for-short-loops
 (which actually is supposed to be
Message-ID: <20150608194020.96D241C0460@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r77969:b0c01840baea
Date: 2015-06-08 21:40 +0200
http://bitbucket.org/pypy/pypy/changeset/b0c01840baea/

Log:	Merge disable-unroll-for-short-loops (which actually is supposed to
	be for *long* looops)

	This disables unrolling if the loops are longer than 100 operations
	(tunable)

diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -103,6 +103,7 @@
 
 # ____________________________________________________________
 
+
 def compile_loop(metainterp, greenkey, start,
                  inputargs, jumpargs,
                  full_preamble_needed=True,
@@ -148,27 +149,28 @@
     if part.quasi_immutable_deps:
         loop.quasi_immutable_deps.update(part.quasi_immutable_deps)
     if part.operations[-1].getopnum() == rop.LABEL:
-        inliner = Inliner(inputargs, jumpargs)
-        part.quasi_immutable_deps = None
-        part.operations = [part.operations[-1]] + \
-                          [inliner.inline_op(h_ops[i]) for i in range(start, len(h_ops))] + \
-                          [ResOperation(rop.JUMP, [inliner.inline_arg(a) for a in jumpargs],
-                                        None, descr=jitcell_token)]
-        target_token = part.operations[0].getdescr()
-        assert isinstance(target_token, TargetToken)
-        all_target_tokens.append(target_token)
-        inputargs = jumpargs
-        jumpargs = part.operations[-1].getarglist()
+        if start_state is not None:
+            inliner = Inliner(inputargs, jumpargs)
+            part.quasi_immutable_deps = None
+            part.operations = [part.operations[-1]] + \
+                              [inliner.inline_op(h_ops[i]) for i in range(start, len(h_ops))] + \
+                              [ResOperation(rop.JUMP, [inliner.inline_arg(a) for a in jumpargs],
+                                            None, descr=jitcell_token)]
+            target_token = part.operations[0].getdescr()
+            assert isinstance(target_token, TargetToken)
+            all_target_tokens.append(target_token)
+            inputargs = jumpargs
+            jumpargs = part.operations[-1].getarglist()
 
-        try:
-            optimize_trace(metainterp_sd, jitdriver_sd, part, enable_opts,
-                           start_state=start_state, export_state=False)
-        except InvalidLoop:
-            return None
+            try:
+                optimize_trace(metainterp_sd, jitdriver_sd, part, enable_opts,
+                               start_state=start_state, export_state=False)
+            except InvalidLoop:
+                return None
 
-        loop.operations = loop.operations[:-1] + part.operations
-        if part.quasi_immutable_deps:
-            loop.quasi_immutable_deps.update(part.quasi_immutable_deps)
+            loop.operations = loop.operations[:-1] + part.operations
+            if part.quasi_immutable_deps:
+                loop.quasi_immutable_deps.update(part.quasi_immutable_deps)
     assert part.operations[-1].getopnum() != rop.LABEL
 
     if not loop.quasi_immutable_deps:
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -154,6 +154,19 @@
 
         loop.operations = self.optimizer.get_newoperations()
         if export_state:
+            jd_sd = self.optimizer.jitdriver_sd
+            threshold = jd_sd.warmstate.disable_unrolling_threshold
+            if 1 or len(loop.operations) > threshold:
+                if loop.operations[0].getopnum() == rop.LABEL:
+                    # abandoning unrolling, too long
+                    new_descr = stop_label.getdescr()
+                    if loop.operations[0].getopnum() == rop.LABEL:
+                        new_descr = loop.operations[0].getdescr()
+                    stop_label = stop_label.copy_and_change(rop.JUMP,
+                                        descr=new_descr)
+                    self.optimizer.send_extra_operation(stop_label)
+                    loop.operations = self.optimizer.get_newoperations()
+                    return None
             final_state = self.export_state(stop_label)
         else:
             final_state = None
diff --git a/rpython/jit/metainterp/warmstate.py b/rpython/jit/metainterp/warmstate.py
--- a/rpython/jit/metainterp/warmstate.py
+++ b/rpython/jit/metainterp/warmstate.py
@@ -256,6 +256,9 @@
     def set_param_inlining(self, value):
         self.inlining = value
 
+    def set_param_disable_unrolling(self, value):
+        self.disable_unrolling_threshold = value
+
     def set_param_enable_opts(self, value):
         from rpython.jit.metainterp.optimizeopt import ALL_OPTS_DICT, ALL_OPTS_NAMES
 
diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py
--- a/rpython/rlib/jit.py
+++ b/rpython/rlib/jit.py
@@ -549,6 +549,7 @@
     'retrace_limit': 'how many times we can try retracing before giving up',
     'max_retrace_guards': 'number of extra guards a retrace can cause',
     'max_unroll_loops': 'number of extra unrollings a loop can cause',
+    'disable_unrolling': 'after how many operations we should not unroll',
     'enable_opts': 'INTERNAL USE ONLY (MAY NOT WORK OR LEAD TO CRASHES): '
                    'optimizations to enable, or all = %s' % ENABLE_ALL_OPTS,
     'max_unroll_recursion': 'how many levels deep to unroll a recursive function'
@@ -564,6 +565,7 @@
               'retrace_limit': 5,
               'max_retrace_guards': 15,
               'max_unroll_loops': 0,
+              'disable_unrolling': 100,
               'enable_opts': 'all',
               'max_unroll_recursion': 7,
               }

From noreply at buildbot.pypy.org  Tue Jun  9 08:34:37 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Tue,  9 Jun 2015 08:34:37 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: scheduler emits packs that are
 interdependent (only within pack and marked accum)
Message-ID: <20150609063437.876E31C1F81@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r77970:c853e8ea2f01
Date: 2015-06-09 08:34 +0200
http://bitbucket.org/pypy/pypy/changeset/c853e8ea2f01/

Log:	scheduler emits packs that are interdependent (only within pack and
	marked accum)

diff --git a/rpython/jit/metainterp/logger.py b/rpython/jit/metainterp/logger.py
--- a/rpython/jit/metainterp/logger.py
+++ b/rpython/jit/metainterp/logger.py
@@ -127,7 +127,8 @@
         elif isinstance(arg, BoxFloat):
             return 'f' + str(mv)
         elif isinstance(arg, BoxVector):
-            return 'v%s[%s%d#%d]' % (str(mv), arg.item_type, arg.item_size, arg.item_count)
+            return 'v%s[%s%d|%d]' % (str(mv), arg.item_type,
+                                     arg.item_size * 8, arg.item_count)
         elif arg is None:
             return 'None'
         else:
diff --git a/rpython/jit/metainterp/optimizeopt/schedule.py b/rpython/jit/metainterp/optimizeopt/schedule.py
--- a/rpython/jit/metainterp/optimizeopt/schedule.py
+++ b/rpython/jit/metainterp/optimizeopt/schedule.py
@@ -44,9 +44,17 @@
 
     def schedulable(self, candidate):
         if candidate.pack:
-            for node in candidate.pack.operations:
-                if node.depends_count() > 0:
-                    return False
+            pack = candidate.pack
+            if pack.is_accumulating():
+                for node in pack.operations:
+                    for dep in node.depends():
+                        if dep.to.pack is not pack:
+                            return False
+                return True
+            else:
+                for node in candidate.pack.operations:
+                    if node.depends_count() > 0:
+                        return False
         return candidate.depends_count() == 0
 
     def schedule(self, candidate, position):
diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/vectorize.py
@@ -552,6 +552,8 @@
                     if not dep.because_of(accum):
                         # not quite ... this is not handlable
                         return None
+            # get the original variable
+            accum = lop.getarg(accum_pos)
 
             # in either of the two cases the arguments are mixed,
             # which is not handled currently

From noreply at buildbot.pypy.org  Tue Jun  9 08:56:06 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue,  9 Jun 2015 08:56:06 +0200 (CEST)
Subject: [pypy-commit] pypy default: Argh :-/
Message-ID: <20150609065606.1B1761C202E@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r77971:be51e18af561
Date: 2015-06-09 08:56 +0200
http://bitbucket.org/pypy/pypy/changeset/be51e18af561/

Log:	Argh :-/

diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -156,7 +156,7 @@
         if export_state:
             jd_sd = self.optimizer.jitdriver_sd
             threshold = jd_sd.warmstate.disable_unrolling_threshold
-            if 1 or len(loop.operations) > threshold:
+            if len(loop.operations) > threshold:
                 if loop.operations[0].getopnum() == rop.LABEL:
                     # abandoning unrolling, too long
                     new_descr = stop_label.getdescr()

From noreply at buildbot.pypy.org  Tue Jun  9 09:01:21 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue,  9 Jun 2015 09:01:21 +0200 (CEST)
Subject: [pypy-commit] pypy default: More test fix
Message-ID: <20150609070121.5BCEF1C202F@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r77972:3ede7f1f26c0
Date: 2015-06-09 09:01 +0200
http://bitbucket.org/pypy/pypy/changeset/3ede7f1f26c0/

Log:	More test fix

diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -155,7 +155,10 @@
         loop.operations = self.optimizer.get_newoperations()
         if export_state:
             jd_sd = self.optimizer.jitdriver_sd
-            threshold = jd_sd.warmstate.disable_unrolling_threshold
+            try:
+                threshold = jd_sd.warmstate.disable_unrolling_threshold
+            except AttributeError:    # tests only
+                threshold = sys.maxint
             if len(loop.operations) > threshold:
                 if loop.operations[0].getopnum() == rop.LABEL:
                     # abandoning unrolling, too long

From noreply at buildbot.pypy.org  Tue Jun  9 09:15:13 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue,  9 Jun 2015 09:15:13 +0200 (CEST)
Subject: [pypy-commit] pypy default: Bump the disable_unrolling threshold in
 tests, because a few of them
Message-ID: <20150609071513.53ABA1C048F@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r77973:3ea814ec1c9e
Date: 2015-06-09 09:15 +0200
http://bitbucket.org/pypy/pypy/changeset/3ea814ec1c9e/

Log:	Bump the disable_unrolling threshold in tests, because a few of them
	produce a lot of operations and are still expecting unrolling

diff --git a/rpython/jit/metainterp/warmspot.py b/rpython/jit/metainterp/warmspot.py
--- a/rpython/jit/metainterp/warmspot.py
+++ b/rpython/jit/metainterp/warmspot.py
@@ -70,7 +70,7 @@
 def jittify_and_run(interp, graph, args, repeat=1, graph_and_interp_only=False,
                     backendopt=False, trace_limit=sys.maxint,
                     inline=False, loop_longevity=0, retrace_limit=5,
-                    function_threshold=4,
+                    function_threshold=4, disable_unrolling=sys.maxint,
                     enable_opts=ALL_OPTS_NAMES, max_retrace_guards=15, 
                     max_unroll_recursion=7, **kwds):
     from rpython.config.config import ConfigError
@@ -95,6 +95,7 @@
         jd.warmstate.set_param_max_retrace_guards(max_retrace_guards)
         jd.warmstate.set_param_enable_opts(enable_opts)
         jd.warmstate.set_param_max_unroll_recursion(max_unroll_recursion)
+        jd.warmstate.set_param_disable_unrolling(disable_unrolling)
     warmrunnerdesc.finish()
     if graph_and_interp_only:
         return interp, graph

From noreply at buildbot.pypy.org  Tue Jun  9 09:34:32 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Tue,  9 Jun 2015 09:34:32 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: generating vector box for accumulation
 before the label and renaming occurances
Message-ID: <20150609073432.D3EEA1C11B6@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r77974:66758cffe3af
Date: 2015-06-09 09:34 +0200
http://bitbucket.org/pypy/pypy/changeset/66758cffe3af/

Log:	generating vector box for accumulation before the label and renaming
	occurances

diff --git a/rpython/jit/metainterp/optimizeopt/schedule.py b/rpython/jit/metainterp/optimizeopt/schedule.py
--- a/rpython/jit/metainterp/optimizeopt/schedule.py
+++ b/rpython/jit/metainterp/optimizeopt/schedule.py
@@ -88,6 +88,49 @@
         node.clear_dependencies()
         node.emitted = True
 
+def vectorbox_outof_box(box, count=-1, size=-1, type='-', clone_signed=True, signed=False):
+    if box.type not in (FLOAT, INT):
+        raise AssertionError("cannot create vector box of type %s" % (box.type))
+    signed = True
+    if box.type == FLOAT:
+        signed = False
+    return BoxVector(box.type, 2, 8, signed)
+
+def vectorbox_clone_set(box, count=-1, size=-1, type='-', clone_signed=True, signed=False):
+    if count == -1:
+        count = box.item_count
+    if size == -1:
+        size = box.item_size
+    if type == '-':
+        type = box.item_type
+    if clone_signed:
+        signed = box.item_signed
+    return BoxVector(type, count, size, signed)
+
+def getpackopnum(type):
+    if type == INT:
+        return rop.VEC_INT_PACK
+    elif type == FLOAT:
+        return rop.VEC_FLOAT_PACK
+    #
+    raise AssertionError("getpackopnum type %s not supported" % (type,))
+
+def getunpackopnum(type):
+    if type == INT:
+        return rop.VEC_INT_UNPACK
+    elif type == FLOAT:
+        return rop.VEC_FLOAT_UNPACK
+    #
+    raise AssertionError("getunpackopnum type %s not supported" % (type,))
+
+def getexpandopnum(type):
+    if type == INT:
+        return rop.VEC_INT_EXPAND
+    elif type == FLOAT:
+        return rop.VEC_FLOAT_EXPAND
+    #
+    raise AssertionError("getexpandopnum type %s not supported" % (type,))
+
 class PackType(object):
     UNKNOWN_TYPE = '-'
 
@@ -163,9 +206,6 @@
         self.input_type = None
         self.output_type = None
 
-    def clone_vbox_set_count(self, box, count):
-        return BoxVector(box.item_type, count, box.item_size, box.item_signed)
-
     def is_vector_arg(self, i):
         if i < 0 or i >= len(self.arg_ptypes):
             return False
@@ -321,10 +361,8 @@
         return vbox_cloned
 
     def unpack(self, vbox, index, count, arg_ptype):
-        vbox_cloned = self.clone_vbox_set_count(vbox, count)
-        opnum = rop.VEC_FLOAT_UNPACK
-        if vbox.item_type == INT:
-            opnum = rop.VEC_INT_UNPACK
+        vbox_cloned = vectorbox_clone_set(vbox, count=count)
+        opnum = getunpackopnum(vbox.item_type)
         op = ResOperation(opnum, [vbox, ConstInt(index), ConstInt(count)], vbox_cloned)
         self.preamble_ops.append(op)
         return vbox_cloned
@@ -336,9 +374,7 @@
           this function creates a box pack instruction to merge them to:
           v1/2 = [A,B,X,Y]
         """
-        opnum = rop.VEC_FLOAT_PACK
-        if tgt_box.item_type == INT:
-            opnum = rop.VEC_INT_PACK
+        opnum = getpackopnum(tgt_box.item_type)
         arg_count = len(args)
         i = index
         while i < arg_count and tgt_box.item_count < packable:
@@ -348,7 +384,7 @@
                 i += 1
                 continue
             count = tgt_box.item_count + src_box.item_count
-            new_box = self.clone_vbox_set_count(tgt_box, count)
+            new_box = vectorbox_clone_set(tgt_box, count=count)
             op = ResOperation(opnum, [tgt_box, src_box, ConstInt(i),
                                       ConstInt(src_box.item_count)], new_box)
             self.preamble_ops.append(op)
@@ -404,9 +440,7 @@
                 break
             i += 1
         else:
-            expand_opnum = rop.VEC_FLOAT_EXPAND
-            if box_type == INT:
-                expand_opnum = rop.VEC_INT_EXPAND
+            expand_opnum = getexpandopnum(box_type)
             op = ResOperation(expand_opnum, [arg], vbox)
             invariant_ops.append(op)
             invariant_vars.append(vbox)
@@ -415,9 +449,7 @@
 
         op = ResOperation(rop.VEC_BOX, [ConstInt(len(nodes))], vbox)
         invariant_ops.append(op)
-        opnum = rop.VEC_FLOAT_PACK
-        if arg.type == INT:
-            opnum = rop.VEC_INT_PACK
+        opnum = getpackopnum(arg.type)
         for i,node in enumerate(nodes):
             op = node.getoperation()
             arg = op.getarg(argidx)
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py b/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
@@ -68,7 +68,7 @@
         opt.analyse_index_calculations()
         if opt.dependency_graph is not None:
             self._write_dot_and_convert_to_svg(opt.dependency_graph, "ee" + self.test_name)
-            opt.schedule()
+            opt.schedule(False)
         opt.unroll_loop_iterations(loop, unroll_factor)
         opt.loop.operations = opt.get_newoperations()
         self.debug_print_operations(opt.loop)
@@ -101,7 +101,7 @@
         opt.find_adjacent_memory_refs()
         opt.extend_packset()
         opt.combine_packset()
-        opt.schedule()
+        opt.schedule(True)
         return opt
 
     def vectorize(self, loop, unroll_factor = -1):
@@ -109,7 +109,7 @@
         opt.find_adjacent_memory_refs()
         opt.extend_packset()
         opt.combine_packset()
-        opt.schedule()
+        opt.schedule(True)
         gso = GuardStrengthenOpt(opt.dependency_graph.index_vars)
         gso.propagate_all_forward(opt.loop)
         return opt
diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/vectorize.py
@@ -10,7 +10,8 @@
 from rpython.jit.metainterp.optimizeopt.util import make_dispatcher_method, Renamer
 from rpython.jit.metainterp.optimizeopt.dependency import (DependencyGraph,
         MemoryRef, Node, IndexVar)
-from rpython.jit.metainterp.optimizeopt.schedule import VecScheduleData, Scheduler, Pack, Pair, AccumPair
+from rpython.jit.metainterp.optimizeopt.schedule import (VecScheduleData,
+        Scheduler, Pack, Pair, AccumPair, vectorbox_outof_box, getpackopnum)
 from rpython.jit.metainterp.optimizeopt.guard import GuardStrengthenOpt
 from rpython.jit.metainterp.resoperation import (rop, ResOperation, GuardResOp)
 from rpython.rlib.objectmodel import we_are_translated
@@ -83,7 +84,6 @@
         self.smallest_type_bytes = 0
         self.early_exit_idx = -1
         self.sched_data = None
-        self.tried_to_pack = False
         self.costmodel = X86_CostModel(cost_threshold)
 
     def propagate_all_forward(self, clear=True):
@@ -107,7 +107,7 @@
         # find index guards and move to the earliest position
         self.analyse_index_calculations()
         if self.dependency_graph is not None:
-            self.schedule() # reorder the trace
+            self.schedule(False) # reorder the trace
 
         # unroll
         self.unroll_count = self.get_unroll_count(vsize)
@@ -122,7 +122,7 @@
         self.combine_packset()
         if not self.costmodel.profitable(self.packset):
             raise NotAProfitableLoop()
-        self.schedule()
+        self.schedule(True)
 
         gso = GuardStrengthenOpt(self.dependency_graph.index_vars)
         gso.propagate_all_forward(self.loop)
@@ -275,8 +275,6 @@
         loop = self.loop
         operations = loop.operations
 
-        self.tried_to_pack = True
-
         self.packset = PackSet(self.dependency_graph, operations,
                                self.unroll_count,
                                self.smallest_type_bytes)
@@ -356,17 +354,21 @@
             if len_before == len(self.packset.packs):
                 break
 
-    def schedule(self):
+    def schedule(self, vector=False):
         self.guard_early_exit = -1
         self.clear_newoperations()
         sched_data = VecScheduleData(self.metainterp_sd.cpu.vector_register_size)
         scheduler = Scheduler(self.dependency_graph, sched_data)
         renamer = Renamer()
+        #
+        if vector:
+            self.packset.accumulate_prepare(sched_data, renamer)
+        #
         while scheduler.has_more():
             position = len(self._newoperations)
             ops = scheduler.next(position)
             for op in ops:
-                if self.tried_to_pack:
+                if vector:
                     self.unpack_from_vector(op, sched_data, renamer)
                 self.emit_operation(op)
 
@@ -534,51 +536,6 @@
             self.accum_vars[pack.accum_variable] = pack.accum_variable
         self.packs.append(pack)
 
-    def accumulates_pair(self, lnode, rnode, origin_pack):
-        # lnode and rnode are isomorphic and dependent
-        assert isinstance(origin_pack, Pair)
-        lop = lnode.getoperation()
-        opnum = lop.getopnum()
-
-        if opnum in (rop.FLOAT_ADD, rop.INT_ADD):
-            roper = rnode.getoperation()
-            assert lop.numargs() == 2 and lop.result is not None
-            accum, accum_pos = self.getaccumulator_variable(lop, roper, origin_pack)
-            if not accum:
-                return None
-            # the dependency exists only because of the result of lnode
-            for dep in lnode.provides():
-                if dep.to is rnode:
-                    if not dep.because_of(accum):
-                        # not quite ... this is not handlable
-                        return None
-            # get the original variable
-            accum = lop.getarg(accum_pos)
-
-            # in either of the two cases the arguments are mixed,
-            # which is not handled currently
-            var_pos = (accum_pos + 1) % 2
-            plop = origin_pack.left.getoperation()
-            if lop.getarg(var_pos) is not plop.result:
-                return None
-            prop = origin_pack.right.getoperation()
-            if roper.getarg(var_pos) is not prop.result:
-                return None
-
-            # this can be handled by accumulation
-            return AccumPair(lnode, rnode, accum, accum_pos)
-
-        return None
-
-    def getaccumulator_variable(self, lop, rop, origin_pack):
-        args = rop.getarglist()
-        for i, arg in enumerate(args):
-            print arg, "is", lop.result
-            if arg is lop.result:
-                return arg, i
-        #
-        return None, -1
-
     def can_be_packed(self, lnode, rnode, origin_pack):
         if isomorphic(lnode.getoperation(), rnode.getoperation()):
             if lnode.independent(rnode):
@@ -645,3 +602,67 @@
             del self.packs[last_pos]
         return last_pos
 
+    def accumulates_pair(self, lnode, rnode, origin_pack):
+        # lnode and rnode are isomorphic and dependent
+        assert isinstance(origin_pack, Pair)
+        lop = lnode.getoperation()
+        opnum = lop.getopnum()
+
+        if opnum in (rop.FLOAT_ADD, rop.INT_ADD):
+            roper = rnode.getoperation()
+            assert lop.numargs() == 2 and lop.result is not None
+            accum, accum_pos = self.getaccumulator_variable(lop, roper, origin_pack)
+            if not accum:
+                return None
+            # the dependency exists only because of the result of lnode
+            for dep in lnode.provides():
+                if dep.to is rnode:
+                    if not dep.because_of(accum):
+                        # not quite ... this is not handlable
+                        return None
+            # get the original variable
+            accum = lop.getarg(accum_pos)
+
+            # in either of the two cases the arguments are mixed,
+            # which is not handled currently
+            var_pos = (accum_pos + 1) % 2
+            plop = origin_pack.left.getoperation()
+            if lop.getarg(var_pos) is not plop.result:
+                return None
+            prop = origin_pack.right.getoperation()
+            if roper.getarg(var_pos) is not prop.result:
+                return None
+
+            # this can be handled by accumulation
+            return AccumPair(lnode, rnode, accum, accum_pos)
+
+        return None
+
+    def getaccumulator_variable(self, lop, rop, origin_pack):
+        args = rop.getarglist()
+        for i, arg in enumerate(args):
+            if arg is lop.result:
+                return arg, i
+        #
+        return None, -1
+
+    def accumulate_prepare(self, sched_data, renamer):
+        for var, pos in self.accum_vars.items():
+            # create a new vector box for the parameters
+            box = vectorbox_outof_box(var)
+            op = ResOperation(rop.VEC_BOX, [ConstInt(0)], box)
+            sched_data.invariant_oplist.append(op)
+            result = box.clonebox()
+            # clear the box to zero
+            op = ResOperation(rop.VEC_INT_XOR, [box, box], result)
+            sched_data.invariant_oplist.append(op)
+            box = result
+            result = box.clonebox()
+            # pack the scalar value
+            op = ResOperation(getpackopnum(box.item_type),
+                              [box, var, ConstInt(0), ConstInt(1)], result)
+            sched_data.invariant_oplist.append(op)
+            # rename the variable with the box
+            renamer.start_renaming(var, result)
+
+

From noreply at buildbot.pypy.org  Tue Jun  9 10:01:19 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Tue,  9 Jun 2015 10:01:19 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: accumulation box is now used instead of
 the scalar box, need to adapt guard exit to produce correct result
Message-ID: <20150609080119.4255A1C1239@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r77975:74976aacf58b
Date: 2015-06-09 10:01 +0200
http://bitbucket.org/pypy/pypy/changeset/74976aacf58b/

Log:	accumulation box is now used instead of the scalar box, need to
	adapt guard exit to produce correct result

diff --git a/rpython/jit/metainterp/optimizeopt/schedule.py b/rpython/jit/metainterp/optimizeopt/schedule.py
--- a/rpython/jit/metainterp/optimizeopt/schedule.py
+++ b/rpython/jit/metainterp/optimizeopt/schedule.py
@@ -1,5 +1,5 @@
 
-from rpython.jit.metainterp.history import (FLOAT,INT,ConstInt,BoxVector,
+from rpython.jit.metainterp.history import (VECTOR,FLOAT,INT,ConstInt,BoxVector,
         BoxFloat,BoxInt,ConstFloat)
 from rpython.jit.metainterp.resoperation import (rop, ResOperation, GuardResOp)
 from rpython.jit.metainterp.optimizeopt.dependency import (DependencyGraph,
@@ -20,12 +20,12 @@
     def has_more(self):
         return len(self.schedulable_nodes) > 0
 
-    def next(self, position):
+    def next(self, renamer, position):
         i = self._next(self.schedulable_nodes)
         if i >= 0:
             candidate = self.schedulable_nodes[i]
             del self.schedulable_nodes[i]
-            return self.schedule(candidate, position)
+            return self.schedule(candidate, renamer, position)
 
         raise AssertionError("schedule failed cannot continue. possible reason: cycle")
 
@@ -57,15 +57,18 @@
                         return False
         return candidate.depends_count() == 0
 
-    def schedule(self, candidate, position):
+    def schedule(self, candidate, renamer, position):
         if candidate.pack:
             pack = candidate.pack
-            vops = self.sched_data.as_vector_operation(pack)
+            for node in pack.operations:
+                renamer.rename(node.getoperation())
+            vops = self.sched_data.as_vector_operation(pack, renamer)
             for node in pack.operations:
                 self.scheduled(node, position)
             return vops
         else:
             self.scheduled(candidate, position)
+            renamer.rename(candidate.getoperation())
             return [candidate.getoperation()]
 
     def scheduled(self, node, position):
@@ -96,6 +99,17 @@
         signed = False
     return BoxVector(box.type, 2, 8, signed)
 
+def packtype_outof_box(box):
+    if box.type == VECTOR:
+        return PackType.of(box)
+    else:
+        if arg.type == INT:
+            return PackType(INT, 8, True, 2)
+        elif arg.type == FLOAT:
+            return PackType(FLOAT, 8, True, 2)
+
+    raise AssertionError("box %s not supported" % (box,))
+
 def vectorbox_clone_set(box, count=-1, size=-1, type='-', clone_signed=True, signed=False):
     if count == -1:
         count = box.item_count
@@ -220,13 +234,7 @@
         if vbox:
             return PackType.of(vbox)
         else:
-            vec_reg_size = self.sched_data.vec_reg_size
-            if isinstance(arg, ConstInt) or isinstance(arg, BoxInt):
-                return PackType(INT, 8, True, 2)
-            elif isinstance(arg, ConstFloat) or isinstance(arg, BoxFloat):
-                return PackType(FLOAT, 8, True, 2)
-            else:
-                raise NotImplementedError("arg %s not supported" % (arg,))
+            return packtype_outof_box(arg)
 
     def determine_output_type(self, op):
         return self.determine_input_type(op)
@@ -283,6 +291,8 @@
         self.before_argument_transform(args)
         #
         for i,arg in enumerate(args):
+            if isinstance(arg, BoxVector):
+                continue
             if self.is_vector_arg(i):
                 args[i] = self.transform_argument(args[i], i, off)
         #
@@ -603,7 +613,7 @@
         self.invariant_vector_vars = []
         self.expanded_map = {}
 
-    def as_vector_operation(self, pack):
+    def as_vector_operation(self, pack, preproc_renamer):
         op_count = len(pack.operations)
         assert op_count > 1
         self.pack = pack
@@ -617,6 +627,15 @@
             raise NotImplementedError("missing vecop for '%s'" % (op0.getopname(),))
         oplist = []
         tovector.as_vector_operation(pack, self, oplist)
+        #
+        if pack.is_accumulating:
+            box = oplist[0].result
+            assert box is not None
+            for node in pack.operations:
+                op = node.getoperation()
+                assert op.result is not None
+                preproc_renamer.start_renaming(op.result, box)
+        #
         return oplist
 
     def getvector_of_box(self, arg):
diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/vectorize.py
@@ -366,7 +366,7 @@
         #
         while scheduler.has_more():
             position = len(self._newoperations)
-            ops = scheduler.next(position)
+            ops = scheduler.next(renamer, position)
             for op in ops:
                 if vector:
                     self.unpack_from_vector(op, sched_data, renamer)

From noreply at buildbot.pypy.org  Tue Jun  9 10:34:44 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue,  9 Jun 2015 10:34:44 +0200 (CEST)
Subject: [pypy-commit] pypy default: Remove some very old code that is not
 useful any more (at least I
Message-ID: <20150609083444.10F0F1C0695@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r77976:8265e76a2734
Date: 2015-06-09 08:18 +0100
http://bitbucket.org/pypy/pypy/changeset/8265e76a2734/

Log:	Remove some very old code that is not useful any more (at least I
	couldn't find a test failing because of that, and translation still
	seems to work).

diff --git a/rpython/translator/c/database.py b/rpython/translator/c/database.py
--- a/rpython/translator/c/database.py
+++ b/rpython/translator/c/database.py
@@ -45,7 +45,6 @@
         self.delayedfunctionptrs = []
         self.completedcontainers = 0
         self.containerstats = {}
-        self.externalfuncs = {}
         self.helper2ptr = {}
 
         # late_initializations is for when the value you want to
diff --git a/rpython/translator/c/extfunc.py b/rpython/translator/c/extfunc.py
--- a/rpython/translator/c/extfunc.py
+++ b/rpython/translator/c/extfunc.py
@@ -62,24 +62,6 @@
                 yield (fname, graph)
 
 
-def predeclare_extfuncs(db, rtyper):
-    modules = {}
-    def module_name(c_name):
-        frags = c_name[3:].split('_')
-        if frags[0] == '':
-            return '_' + frags[1]
-        else:
-            return frags[0]
-
-    for func, funcobj in db.externalfuncs.items():
-        # construct a define LL_NEED_<modname> to make it possible to isolate in-development externals and headers
-        modname = module_name(func)
-        if modname not in modules:
-            modules[modname] = True
-            yield 'LL_NEED_%s' % modname.upper(), 1
-        funcptr = funcobj._as_ptr()
-        yield func, funcptr
-
 def predeclare_exception_data(db, rtyper):
     # Exception-related types and constants
     exceptiondata = rtyper.exceptiondata
@@ -112,7 +94,6 @@
     for fn in [predeclare_common_types,
                predeclare_utility_functions,
                predeclare_exception_data,
-               predeclare_extfuncs,
                ]:
         for t in fn(db, rtyper):
             yield t
@@ -122,7 +103,6 @@
     for fn in [predeclare_common_types,
                predeclare_utility_functions,
                predeclare_exception_data,
-               predeclare_extfuncs,
                ]:
         for t in fn(db, rtyper):
             yield t[1]
diff --git a/rpython/translator/c/node.py b/rpython/translator/c/node.py
--- a/rpython/translator/c/node.py
+++ b/rpython/translator/c/node.py
@@ -915,15 +915,8 @@
     return [FunctionCodeGenerator(graph, db)]
 
 def select_function_code_generators(fnobj, db, functionname):
-    # XXX this logic is completely broken nowadays
-    #     _external_name does not mean that this is done oldstyle
     sandbox = db.need_sandboxing(fnobj)
-    if hasattr(fnobj, '_external_name'):
-        if sandbox:
-            return sandbox_stub(fnobj, db)
-        db.externalfuncs[fnobj._external_name] = fnobj
-        return []
-    elif hasattr(fnobj, 'graph'):
+    if hasattr(fnobj, 'graph'):
         if sandbox and sandbox != "if_external":
             # apply the sandbox transformation
             return sandbox_transform(fnobj, db)

From noreply at buildbot.pypy.org  Tue Jun  9 10:34:45 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue,  9 Jun 2015 10:34:45 +0200 (CEST)
Subject: [pypy-commit] pypy default: Document who is using this case
Message-ID: <20150609083445.615541C0695@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r77977:f5c8f6b5aca5
Date: 2015-06-09 08:29 +0100
http://bitbucket.org/pypy/pypy/changeset/f5c8f6b5aca5/

Log:	Document who is using this case

diff --git a/rpython/translator/c/node.py b/rpython/translator/c/node.py
--- a/rpython/translator/c/node.py
+++ b/rpython/translator/c/node.py
@@ -932,7 +932,7 @@
             assert fnobj.external == 'CPython'
             return [CExternalFunctionCodeGenerator(fnobj, db)]
     elif hasattr(fnobj._callable, "c_name"):
-        return []
+        return []    # this case should only be used for entrypoints
     else:
         raise ValueError("don't know how to generate code for %r" % (fnobj,))
 

From noreply at buildbot.pypy.org  Tue Jun  9 10:54:19 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Tue,  9 Jun 2015 10:54:19 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: need to call method to get the result :)
Message-ID: <20150609085419.70F181C1239@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r77978:7aba38455439
Date: 2015-06-09 10:08 +0200
http://bitbucket.org/pypy/pypy/changeset/7aba38455439/

Log:	need to call method to get the result :)

diff --git a/rpython/jit/metainterp/optimizeopt/schedule.py b/rpython/jit/metainterp/optimizeopt/schedule.py
--- a/rpython/jit/metainterp/optimizeopt/schedule.py
+++ b/rpython/jit/metainterp/optimizeopt/schedule.py
@@ -628,7 +628,7 @@
         oplist = []
         tovector.as_vector_operation(pack, self, oplist)
         #
-        if pack.is_accumulating:
+        if pack.is_accumulating():
             box = oplist[0].result
             assert box is not None
             for node in pack.operations:
diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/vectorize.py
@@ -508,8 +508,6 @@
                     return 2
         return 1
 
-
-
 def isomorphic(l_op, r_op):
     """ Subject of definition """
     if l_op.getopnum() == r_op.getopnum():

From noreply at buildbot.pypy.org  Tue Jun  9 10:54:20 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Tue,  9 Jun 2015 10:54:20 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: added test to ensure sum is not
 prohibited by cost calculation
Message-ID: <20150609085420.ACF471C1239@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r77979:450cfd82b800
Date: 2015-06-09 10:17 +0200
http://bitbucket.org/pypy/pypy/changeset/450cfd82b800/

Log:	added test to ensure sum is not prohibited by cost calculation

diff --git a/rpython/jit/metainterp/optimizeopt/test/test_costmodel.py b/rpython/jit/metainterp/optimizeopt/test/test_costmodel.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_costmodel.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_costmodel.py
@@ -121,5 +121,15 @@
         savings = self.savings(loop1)
         assert savings == 1
 
+    def test_sum(self):
+        loop1 = self.parse("""
+        f10 = raw_load(p0, i0, descr=double)
+        f11 = raw_load(p0, i1, descr=double)
+        f12 = float_add(f1, f10)
+        f13 = float_add(f12, f11)
+        """)
+        savings = self.savings(loop1)
+        assert savings == 2
+
 class Test(CostModelBaseTest, LLtypeMixin):
     pass

From noreply at buildbot.pypy.org  Tue Jun  9 10:54:21 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Tue,  9 Jun 2015 10:54:21 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: updated tests to provide arguments for
 some changed methods. float vector box signed is always false
Message-ID: <20150609085421.C8AD21C1239@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r77980:716facf22bdc
Date: 2015-06-09 10:54 +0200
http://bitbucket.org/pypy/pypy/changeset/716facf22bdc/

Log:	updated tests to provide arguments for some changed methods. float
	vector box signed is always false

diff --git a/rpython/jit/metainterp/optimizeopt/schedule.py b/rpython/jit/metainterp/optimizeopt/schedule.py
--- a/rpython/jit/metainterp/optimizeopt/schedule.py
+++ b/rpython/jit/metainterp/optimizeopt/schedule.py
@@ -91,7 +91,7 @@
         node.clear_dependencies()
         node.emitted = True
 
-def vectorbox_outof_box(box, count=-1, size=-1, type='-', clone_signed=True, signed=False):
+def vectorbox_outof_box(box, count=-1, size=-1, type='-'):
     if box.type not in (FLOAT, INT):
         raise AssertionError("cannot create vector box of type %s" % (box.type))
     signed = True
@@ -103,10 +103,10 @@
     if box.type == VECTOR:
         return PackType.of(box)
     else:
-        if arg.type == INT:
+        if box.type == INT:
             return PackType(INT, 8, True, 2)
-        elif arg.type == FLOAT:
-            return PackType(FLOAT, 8, True, 2)
+        elif box.type == FLOAT:
+            return PackType(FLOAT, 8, False, 2)
 
     raise AssertionError("box %s not supported" % (box,))
 
@@ -146,6 +146,8 @@
     raise AssertionError("getexpandopnum type %s not supported" % (type,))
 
 class PackType(object):
+    # TODO merge with vector box? the save the same fields
+    # difference: this is more of a type specification
     UNKNOWN_TYPE = '-'
 
     def __init__(self, type, size, signed, count=-1):
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_schedule.py b/rpython/jit/metainterp/optimizeopt/test/test_schedule.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_schedule.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_schedule.py
@@ -1,7 +1,7 @@
 import py
 
 from rpython.jit.metainterp.history import TargetToken, JitCellToken, TreeLoop
-from rpython.jit.metainterp.optimizeopt.util import equaloplists
+from rpython.jit.metainterp.optimizeopt.util import equaloplists, Renamer
 from rpython.jit.metainterp.optimizeopt.vectorize import (VecScheduleData,
         Pack, NotAProfitableLoop, VectorizingOptimizer)
 from rpython.jit.metainterp.optimizeopt.dependency import Node
@@ -48,11 +48,12 @@
         vsd = VecScheduleData(vec_reg_size)
         if getvboxfunc is not None:
             vsd.getvector_of_box = getvboxfunc
+        renamer = Renamer()
         for pack in packs:
             if len(pack) == 1:
                 ops.append(pack[0].getoperation())
             else:
-                for op in vsd.as_vector_operation(Pack(pack)):
+                for op in vsd.as_vector_operation(Pack(pack), renamer):
                     ops.append(op)
         loop.operations = ops
         if prepend_invariant:
diff --git a/rpython/jit/tool/oparser.py b/rpython/jit/tool/oparser.py
--- a/rpython/jit/tool/oparser.py
+++ b/rpython/jit/tool/oparser.py
@@ -130,6 +130,8 @@
                 item_size = int(match.group(3)) // 8
                 item_count = int(match.group(5))
                 item_signed = not (match.group(1) == 'u')
+                if item_type == 'f':
+                    item_signed = False
                 box = self.model.BoxVector(item_type, item_count, item_size, item_signed)
                 lbracket = elem.find('[')
                 number = elem[1:lbracket]

From noreply at buildbot.pypy.org  Tue Jun  9 11:06:09 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Tue,  9 Jun 2015 11:06:09 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: This can be NULL here
Message-ID: <20150609090609.D73811C130E@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77981:0aa906ef32e5
Date: 2015-06-09 11:06 +0200
http://bitbucket.org/pypy/pypy/changeset/0aa906ef32e5/

Log:	This can be NULL here

diff --git a/rpython/jit/metainterp/optimizeopt/heap.py b/rpython/jit/metainterp/optimizeopt/heap.py
--- a/rpython/jit/metainterp/optimizeopt/heap.py
+++ b/rpython/jit/metainterp/optimizeopt/heap.py
@@ -447,7 +447,7 @@
                 assert not opinfo.is_virtual()      # it must be a non-virtual
                 if op.getarg(2).type == 'r':
                     fieldinfo = self.getptrinfo(op.getarg(2))
-                    if fieldinfo.is_virtual():
+                    if fieldinfo and fieldinfo.is_virtual():
                         pendingfields.append(op)
                     else:
                         cf.force_lazy_setfield(self, descr)

From noreply at buildbot.pypy.org  Tue Jun  9 11:13:05 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Tue,  9 Jun 2015 11:13:05 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: disable this import,
	again for --fork-before
Message-ID: <20150609091305.43DB91C0460@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77982:42cedb5ee41e
Date: 2015-06-09 11:13 +0200
http://bitbucket.org/pypy/pypy/changeset/42cedb5ee41e/

Log:	disable this import, again for --fork-before

diff --git a/pypy/module/pypyjit/interp_resop.py b/pypy/module/pypyjit/interp_resop.py
--- a/pypy/module/pypyjit/interp_resop.py
+++ b/pypy/module/pypyjit/interp_resop.py
@@ -8,7 +8,7 @@
 from rpython.rtyper.lltypesystem import lltype
 from rpython.rtyper.annlowlevel import cast_base_ptr_to_instance, hlstr
 from rpython.rtyper.rclass import OBJECT
-from rpython.jit.metainterp.resoperation import rop
+#from rpython.jit.metainterp.resoperation import rop
 from rpython.rlib.nonconst import NonConstant
 from rpython.rlib import jit_hooks
 from rpython.rlib.jit import Counters

From noreply at buildbot.pypy.org  Tue Jun  9 11:15:19 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Tue,  9 Jun 2015 11:15:19 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: fix pypyjit.py
Message-ID: <20150609091519.D25411C0579@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77983:649600014b9a
Date: 2015-06-09 11:15 +0200
http://bitbucket.org/pypy/pypy/changeset/649600014b9a/

Log:	fix pypyjit.py

diff --git a/pypy/tool/pypyjit.py b/pypy/tool/pypyjit.py
--- a/pypy/tool/pypyjit.py
+++ b/pypy/tool/pypyjit.py
@@ -14,6 +14,9 @@
     print >> sys.stderr, __doc__
     sys.exit(2)
 
+import sys
+sys.setrecursionlimit(100000000)
+
 from pypy.objspace.std import Space
 from rpython.config.translationoption import set_opt_level
 from pypy.config.pypyoption import get_pypy_config, set_pypy_opt_level
@@ -22,6 +25,7 @@
 from rpython.rtyper.lltypesystem import lltype
 from pypy.interpreter.pycode import PyCode
 from rpython.translator.goal import unixcheckpoint
+import pypy.module.pypyjit.interp_jit
 
 config = get_pypy_config(translating=True)
 config.translation.backendopt.inline_threshold = 0.1
@@ -33,6 +37,8 @@
 config.objspace.usemodules.pypyjit = True
 config.objspace.usemodules.array = False
 config.objspace.usemodules._weakref = False
+config.objspace.usemodules.struct = True
+config.objspace.usemodules.time = True
 config.objspace.usemodules._sre = False
 config.objspace.usemodules._lsprof = False
 #
@@ -73,6 +79,7 @@
 read_code_ptr = llhelper(FPTR, read_code)
 
 def entry_point():
+    space.startup()
     from pypy.module.marshal.interp_marshal import loads
     code = loads(space, space.wrap(hlstr(read_code_ptr())))
     assert isinstance(code, PyCode)
diff --git a/pypy/tool/pypyjit_demo.py b/pypy/tool/pypyjit_demo.py
--- a/pypy/tool/pypyjit_demo.py
+++ b/pypy/tool/pypyjit_demo.py
@@ -1,8 +1,31 @@
-def f():
-    i = 0
-    while i < 1303:
-        i += 1
-    return i
 
+import time
+l = []
 
-f()
+for i in range(100):
+    print i
+    t0 = time.time()
+    exec """
+def k(a, b, c):
+    pass
+
+def g(a, b, c):
+    k(a, b + 1, c + 2)
+    k(a, b + 1, c + 2)
+    k(a, b + 1, c + 2)
+    k(a, b + 1, c + 2)
+    k(a, b + 1, c + 2)
+
+def f(i):
+    g(i, i + 1, i + 2)
+    g(i, i + 1, i + 2)
+    g(i, i + 1, i + 2)
+    g(i, i + 1, i + 2)
+    g(i, i + 1, i + 2)
+    g(i, i + 1, i + 2)
+for i in range(1000):
+    f(i)
+"""
+    l.append(time.time() - t0)
+
+print l

From noreply at buildbot.pypy.org  Tue Jun  9 11:30:09 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Tue,  9 Jun 2015 11:30:09 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: there is no point in trying to
	preserve the Const here
Message-ID: <20150609093009.299871C130E@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77984:1d6811565946
Date: 2015-06-09 11:30 +0200
http://bitbucket.org/pypy/pypy/changeset/1d6811565946/

Log:	there is no point in trying to preserve the Const here

diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -349,7 +349,8 @@
         if opinfo is not None:
             assert isinstance(opinfo, info.AbstractInfo)
             op.set_forwarded(newop)
-            newop.set_forwarded(opinfo)
+            if not isinstance(newop, Const):
+                newop.set_forwarded(opinfo)
         else:
             op.set_forwarded(newop)
 

From noreply at buildbot.pypy.org  Tue Jun  9 11:30:21 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue,  9 Jun 2015 11:30:21 +0200 (CEST)
Subject: [pypy-commit] cffi default: Issue #206: the rewrite of gc_weakref
 was not multithread-safe
Message-ID: <20150609093021.289691C130E@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2174:471a9d09c7f3
Date: 2015-06-09 11:29 +0200
http://bitbucket.org/cffi/cffi/changeset/471a9d09c7f3/

Log:	Issue #206: the rewrite of gc_weakref was not multithread-safe

diff --git a/cffi/gc_weakref.py b/cffi/gc_weakref.py
--- a/cffi/gc_weakref.py
+++ b/cffi/gc_weakref.py
@@ -4,25 +4,21 @@
 class GcWeakrefs(object):
     def __init__(self, ffi):
         self.ffi = ffi
-        self.data = []
-        self.freelist = None
+        self.data = {}
+        self.nextindex = 0
 
     def build(self, cdata, destructor):
         # make a new cdata of the same type as the original one
         new_cdata = self.ffi.cast(self.ffi._backend.typeof(cdata), cdata)
         #
         def remove(key):
-            assert self.data[index] is key
-            self.data[index] = self.freelist
-            self.freelist = index
+            # careful, this function is not protected by any lock
+            old_key = self.data.pop(index)
+            assert old_key is key
             destructor(cdata)
         #
         key = ref(new_cdata, remove)
-        index = self.freelist
-        if index is None:
-            index = len(self.data)
-            self.data.append(key)
-        else:
-            self.freelist = self.data[index]
-            self.data[index] = key
+        index = self.nextindex
+        self.nextindex = index + 1     # we're protected by the lock here
+        self.data[index] = key
         return new_cdata

From noreply at buildbot.pypy.org  Tue Jun  9 11:31:27 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Tue,  9 Jun 2015 11:31:27 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: be more specific,
	I'm sure we have more of those
Message-ID: <20150609093127.D734D1C1333@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77985:9e9f0e4f0d29
Date: 2015-06-09 11:31 +0200
http://bitbucket.org/pypy/pypy/changeset/9e9f0e4f0d29/

Log:	be more specific, I'm sure we have more of those

diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -1,9 +1,12 @@
-import weakref
+import weakref, os
 from rpython.rlib.objectmodel import we_are_translated, specialize
 from rpython.rlib.objectmodel import compute_identity_hash
 from rpython.rtyper.lltypesystem import lltype, llmemory
 from rpython.jit.codewriter import longlong
 
+class SettingForwardedOnAbstractValue(Exception):
+    pass
+
 class AbstractValue(object):
     _repr_memo = weakref.WeakKeyDictionary()
     is_info_class = False
@@ -25,7 +28,8 @@
         return None
 
     def set_forwarded(self, forwarded_to):
-        raise Exception("oups")
+        os.write(2, "setting forwarded on: " + self.__class__.__name__)
+        raise SettingForwardedOnAbstractValue()
 
     def get_box_replacement(op):
         orig_op = op

From noreply at buildbot.pypy.org  Tue Jun  9 11:53:46 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Tue,  9 Jun 2015 11:53:46 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: in the case #pack > 2 the accum is not
	split
Message-ID: <20150609095346.7388A1C14AA@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r77986:f7b571afa033
Date: 2015-06-09 11:53 +0200
http://bitbucket.org/pypy/pypy/changeset/f7b571afa033/

Log:	in the case #pack > 2 the accum is not split

diff --git a/rpython/jit/metainterp/optimizeopt/dependency.py b/rpython/jit/metainterp/optimizeopt/dependency.py
--- a/rpython/jit/metainterp/optimizeopt/dependency.py
+++ b/rpython/jit/metainterp/optimizeopt/dependency.py
@@ -312,7 +312,7 @@
         return None
 
     def __repr__(self):
-        return "Node(opidx: %d)"%self.opidx
+        return "Node(opidx: %d)" % self.opidx
 
     def __ne__(self, other):
         return not self.__eq__(other)
diff --git a/rpython/jit/metainterp/optimizeopt/schedule.py b/rpython/jit/metainterp/optimizeopt/schedule.py
--- a/rpython/jit/metainterp/optimizeopt/schedule.py
+++ b/rpython/jit/metainterp/optimizeopt/schedule.py
@@ -107,7 +107,7 @@
             return PackType(INT, 8, True, 2)
         elif box.type == FLOAT:
             return PackType(FLOAT, 8, False, 2)
-
+    #
     raise AssertionError("box %s not supported" % (box,))
 
 def vectorbox_clone_set(box, count=-1, size=-1, type='-', clone_signed=True, signed=False):
@@ -695,9 +695,21 @@
         assert isinstance(other, Pack)
         rightmost = self.operations[-1]
         leftmost = other.operations[0]
-        return rightmost == leftmost and \
-               self.accum_variable == other.accum_variable and \
-               self.accum_position == other.accum_position
+        # if it is not accumulating it is valid
+        accum = True
+        if self.is_accumulating():
+            if not other.is_accumulating():
+                accum = False
+            elif self.accum_position != other.accum_position:
+                accum = False
+            # aa
+            #else:
+            #    i = self.accum_position
+            #    lop = leftmost.getoperation()
+            #    roper = rightmost.getoperation()
+            #    if lop.getarg(i) is not roper.result:
+            #        accum = False
+        return rightmost is leftmost and accum
 
     def __repr__(self):
         return "Pack(%r)" % self.operations
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_util.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py
@@ -156,6 +156,7 @@
     floatarraydescr = cpu.arraydescrof(lltype.GcArray(lltype.Float))
     intarraydescr = cpu.arraydescrof(lltype.GcArray(lltype.Signed))
     int32arraydescr = cpu.arraydescrof(lltype.GcArray(rffi.INT))
+    int16arraydescr = cpu.arraydescrof(lltype.GcArray(rffi.SHORT))
     uintarraydescr = cpu.arraydescrof(lltype.GcArray(lltype.Unsigned))
     chararraydescr = cpu.arraydescrof(lltype.GcArray(lltype.Char))
     singlefloatarraydescr = cpu.arraydescrof(lltype.GcArray(lltype.SingleFloat))
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py b/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
@@ -1069,6 +1069,25 @@
         assert opt.loop.inputargs[2] in opt.packset.accum_vars
         self.debug_print_operations(opt.loop)
 
+    def test_accumulate_int16(self):
+        trace = """
+        [p3, i4, p1, i5, i6, i7, i8]
+        guard_early_exit() [p1, i4, i5, i6, p3]
+        i9 = raw_load(i7, i5, descr=int16arraydescr)
+        guard_not_invalidated() [p1, i9, i4, i5, i6, p3]
+        i10 = int_add(i6, i9)
+        i12 = int_add(i4, 1)
+        i14 = int_add(i5, 2)
+        i15 = int_ge(i12, i8)
+        guard_false(i15) [p1, i14, i10, i12, None, None, None, p3]
+        jump(p3, i12, p1, i14, i10, i7, i8)
+        """
+        opt = self.vectorize(self.parse_loop(trace))
+        assert len(opt.packset.packs) == 2
+        assert len(opt.packset.accum_vars) == 1
+        assert opt.loop.inputargs[4] in opt.packset.accum_vars
+        self.debug_print_operations(opt.loop)
+
 
     def test_element_f45_in_guard_failargs(self):
         ops = """
diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/vectorize.py
@@ -354,6 +354,15 @@
             if len_before == len(self.packset.packs):
                 break
 
+        if not we_are_translated():
+            # some test cases check the accumulation variables
+            self.packset.accum_vars = {}
+            for pack in self.packset.packs:
+                var = pack.accum_variable
+                pos = pack.accum_position
+                if var:
+                    self.packset.accum_vars[var] = pos
+
     def schedule(self, vector=False):
         self.guard_early_exit = -1
         self.clear_newoperations()
@@ -523,15 +532,11 @@
         self.operations = operations
         self.unroll_count = unroll_count
         self.smallest_type_bytes = smallest_type_bytes
-        self.accum_vars = {}
 
     def pack_count(self):
         return len(self.packs)
 
     def add_pack(self, pack):
-        if pack.is_accumulating():
-            # remember the variable and the position in this map
-            self.accum_vars[pack.accum_variable] = pack.accum_variable
         self.packs.append(pack)
 
     def can_be_packed(self, lnode, rnode, origin_pack):
@@ -586,6 +591,9 @@
         for op in pack_j.operations[1:]:
             operations.append(op)
         self.packs[i] = pack = Pack(operations)
+        # preserve the accum variable (if present) of the
+        # left most pack, that is the pack with the earliest
+        # operation at index 0 in the trace
         pack.accum_variable = pack_i.accum_variable
         pack.accum_position = pack_i.accum_position
 
@@ -645,7 +653,11 @@
         return None, -1
 
     def accumulate_prepare(self, sched_data, renamer):
-        for var, pos in self.accum_vars.items():
+        for pack in self.packs:
+            if pack.accum_variable is None:
+                continue
+            var = pack.accum_variable
+            pos = pack.accum_position
             # create a new vector box for the parameters
             box = vectorbox_outof_box(var)
             op = ResOperation(rop.VEC_BOX, [ConstInt(0)], box)

From noreply at buildbot.pypy.org  Tue Jun  9 12:08:27 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue,  9 Jun 2015 12:08:27 +0200 (CEST)
Subject: [pypy-commit] cffi release-1.1: Issue #206: the rewrite of
 gc_weakref was not multithread-safe
Message-ID: <20150609100827.61D9F1C14C1@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: release-1.1
Changeset: r2175:85e32070b9f7
Date: 2015-06-09 11:29 +0200
http://bitbucket.org/cffi/cffi/changeset/85e32070b9f7/

Log:	Issue #206: the rewrite of gc_weakref was not multithread-safe

diff --git a/cffi/gc_weakref.py b/cffi/gc_weakref.py
--- a/cffi/gc_weakref.py
+++ b/cffi/gc_weakref.py
@@ -4,25 +4,21 @@
 class GcWeakrefs(object):
     def __init__(self, ffi):
         self.ffi = ffi
-        self.data = []
-        self.freelist = None
+        self.data = {}
+        self.nextindex = 0
 
     def build(self, cdata, destructor):
         # make a new cdata of the same type as the original one
         new_cdata = self.ffi.cast(self.ffi._backend.typeof(cdata), cdata)
         #
         def remove(key):
-            assert self.data[index] is key
-            self.data[index] = self.freelist
-            self.freelist = index
+            # careful, this function is not protected by any lock
+            old_key = self.data.pop(index)
+            assert old_key is key
             destructor(cdata)
         #
         key = ref(new_cdata, remove)
-        index = self.freelist
-        if index is None:
-            index = len(self.data)
-            self.data.append(key)
-        else:
-            self.freelist = self.data[index]
-            self.data[index] = key
+        index = self.nextindex
+        self.nextindex = index + 1     # we're protected by the lock here
+        self.data[index] = key
         return new_cdata

From noreply at buildbot.pypy.org  Tue Jun  9 12:08:28 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue,  9 Jun 2015 12:08:28 +0200 (CEST)
Subject: [pypy-commit] cffi release-1.1: Whatsnew
Message-ID: <20150609100828.7A74F1C14C1@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: release-1.1
Changeset: r2176:68fd76822813
Date: 2015-06-09 11:33 +0200
http://bitbucket.org/cffi/cffi/changeset/68fd76822813/

Log:	Whatsnew

diff --git a/doc/source/whatsnew.rst b/doc/source/whatsnew.rst
--- a/doc/source/whatsnew.rst
+++ b/doc/source/whatsnew.rst
@@ -3,6 +3,13 @@
 ======================
 
 
+1.1.2
+=====
+
+* ``ffi.gc()``: fixed a race condition in multithreaded programs
+  introduced in 1.1.1
+
+
 1.1.1
 =====
 

From noreply at buildbot.pypy.org  Tue Jun  9 12:08:29 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue,  9 Jun 2015 12:08:29 +0200 (CEST)
Subject: [pypy-commit] cffi release-1.1: Bump version to 1.1.2
Message-ID: <20150609100829.88A591C14C1@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: release-1.1
Changeset: r2177:e7fe2b39ba90
Date: 2015-06-09 11:35 +0200
http://bitbucket.org/cffi/cffi/changeset/e7fe2b39ba90/

Log:	Bump version to 1.1.2

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -6063,7 +6063,7 @@
     if (v == NULL || PyModule_AddObject(m, "_C_API", v) < 0)
         INITERROR;
 
-    v = PyText_FromString("1.1.1");
+    v = PyText_FromString("1.1.2");
     if (v == NULL || PyModule_AddObject(m, "__version__", v) < 0)
         INITERROR;
 
diff --git a/c/test_c.py b/c/test_c.py
--- a/c/test_c.py
+++ b/c/test_c.py
@@ -3346,4 +3346,4 @@
 
 def test_version():
     # this test is here mostly for PyPy
-    assert __version__ == "1.1.1"
+    assert __version__ == "1.1.2"
diff --git a/cffi/__init__.py b/cffi/__init__.py
--- a/cffi/__init__.py
+++ b/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI, CDefError, FFIError
 from .ffiplatform import VerificationError, VerificationMissing
 
-__version__ = "1.1.1"
-__version_info__ = (1, 1, 1)
+__version__ = "1.1.2"
+__version_info__ = (1, 1, 2)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/doc/source/conf.py b/doc/source/conf.py
--- a/doc/source/conf.py
+++ b/doc/source/conf.py
@@ -47,7 +47,7 @@
 # The short X.Y version.
 version = '1.1'
 # The full version, including alpha/beta/rc tags.
-release = '1.1.1'
+release = '1.1.2'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
diff --git a/doc/source/installation.rst b/doc/source/installation.rst
--- a/doc/source/installation.rst
+++ b/doc/source/installation.rst
@@ -51,13 +51,13 @@
 
 Download and Installation:
 
-* http://pypi.python.org/packages/source/c/cffi/cffi-1.1.1.tar.gz
+* http://pypi.python.org/packages/source/c/cffi/cffi-1.1.2.tar.gz
 
    - Or grab the most current version by following the instructions below.
 
-   - MD5: f397363bfbf99048accb0498ffc3e72b
+   - MD5: ...
 
-   - SHA: 8c4f4d1078d05c796c12fc6d8f8cea25aaff0148
+   - SHA: ...
 
 * Or get it from the `Bitbucket page`_:
   ``hg clone https://bitbucket.org/cffi/cffi``
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -144,7 +144,7 @@
 
 `Mailing list <https://groups.google.com/forum/#!forum/python-cffi>`_
 """,
-        version='1.1.1',
+        version='1.1.2',
         packages=['cffi'] if cpython else [],
         package_data={'cffi': ['_cffi_include.h', 'parse_c_type.h']}
                      if cpython else {},

From noreply at buildbot.pypy.org  Tue Jun  9 12:08:30 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue,  9 Jun 2015 12:08:30 +0200 (CEST)
Subject: [pypy-commit] cffi release-1.1: md5/sha1
Message-ID: <20150609100830.841521C14C1@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: release-1.1
Changeset: r2178:307e778060a2
Date: 2015-06-09 12:05 +0200
http://bitbucket.org/cffi/cffi/changeset/307e778060a2/

Log:	md5/sha1

diff --git a/doc/source/installation.rst b/doc/source/installation.rst
--- a/doc/source/installation.rst
+++ b/doc/source/installation.rst
@@ -55,9 +55,9 @@
 
    - Or grab the most current version by following the instructions below.
 
-   - MD5: ...
+   - MD5: ca6e6c45b45caa87aee9adc7c796eaea
 
-   - SHA: ...
+   - SHA: 6d6203bf7d390560ac50943da4a3d2c96ab29756
 
 * Or get it from the `Bitbucket page`_:
   ``hg clone https://bitbucket.org/cffi/cffi``

From noreply at buildbot.pypy.org  Tue Jun  9 12:08:31 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue,  9 Jun 2015 12:08:31 +0200 (CEST)
Subject: [pypy-commit] cffi default: hg merge release-1.1
Message-ID: <20150609100831.889191C14C1@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2179:0c34f9e4f4c6
Date: 2015-06-09 12:08 +0200
http://bitbucket.org/cffi/cffi/changeset/0c34f9e4f4c6/

Log:	hg merge release-1.1

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -6063,7 +6063,7 @@
     if (v == NULL || PyModule_AddObject(m, "_C_API", v) < 0)
         INITERROR;
 
-    v = PyText_FromString("1.1.1");
+    v = PyText_FromString("1.1.2");
     if (v == NULL || PyModule_AddObject(m, "__version__", v) < 0)
         INITERROR;
 
diff --git a/c/test_c.py b/c/test_c.py
--- a/c/test_c.py
+++ b/c/test_c.py
@@ -3346,4 +3346,4 @@
 
 def test_version():
     # this test is here mostly for PyPy
-    assert __version__ == "1.1.1"
+    assert __version__ == "1.1.2"
diff --git a/cffi/__init__.py b/cffi/__init__.py
--- a/cffi/__init__.py
+++ b/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI, CDefError, FFIError
 from .ffiplatform import VerificationError, VerificationMissing
 
-__version__ = "1.1.1"
-__version_info__ = (1, 1, 1)
+__version__ = "1.1.2"
+__version_info__ = (1, 1, 2)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/doc/source/conf.py b/doc/source/conf.py
--- a/doc/source/conf.py
+++ b/doc/source/conf.py
@@ -47,7 +47,7 @@
 # The short X.Y version.
 version = '1.1'
 # The full version, including alpha/beta/rc tags.
-release = '1.1.1'
+release = '1.1.2'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
diff --git a/doc/source/installation.rst b/doc/source/installation.rst
--- a/doc/source/installation.rst
+++ b/doc/source/installation.rst
@@ -51,13 +51,13 @@
 
 Download and Installation:
 
-* http://pypi.python.org/packages/source/c/cffi/cffi-1.1.1.tar.gz
+* http://pypi.python.org/packages/source/c/cffi/cffi-1.1.2.tar.gz
 
    - Or grab the most current version by following the instructions below.
 
-   - MD5: f397363bfbf99048accb0498ffc3e72b
+   - MD5: ca6e6c45b45caa87aee9adc7c796eaea
 
-   - SHA: 8c4f4d1078d05c796c12fc6d8f8cea25aaff0148
+   - SHA: 6d6203bf7d390560ac50943da4a3d2c96ab29756
 
 * Or get it from the `Bitbucket page`_:
   ``hg clone https://bitbucket.org/cffi/cffi``
diff --git a/doc/source/whatsnew.rst b/doc/source/whatsnew.rst
--- a/doc/source/whatsnew.rst
+++ b/doc/source/whatsnew.rst
@@ -15,6 +15,13 @@
   that in C it means ``int (a[5])[...];``).
 
 
+1.1.2
+=====
+
+* ``ffi.gc()``: fixed a race condition in multithreaded programs
+  introduced in 1.1.1
+
+
 1.1.1
 =====
 
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -144,7 +144,7 @@
 
 `Mailing list <https://groups.google.com/forum/#!forum/python-cffi>`_
 """,
-        version='1.1.1',
+        version='1.1.2',
         packages=['cffi'] if cpython else [],
         package_data={'cffi': ['_cffi_include.h', 'parse_c_type.h']}
                      if cpython else {},

From noreply at buildbot.pypy.org  Tue Jun  9 12:09:10 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue,  9 Jun 2015 12:09:10 +0200 (CEST)
Subject: [pypy-commit] pypy default: Update to cffi 1.1.2
Message-ID: <20150609100910.764B91C0460@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r77987:531b9dea9776
Date: 2015-06-09 12:09 +0200
http://bitbucket.org/pypy/pypy/changeset/531b9dea9776/

Log:	Update to cffi 1.1.2

diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: cffi
-Version: 1.1.1
+Version: 1.1.2
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI, CDefError, FFIError
 from .ffiplatform import VerificationError, VerificationMissing
 
-__version__ = "1.1.1"
-__version_info__ = (1, 1, 1)
+__version__ = "1.1.2"
+__version_info__ = (1, 1, 2)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/lib_pypy/cffi/gc_weakref.py b/lib_pypy/cffi/gc_weakref.py
--- a/lib_pypy/cffi/gc_weakref.py
+++ b/lib_pypy/cffi/gc_weakref.py
@@ -4,25 +4,21 @@
 class GcWeakrefs(object):
     def __init__(self, ffi):
         self.ffi = ffi
-        self.data = []
-        self.freelist = None
+        self.data = {}
+        self.nextindex = 0
 
     def build(self, cdata, destructor):
         # make a new cdata of the same type as the original one
         new_cdata = self.ffi.cast(self.ffi._backend.typeof(cdata), cdata)
         #
         def remove(key):
-            assert self.data[index] is key
-            self.data[index] = self.freelist
-            self.freelist = index
+            # careful, this function is not protected by any lock
+            old_key = self.data.pop(index)
+            assert old_key is key
             destructor(cdata)
         #
         key = ref(new_cdata, remove)
-        index = self.freelist
-        if index is None:
-            index = len(self.data)
-            self.data.append(key)
-        else:
-            self.freelist = self.data[index]
-            self.data[index] = key
+        index = self.nextindex
+        self.nextindex = index + 1     # we're protected by the lock here
+        self.data[index] = key
         return new_cdata
diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -2,7 +2,7 @@
 from pypy.interpreter.mixedmodule import MixedModule
 from rpython.rlib import rdynload
 
-VERSION = "1.1.1"
+VERSION = "1.1.2"
 
 
 class Module(MixedModule):
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -3335,4 +3335,4 @@
 
 def test_version():
     # this test is here mostly for PyPy
-    assert __version__ == "1.1.1"
+    assert __version__ == "1.1.2"
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zintegration.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zintegration.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zintegration.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zintegration.py
@@ -1,6 +1,5 @@
 # Generated by pypy/tool/import_cffi.py
 import py, os, sys, shutil
-import imp
 import subprocess
 from pypy.module.test_lib_pypy.cffi_tests.udir import udir
 
@@ -16,28 +15,12 @@
     except OSError as e:
         py.test.skip("Cannot execute virtualenv: %s" % (e,))
 
-    try:
-        deepcopy = os.symlink
-    except:
-        import shutil, errno
-        def deepcopy(src, dst):
-            try:
-                shutil.copytree(src, dst)
-            except OSError as e:
-                if e.errno in (errno.ENOTDIR, errno.EINVAL):
-                    shutil.copy(src, dst)
-                else:
-                    print('got errno')
-                    print(e.errno)
-                    print('not')
-                    print(errno.ENOTDIR)
-                    raise
-
     site_packages = None
     for dirpath, dirnames, filenames in os.walk(str(tmpdir)):
         if os.path.basename(dirpath) == 'site-packages':
             site_packages = dirpath
             break
+    paths = ""
     if site_packages:
         try:
             from cffi import _pycparser
@@ -50,15 +33,22 @@
                 pass
             else:
                 modules += ('ply',)   # needed for older versions of pycparser
+        paths = []
         for module in modules:
-            target = imp.find_module(module)[1]
-            deepcopy(target, os.path.join(site_packages,
-                                            os.path.basename(target)))
-    return tmpdir
+            target = __import__(module, None, None, [])
+            src = os.path.abspath(target.__file__)
+            for end in ['__init__.pyc', '__init__.pyo', '__init__.py']:
+                if src.lower().endswith(end):
+                    src = src[:-len(end)-1]
+                    break
+            paths.append(os.path.dirname(src))
+        paths = os.pathsep.join(paths)
+    return tmpdir, paths
 
 SNIPPET_DIR = py.path.local(__file__).join('..', 'snippets')
 
-def really_run_setup_and_program(dirname, venv_dir, python_snippet):
+def really_run_setup_and_program(dirname, venv_dir_and_paths, python_snippet):
+    venv_dir, paths = venv_dir_and_paths
     def remove(dir):
         dir = str(SNIPPET_DIR.join(dirname, dir))
         shutil.rmtree(dir, ignore_errors=True)
@@ -76,9 +66,11 @@
         else:
             bindir = 'bin'
         vp = str(venv_dir.join(bindir).join('python'))
-        subprocess.check_call((vp, 'setup.py', 'clean'))
-        subprocess.check_call((vp, 'setup.py', 'install'))
-        subprocess.check_call((vp, str(python_f)))
+        env = os.environ.copy()
+        env['PYTHONPATH'] = paths
+        subprocess.check_call((vp, 'setup.py', 'clean'), env=env)
+        subprocess.check_call((vp, 'setup.py', 'install'), env=env)
+        subprocess.check_call((vp, str(python_f)), env=env)
     finally:
         os.chdir(olddir)
 
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_ffi_obj.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_ffi_obj.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_ffi_obj.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_ffi_obj.py
@@ -160,9 +160,10 @@
                             "struct never_heard_of_s\n"
                             "       ^")
     e = py.test.raises(ffi.error, ffi.cast, "\t\n\x01\x1f~\x7f\x80\xff", 0)
+    marks = "?" if sys.version_info < (3,) else "??"
     assert str(e.value) == ("identifier expected\n"
-                            "  ??~???\n"
-                            "  ^")
+                            "  ??~?%s%s\n"
+                            "  ^" % (marks, marks))
     e = py.test.raises(ffi.error, ffi.cast, "X" * 600, 0)
     assert str(e.value) == ("undefined type name")
 

From noreply at buildbot.pypy.org  Tue Jun  9 12:44:00 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Tue,  9 Jun 2015 12:44:00 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: gah, dont release the GIL here
Message-ID: <20150609104400.4F0741C15BC@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r77988:642c82bdc9cb
Date: 2015-06-09 12:44 +0200
http://bitbucket.org/pypy/pypy/changeset/642c82bdc9cb/

Log:	gah, dont release the GIL here

diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -1,5 +1,6 @@
 import weakref, os
 from rpython.rlib.objectmodel import we_are_translated, specialize
+from rpython.rtyper.lltypesystem.lloperation import llop
 from rpython.rlib.objectmodel import compute_identity_hash
 from rpython.rtyper.lltypesystem import lltype, llmemory
 from rpython.jit.codewriter import longlong
@@ -28,7 +29,7 @@
         return None
 
     def set_forwarded(self, forwarded_to):
-        os.write(2, "setting forwarded on: " + self.__class__.__name__)
+        llop.debug_print(lltype.Void, "setting forwarded on:", self.__class__.__name__)
         raise SettingForwardedOnAbstractValue()
 
     def get_box_replacement(op):

From noreply at buildbot.pypy.org  Tue Jun  9 13:53:05 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue,  9 Jun 2015 13:53:05 +0200 (CEST)
Subject: [pypy-commit] cffi default: Don't need an integer, actually,
	just object() is fine
Message-ID: <20150609115305.B50351C048F@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2180:325525b02433
Date: 2015-06-09 13:53 +0200
http://bitbucket.org/cffi/cffi/changeset/325525b02433/

Log:	Don't need an integer, actually, just object() is fine

diff --git a/cffi/gc_weakref.py b/cffi/gc_weakref.py
--- a/cffi/gc_weakref.py
+++ b/cffi/gc_weakref.py
@@ -5,7 +5,6 @@
     def __init__(self, ffi):
         self.ffi = ffi
         self.data = {}
-        self.nextindex = 0
 
     def build(self, cdata, destructor):
         # make a new cdata of the same type as the original one
@@ -18,7 +17,6 @@
             destructor(cdata)
         #
         key = ref(new_cdata, remove)
-        index = self.nextindex
-        self.nextindex = index + 1     # we're protected by the lock here
+        index = object()
         self.data[index] = key
         return new_cdata

From noreply at buildbot.pypy.org  Tue Jun  9 14:18:45 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Tue,  9 Jun 2015 14:18:45 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: moving the type determination from the
 scheduling transformation to the find pack routines, 
Message-ID: <20150609121845.CC6001C1239@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r77989:2982e1a68dcf
Date: 2015-06-09 13:36 +0200
http://bitbucket.org/pypy/pypy/changeset/2982e1a68dcf/

Log:	moving the type determination from the scheduling transformation to
	the find pack routines, this helps to better determine the costs of
	unpacking and is needed for accumulation (vector type,size,count)
	must be known before scheduling begins

diff --git a/rpython/jit/metainterp/optimizeopt/schedule.py b/rpython/jit/metainterp/optimizeopt/schedule.py
--- a/rpython/jit/metainterp/optimizeopt/schedule.py
+++ b/rpython/jit/metainterp/optimizeopt/schedule.py
@@ -184,7 +184,9 @@
     def is_valid(self):
         return self.type != PackType.UNKNOWN_TYPE and self.size > 0
 
-    def new_vector_box(self, count):
+    def new_vector_box(self, count = -1):
+        if count == -1:
+            count = self.count
         return BoxVector(self.type, count, self.size, self.signed)
 
     def __repr__(self):
@@ -262,7 +264,7 @@
                 off += 1
                 continue
             ops = pack.operations[off:off+stride]
-            self.pack = Pack(ops)
+            self.pack = Pack(ops, pack.input_type, pack.output_type)
             self.transform_pack(ops, off, stride)
             off += stride
             left -= stride
@@ -671,13 +673,15 @@
         * isomorphic
         * independent
     """
-    def __init__(self, ops):
+    def __init__(self, ops, input_type, output_type):
         self.operations = ops
         for i,node in enumerate(self.operations):
             node.pack = self
             node.pack_position = i
         self.accum_variable = None
         self.accum_position = -1
+        self.input_type = input_type
+        self.output_type = output_type
 
     def opcount(self):
         return len(self.operations)
@@ -719,12 +723,12 @@
 
 class Pair(Pack):
     """ A special Pack object with only two statements. """
-    def __init__(self, left, right):
+    def __init__(self, left, right, input_type, output_type):
         assert isinstance(left, Node)
         assert isinstance(right, Node)
         self.left = left
         self.right = right
-        Pack.__init__(self, [left, right])
+        Pack.__init__(self, [left, right], input_type, output_type)
 
     def __eq__(self, other):
         if isinstance(other, Pair):
@@ -732,10 +736,10 @@
                    self.right is other.right
 
 class AccumPair(Pair):
-    def __init__(self, left, right, accum_var, accum_pos):
+    def __init__(self, left, right, input_type, output_type, accum_var, accum_pos):
         assert isinstance(left, Node)
         assert isinstance(right, Node)
-        Pair.__init__(self, left, right)
+        Pair.__init__(self, left, right, input_type, output_type)
         self.left = left
         self.right = right
         self.accum_variable = accum_var
diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/vectorize.py
@@ -11,7 +11,8 @@
 from rpython.jit.metainterp.optimizeopt.dependency import (DependencyGraph,
         MemoryRef, Node, IndexVar)
 from rpython.jit.metainterp.optimizeopt.schedule import (VecScheduleData,
-        Scheduler, Pack, Pair, AccumPair, vectorbox_outof_box, getpackopnum)
+        Scheduler, Pack, Pair, AccumPair, vectorbox_outof_box, getpackopnum,
+        getunpackopnum, PackType)
 from rpython.jit.metainterp.optimizeopt.guard import GuardStrengthenOpt
 from rpython.jit.metainterp.resoperation import (rop, ResOperation, GuardResOp)
 from rpython.rlib.objectmodel import we_are_translated
@@ -275,9 +276,10 @@
         loop = self.loop
         operations = loop.operations
 
+        vsize = self.metainterp_sd.cpu.vector_register_size
         self.packset = PackSet(self.dependency_graph, operations,
-                               self.unroll_count,
-                               self.smallest_type_bytes)
+                               self.unroll_count, self.smallest_type_bytes,
+                               vsize)
         graph = self.dependency_graph
         memory_refs = graph.memory_refs.items()
         # initialize the pack set
@@ -380,7 +382,7 @@
                 if vector:
                     self.unpack_from_vector(op, sched_data, renamer)
                 self.emit_operation(op)
-
+        #
         if not we_are_translated():
             for node in self.dependency_graph.nodes:
                 assert node.emitted
@@ -411,9 +413,7 @@
             renamer.start_renaming(arg, arg_cloned)
             cj = ConstInt(j)
             ci = ConstInt(1)
-            opnum = rop.VEC_FLOAT_UNPACK
-            if vbox.item_type == INT:
-                opnum = rop.VEC_INT_UNPACK
+            opnum = getunpackopnum(vbox.item_type)
             unpack_op = ResOperation(opnum, [vbox, cj, ci], arg_cloned)
             self.emit_operation(unpack_op)
             return arg_cloned
@@ -526,12 +526,13 @@
 class PackSet(object):
 
     def __init__(self, dependency_graph, operations, unroll_count,
-                 smallest_type_bytes):
+                 smallest_type_bytes, vec_reg_size):
         self.packs = []
         self.dependency_graph = dependency_graph
         self.operations = operations
         self.unroll_count = unroll_count
         self.smallest_type_bytes = smallest_type_bytes
+        self.vec_reg_size = vec_reg_size
 
     def pack_count(self):
         return len(self.packs)
@@ -545,9 +546,12 @@
                 if self.contains_pair(lnode, rnode):
                     return None
                 if origin_pack is None:
-                    return Pair(lnode, rnode)
+                    descr = lnode.getoperation().getdescr()
+                    input_type = PackType.by_descr(descr, self.vec_reg_size)
+                    return Pair(lnode, rnode, input_type, None)
                 if self.profitable_pack(lnode, rnode, origin_pack):
-                    return Pair(lnode, rnode)
+                    ptype = origin_pack.output_type
+                    return Pair(lnode, rnode, ptype, ptype)
             else:
                 if self.contains_pair(lnode, rnode):
                     return None
@@ -590,7 +594,8 @@
         operations = pack_i.operations
         for op in pack_j.operations[1:]:
             operations.append(op)
-        self.packs[i] = pack = Pack(operations)
+        pack = Pack(operations, pack_i.input_type, pack_i.output_type)
+        self.packs[i] = pack
         # preserve the accum variable (if present) of the
         # left most pack, that is the pack with the earliest
         # operation at index 0 in the trace
@@ -640,7 +645,8 @@
                 return None
 
             # this can be handled by accumulation
-            return AccumPair(lnode, rnode, accum, accum_pos)
+            ptype = origin_pack.output_type
+            return AccumPair(lnode, rnode, ptype, ptype, accum, accum_pos)
 
         return None
 
@@ -659,7 +665,7 @@
             var = pack.accum_variable
             pos = pack.accum_position
             # create a new vector box for the parameters
-            box = vectorbox_outof_box(var)
+            box = pack.input_type.new_vector_box(0)
             op = ResOperation(rop.VEC_BOX, [ConstInt(0)], box)
             sched_data.invariant_oplist.append(op)
             result = box.clonebox()
@@ -675,4 +681,3 @@
             # rename the variable with the box
             renamer.start_renaming(var, result)
 
-

From noreply at buildbot.pypy.org  Tue Jun  9 14:18:47 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Tue,  9 Jun 2015 14:18:47 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: works now for the int16 case
Message-ID: <20150609121847.010491C1239@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r77990:ec09c339cc68
Date: 2015-06-09 13:55 +0200
http://bitbucket.org/pypy/pypy/changeset/ec09c339cc68/

Log:	works now for the int16 case

diff --git a/rpython/jit/metainterp/optimizeopt/schedule.py b/rpython/jit/metainterp/optimizeopt/schedule.py
--- a/rpython/jit/metainterp/optimizeopt/schedule.py
+++ b/rpython/jit/metainterp/optimizeopt/schedule.py
@@ -609,6 +609,15 @@
     rop.VEC_CAST_INT_TO_FLOAT: OpToVectorOpConv(PT_INT32_2, PT_DOUBLE_2),
 }
 
+def determine_output_type(node, input_type):
+    op = node.getoperation()
+    op2vecop = ROP_ARG_RES_VECTOR.get(op.vector, None)
+    if op2vecop is None:
+        raise NotImplementedError("missing vecop for '%s'" % (op.getopname(),))
+    if isinstance(op2vecop, OpToVectorOpConv):
+        return op2vecop.determine_output_type(op)
+    return input_type
+
 class VecScheduleData(SchedulerData):
     def __init__(self, vec_reg_size):
         self.box_to_vbox = {}
@@ -681,7 +690,11 @@
         self.accum_variable = None
         self.accum_position = -1
         self.input_type = input_type
+        if input_type:
+            self.input_type.count = len(ops)
         self.output_type = output_type
+        if output_type:
+            self.output_type.count = len(ops)
 
     def opcount(self):
         return len(self.operations)
diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/vectorize.py
@@ -547,11 +547,17 @@
                     return None
                 if origin_pack is None:
                     descr = lnode.getoperation().getdescr()
-                    input_type = PackType.by_descr(descr, self.vec_reg_size)
-                    return Pair(lnode, rnode, input_type, None)
+                    ptype = PackType.by_descr(descr, self.vec_reg_size)
+                    if lnode.getoperation().is_raw_load():
+                        # load outputs value, no input
+                        return Pair(lnode, rnode, None, ptype)
+                    else:
+                        # store only has an input
+                        return Pair(lnode, rnode, ptype, None)
                 if self.profitable_pack(lnode, rnode, origin_pack):
-                    ptype = origin_pack.output_type
-                    return Pair(lnode, rnode, ptype, ptype)
+                    input_type = origin_pack.output_type
+                    output_type = determine_output_type(lnode, input_type)
+                    return Pair(lnode, rnode, input_type, output_type)
             else:
                 if self.contains_pair(lnode, rnode):
                     return None
@@ -665,7 +671,7 @@
             var = pack.accum_variable
             pos = pack.accum_position
             # create a new vector box for the parameters
-            box = pack.input_type.new_vector_box(0)
+            box = pack.input_type.new_vector_box()
             op = ResOperation(rop.VEC_BOX, [ConstInt(0)], box)
             sched_data.invariant_oplist.append(op)
             result = box.clonebox()

From noreply at buildbot.pypy.org  Tue Jun  9 14:18:48 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Tue,  9 Jun 2015 14:18:48 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: preventing accumulator flush within the
 trace. the plan is only to support in in guard exits
Message-ID: <20150609121848.203501C1239@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r77991:be2247303111
Date: 2015-06-09 14:18 +0200
http://bitbucket.org/pypy/pypy/changeset/be2247303111/

Log:	preventing accumulator flush within the trace. the plan is only to
	support in in guard exits

diff --git a/rpython/jit/metainterp/optimizeopt/schedule.py b/rpython/jit/metainterp/optimizeopt/schedule.py
--- a/rpython/jit/metainterp/optimizeopt/schedule.py
+++ b/rpython/jit/metainterp/optimizeopt/schedule.py
@@ -690,11 +690,7 @@
         self.accum_variable = None
         self.accum_position = -1
         self.input_type = input_type
-        if input_type:
-            self.input_type.count = len(ops)
         self.output_type = output_type
-        if output_type:
-            self.output_type.count = len(ops)
 
     def opcount(self):
         return len(self.operations)
diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/vectorize.py
@@ -543,8 +543,14 @@
     def can_be_packed(self, lnode, rnode, origin_pack):
         if isomorphic(lnode.getoperation(), rnode.getoperation()):
             if lnode.independent(rnode):
+                if isinstance(origin_pack, AccumPair):
+                    # in this case the splitted accumulator must
+                    # be combined. This case is not supported
+                    raise NotAVectorizeableLoop()
+                #
                 if self.contains_pair(lnode, rnode):
                     return None
+                #
                 if origin_pack is None:
                     descr = lnode.getoperation().getdescr()
                     ptype = PackType.by_descr(descr, self.vec_reg_size)
@@ -620,6 +626,11 @@
         return last_pos
 
     def accumulates_pair(self, lnode, rnode, origin_pack):
+        if isinstance(origin_pack, AccumPair):
+            # in this case the splitted accumulator must
+            # be combined. This case is not supported
+            raise NotAVectorizeableLoop()
+        #
         # lnode and rnode are isomorphic and dependent
         assert isinstance(origin_pack, Pair)
         lop = lnode.getoperation()

From noreply at buildbot.pypy.org  Tue Jun  9 15:24:18 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue,  9 Jun 2015 15:24:18 +0200 (CEST)
Subject: [pypy-commit] pypy default: Fix the ".pypy-26.so" extension,
 i.e. stop updating it to ".pypy-XY.so".
Message-ID: <20150609132418.C43DF1C11B3@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r77992:97d59353f34e
Date: 2015-06-09 15:24 +0200
http://bitbucket.org/pypy/pypy/changeset/97d59353f34e/

Log:	Fix the ".pypy-26.so" extension, i.e. stop updating it to ".pypy-
	XY.so".

diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py
--- a/pypy/module/imp/importing.py
+++ b/pypy/module/imp/importing.py
@@ -30,7 +30,15 @@
 IMP_HOOK = 9
 
 SO = '.pyd' if _WIN32 else '.so'
-DEFAULT_SOABI = 'pypy-%d%d' % PYPY_VERSION[:2]
+
+# this used to change for every minor version, but no longer does: there
+# is little point any more, as the so's tend to be cross-version-
+# compatible, more so than between various versions of CPython.  Be
+# careful if we need to update it again: it is now used for both cpyext
+# and cffi so's.  If we do have to update it, we'd likely need a way to
+# split the two usages again.
+#DEFAULT_SOABI = 'pypy-%d%d' % PYPY_VERSION[:2]
+DEFAULT_SOABI = 'pypy-26'
 
 @specialize.memo()
 def get_so_extension(space):

From noreply at buildbot.pypy.org  Tue Jun  9 15:42:02 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Tue,  9 Jun 2015 15:42:02 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: preventing the transformation of cumsum
 (some problems that followed from last changes)
Message-ID: <20150609134202.1DA321C0460@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r77993:652fca81f3d1
Date: 2015-06-09 15:41 +0200
http://bitbucket.org/pypy/pypy/changeset/652fca81f3d1/

Log:	preventing the transformation of cumsum (some problems that followed
	from last changes)

diff --git a/rpython/jit/metainterp/optimizeopt/schedule.py b/rpython/jit/metainterp/optimizeopt/schedule.py
--- a/rpython/jit/metainterp/optimizeopt/schedule.py
+++ b/rpython/jit/metainterp/optimizeopt/schedule.py
@@ -715,13 +715,6 @@
                 accum = False
             elif self.accum_position != other.accum_position:
                 accum = False
-            # aa
-            #else:
-            #    i = self.accum_position
-            #    lop = leftmost.getoperation()
-            #    roper = rightmost.getoperation()
-            #    if lop.getarg(i) is not roper.result:
-            #        accum = False
         return rightmost is leftmost and accum
 
     def __repr__(self):
diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/vectorize.py
@@ -292,7 +292,7 @@
                 # that point forward:
                 if node_a.is_before(node_b):
                     if memref_a.is_adjacent_to(memref_b):
-                        pair = self.packset.can_be_packed(node_a, node_b, None)
+                        pair = self.packset.can_be_packed(node_a, node_b, None, False)
                         if pair:
                             self.packset.add_pack(pair)
 
@@ -314,7 +314,7 @@
                 rnode = rdep.to
                 isomorph = isomorphic(lnode.getoperation(), rnode.getoperation())
                 if isomorph and lnode.is_before(rnode):
-                    pair = self.packset.can_be_packed(lnode, rnode, pack)
+                    pair = self.packset.can_be_packed(lnode, rnode, pack, False)
                     if pair:
                         self.packset.add_pack(pair)
 
@@ -326,7 +326,7 @@
                 rnode = rdep.to
                 isomorph = isomorphic(lnode.getoperation(), rnode.getoperation())
                 if isomorph and lnode.is_before(rnode):
-                    pair = self.packset.can_be_packed(lnode, rnode, pack)
+                    pair = self.packset.can_be_packed(lnode, rnode, pack, True)
                     if pair:
                         self.packset.add_pack(pair)
 
@@ -540,10 +540,10 @@
     def add_pack(self, pack):
         self.packs.append(pack)
 
-    def can_be_packed(self, lnode, rnode, origin_pack):
+    def can_be_packed(self, lnode, rnode, origin_pack, forward):
         if isomorphic(lnode.getoperation(), rnode.getoperation()):
             if lnode.independent(rnode):
-                if isinstance(origin_pack, AccumPair):
+                if forward and isinstance(origin_pack, AccumPair):
                     # in this case the splitted accumulator must
                     # be combined. This case is not supported
                     raise NotAVectorizeableLoop()
@@ -626,11 +626,6 @@
         return last_pos
 
     def accumulates_pair(self, lnode, rnode, origin_pack):
-        if isinstance(origin_pack, AccumPair):
-            # in this case the splitted accumulator must
-            # be combined. This case is not supported
-            raise NotAVectorizeableLoop()
-        #
         # lnode and rnode are isomorphic and dependent
         assert isinstance(origin_pack, Pair)
         lop = lnode.getoperation()
diff --git a/rpython/jit/tool/oparser.py b/rpython/jit/tool/oparser.py
--- a/rpython/jit/tool/oparser.py
+++ b/rpython/jit/tool/oparser.py
@@ -239,7 +239,7 @@
         args, descr = self.parse_args(opname, line[num + 1:endnum])
         if rop._GUARD_FIRST <= opnum <= rop._GUARD_LAST:
             i = line.find('[', endnum) + 1
-            j = line.find(']', i)
+            j = line.rfind(']', i)
             if (i <= 0 or j <= 0) and not self.nonstrict:
                 raise ParseError("missing fail_args for guard operation")
             fail_args = []

From noreply at buildbot.pypy.org  Tue Jun  9 17:13:07 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue,  9 Jun 2015 17:13:07 +0200 (CEST)
Subject: [pypy-commit] stmgc c8-gil-like: hg merge default
Message-ID: <20150609151307.8E1831C048F@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: c8-gil-like
Changeset: r1793:f85a069a561e
Date: 2015-06-07 11:57 +0200
http://bitbucket.org/pypy/stmgc/changeset/f85a069a561e/

Log:	hg merge default

diff --git a/c8/stm/core.h b/c8/stm/core.h
--- a/c8/stm/core.h
+++ b/c8/stm/core.h
@@ -170,6 +170,10 @@
     TS_INEVITABLE,
 };
 
+#define in_transaction(tl)                                              \
+    (get_segment((tl)->last_associated_segment_num)->running_thread == (tl))
+
+
 /* Commit Log things */
 struct stm_undo_s {
   union {
diff --git a/c8/stm/extra.c b/c8/stm/extra.c
--- a/c8/stm/extra.c
+++ b/c8/stm/extra.c
@@ -8,7 +8,7 @@
 {
     dprintf(("register_callbacks: tl=%p key=%p callback=%p index=%ld\n",
              tl, key, callback, index));
-    if (tl->associated_segment_num == -1) {
+    if (!in_transaction(tl)) {
         /* check that the provided thread-local is really running a
            transaction, and do nothing otherwise. */
         dprintf(("  NOT IN TRANSACTION\n"));
diff --git a/c8/stm/forksupport.c b/c8/stm/forksupport.c
--- a/c8/stm/forksupport.c
+++ b/c8/stm/forksupport.c
@@ -41,6 +41,7 @@
     bool was_in_transaction = _stm_in_transaction(this_tl);
     if (!was_in_transaction)
         stm_start_transaction(this_tl);
+    assert(in_transaction(this_tl));
 
     stm_become_inevitable(this_tl, "fork");
     /* Note that the line above can still fail and abort, which should
@@ -83,7 +84,8 @@
     struct stm_priv_segment_info_s *pr = get_priv_segment(i);
     stm_thread_local_t *tl = pr->pub.running_thread;
     dprintf(("forksupport_child: abort in seg%ld\n", i));
-    assert(tl->associated_segment_num == i);
+    assert(tl->last_associated_segment_num == i);
+    assert(in_transaction(tl));
     assert(pr->transaction_state != TS_INEVITABLE);
     set_gs_register(get_segment_base(i));
     assert(STM_SEGMENT->segment_num == i);
@@ -150,7 +152,7 @@
 
     /* Restore a few things: the new pthread_self(), and the %gs
        register */
-    int segnum = fork_this_tl->associated_segment_num;
+    int segnum = fork_this_tl->last_associated_segment_num;
     assert(1 <= segnum && segnum < NB_SEGMENTS);
     *_get_cpth(fork_this_tl) = pthread_self();
     set_gs_register(get_segment_base(segnum));
diff --git a/c8/stm/setup.c b/c8/stm/setup.c
--- a/c8/stm/setup.c
+++ b/c8/stm/setup.c
@@ -244,7 +244,6 @@
     /* assign numbers consecutively, but that's for tests; we could also
        assign the same number to all of them and they would get their own
        numbers automatically. */
-    tl->associated_segment_num = -1;
     tl->last_associated_segment_num = num + 1;
     tl->thread_local_counter = ++thread_local_counters;
     *_get_cpth(tl) = pthread_self();
diff --git a/c8/stm/sync.c b/c8/stm/sync.c
--- a/c8/stm/sync.c
+++ b/c8/stm/sync.c
@@ -176,8 +176,10 @@
     sync_ctl.in_use1[num+1] = 1;
     assert(STM_SEGMENT->segment_num == num+1);
     assert(STM_SEGMENT->running_thread == NULL);
-    tl->associated_segment_num = tl->last_associated_segment_num;
+    assert(tl->last_associated_segment_num == STM_SEGMENT->segment_num);
+    assert(!in_transaction(tl));
     STM_SEGMENT->running_thread = tl;
+    assert(in_transaction(tl));
     return true;
 }
 
@@ -188,9 +190,10 @@
     cond_signal(C_SEGMENT_FREE);
 
     assert(STM_SEGMENT->running_thread == tl);
-    assert(tl->associated_segment_num == tl->last_associated_segment_num);
-    tl->associated_segment_num = -1;
+    assert(tl->last_associated_segment_num == STM_SEGMENT->segment_num);
+    assert(in_transaction(tl));
     STM_SEGMENT->running_thread = NULL;
+    assert(!in_transaction(tl));
 
     assert(sync_ctl.in_use1[tl->last_associated_segment_num] == 1);
     sync_ctl.in_use1[tl->last_associated_segment_num] = 0;
@@ -204,22 +207,15 @@
 
 bool _stm_in_transaction(stm_thread_local_t *tl)
 {
-    if (tl->associated_segment_num == -1) {
-        return false;
-    }
-    else {
-        int num = tl->associated_segment_num;
-        OPT_ASSERT(1 <= num && num < NB_SEGMENTS);
-        OPT_ASSERT(num == tl->last_associated_segment_num);
-        OPT_ASSERT(get_segment(num)->running_thread == tl);
-        return true;
-    }
+    int num = tl->last_associated_segment_num;
+    OPT_ASSERT(1 <= num && num < NB_SEGMENTS);
+    return in_transaction(tl);
 }
 
 void _stm_test_switch(stm_thread_local_t *tl)
 {
     assert(_stm_in_transaction(tl));
-    set_gs_register(get_segment_base(tl->associated_segment_num));
+    set_gs_register(get_segment_base(tl->last_associated_segment_num));
     assert(STM_SEGMENT->running_thread == tl);
     exec_local_finalizers();
 }
diff --git a/c8/stmgc.h b/c8/stmgc.h
--- a/c8/stmgc.h
+++ b/c8/stmgc.h
@@ -69,8 +69,7 @@
        (this field is not modified on a successful commit) */
     long last_abort__bytes_in_nursery;
     /* the next fields are handled internally by the library */
-    int associated_segment_num;
-    int last_associated_segment_num;
+    int last_associated_segment_num;   /* always a valid seg num */
     int thread_local_counter;
     struct stm_thread_local_s *prev, *next;
     void *creating_pthread[2];
diff --git a/c8/test/support.py b/c8/test/support.py
--- a/c8/test/support.py
+++ b/c8/test/support.py
@@ -29,7 +29,6 @@
     char *mem_clear_on_abort;
     size_t mem_bytes_to_clear_on_abort;
     long last_abort__bytes_in_nursery;
-    int associated_segment_num;
     int last_associated_segment_num;
     struct stm_thread_local_s *prev, *next;
     void *creating_pthread[2];
@@ -798,8 +797,8 @@
         seen = set()
         for tl1 in self.tls:
             if lib._stm_in_transaction(tl1):
-                assert tl1.associated_segment_num not in seen
-                seen.add(tl1.associated_segment_num)
+                assert tl1.last_associated_segment_num not in seen
+                seen.add(tl1.last_associated_segment_num)
 
     def commit_transaction(self):
         tl = self.tls[self.current_thread]
diff --git a/c8/test/test_finalizer.py b/c8/test/test_finalizer.py
--- a/c8/test/test_finalizer.py
+++ b/c8/test/test_finalizer.py
@@ -13,9 +13,10 @@
             segnum = lib.current_segment_num()
             tlnum = '?'
             for n, tl in enumerate(self.tls):
-                if tl.associated_segment_num == segnum:
-                    tlnum = n
-                    break
+                if lib._stm_in_transaction(tl):
+                    if tl.last_associated_segment_num == segnum:
+                        tlnum = n
+                        break
             self.light_finalizers_called.append((obj, tlnum))
         self.light_finalizers_called = []
         lib.stmcb_light_finalizer = light_finalizer

From noreply at buildbot.pypy.org  Tue Jun  9 17:13:08 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue,  9 Jun 2015 17:13:08 +0200 (CEST)
Subject: [pypy-commit] stmgc c8-gil-like: in-progress
Message-ID: <20150609151308.A49911C048F@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: c8-gil-like
Changeset: r1794:2545c3033c9b
Date: 2015-06-09 17:13 +0200
http://bitbucket.org/pypy/stmgc/changeset/2545c3033c9b/

Log:	in-progress

diff --git a/c8/CALL_RELEASE_GIL b/c8/CALL_RELEASE_GIL
--- a/c8/CALL_RELEASE_GIL
+++ b/c8/CALL_RELEASE_GIL
@@ -50,12 +50,12 @@
   same or a different thread.
 
 - we add a global variable, "stm_detached_inevitable_from_thread".  It
-  is equal to the shadowstack pointer of the thread that detached
+  is equal to the stm_thread_local pointer of the thread that detached
   inevitable transaction (like rpy_fastgil == 0), or NULL if there is
   no detached inevitable transaction (like rpy_fastgil == 1).
 
 - the macro stm_detach_inevitable_transaction() simply writes the
-  current thread's shadowstack pointer into the global variable
+  current thread's stm_thread_local pointer into the global variable
   stm_detached_inevitable_from_thread.  It can only be used if the
   current transaction is inevitable (and in particular the inevitable
   transaction was not detached already, because we're running it).
@@ -65,7 +65,7 @@
 
 - the macro stm_reattach_transaction() does an atomic swap on
   stm_detached_inevitable_from_thread to change it to NULL.  If the
-  old value was equal to our own shadowstack pointer, we are done.  If
+  old value was equal to our own stm_thread_local pointer, we are done.  If
   not, we call a helper, _stm_reattach_transaction().
 
 - we also add the macro stm_detach_transation().  If the current
@@ -76,7 +76,7 @@
   stm_detached_inevitable_from_thread (which was swapped to be NULL just
   now).  If old != NULL, this swap had the effect that we took over
   the inevitable transaction originally detached from a different
-  thread; we need to fix a few things like the shadowstack and %gs but
+  thread; we need to fix a few things like the stm_thread_local and %gs but
   then we can continue running this reattached inevitable transaction.
   If old == NULL, we need to fall back to the current
   stm_start_transaction().  (A priori, there is no need to wait at
diff --git a/c8/stm/core.c b/c8/stm/core.c
--- a/c8/stm/core.c
+++ b/c8/stm/core.c
@@ -509,7 +509,8 @@
 static void readd_wb_executed_flags(void);
 static void check_all_write_barrier_flags(char *segbase, struct list_s *list);
 
-static void _validate_and_attach(struct stm_commit_log_entry_s *new)
+static bool _validate_and_attach(struct stm_commit_log_entry_s *new,
+                                 bool can_sleep)
 {
     struct stm_commit_log_entry_s *old;
 
@@ -571,6 +572,8 @@
             /* XXXXXX for now just sleep.  We should really ask to inev
                transaction to do the commit for us, and then we can
                continue running. */
+            if (!can_sleep)
+                return false;
             dprintf(("_validate_and_attach(%p) failed, "
                      "waiting for inevitable\n", new));
             wait_for_other_inevitable(old);
@@ -598,11 +601,13 @@
         STM_PSEGMENT->last_commit_log_entry = new;
         release_modification_lock_wr(STM_SEGMENT->segment_num);
     }
+    return true;
 }
 
-static void _validate_and_turn_inevitable(void)
+static bool _validate_and_turn_inevitable(bool can_sleep)
 {
-    _validate_and_attach((struct stm_commit_log_entry_s *)INEV_RUNNING);
+    return _validate_and_attach((struct stm_commit_log_entry_s *)INEV_RUNNING,
+                                can_sleep);
 }
 
 static void _validate_and_add_to_commit_log(void)
@@ -631,7 +636,7 @@
         OPT_ASSERT(yes);
     }
     else {
-        _validate_and_attach(new);
+        _validate_and_attach(new, /*can_sleep=*/true);
     }
 }
 
@@ -1123,7 +1128,7 @@
 
 
 
-static void _stm_start_transaction(stm_thread_local_t *tl)
+static void _do_start_transaction(stm_thread_local_t *tl)
 {
     assert(!_stm_in_transaction(tl));
 
@@ -1181,7 +1186,7 @@
     stm_validate();
 }
 
-long stm_start_transaction(stm_thread_local_t *tl)
+long _stm_start_transaction(stm_thread_local_t *tl)
 {
     s_mutex_lock();
 #ifdef STM_NO_AUTOMATIC_SETJMP
@@ -1189,23 +1194,10 @@
 #else
     long repeat_count = stm_rewind_jmp_setjmp(tl);
 #endif
-    _stm_start_transaction(tl);
+    _do_start_transaction(tl);
     return repeat_count;
 }
 
-void stm_start_inevitable_transaction(stm_thread_local_t *tl)
-{
-    /* used to be more efficient, starting directly an inevitable transaction,
-       but there is no real point any more, I believe */
-    rewind_jmp_buf rjbuf;
-    stm_rewind_jmp_enterframe(tl, &rjbuf);
-
-    stm_start_transaction(tl);
-    stm_become_inevitable(tl, "start_inevitable_transaction");
-
-    stm_rewind_jmp_leaveframe(tl, &rjbuf);
-}
-
 #ifdef STM_NO_AUTOMATIC_SETJMP
 void _test_run_abort(stm_thread_local_t *tl) __attribute__((noreturn));
 int stm_is_inevitable(void)
@@ -1280,7 +1272,7 @@
 }
 
 
-void stm_commit_transaction(void)
+void _stm_commit_transaction(void)
 {
     exec_local_finalizers();
 
@@ -1502,20 +1494,23 @@
 
 void _stm_become_inevitable(const char *msg)
 {
-    if (STM_PSEGMENT->transaction_state == TS_REGULAR) {
+    assert(STM_PSEGMENT->transaction_state == TS_REGULAR);
+    _stm_collectable_safe_point();
+
+    if (msg != MSG_INEV_DONT_SLEEP) {
         dprintf(("become_inevitable: %s\n", msg));
-        _stm_collectable_safe_point();
         timing_become_inevitable();
-
-        _validate_and_turn_inevitable();
-        STM_PSEGMENT->transaction_state = TS_INEVITABLE;
-
-        stm_rewind_jmp_forget(STM_SEGMENT->running_thread);
-        invoke_and_clear_user_callbacks(0);   /* for commit */
+        _validate_and_turn_inevitable(/*can_sleep=*/true);
     }
     else {
-        assert(STM_PSEGMENT->transaction_state == TS_INEVITABLE);
+        if (!_validate_and_turn_inevitable(/*can_sleep=*/false))
+            return;
+        timing_become_inevitable();
     }
+    STM_PSEGMENT->transaction_state = TS_INEVITABLE;
+
+    stm_rewind_jmp_forget(STM_SEGMENT->running_thread);
+    invoke_and_clear_user_callbacks(0);   /* for commit */
 }
 
 void stm_become_globally_unique_transaction(stm_thread_local_t *tl,
diff --git a/c8/stm/core.h b/c8/stm/core.h
--- a/c8/stm/core.h
+++ b/c8/stm/core.h
@@ -170,6 +170,8 @@
     TS_INEVITABLE,
 };
 
+#define MSG_INEV_DONT_SLEEP  ((const char *)1)
+
 #define in_transaction(tl)                                              \
     (get_segment((tl)->last_associated_segment_num)->running_thread == (tl))
 
diff --git a/c8/stm/detach.c b/c8/stm/detach.c
new file mode 100644
--- /dev/null
+++ b/c8/stm/detach.c
@@ -0,0 +1,76 @@
+#ifndef _STM_CORE_H_
+# error "must be compiled via stmgc.c"
+#endif
+
+
+#define DETACHED_NO_THREAD  ((stm_thread_local_t *)-1)
+
+
+stm_thread_local_t *volatile _stm_detached_inevitable_from_thread;
+
+
+static void setup_detach(void)
+{
+    _stm_detached_inevitable_from_thread = NULL;
+}
+
+
+void _stm_leave_noninevitable_transactional_zone(void)
+{
+    _stm_become_inevitable(MSG_INEV_DONT_SLEEP);
+
+    /* did it work? */
+    if (STM_PSEGMENT->transaction_state == TS_INEVITABLE) {   /* yes */
+        _stm_detach_inevitable_transaction(STM_SEGMENT->running_thread);
+    }
+    else {   /* no */
+        _stm_commit_transaction();
+    }
+}
+
+void _stm_reattach_transaction(stm_thread_local_t *old, stm_thread_local_t *tl)
+{
+    if (old != NULL) {
+        /* We took over the inevitable transaction originally detached
+           from a different thread.  We have to fix the %gs register if
+           it is incorrect.  Careful, 'old' might be DETACHED_NO_THREAD.
+        */
+        int mysegnum = tl->last_associated_segment_num;
+
+        if (STM_SEGMENT->segment_num != mysegnum) {
+            set_gs_register(get_segment_base(mysegnum));
+            assert(STM_SEGMENT->segment_num == mysegnum);
+        }
+        assert(old == DETACHED_NO_THREAD || STM_SEGMENT->running_thread == old);
+        STM_SEGMENT->running_thread = tl;
+
+        stm_safe_point();
+    }
+    else {
+        /* there was no detached inevitable transaction */
+        _stm_start_transaction(tl);
+    }
+}
+
+static void fully_detach_thread(void)
+{
+    /* If there is a detached inevitable transaction, then make sure
+       that it is "fully" detached.  The point is to make sure that
+       the fast path of stm_enter_transactional_zone() will fail, and
+       we'll call _stm_reattach_transaction(), which will in turn call
+       stm_safe_point().  So a "fully detached" transaction will enter
+       a safe point as soon as it is reattached.
+
+       XXX THINK about concurrent threads here!
+    */
+    assert(_has_mutex());
+
+ restart:
+    stm_thread_local_t *old = stm_detached_inevitable_from_thread;
+    if (old == NULL || old == DETACHED_NO_THREAD)
+        return;
+
+    if (!__sync_bool_compare_and_swap(&stm_detached_inevitable_from_thread,
+                                      old, DETACHED_NO_THREAD))
+        goto restart;
+}
diff --git a/c8/stm/detach.h b/c8/stm/detach.h
new file mode 100644
--- /dev/null
+++ b/c8/stm/detach.h
@@ -0,0 +1,3 @@
+
+static void setup_detach(void);
+static void fully_detach_thread(void);
diff --git a/c8/stm/finalizer.c b/c8/stm/finalizer.c
--- a/c8/stm/finalizer.c
+++ b/c8/stm/finalizer.c
@@ -494,11 +494,11 @@
 
     rewind_jmp_buf rjbuf;
     stm_rewind_jmp_enterframe(tl, &rjbuf);
-    stm_start_transaction(tl);
+    _stm_start_transaction(tl);
 
     _execute_finalizers(&g_finalizers);
 
-    stm_commit_transaction();
+    _stm_commit_transaction();
     stm_rewind_jmp_leaveframe(tl, &rjbuf);
 
     __sync_lock_release(&lock);
diff --git a/c8/stm/forksupport.c b/c8/stm/forksupport.c
--- a/c8/stm/forksupport.c
+++ b/c8/stm/forksupport.c
@@ -40,7 +40,7 @@
 
     bool was_in_transaction = _stm_in_transaction(this_tl);
     if (!was_in_transaction)
-        stm_start_transaction(this_tl);
+        _stm_start_transaction(this_tl);
     assert(in_transaction(this_tl));
 
     stm_become_inevitable(this_tl, "fork");
@@ -73,7 +73,7 @@
     s_mutex_unlock();
 
     if (!was_in_transaction) {
-        stm_commit_transaction();
+        _stm_commit_transaction();
     }
 
     dprintf(("forksupport_parent: continuing to run\n"));
@@ -159,7 +159,7 @@
     assert(STM_SEGMENT->segment_num == segnum);
 
     if (!fork_was_in_transaction) {
-        stm_commit_transaction();
+        _stm_commit_transaction();
     }
 
     /* Done */
diff --git a/c8/stm/setup.c b/c8/stm/setup.c
--- a/c8/stm/setup.c
+++ b/c8/stm/setup.c
@@ -134,6 +134,7 @@
     setup_pages();
     setup_forksupport();
     setup_finalizer();
+    setup_detach();
 
     set_gs_register(get_segment_base(0));
 }
diff --git a/c8/stm/sync.c b/c8/stm/sync.c
--- a/c8/stm/sync.c
+++ b/c8/stm/sync.c
@@ -103,6 +103,7 @@
 /************************************************************/
 
 
+#if 0
 void stm_wait_for_current_inevitable_transaction(void)
 {
  restart:
@@ -125,7 +126,7 @@
     }
     s_mutex_unlock();
 }
-
+#endif
 
 
 static bool acquire_thread_segment(stm_thread_local_t *tl)
@@ -263,6 +264,7 @@
     }
     assert(!pause_signalled);
     pause_signalled = true;
+    fully_detach_thread();
 }
 
 static inline long count_other_threads_sp_running(void)
diff --git a/c8/stmgc.c b/c8/stmgc.c
--- a/c8/stmgc.c
+++ b/c8/stmgc.c
@@ -18,6 +18,7 @@
 #include "stm/rewind_setjmp.h"
 #include "stm/finalizer.h"
 #include "stm/locks.h"
+#include "stm/detach.h"
 
 #include "stm/misc.c"
 #include "stm/list.c"
@@ -41,3 +42,4 @@
 #include "stm/rewind_setjmp.c"
 #include "stm/finalizer.c"
 #include "stm/hashtable.c"
+#include "stm/detach.c"
diff --git a/c8/stmgc.h b/c8/stmgc.h
--- a/c8/stmgc.h
+++ b/c8/stmgc.h
@@ -13,6 +13,7 @@
 #include <limits.h>
 #include <unistd.h>
 
+#include "stm/atomic.h"
 #include "stm/rewind_setjmp.h"
 
 #if LONG_MAX == 2147483647
@@ -82,6 +83,16 @@
 void _stm_write_slowpath_card(object_t *, uintptr_t);
 object_t *_stm_allocate_slowpath(ssize_t);
 object_t *_stm_allocate_external(ssize_t);
+
+extern stm_thread_local_t *volatile _stm_detached_inevitable_from_thread;
+long _stm_start_transaction(stm_thread_local_t *tl);
+void _stm_commit_transaction(void);
+void _stm_leave_noninevitable_transactional_zone(void);
+#define _stm_detach_inevitable_transaction(tl)  do {    \
+    write_fence();                                      \
+    _stm_detached_inevitable_from_thread = (tl);        \
+} while (0)
+void _stm_reattach_transaction(stm_thread_local_t *old, stm_thread_local_t *tl);
 void _stm_become_inevitable(const char*);
 void _stm_collectable_safe_point(void);
 
@@ -379,23 +390,6 @@
     rewind_jmp_enum_shadowstack(&(tl)->rjthread, callback)
 
 
-/* Starting and ending transactions.  stm_read(), stm_write() and
-   stm_allocate() should only be called from within a transaction.
-   The stm_start_transaction() call returns the number of times it
-   returned, starting at 0.  If it is > 0, then the transaction was
-   aborted and restarted this number of times. */
-long stm_start_transaction(stm_thread_local_t *tl);
-void stm_start_inevitable_transaction(stm_thread_local_t *tl);
-void stm_commit_transaction(void);
-
-/* Temporary fix?  Call this outside a transaction.  If there is an
-   inevitable transaction running somewhere else, wait until it finishes. */
-void stm_wait_for_current_inevitable_transaction(void);
-
-/* Abort the currently running transaction.  This function never
-   returns: it jumps back to the stm_start_transaction(). */
-void stm_abort_transaction(void) __attribute__((noreturn));
-
 #ifdef STM_NO_AUTOMATIC_SETJMP
 int stm_is_inevitable(void);
 #else
@@ -404,6 +398,54 @@
 }
 #endif
 
+
+/* Entering and leaving a "transactional code zone": a (typically very
+   large) section in the code where we are running a transaction.
+   This is the STM equivalent to "acquire the GIL" and "release the
+   GIL", respectively.  stm_read(), stm_write(), stm_allocate(), and
+   other functions should only be called from within a transaction.
+
+   Note that transactions, in the STM sense, cover _at least_ one
+   transactional code zone.  They may be longer; for example, if one
+   thread does a lot of stm_enter_transactional_zone() +
+   stm_become_inevitable() + stm_leave_transactional_zone(), as is
+   typical in a thread that does a lot of C function calls, then we
+   get only a few bigger inevitable transactions that cover the many
+   short transactional zones.  This is done by having
+   stm_leave_transactional_zone() turn the current transaction
+   inevitable and detach it from the running thread (if there is no
+   other inevitable transaction running so far).  Then
+   stm_enter_transactional_zone() will try to reattach to it.  This is
+   far more efficient than constantly starting and committing
+   transactions.
+*/
+inline void stm_enter_transactional_zone(stm_thread_local_t *tl) {
+    stm_thread_local_t *old = __sync_lock_test_and_set(    /* XCHG */
+        &_stm_detached_inevitable_from_thread, NULL);
+    if (old != (tl))
+        _stm_reattach_transaction(old, tl);
+}
+inline void stm_leave_transactional_zone(stm_thread_local_t *tl) {
+    assert(STM_SEGMENT->running_thread == tl);
+    if (stm_is_inevitable())
+        _stm_detach_inevitable_transaction(tl);
+    else
+        _stm_leave_noninevitable_transactional_zone();
+}
+
+/* stm_break_transaction() is in theory equivalent to
+   stm_leave_transactional_zone() immediately followed by
+   stm_enter_transactional_zone(); however, it is supposed to be
+   called in CPU-heavy threads that had a transaction run for a while,
+   and so it *always* forces a commit and starts the next transaction.
+   The new transaction is never inevitable. */
+void stm_break_transaction(stm_thread_local_t *tl);
+
+/* Abort the currently running transaction.  This function never
+   returns: it jumps back to the start of the transaction (which must
+   not be inevitable). */
+void stm_abort_transaction(void) __attribute__((noreturn));
+
 /* Turn the current transaction inevitable.
    stm_become_inevitable() itself may still abort the transaction instead
    of returning. */
@@ -412,6 +454,8 @@
     assert(STM_SEGMENT->running_thread == tl);
     if (!stm_is_inevitable())
         _stm_become_inevitable(msg);
+    /* now, we're running the inevitable transaction, so: */
+    assert(_stm_detached_inevitable_from_thread == NULL);
 }
 
 /* Forces a safe-point if needed.  Normally not needed: this is
diff --git a/c8/test/support.py b/c8/test/support.py
--- a/c8/test/support.py
+++ b/c8/test/support.py
@@ -275,7 +275,7 @@
 }
 
 bool _check_commit_transaction(void) {
-    CHECKED(stm_commit_transaction());
+    CHECKED(_stm_commit_transaction());
 }
 
 bool _check_stm_collect(long level) {
@@ -285,7 +285,7 @@
 long _check_start_transaction(stm_thread_local_t *tl) {
    void **jmpbuf = tl->rjthread.jmpbuf;                         \
     if (__builtin_setjmp(jmpbuf) == 0) { /* returned directly */\
-        stm_start_transaction(tl);                              \
+        stm_enter_transactional_zone(tl);                       \
         clear_jmpbuf(tl);                                       \
         return 0;                                               \
     }                                                           \

From noreply at buildbot.pypy.org  Tue Jun  9 17:15:28 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue,  9 Jun 2015 17:15:28 +0200 (CEST)
Subject: [pypy-commit] stmgc c8-gil-like: tweaks, some tests pass again
Message-ID: <20150609151528.DD1381C048F@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: c8-gil-like
Changeset: r1795:c8581bc6bc2a
Date: 2015-06-09 17:16 +0200
http://bitbucket.org/pypy/stmgc/changeset/c8581bc6bc2a/

Log:	tweaks, some tests pass again

diff --git a/c8/stm/detach.c b/c8/stm/detach.c
--- a/c8/stm/detach.c
+++ b/c8/stm/detach.c
@@ -65,12 +65,12 @@
     */
     assert(_has_mutex());
 
- restart:
-    stm_thread_local_t *old = stm_detached_inevitable_from_thread;
+ restart:;
+    stm_thread_local_t *old = _stm_detached_inevitable_from_thread;
     if (old == NULL || old == DETACHED_NO_THREAD)
         return;
 
-    if (!__sync_bool_compare_and_swap(&stm_detached_inevitable_from_thread,
+    if (!__sync_bool_compare_and_swap(&_stm_detached_inevitable_from_thread,
                                       old, DETACHED_NO_THREAD))
         goto restart;
 }
diff --git a/c8/stmgc.h b/c8/stmgc.h
--- a/c8/stmgc.h
+++ b/c8/stmgc.h
@@ -419,13 +419,13 @@
    far more efficient than constantly starting and committing
    transactions.
 */
-inline void stm_enter_transactional_zone(stm_thread_local_t *tl) {
+static inline void stm_enter_transactional_zone(stm_thread_local_t *tl) {
     stm_thread_local_t *old = __sync_lock_test_and_set(    /* XCHG */
         &_stm_detached_inevitable_from_thread, NULL);
     if (old != (tl))
         _stm_reattach_transaction(old, tl);
 }
-inline void stm_leave_transactional_zone(stm_thread_local_t *tl) {
+static inline void stm_leave_transactional_zone(stm_thread_local_t *tl) {
     assert(STM_SEGMENT->running_thread == tl);
     if (stm_is_inevitable())
         _stm_detach_inevitable_transaction(tl);

From noreply at buildbot.pypy.org  Tue Jun  9 17:56:57 2015
From: noreply at buildbot.pypy.org (cfbolz)
Date: Tue,  9 Jun 2015 17:56:57 +0200 (CEST)
Subject: [pypy-commit] benchmarks default: add a synthetic sqlite benchmark,
 mostly to stress CFFI
Message-ID: <20150609155657.A62711C0460@cobra.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r329:743b6eafd02d
Date: 2015-06-09 16:57 +0200
http://bitbucket.org/pypy/benchmarks/changeset/743b6eafd02d/

Log:	add a synthetic sqlite benchmark, mostly to stress CFFI

diff --git a/benchmarks.py b/benchmarks.py
--- a/benchmarks.py
+++ b/benchmarks.py
@@ -83,7 +83,7 @@
              'raytrace-simple', 'crypto_pyaes', 'bm_mako', 'bm_chameleon',
              'json_bench', 'pidigits', 'hexiom2', 'eparse', 'deltablue',
              'bm_dulwich_log', 'bm_krakatau', 'bm_mdp', 'pypy_interp',
-             'bm_icbd']:
+             'sqlitesynth]:
     _register_new_bm(name, name, globals(), **opts.get(name, {}))
 
 for name in ['names', 'iteration', 'tcp', 'pb', ]:#'web']:#, 'accepts']:
diff --git a/own/sqlitesynth.py b/own/sqlitesynth.py
new file mode 100644
--- /dev/null
+++ b/own/sqlitesynth.py
@@ -0,0 +1,58 @@
+import sqlite3
+import math
+
+# the goal of the benchmark is to test CFFI performance and going back and
+# forth between SQLite and Python a lot. Therefore the queries themselves are
+# really simple
+
+class AvgLength(object):
+    def __init__(self):
+        self.sum = 0
+        self.count = 0
+
+    def step(self, x):
+        if x is not None:
+            self.count += 1
+            self.sum += len(x)
+
+    def finalize(self):
+        return self.sum / float(self.count)
+
+def _main():
+    conn = sqlite3.connect(":memory:")
+    conn.execute('create table cos (x, y, z);')
+    for i in range(300000):
+        conn.execute('insert into cos values (?, ?, ?)', [i, math.cos(i), str(i)])
+    conn.create_function("cos", 1, math.cos)
+    for x, cosx1, cosx2 in conn.execute("select x, cos(x), y from cos"):
+        assert math.cos(x) == cosx1 == cosx2
+
+    conn.create_aggregate("avglength", 1, AvgLength)
+    avglen, = conn.execute("select avglength(z) from cos;").next()
+    conn.execute("delete from cos;")
+    conn.close()
+
+
+
+def main(n):
+    import time
+    times = []
+    for i in range(6):
+        _main() # warmup
+    for i in range(n):
+        t1 = time.time()
+        _main()
+        t2 = time.time()
+        times.append(t2 - t1)
+    return times
+
+if __name__ == "__main__":
+    import util, optparse
+    parser = optparse.OptionParser(
+        usage="%prog [options]",
+        description="Test the performance of the SqliteSynth benchmark")
+    util.add_standard_options_to(parser)
+    options, args = parser.parse_args()
+
+    util.run_benchmark(options, options.num_runs, main)
+

From noreply at buildbot.pypy.org  Tue Jun  9 19:01:26 2015
From: noreply at buildbot.pypy.org (mgedmin)
Date: Tue,  9 Jun 2015 19:01:26 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Make curses.tigetstr/tigetnum/tigetflag
 handle Unicode strings
Message-ID: <20150609170126.CD2ED1C130E@cobra.cs.uni-duesseldorf.de>

Author: Marius Gedminas <marius at gedmin.as>
Branch: py3k
Changeset: r77994:d07170655326
Date: 2015-04-02 11:07 +0300
http://bitbucket.org/pypy/pypy/changeset/d07170655326/

Log:	Make curses.tigetstr/tigetnum/tigetflag handle Unicode strings

	Fixes https://bitbucket.org/pypy/pypy/issue/1997/pypy3
	-cursestigetstr-raises-ctype and
	https://bitbucket.org/pypy/pypy/issue/2016/pypy3-cursestigetnum-
	raises-ctype

	All credit belongs to Thomas Ballinger for creating the original fix
	for tigetstr() that I copied and pasted to fix tigetnum() and
	tigetflag() as well.

diff --git a/lib_pypy/_curses.py b/lib_pypy/_curses.py
--- a/lib_pypy/_curses.py
+++ b/lib_pypy/_curses.py
@@ -1347,16 +1347,22 @@
 
 def tigetflag(capname):
     _ensure_initialised_setupterm()
+    if isinstance(capname, str):
+        capname = capname.encode('utf-8')
     return lib.tigetflag(capname)
 
 
 def tigetnum(capname):
     _ensure_initialised_setupterm()
+    if isinstance(capname, str):
+        capname = capname.encode('utf-8')
     return lib.tigetnum(capname)
 
 
 def tigetstr(capname):
     _ensure_initialised_setupterm()
+    if isinstance(capname, str):
+        capname = capname.encode('utf-8')
     val = lib.tigetstr(capname)
     if int(ffi.cast("intptr_t", val)) in (0, -1):
         return None
@@ -1365,6 +1371,13 @@
 
 def tparm(fmt, i1=0, i2=0, i3=0, i4=0, i5=0, i6=0, i7=0, i8=0, i9=0):
     args = [ffi.cast("int", i) for i in (i1, i2, i3, i4, i5, i6, i7, i8, i9)]
+    # fmt is expected to be a byte string; CPython 3.x complains
+    # "TypeError: 'str' does not support the buffer interface", but we
+    # can do better.
+    if isinstance(fmt, str):
+        # error message modeled on "TypeError: must be str, not bytes"
+        # that you get if you call curses.tigetstr(b'...') on CPython 3.x
+        raise TypeError('must be bytes, not str')
     result = lib.tparm(fmt, *args)
     if result == ffi.NULL:
         raise error("tparm() returned NULL")

From noreply at buildbot.pypy.org  Tue Jun  9 19:01:27 2015
From: noreply at buildbot.pypy.org (mgedmin)
Date: Tue,  9 Jun 2015 19:01:27 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Fix curses text-output functions and
	methods
Message-ID: <20150609170127.F12031C130E@cobra.cs.uni-duesseldorf.de>

Author: Marius Gedminas <marius at gedmin.as>
Branch: py3k
Changeset: r77995:b110a435b27a
Date: 2015-04-02 11:23 +0300
http://bitbucket.org/pypy/pypy/changeset/b110a435b27a/

Log:	Fix curses text-output functions and methods

	The fix works for pure ASCII text only. I'm still investigating why
	Unicode text results in output like 'M-D~EM-D~MM-D~Y'.

diff --git a/lib_pypy/_curses.py b/lib_pypy/_curses.py
--- a/lib_pypy/_curses.py
+++ b/lib_pypy/_curses.py
@@ -485,12 +485,12 @@
     return int(ffi.cast("chtype", ch))
 
 def _texttype(text):
-    if isinstance(text, str):
+    if isinstance(text, bytes):
         return text
-    elif isinstance(text, unicode):
-        return str(text)   # default encoding
+    elif isinstance(text, str):
+        return text.encode('utf-8')
     else:
-        raise TypeError("str or unicode expected, got a '%s' object"
+        raise TypeError("str or bytes expected, got a '%s' object"
                         % (type(text).__name__,))
 
 

From noreply at buildbot.pypy.org  Tue Jun  9 19:01:29 2015
From: noreply at buildbot.pypy.org (mgedmin)
Date: Tue,  9 Jun 2015 19:01:29 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Link against libncursesw,
	for Unicode support
Message-ID: <20150609170129.25EB71C130E@cobra.cs.uni-duesseldorf.de>

Author: Marius Gedminas <marius at gedmin.as>
Branch: py3k
Changeset: r77996:321f5dfaf308
Date: 2015-04-02 11:30 +0300
http://bitbucket.org/pypy/pypy/changeset/321f5dfaf308/

Log:	Link against libncursesw, for Unicode support

diff --git a/lib_pypy/_curses.py b/lib_pypy/_curses.py
--- a/lib_pypy/_curses.py
+++ b/lib_pypy/_curses.py
@@ -324,7 +324,7 @@
 void _m_getsyx(int *yx) {
     getsyx(yx[0], yx[1]);
 }
-""", libraries=['ncurses', 'panel'])
+""", libraries=['ncursesw', 'panel'])
 
 
 def _copy_to_globals(name):

From noreply at buildbot.pypy.org  Tue Jun  9 19:01:42 2015
From: noreply at buildbot.pypy.org (mgedmin)
Date: Tue,  9 Jun 2015 19:01:42 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Merge with upstream
Message-ID: <20150609170142.469F71C130E@cobra.cs.uni-duesseldorf.de>

Author: Marius Gedminas <marius at gedmin.as>
Branch: py3k
Changeset: r77997:404f7178d54f
Date: 2015-06-09 19:45 +0300
http://bitbucket.org/pypy/pypy/changeset/404f7178d54f/

Log:	Merge with upstream

diff too long, truncating to 2000 out of 61469 lines

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -3,11 +3,15 @@
 d8ac7d23d3ec5f9a0fa1264972f74a010dbfd07f release-1.6
 ff4af8f318821f7f5ca998613a60fca09aa137da release-1.7
 07e08e9c885ca67d89bcc304e45a32346daea2fa release-2.0-beta-1
-9b623bc48b5950cf07184462a0e48f2c4df0d720 pypy-2.1-beta1-arm
-9b623bc48b5950cf07184462a0e48f2c4df0d720 pypy-2.1-beta1-arm
 ab0dd631c22015ed88e583d9fdd4c43eebf0be21 pypy-2.1-beta1-arm
 20e51c4389ed4469b66bb9d6289ce0ecfc82c4b9 release-2.3.0
 394146e9bb673514c61f0150ab2013ccf78e8de7 release-2.3
 32f35069a16d819b58c1b6efb17c44e3e53397b2 release-2.2=3.1
 32f35069a16d819b58c1b6efb17c44e3e53397b2 release-2.3.1
 10f1b29a2bd21f837090286174a9ca030b8680b2 release-2.5.0
+9c4588d731b7fe0b08669bd732c2b676cb0a8233 release-2.5.1
+fcdb941565156385cbac04cfb891f8f4c7a92ef6 release-2.6.0
+fcdb941565156385cbac04cfb891f8f4c7a92ef6 release-2.6.0
+e03971291f3a0729ecd3ee7fae7ddb0bb82d476c release-2.6.0
+e03971291f3a0729ecd3ee7fae7ddb0bb82d476c release-2.6.0
+295ee98b69288471b0fcf2e0ede82ce5209eb90b release-2.6.0
diff --git a/.tddium.requirements.txt b/.tddium.requirements.txt
deleted file mode 100644
--- a/.tddium.requirements.txt
+++ /dev/null
@@ -1,1 +0,0 @@
-pytest
diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -38,8 +38,8 @@
   Armin Rigo
   Maciej Fijalkowski
   Carl Friedrich Bolz
+  Amaury Forgeot d'Arc
   Antonio Cuni
-  Amaury Forgeot d'Arc
   Samuele Pedroni
   Alex Gaynor
   Brian Kearns
@@ -50,9 +50,9 @@
   Holger Krekel
   Christian Tismer
   Hakan Ardo
-  Benjamin Peterson
   Manuel Jacob
   Ronan Lamy
+  Benjamin Peterson
   Anders Chrigstrom
   Eric van Riet Paap
   Wim Lavrijsen
@@ -63,8 +63,8 @@
   Sven Hager
   Anders Lehmann
   Aurelien Campeas
+  Remi Meier
   Niklaus Haldimann
-  Remi Meier
   Camillo Bruni
   Laura Creighton
   Toon Verwaest
@@ -76,10 +76,10 @@
   David Edelsohn
   Anders Hammarquist
   Jakub Gustak
+  Gregor Wegberg
   Guido Wesdorp
   Lawrence Oluyede
   Bartosz Skowron
-  Gregor Wegberg
   Daniel Roberts
   Niko Matsakis
   Adrien Di Mascio
@@ -87,10 +87,11 @@
   Ludovic Aubry
   Jacob Hallen
   Jason Creighton
+  Richard Plangger
   Alex Martelli
   Michal Bendowski
+  stian
   Jan de Mooij
-  stian
   Tyler Wade
   Michael Foord
   Stephan Diehl
@@ -133,15 +134,15 @@
   Georg Brandl
   Bert Freudenberg
   Stian Andreassen
+  Edd Barrett
   Wanja Saatkamp
   Gerald Klix
   Mike Blume
+  Tobias Pape
   Oscar Nierstrasz
   Stefan H. Muller
-  Edd Barrett
   Jeremy Thurgood
   Rami Chowdhury
-  Tobias Pape
   Eugene Oden
   Henry Mason
   Vasily Kuznetsov
@@ -167,11 +168,13 @@
   Michael Twomey
   Lucian Branescu Mihaila
   Yichao Yu
+  Anton Gulenko
   Gabriel Lavoie
   Olivier Dormond
   Jared Grubb
   Karl Bartel
   Wouter van Heyst
+  Sebastian Pawlu&#347;
   Brian Dorsey
   Victor Stinner
   Andrews Medina
@@ -188,6 +191,7 @@
   Neil Shepperd
   Stanislaw Halik
   Mikael Sch&#246;nenberg
+  Berkin Ilbeyi
   Elmo M?ntynen
   Jonathan David Riehl
   Anders Qvist
@@ -211,11 +215,11 @@
   Carl Meyer
   Karl Ramm
   Pieter Zieschang
-  Sebastian Pawlu&#347;
   Gabriel
   Lukas Vacek
   Andrew Dalke
   Sylvain Thenault
+  Jakub Stasiak
   Nathan Taylor
   Vladimir Kryachko
   Jacek Generowicz
@@ -242,6 +246,7 @@
   Tomo Cocoa
   Toni Mattis
   Lucas Stadler
+  Julian Berman
   roberto at goyle
   Yury V. Zaytsev
   Anna Katrina Dominguez
@@ -253,6 +258,8 @@
   Ben Darnell
   Roberto De Ioris
   Juan Francisco Cantero Hurtado
+  Ruochen Huang
+  Jeong YunWon
   Godefroid Chappelle
   Joshua Gilbert
   Dan Colish
@@ -271,6 +278,7 @@
   Christian Muirhead
   Berker Peksag
   James Lan
+  Volodymyr Vladymyrov
   shoma hosaka
   Daniel Neuh&#228;user
   Ben Mather
@@ -316,6 +324,7 @@
   yasirs
   Michael Chermside
   Anna Ravencroft
+  Andrey Churin
   Dan Crosta
   Julien Phalip
   Roman Podoliaka
@@ -420,3 +429,10 @@
 the terms of the GPL license version 2 or any later version.  Thus the
 _gdbm module, provided in the file lib_pypy/_gdbm.py, is redistributed
 under the terms of the GPL license as well.
+
+License for 'pypy/module/_vmprof/src'
+--------------------------------------
+
+The code is based on gperftools. You may see a copy of the License for it at
+
+    https://code.google.com/p/gperftools/source/browse/COPYING
diff --git a/lib-python/2.7/socket.py b/lib-python/2.7/socket.py
--- a/lib-python/2.7/socket.py
+++ b/lib-python/2.7/socket.py
@@ -145,6 +145,34 @@
             name = hostname
     return name
 
+class RefCountingWarning(UserWarning):
+    pass
+
+def _do_reuse_or_drop(socket, methname):
+    try:
+        method = getattr(socket, methname)
+    except (AttributeError, TypeError):
+        warnings.warn("""'%s' object has no _reuse/_drop methods
+{{
+    You make use (or a library you are using makes use) of the internal
+    classes '_socketobject' and '_fileobject' in socket.py, initializing
+    them with custom objects.  On PyPy, these custom objects need two
+    extra methods, _reuse() and _drop(), that maintain an explicit
+    reference counter.  When _drop() has been called as many times as
+    _reuse(), then the object should be freed.
+
+    Without these methods, you get the warning here.  This is to
+    prevent the following situation: if your (or the library's) code
+    relies on reference counting for prompt closing, then on PyPy, the
+    __del__ method will be called later than on CPython.  You can
+    easily end up in a situation where you open and close a lot of
+    (high-level) '_socketobject' or '_fileobject', but the (low-level)
+    custom objects will accumulate before their __del__ are called.
+    You quickly risk running out of file descriptors, for example.
+}}""" % (socket.__class__.__name__,), RefCountingWarning, stacklevel=3)
+    else:
+        method()
+
 
 _socketmethods = (
     'bind', 'connect', 'connect_ex', 'fileno', 'listen',
@@ -182,19 +210,7 @@
         if _sock is None:
             _sock = _realsocket(family, type, proto)
         else:
-            # PyPy note about refcounting: implemented with _reuse()/_drop()
-            # on the class '_socket.socket'.  Python 3 did it differently
-            # with a reference counter on this class 'socket._socketobject'
-            # instead, but it is a less compatible change.
-            
-            # Note that a few libraries (like eventlet) poke at the
-            # private implementation of socket.py, passing custom
-            # objects to _socketobject().  These libraries need the
-            # following fix for use on PyPy: the custom objects need
-            # methods _reuse() and _drop() that maintains an explicit
-            # reference counter, starting at 0.  When it drops back to
-            # zero, close() must be called.
-            _sock._reuse()
+            _do_reuse_or_drop(_sock, '_reuse')
 
         self._sock = _sock
 
@@ -228,13 +244,13 @@
     def close(self):
         s = self._sock
         self._sock = _closedsocket()
-        s._drop()
+        _do_reuse_or_drop(s, '_drop')
     close.__doc__ = _realsocket.close.__doc__
 
     def accept(self):
         sock, addr = self._sock.accept()
         sockobj = _socketobject(_sock=sock)
-        sock._drop()    # already a copy in the _socketobject()
+        _do_reuse_or_drop(sock, '_drop') # already a copy in the _socketobject()
         return sockobj, addr
     accept.__doc__ = _realsocket.accept.__doc__
 
@@ -290,14 +306,7 @@
                  "_close"]
 
     def __init__(self, sock, mode='rb', bufsize=-1, close=False):
-        # Note that a few libraries (like eventlet) poke at the
-        # private implementation of socket.py, passing custom
-        # objects to _fileobject().  These libraries need the
-        # following fix for use on PyPy: the custom objects need
-        # methods _reuse() and _drop() that maintains an explicit
-        # reference counter, starting at 0.  When it drops back to
-        # zero, close() must be called.
-        sock._reuse()
+        _do_reuse_or_drop(sock, '_reuse')
         self._sock = sock
         self.mode = mode # Not actually used in this version
         if bufsize < 0:
@@ -338,7 +347,7 @@
                 if self._close:
                     s.close()
                 else:
-                    s._drop()
+                    _do_reuse_or_drop(s, '_drop')
 
     def __del__(self):
         try:
diff --git a/lib-python/2.7/test/test_urllib2net.py b/lib-python/2.7/test/test_urllib2net.py
--- a/lib-python/2.7/test/test_urllib2net.py
+++ b/lib-python/2.7/test/test_urllib2net.py
@@ -102,11 +102,8 @@
 
     def test_ftp(self):
         urls = [
-            'ftp://ftp.kernel.org/pub/linux/kernel/README',
-            'ftp://ftp.kernel.org/pub/linux/kernel/non-existent-file',
-            #'ftp://ftp.kernel.org/pub/leenox/kernel/test',
-            'ftp://gatekeeper.research.compaq.com/pub/DEC/SRC'
-                '/research-reports/00README-Legal-Rules-Regs',
+            'ftp://ftp.debian.org/debian/README',
+            'ftp://ftp.debian.org/debian/non-existent-file',
             ]
         self._test_urls(urls, self._extra_handlers())
 
@@ -255,6 +252,7 @@
         with test_support.transient_internet(url, timeout=None):
             u = _urlopen_with_retry(url)
             self.assertIsNone(u.fp._sock.fp._sock.gettimeout())
+            u.close()
 
     def test_http_default_timeout(self):
         self.assertIsNone(socket.getdefaulttimeout())
@@ -266,6 +264,7 @@
             finally:
                 socket.setdefaulttimeout(None)
             self.assertEqual(u.fp._sock.fp._sock.gettimeout(), 60)
+            u.close()
 
     def test_http_no_timeout(self):
         self.assertIsNone(socket.getdefaulttimeout())
@@ -277,20 +276,23 @@
             finally:
                 socket.setdefaulttimeout(None)
             self.assertIsNone(u.fp._sock.fp._sock.gettimeout())
+            u.close()
 
     def test_http_timeout(self):
         url = "http://www.example.com"
         with test_support.transient_internet(url):
             u = _urlopen_with_retry(url, timeout=120)
             self.assertEqual(u.fp._sock.fp._sock.gettimeout(), 120)
+            u.close()
 
-    FTP_HOST = "ftp://ftp.mirror.nl/pub/gnu/"
+    FTP_HOST = 'ftp://ftp.debian.org/debian/'
 
     def test_ftp_basic(self):
         self.assertIsNone(socket.getdefaulttimeout())
         with test_support.transient_internet(self.FTP_HOST, timeout=None):
             u = _urlopen_with_retry(self.FTP_HOST)
             self.assertIsNone(u.fp.fp._sock.gettimeout())
+            u.close()
 
     def test_ftp_default_timeout(self):
         self.assertIsNone(socket.getdefaulttimeout())
@@ -301,6 +303,7 @@
             finally:
                 socket.setdefaulttimeout(None)
             self.assertEqual(u.fp.fp._sock.gettimeout(), 60)
+            u.close()
 
     def test_ftp_no_timeout(self):
         self.assertIsNone(socket.getdefaulttimeout(),)
@@ -311,11 +314,16 @@
             finally:
                 socket.setdefaulttimeout(None)
             self.assertIsNone(u.fp.fp._sock.gettimeout())
+            u.close()
 
     def test_ftp_timeout(self):
         with test_support.transient_internet(self.FTP_HOST):
-            u = _urlopen_with_retry(self.FTP_HOST, timeout=60)
+            try:
+                u = _urlopen_with_retry(self.FTP_HOST, timeout=60)
+            except:
+                raise
             self.assertEqual(u.fp.fp._sock.gettimeout(), 60)
+            u.close()
 
 
 def test_main():
diff --git a/lib-python/3/test/test_curses.py b/lib-python/3/test/test_curses.py
--- a/lib-python/3/test/test_curses.py
+++ b/lib-python/3/test/test_curses.py
@@ -115,8 +115,8 @@
     stdscr.notimeout(1)
     win2.overlay(win)
     win2.overwrite(win)
-    win2.overlay(win, 1, 2, 3, 3, 2, 1)
-    win2.overwrite(win, 1, 2, 3, 3, 2, 1)
+    win2.overlay(win, 1, 2, 2, 1, 3, 3)
+    win2.overwrite(win, 1, 2, 2, 1, 3, 3)
     stdscr.redrawln(1,2)
 
     stdscr.scrollok(1)
diff --git a/lib-python/3/trace.py b/lib-python/3/trace.py
--- a/lib-python/3/trace.py
+++ b/lib-python/3/trace.py
@@ -245,7 +245,12 @@
         we want to have reported.
         """
         return (filename == "<string>" or
-                filename.startswith("<doctest "))
+                filename.startswith("<doctest ") or
+                # XXX PyPy freezes some (pure-Python) modules at
+                # translation-time.  These contain filenames starting with
+                # "<builtin>/" instead of their actual filenames.  Ignore them
+                # for now.
+                filename.startswith("<builtin>/"))
 
     def update(self, other):
         """Merge in the data from another CoverageResults"""
diff --git a/lib_pypy/_audioop_build.py b/lib_pypy/_audioop_build.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_audioop_build.py
@@ -0,0 +1,621 @@
+from cffi import FFI
+
+ffi = FFI()
+ffi.cdef("""
+typedef short PyInt16;
+
+int ratecv(char* rv, char* cp, size_t len, int size,
+           int nchannels, int inrate, int outrate,
+           int* state_d, int* prev_i, int* cur_i,
+           int weightA, int weightB);
+
+void tostereo(char* rv, char* cp, size_t len, int size,
+              double fac1, double fac2);
+void add(char* rv, char* cp1, char* cp2, size_t len1, int size);
+
+/* 2's complement (14-bit range) */
+unsigned char
+st_14linear2ulaw(PyInt16 pcm_val);
+PyInt16 st_ulaw2linear16(unsigned char);
+
+/* 2's complement (13-bit range) */
+unsigned char
+st_linear2alaw(PyInt16 pcm_val);
+PyInt16 st_alaw2linear16(unsigned char);
+
+
+void lin2adcpm(unsigned char* rv, unsigned char* cp, size_t len,
+               size_t size, int* state);
+void adcpm2lin(unsigned char* rv, unsigned char* cp, size_t len,
+               size_t size, int* state);
+""")
+
+# This code is directly copied from CPython file: Modules/audioop.c
+_AUDIOOP_C_MODULE = r"""
+typedef short PyInt16;
+typedef int Py_Int32;
+
+/* Code shamelessly stolen from sox, 12.17.7, g711.c
+** (c) Craig Reese, Joe Campbell and Jeff Poskanzer 1989 */
+
+/* From g711.c:
+ *
+ * December 30, 1994:
+ * Functions linear2alaw, linear2ulaw have been updated to correctly
+ * convert unquantized 16 bit values.
+ * Tables for direct u- to A-law and A- to u-law conversions have been
+ * corrected.
+ * Borge Lindberg, Center for PersonKommunikation, Aalborg University.
+ * bli at cpk.auc.dk
+ *
+ */
+#define BIAS 0x84   /* define the add-in bias for 16 bit samples */
+#define CLIP 32635
+#define SIGN_BIT        (0x80)          /* Sign bit for a A-law byte. */
+#define QUANT_MASK      (0xf)           /* Quantization field mask. */
+#define SEG_SHIFT       (4)             /* Left shift for segment number. */
+#define SEG_MASK        (0x70)          /* Segment field mask. */
+
+static PyInt16 seg_aend[8] = {0x1F, 0x3F, 0x7F, 0xFF,
+                              0x1FF, 0x3FF, 0x7FF, 0xFFF};
+static PyInt16 seg_uend[8] = {0x3F, 0x7F, 0xFF, 0x1FF,
+                              0x3FF, 0x7FF, 0xFFF, 0x1FFF};
+
+static PyInt16
+search(PyInt16 val, PyInt16 *table, int size)
+{
+    int i;
+
+    for (i = 0; i < size; i++) {
+        if (val <= *table++)
+            return (i);
+    }
+    return (size);
+}
+#define st_ulaw2linear16(uc) (_st_ulaw2linear16[uc])
+#define st_alaw2linear16(uc) (_st_alaw2linear16[uc])
+
+static PyInt16 _st_ulaw2linear16[256] = {
+    -32124,  -31100,  -30076,  -29052,  -28028,  -27004,  -25980,
+    -24956,  -23932,  -22908,  -21884,  -20860,  -19836,  -18812,
+    -17788,  -16764,  -15996,  -15484,  -14972,  -14460,  -13948,
+    -13436,  -12924,  -12412,  -11900,  -11388,  -10876,  -10364,
+     -9852,   -9340,   -8828,   -8316,   -7932,   -7676,   -7420,
+     -7164,   -6908,   -6652,   -6396,   -6140,   -5884,   -5628,
+     -5372,   -5116,   -4860,   -4604,   -4348,   -4092,   -3900,
+     -3772,   -3644,   -3516,   -3388,   -3260,   -3132,   -3004,
+     -2876,   -2748,   -2620,   -2492,   -2364,   -2236,   -2108,
+     -1980,   -1884,   -1820,   -1756,   -1692,   -1628,   -1564,
+     -1500,   -1436,   -1372,   -1308,   -1244,   -1180,   -1116,
+     -1052,    -988,    -924,    -876,    -844,    -812,    -780,
+      -748,    -716,    -684,    -652,    -620,    -588,    -556,
+      -524,    -492,    -460,    -428,    -396,    -372,    -356,
+      -340,    -324,    -308,    -292,    -276,    -260,    -244,
+      -228,    -212,    -196,    -180,    -164,    -148,    -132,
+      -120,    -112,    -104,     -96,     -88,     -80,     -72,
+       -64,     -56,     -48,     -40,     -32,     -24,     -16,
+    -8,       0,   32124,   31100,   30076,   29052,   28028,
+     27004,   25980,   24956,   23932,   22908,   21884,   20860,
+     19836,   18812,   17788,   16764,   15996,   15484,   14972,
+     14460,   13948,   13436,   12924,   12412,   11900,   11388,
+     10876,   10364,    9852,    9340,    8828,    8316,    7932,
+      7676,    7420,    7164,    6908,    6652,    6396,    6140,
+      5884,    5628,    5372,    5116,    4860,    4604,    4348,
+      4092,    3900,    3772,    3644,    3516,    3388,    3260,
+      3132,    3004,    2876,    2748,    2620,    2492,    2364,
+      2236,    2108,    1980,    1884,    1820,    1756,    1692,
+      1628,    1564,    1500,    1436,    1372,    1308,    1244,
+      1180,    1116,    1052,     988,     924,     876,     844,
+       812,     780,     748,     716,     684,     652,     620,
+       588,     556,     524,     492,     460,     428,     396,
+       372,     356,     340,     324,     308,     292,     276,
+       260,     244,     228,     212,     196,     180,     164,
+       148,     132,     120,     112,     104,      96,      88,
+    80,      72,      64,      56,      48,      40,      32,
+    24,      16,       8,       0
+};
+
+/*
+ * linear2ulaw() accepts a 14-bit signed integer and encodes it as u-law data
+ * stored in a unsigned char.  This function should only be called with
+ * the data shifted such that it only contains information in the lower
+ * 14-bits.
+ *
+ * In order to simplify the encoding process, the original linear magnitude
+ * is biased by adding 33 which shifts the encoding range from (0 - 8158) to
+ * (33 - 8191). The result can be seen in the following encoding table:
+ *
+ *      Biased Linear Input Code        Compressed Code
+ *      ------------------------        ---------------
+ *      00000001wxyza                   000wxyz
+ *      0000001wxyzab                   001wxyz
+ *      000001wxyzabc                   010wxyz
+ *      00001wxyzabcd                   011wxyz
+ *      0001wxyzabcde                   100wxyz
+ *      001wxyzabcdef                   101wxyz
+ *      01wxyzabcdefg                   110wxyz
+ *      1wxyzabcdefgh                   111wxyz
+ *
+ * Each biased linear code has a leading 1 which identifies the segment
+ * number. The value of the segment number is equal to 7 minus the number
+ * of leading 0's. The quantization interval is directly available as the
+ * four bits wxyz.  * The trailing bits (a - h) are ignored.
+ *
+ * Ordinarily the complement of the resulting code word is used for
+ * transmission, and so the code word is complemented before it is returned.
+ *
+ * For further information see John C. Bellamy's Digital Telephony, 1982,
+ * John Wiley & Sons, pps 98-111 and 472-476.
+ */
+static unsigned char
+st_14linear2ulaw(PyInt16 pcm_val)       /* 2's complement (14-bit range) */
+{
+    PyInt16         mask;
+    PyInt16         seg;
+    unsigned char   uval;
+
+    /* The original sox code does this in the calling function, not here */
+    pcm_val = pcm_val >> 2;
+
+    /* u-law inverts all bits */
+    /* Get the sign and the magnitude of the value. */
+    if (pcm_val < 0) {
+        pcm_val = -pcm_val;
+        mask = 0x7F;
+    } else {
+        mask = 0xFF;
+    }
+    if ( pcm_val > CLIP ) pcm_val = CLIP;           /* clip the magnitude */
+    pcm_val += (BIAS >> 2);
+
+    /* Convert the scaled magnitude to segment number. */
+    seg = search(pcm_val, seg_uend, 8);
+
+    /*
+     * Combine the sign, segment, quantization bits;
+     * and complement the code word.
+     */
+    if (seg >= 8)           /* out of range, return maximum value. */
+        return (unsigned char) (0x7F ^ mask);
+    else {
+        uval = (unsigned char) (seg << 4) | ((pcm_val >> (seg + 1)) & 0xF);
+        return (uval ^ mask);
+    }
+
+}
+
+static PyInt16 _st_alaw2linear16[256] = {
+     -5504,   -5248,   -6016,   -5760,   -4480,   -4224,   -4992,
+     -4736,   -7552,   -7296,   -8064,   -7808,   -6528,   -6272,
+     -7040,   -6784,   -2752,   -2624,   -3008,   -2880,   -2240,
+     -2112,   -2496,   -2368,   -3776,   -3648,   -4032,   -3904,
+     -3264,   -3136,   -3520,   -3392,  -22016,  -20992,  -24064,
+    -23040,  -17920,  -16896,  -19968,  -18944,  -30208,  -29184,
+    -32256,  -31232,  -26112,  -25088,  -28160,  -27136,  -11008,
+    -10496,  -12032,  -11520,   -8960,   -8448,   -9984,   -9472,
+    -15104,  -14592,  -16128,  -15616,  -13056,  -12544,  -14080,
+    -13568,    -344,    -328,    -376,    -360,    -280,    -264,
+      -312,    -296,    -472,    -456,    -504,    -488,    -408,
+      -392,    -440,    -424,     -88,     -72,    -120,    -104,
+       -24,      -8,     -56,     -40,    -216,    -200,    -248,
+      -232,    -152,    -136,    -184,    -168,   -1376,   -1312,
+     -1504,   -1440,   -1120,   -1056,   -1248,   -1184,   -1888,
+     -1824,   -2016,   -1952,   -1632,   -1568,   -1760,   -1696,
+      -688,    -656,    -752,    -720,    -560,    -528,    -624,
+      -592,    -944,    -912,   -1008,    -976,    -816,    -784,
+      -880,    -848,    5504,    5248,    6016,    5760,    4480,
+      4224,    4992,    4736,    7552,    7296,    8064,    7808,
+      6528,    6272,    7040,    6784,    2752,    2624,    3008,
+      2880,    2240,    2112,    2496,    2368,    3776,    3648,
+      4032,    3904,    3264,    3136,    3520,    3392,   22016,
+     20992,   24064,   23040,   17920,   16896,   19968,   18944,
+     30208,   29184,   32256,   31232,   26112,   25088,   28160,
+     27136,   11008,   10496,   12032,   11520,    8960,    8448,
+      9984,    9472,   15104,   14592,   16128,   15616,   13056,
+     12544,   14080,   13568,     344,     328,     376,     360,
+       280,     264,     312,     296,     472,     456,     504,
+       488,     408,     392,     440,     424,      88,      72,
+       120,     104,      24,       8,      56,      40,     216,
+       200,     248,     232,     152,     136,     184,     168,
+      1376,    1312,    1504,    1440,    1120,    1056,    1248,
+      1184,    1888,    1824,    2016,    1952,    1632,    1568,
+      1760,    1696,     688,     656,     752,     720,     560,
+       528,     624,     592,     944,     912,    1008,     976,
+       816,     784,     880,     848
+};
+
+/*
+ * linear2alaw() accepts an 13-bit signed integer and encodes it as A-law data
+ * stored in a unsigned char.  This function should only be called with
+ * the data shifted such that it only contains information in the lower
+ * 13-bits.
+ *
+ *              Linear Input Code       Compressed Code
+ *      ------------------------        ---------------
+ *      0000000wxyza                    000wxyz
+ *      0000001wxyza                    001wxyz
+ *      000001wxyzab                    010wxyz
+ *      00001wxyzabc                    011wxyz
+ *      0001wxyzabcd                    100wxyz
+ *      001wxyzabcde                    101wxyz
+ *      01wxyzabcdef                    110wxyz
+ *      1wxyzabcdefg                    111wxyz
+ *
+ * For further information see John C. Bellamy's Digital Telephony, 1982,
+ * John Wiley & Sons, pps 98-111 and 472-476.
+ */
+static unsigned char
+st_linear2alaw(PyInt16 pcm_val) /* 2's complement (13-bit range) */
+{
+    PyInt16         mask;
+    short           seg;
+    unsigned char   aval;
+
+    /* The original sox code does this in the calling function, not here */
+    pcm_val = pcm_val >> 3;
+
+    /* A-law using even bit inversion */
+    if (pcm_val >= 0) {
+        mask = 0xD5;            /* sign (7th) bit = 1 */
+    } else {
+        mask = 0x55;            /* sign bit = 0 */
+        pcm_val = -pcm_val - 1;
+    }
+
+    /* Convert the scaled magnitude to segment number. */
+    seg = search(pcm_val, seg_aend, 8);
+
+    /* Combine the sign, segment, and quantization bits. */
+
+    if (seg >= 8)           /* out of range, return maximum value. */
+        return (unsigned char) (0x7F ^ mask);
+    else {
+        aval = (unsigned char) seg << SEG_SHIFT;
+        if (seg < 2)
+            aval |= (pcm_val >> 1) & QUANT_MASK;
+        else
+            aval |= (pcm_val >> seg) & QUANT_MASK;
+        return (aval ^ mask);
+    }
+}
+/* End of code taken from sox */
+
+/* Intel ADPCM step variation table */
+static int indexTable[16] = {
+    -1, -1, -1, -1, 2, 4, 6, 8,
+    -1, -1, -1, -1, 2, 4, 6, 8,
+};
+
+static int stepsizeTable[89] = {
+    7, 8, 9, 10, 11, 12, 13, 14, 16, 17,
+    19, 21, 23, 25, 28, 31, 34, 37, 41, 45,
+    50, 55, 60, 66, 73, 80, 88, 97, 107, 118,
+    130, 143, 157, 173, 190, 209, 230, 253, 279, 307,
+    337, 371, 408, 449, 494, 544, 598, 658, 724, 796,
+    876, 963, 1060, 1166, 1282, 1411, 1552, 1707, 1878, 2066,
+    2272, 2499, 2749, 3024, 3327, 3660, 4026, 4428, 4871, 5358,
+    5894, 6484, 7132, 7845, 8630, 9493, 10442, 11487, 12635, 13899,
+    15289, 16818, 18500, 20350, 22385, 24623, 27086, 29794, 32767
+};
+
+#define CHARP(cp, i) ((signed char *)(cp+i))
+#define SHORTP(cp, i) ((short *)(cp+i))
+#define LONGP(cp, i) ((Py_Int32 *)(cp+i))
+"""
+
+C_SOURCE = _AUDIOOP_C_MODULE + r"""
+#include <math.h>
+
+static const int maxvals[] = {0, 0x7F, 0x7FFF, 0x7FFFFF, 0x7FFFFFFF};
+/* -1 trick is needed on Windows to support -0x80000000 without a warning */
+static const int minvals[] = {0, -0x80, -0x8000, -0x800000, -0x7FFFFFFF-1};
+
+static int
+fbound(double val, double minval, double maxval)
+{
+    if (val > maxval)
+        val = maxval;
+    else if (val < minval + 1)
+        val = minval;
+    return val;
+}
+
+static int
+gcd(int a, int b)
+{
+    while (b > 0) {
+        int tmp = a % b;
+        a = b;
+        b = tmp;
+    }
+    return a;
+}
+
+int ratecv(char* rv, char* cp, size_t len, int size,
+           int nchannels, int inrate, int outrate,
+           int* state_d, int* prev_i, int* cur_i,
+           int weightA, int weightB)
+{
+    char *ncp = rv;
+    int d, chan;
+
+    /* divide inrate and outrate by their greatest common divisor */
+    d = gcd(inrate, outrate);
+    inrate /= d;
+    outrate /= d;
+    /* divide weightA and weightB by their greatest common divisor */
+    d = gcd(weightA, weightB);
+    weightA /= d;
+    weightA /= d;
+
+    d = *state_d;
+
+    for (;;) {
+        while (d < 0) {
+            if (len == 0) {
+                *state_d = d;
+                return ncp - rv;
+            }
+            for (chan = 0; chan < nchannels; chan++) {
+                prev_i[chan] = cur_i[chan];
+                if (size == 1)
+                    cur_i[chan] = ((int)*CHARP(cp, 0)) << 24;
+                else if (size == 2)
+                    cur_i[chan] = ((int)*SHORTP(cp, 0)) << 16;
+                else if (size == 4)
+                    cur_i[chan] = (int)*LONGP(cp, 0);
+                cp += size;
+                /* implements a simple digital filter */
+                cur_i[chan] = (int)(
+                    ((double)weightA * (double)cur_i[chan] +
+                     (double)weightB * (double)prev_i[chan]) /
+                    ((double)weightA + (double)weightB));
+            }
+            len--;
+            d += outrate;
+        }
+        while (d >= 0) {
+            for (chan = 0; chan < nchannels; chan++) {
+                int cur_o;
+                cur_o = (int)(((double)prev_i[chan] * (double)d +
+                         (double)cur_i[chan] * (double)(outrate - d)) /
+                    (double)outrate);
+                if (size == 1)
+                    *CHARP(ncp, 0) = (signed char)(cur_o >> 24);
+                else if (size == 2)
+                    *SHORTP(ncp, 0) = (short)(cur_o >> 16);
+                else if (size == 4)
+                    *LONGP(ncp, 0) = (Py_Int32)(cur_o);
+                ncp += size;
+            }
+            d -= inrate;
+        }
+    }
+}
+
+void tostereo(char* rv, char* cp, size_t len, int size,
+              double fac1, double fac2)
+{
+    int val1, val2, val = 0;
+    double fval, maxval, minval;
+    char *ncp = rv;
+    int i;
+
+    maxval = (double) maxvals[size];
+    minval = (double) minvals[size];
+
+    for ( i=0; i < len; i += size ) {
+        if ( size == 1 )      val = (int)*CHARP(cp, i);
+        else if ( size == 2 ) val = (int)*SHORTP(cp, i);
+        else if ( size == 4 ) val = (int)*LONGP(cp, i);
+
+        fval = (double)val*fac1;
+        val1 = (int)floor(fbound(fval, minval, maxval));
+
+        fval = (double)val*fac2;
+        val2 = (int)floor(fbound(fval, minval, maxval));
+
+        if ( size == 1 )      *CHARP(ncp, i*2) = (signed char)val1;
+        else if ( size == 2 ) *SHORTP(ncp, i*2) = (short)val1;
+        else if ( size == 4 ) *LONGP(ncp, i*2) = (Py_Int32)val1;
+
+        if ( size == 1 )      *CHARP(ncp, i*2+1) = (signed char)val2;
+        else if ( size == 2 ) *SHORTP(ncp, i*2+2) = (short)val2;
+        else if ( size == 4 ) *LONGP(ncp, i*2+4) = (Py_Int32)val2;
+    }
+}
+
+void add(char* rv, char* cp1, char* cp2, size_t len1, int size)
+{
+    int i;
+    int val1 = 0, val2 = 0, minval, maxval, newval;
+    char* ncp = rv;
+
+    maxval = maxvals[size];
+    minval = minvals[size];
+
+    for ( i=0; i < len1; i += size ) {
+        if ( size == 1 )      val1 = (int)*CHARP(cp1, i);
+        else if ( size == 2 ) val1 = (int)*SHORTP(cp1, i);
+        else if ( size == 4 ) val1 = (int)*LONGP(cp1, i);
+
+        if ( size == 1 )      val2 = (int)*CHARP(cp2, i);
+        else if ( size == 2 ) val2 = (int)*SHORTP(cp2, i);
+        else if ( size == 4 ) val2 = (int)*LONGP(cp2, i);
+
+        if (size < 4) {
+            newval = val1 + val2;
+            /* truncate in case of overflow */
+            if (newval > maxval)
+                newval = maxval;
+            else if (newval < minval)
+                newval = minval;
+        }
+        else {
+            double fval = (double)val1 + (double)val2;
+            /* truncate in case of overflow */
+            newval = (int)floor(fbound(fval, minval, maxval));
+        }
+
+        if ( size == 1 )      *CHARP(ncp, i) = (signed char)newval;
+        else if ( size == 2 ) *SHORTP(ncp, i) = (short)newval;
+        else if ( size == 4 ) *LONGP(ncp, i) = (Py_Int32)newval;
+    }
+}
+
+void lin2adcpm(unsigned char* ncp, unsigned char* cp, size_t len,
+               size_t size, int* state)
+{
+    int step, outputbuffer = 0, bufferstep;
+    int val = 0;
+    int diff, vpdiff, sign, delta;
+    size_t i;
+    int valpred = state[0];
+    int index = state[1];
+
+    step = stepsizeTable[index];
+    bufferstep = 1;
+
+    for ( i=0; i < len; i += size ) {
+        if ( size == 1 )      val = ((int)*CHARP(cp, i)) << 8;
+        else if ( size == 2 ) val = (int)*SHORTP(cp, i);
+        else if ( size == 4 ) val = ((int)*LONGP(cp, i)) >> 16;
+
+        /* Step 1 - compute difference with previous value */
+        diff = val - valpred;
+        sign = (diff < 0) ? 8 : 0;
+        if ( sign ) diff = (-diff);
+
+        /* Step 2 - Divide and clamp */
+        /* Note:
+        ** This code *approximately* computes:
+        **    delta = diff*4/step;
+        **    vpdiff = (delta+0.5)*step/4;
+        ** but in shift step bits are dropped. The net result of this
+        ** is that even if you have fast mul/div hardware you cannot
+        ** put it to good use since the fixup would be too expensive.
+        */
+        delta = 0;
+        vpdiff = (step >> 3);
+
+        if ( diff >= step ) {
+            delta = 4;
+            diff -= step;
+            vpdiff += step;
+        }
+        step >>= 1;
+        if ( diff >= step  ) {
+            delta |= 2;
+            diff -= step;
+            vpdiff += step;
+        }
+        step >>= 1;
+        if ( diff >= step ) {
+            delta |= 1;
+            vpdiff += step;
+        }
+
+        /* Step 3 - Update previous value */
+        if ( sign )
+            valpred -= vpdiff;
+        else
+            valpred += vpdiff;
+
+        /* Step 4 - Clamp previous value to 16 bits */
+        if ( valpred > 32767 )
+            valpred = 32767;
+        else if ( valpred < -32768 )
+            valpred = -32768;
+
+        /* Step 5 - Assemble value, update index and step values */
+        delta |= sign;
+
+        index += indexTable[delta];
+        if ( index < 0 ) index = 0;
+        if ( index > 88 ) index = 88;
+        step = stepsizeTable[index];
+
+        /* Step 6 - Output value */
+        if ( bufferstep ) {
+            outputbuffer = (delta << 4) & 0xf0;
+        } else {
+            *ncp++ = (delta & 0x0f) | outputbuffer;
+        }
+        bufferstep = !bufferstep;
+    }
+    state[0] = valpred;
+    state[1] = index;
+}
+
+
+void adcpm2lin(unsigned char* ncp, unsigned char* cp, size_t len,
+               size_t size, int* state)
+{
+    int step, inputbuffer = 0, bufferstep;
+    int val = 0;
+    int diff, vpdiff, sign, delta;
+    size_t i;
+    int valpred = state[0];
+    int index = state[1];
+
+    step = stepsizeTable[index];
+    bufferstep = 0;
+
+    for ( i=0; i < len*size*2; i += size ) {
+        /* Step 1 - get the delta value and compute next index */
+        if ( bufferstep ) {
+            delta = inputbuffer & 0xf;
+        } else {
+            inputbuffer = *cp++;
+            delta = (inputbuffer >> 4) & 0xf;
+        }
+
+        bufferstep = !bufferstep;
+
+        /* Step 2 - Find new index value (for later) */
+        index += indexTable[delta];
+        if ( index < 0 ) index = 0;
+        if ( index > 88 ) index = 88;
+
+        /* Step 3 - Separate sign and magnitude */
+        sign = delta & 8;
+        delta = delta & 7;
+
+        /* Step 4 - Compute difference and new predicted value */
+        /*
+        ** Computes 'vpdiff = (delta+0.5)*step/4', but see comment
+        ** in adpcm_coder.
+        */
+        vpdiff = step >> 3;
+        if ( delta & 4 ) vpdiff += step;
+        if ( delta & 2 ) vpdiff += step>>1;
+        if ( delta & 1 ) vpdiff += step>>2;
+
+        if ( sign )
+            valpred -= vpdiff;
+        else
+            valpred += vpdiff;
+
+        /* Step 5 - clamp output value */
+        if ( valpred > 32767 )
+            valpred = 32767;
+        else if ( valpred < -32768 )
+            valpred = -32768;
+
+        /* Step 6 - Update step value */
+        step = stepsizeTable[index];
+
+        /* Step 6 - Output value */
+        if ( size == 1 ) *CHARP(ncp, i) = (signed char)(valpred >> 8);
+        else if ( size == 2 ) *SHORTP(ncp, i) = (short)(valpred);
+        else if ( size == 4 ) *LONGP(ncp, i) = (Py_Int32)(valpred<<16);
+    }
+    state[0] = valpred;
+    state[1] = index;
+}
+"""
+
+ffi.set_source("_audioop_cffi", C_SOURCE)
+
+if __name__ == "__main__":
+    ffi.compile()
diff --git a/lib_pypy/_ctypes/function.py b/lib_pypy/_ctypes/function.py
--- a/lib_pypy/_ctypes/function.py
+++ b/lib_pypy/_ctypes/function.py
@@ -275,7 +275,11 @@
             if argtypes:
                 args = [argtype._CData_retval(argtype.from_address(arg)._buffer)
                         for argtype, arg in zip(argtypes, args)]
-            return to_call(*args)
+            try:
+                return to_call(*args)
+            except SystemExit as e:
+                handle_system_exit(e)
+                raise
         return f
 
     def __call__(self, *args, **kwargs):
@@ -304,7 +308,11 @@
             except (UnicodeError, TypeError, ValueError) as e:
                 raise ArgumentError(str(e))
             try:
-                res = self.callable(*newargs)
+                try:
+                    res = self.callable(*newargs)
+                except SystemExit as e:
+                    handle_system_exit(e)
+                    raise
             except:
                 exc_info = sys.exc_info()
                 traceback.print_tb(exc_info[2], file=sys.stderr)
@@ -715,3 +723,22 @@
     make_fastpath_subclass.memo[CFuncPtr] = CFuncPtrFast
     return CFuncPtrFast
 make_fastpath_subclass.memo = {}
+
+
+def handle_system_exit(e):
+    # issue #1194: if we get SystemExit here, then exit the interpreter.
+    # Highly obscure imho but some people seem to depend on it.
+    if sys.flags.inspect:
+        return   # Don't exit if -i flag was given.
+    else:
+        code = e.code
+        if isinstance(code, int):
+            exitcode = code
+        else:
+            f = getattr(sys, 'stderr', None)
+            if f is None:
+                f = sys.__stderr__
+            print >> f, code
+            exitcode = 1
+
+        _rawffi.exit(exitcode)
diff --git a/lib_pypy/_curses.py b/lib_pypy/_curses.py
--- a/lib_pypy/_curses.py
+++ b/lib_pypy/_curses.py
@@ -6,326 +6,7 @@
     raise ImportError('No module named _curses')
 from functools import wraps
 
-from cffi import FFI
-
-ffi = FFI()
-
-ffi.cdef("""
-typedef ... WINDOW;
-typedef ... SCREEN;
-typedef unsigned long mmask_t;
-typedef unsigned char bool;
-typedef unsigned long chtype;
-typedef chtype attr_t;
-
-typedef struct
-{
-    short id;           /* ID to distinguish multiple devices */
-    int x, y, z;        /* event coordinates (character-cell) */
-    mmask_t bstate;     /* button state bits */
-}
-MEVENT;
-
-static const int ERR, OK;
-static const int TRUE, FALSE;
-static const int KEY_MIN, KEY_MAX;
-
-static const int COLOR_BLACK;
-static const int COLOR_RED;
-static const int COLOR_GREEN;
-static const int COLOR_YELLOW;
-static const int COLOR_BLUE;
-static const int COLOR_MAGENTA;
-static const int COLOR_CYAN;
-static const int COLOR_WHITE;
-
-static const chtype A_ATTRIBUTES;
-static const chtype A_NORMAL;
-static const chtype A_STANDOUT;
-static const chtype A_UNDERLINE;
-static const chtype A_REVERSE;
-static const chtype A_BLINK;
-static const chtype A_DIM;
-static const chtype A_BOLD;
-static const chtype A_ALTCHARSET;
-static const chtype A_INVIS;
-static const chtype A_PROTECT;
-static const chtype A_CHARTEXT;
-static const chtype A_COLOR;
-
-static const int BUTTON1_RELEASED;
-static const int BUTTON1_PRESSED;
-static const int BUTTON1_CLICKED;
-static const int BUTTON1_DOUBLE_CLICKED;
-static const int BUTTON1_TRIPLE_CLICKED;
-static const int BUTTON2_RELEASED;
-static const int BUTTON2_PRESSED;
-static const int BUTTON2_CLICKED;
-static const int BUTTON2_DOUBLE_CLICKED;
-static const int BUTTON2_TRIPLE_CLICKED;
-static const int BUTTON3_RELEASED;
-static const int BUTTON3_PRESSED;
-static const int BUTTON3_CLICKED;
-static const int BUTTON3_DOUBLE_CLICKED;
-static const int BUTTON3_TRIPLE_CLICKED;
-static const int BUTTON4_RELEASED;
-static const int BUTTON4_PRESSED;
-static const int BUTTON4_CLICKED;
-static const int BUTTON4_DOUBLE_CLICKED;
-static const int BUTTON4_TRIPLE_CLICKED;
-static const int BUTTON_SHIFT;
-static const int BUTTON_CTRL;
-static const int BUTTON_ALT;
-static const int ALL_MOUSE_EVENTS;
-static const int REPORT_MOUSE_POSITION;
-
-int setupterm(char *, int, int *);
-
-WINDOW *stdscr;
-int COLORS;
-int COLOR_PAIRS;
-int COLS;
-int LINES;
-
-int baudrate(void);
-int beep(void);
-int box(WINDOW *, chtype, chtype);
-bool can_change_color(void);
-int cbreak(void);
-int clearok(WINDOW *, bool);
-int color_content(short, short*, short*, short*);
-int copywin(const WINDOW*, WINDOW*, int, int, int, int, int, int, int);
-int curs_set(int);
-int def_prog_mode(void);
-int def_shell_mode(void);
-int delay_output(int);
-int delwin(WINDOW *);
-WINDOW * derwin(WINDOW *, int, int, int, int);
-int doupdate(void);
-int echo(void);
-int endwin(void);
-char erasechar(void);
-void filter(void);
-int flash(void);
-int flushinp(void);
-chtype getbkgd(WINDOW *);
-WINDOW * getwin(FILE *);
-int halfdelay(int);
-bool has_colors(void);
-bool has_ic(void);
-bool has_il(void);
-void idcok(WINDOW *, bool);
-int idlok(WINDOW *, bool);
-void immedok(WINDOW *, bool);
-WINDOW * initscr(void);
-int init_color(short, short, short, short);
-int init_pair(short, short, short);
-int intrflush(WINDOW *, bool);
-bool isendwin(void);
-bool is_linetouched(WINDOW *, int);
-bool is_wintouched(WINDOW *);
-const char * keyname(int);
-int keypad(WINDOW *, bool);
-char killchar(void);
-int leaveok(WINDOW *, bool);
-char * longname(void);
-int meta(WINDOW *, bool);
-int mvderwin(WINDOW *, int, int);
-int mvwaddch(WINDOW *, int, int, const chtype);
-int mvwaddnstr(WINDOW *, int, int, const char *, int);
-int mvwaddstr(WINDOW *, int, int, const char *);
-int mvwchgat(WINDOW *, int, int, int, attr_t, short, const void *);
-int mvwdelch(WINDOW *, int, int);
-int mvwgetch(WINDOW *, int, int);
-int mvwgetnstr(WINDOW *, int, int, char *, int);
-int mvwin(WINDOW *, int, int);
-chtype mvwinch(WINDOW *, int, int);
-int mvwinnstr(WINDOW *, int, int, char *, int);
-int mvwinsch(WINDOW *, int, int, chtype);
-int mvwinsnstr(WINDOW *, int, int, const char *, int);
-int mvwinsstr(WINDOW *, int, int, const char *);
-int napms(int);
-WINDOW * newpad(int, int);
-WINDOW * newwin(int, int, int, int);
-int nl(void);
-int nocbreak(void);
-int nodelay(WINDOW *, bool);
-int noecho(void);
-int nonl(void);
-void noqiflush(void);
-int noraw(void);
-int notimeout(WINDOW *, bool);
-int overlay(const WINDOW*, WINDOW *);
-int overwrite(const WINDOW*, WINDOW *);
-int pair_content(short, short*, short*);
-int pechochar(WINDOW *, const chtype);
-int pnoutrefresh(WINDOW*, int, int, int, int, int, int);
-int prefresh(WINDOW *, int, int, int, int, int, int);
-int putwin(WINDOW *, FILE *);
-void qiflush(void);
-int raw(void);
-int redrawwin(WINDOW *);
-int resetty(void);
-int reset_prog_mode(void);
-int reset_shell_mode(void);
-int savetty(void);
-int scroll(WINDOW *);
-int scrollok(WINDOW *, bool);
-int start_color(void);
-WINDOW * subpad(WINDOW *, int, int, int, int);
-WINDOW * subwin(WINDOW *, int, int, int, int);
-int syncok(WINDOW *, bool);
-chtype termattrs(void);
-char * termname(void);
-int touchline(WINDOW *, int, int);
-int touchwin(WINDOW *);
-int typeahead(int);
-int ungetch(int);
-int untouchwin(WINDOW *);
-void use_env(bool);
-int waddch(WINDOW *, const chtype);
-int waddnstr(WINDOW *, const char *, int);
-int waddstr(WINDOW *, const char *);
-int wattron(WINDOW *, int);
-int wattroff(WINDOW *, int);
-int wattrset(WINDOW *, int);
-int wbkgd(WINDOW *, chtype);
-void wbkgdset(WINDOW *, chtype);
-int wborder(WINDOW *, chtype, chtype, chtype, chtype,
-            chtype, chtype, chtype, chtype);
-int wchgat(WINDOW *, int, attr_t, short, const void *);
-int wclear(WINDOW *);
-int wclrtobot(WINDOW *);
-int wclrtoeol(WINDOW *);
-void wcursyncup(WINDOW *);
-int wdelch(WINDOW *);
-int wdeleteln(WINDOW *);
-int wechochar(WINDOW *, const chtype);
-int werase(WINDOW *);
-int wgetch(WINDOW *);
-int wgetnstr(WINDOW *, char *, int);
-int whline(WINDOW *, chtype, int);
-chtype winch(WINDOW *);
-int winnstr(WINDOW *, char *, int);
-int winsch(WINDOW *, chtype);
-int winsdelln(WINDOW *, int);
-int winsertln(WINDOW *);
-int winsnstr(WINDOW *, const char *, int);
-int winsstr(WINDOW *, const char *);
-int wmove(WINDOW *, int, int);
-int wresize(WINDOW *, int, int);
-int wnoutrefresh(WINDOW *);
-int wredrawln(WINDOW *, int, int);
-int wrefresh(WINDOW *);
-int wscrl(WINDOW *, int);
-int wsetscrreg(WINDOW *, int, int);
-int wstandout(WINDOW *);
-int wstandend(WINDOW *);
-void wsyncdown(WINDOW *);
-void wsyncup(WINDOW *);
-void wtimeout(WINDOW *, int);
-int wtouchln(WINDOW *, int, int, int);
-int wvline(WINDOW *, chtype, int);
-int tigetflag(char *);
-int tigetnum(char *);
-char * tigetstr(char *);
-int putp(const char *);
-char * tparm(const char *, ...);
-int getattrs(const WINDOW *);
-int getcurx(const WINDOW *);
-int getcury(const WINDOW *);
-int getbegx(const WINDOW *);
-int getbegy(const WINDOW *);
-int getmaxx(const WINDOW *);
-int getmaxy(const WINDOW *);
-int getparx(const WINDOW *);
-int getpary(const WINDOW *);
-
-int getmouse(MEVENT *);
-int ungetmouse(MEVENT *);
-mmask_t mousemask(mmask_t, mmask_t *);
-bool wenclose(const WINDOW *, int, int);
-int mouseinterval(int);
-
-void setsyx(int y, int x);
-const char *unctrl(chtype);
-int use_default_colors(void);
-
-int has_key(int);
-bool is_term_resized(int, int);
-
-#define _m_STRICT_SYSV_CURSES ...
-#define _m_NCURSES_MOUSE_VERSION ...
-#define _m_NetBSD ...
-int _m_ispad(WINDOW *);
-
-chtype acs_map[];
-
-// For _curses_panel:
-
-typedef ... PANEL;
-
-WINDOW *panel_window(const PANEL *);
-void update_panels(void);
-int hide_panel(PANEL *);
-int show_panel(PANEL *);
-int del_panel(PANEL *);
-int top_panel(PANEL *);
-int bottom_panel(PANEL *);
-PANEL *new_panel(WINDOW *);
-PANEL *panel_above(const PANEL *);
-PANEL *panel_below(const PANEL *);
-int set_panel_userptr(PANEL *, void *);
-const void *panel_userptr(const PANEL *);
-int move_panel(PANEL *, int, int);
-int replace_panel(PANEL *,WINDOW *);
-int panel_hidden(const PANEL *);
-
-void _m_getsyx(int *yx);
-""")
-
-
-lib = ffi.verify("""
-#ifdef __APPLE__
-/* the following define is necessary for OS X 10.6+; without it, the
-   Apple-supplied ncurses.h sets NCURSES_OPAQUE to 1, and then Python
-   can't get at the WINDOW flags field. */
-#define NCURSES_OPAQUE 0
-#endif
-
-#include <ncurses.h>
-#include <panel.h>
-#include <term.h>
-
-#if defined STRICT_SYSV_CURSES
-#define _m_STRICT_SYSV_CURSES TRUE
-#else
-#define _m_STRICT_SYSV_CURSES FALSE
-#endif
-
-#if defined NCURSES_MOUSE_VERSION
-#define _m_NCURSES_MOUSE_VERSION TRUE
-#else
-#define _m_NCURSES_MOUSE_VERSION FALSE
-#endif
-
-#if defined __NetBSD__
-#define _m_NetBSD TRUE
-#else
-#define _m_NetBSD FALSE
-#endif
-
-int _m_ispad(WINDOW *win) {
-    // <curses.h> may not have _flags (and possibly _ISPAD),
-    // but for now let's assume that <ncurses.h> always has it
-    return (win->_flags & _ISPAD);
-}
-
-void _m_getsyx(int *yx) {
-    getsyx(yx[0], yx[1]);
-}
-""", libraries=['ncursesw', 'panel'])
-
+from _curses_cffi import ffi, lib
 
 def _copy_to_globals(name):
     globals()[name] = getattr(lib, name)
@@ -484,13 +165,13 @@
 def _chtype(ch):
     return int(ffi.cast("chtype", ch))
 
-def _texttype(text):
+def _bytestype(text):
     if isinstance(text, bytes):
         return text
     elif isinstance(text, str):
         return text.encode('utf-8')
     else:
-        raise TypeError("str or bytes expected, got a '%s' object"
+        raise TypeError("bytes or str expected, got a '%s' object"
                         % (type(text).__name__,))
 
 
@@ -606,7 +287,7 @@
 
     @_argspec(1, 1, 2)
     def addstr(self, y, x, text, attr=None):
-        text = _texttype(text)
+        text = _bytestype(text)
         if attr is not None:
             attr_old = lib.getattrs(self._win)
             lib.wattrset(self._win, attr)
@@ -620,7 +301,7 @@
 
     @_argspec(2, 1, 2)
     def addnstr(self, y, x, text, n, attr=None):
-        text = _texttype(text)
+        text = _bytestype(text)
         if attr is not None:
             attr_old = lib.getattrs(self._win)
             lib.wattrset(self._win, attr)
@@ -799,7 +480,7 @@
 
     @_argspec(1, 1, 2)
     def insstr(self, y, x, text, attr=None):
-        text = _texttype(text)
+        text = _bytestype(text)
         if attr is not None:
             attr_old = lib.getattrs(self._win)
             lib.wattrset(self._win, attr)
@@ -813,7 +494,7 @@
 
     @_argspec(2, 1, 2)
     def insnstr(self, y, x, text, n, attr=None):
-        text = _texttype(text)
+        text = _bytestype(text)
         if attr is not None:
             attr_old = lib.getattrs(self._win)
             lib.wattrset(self._win, attr)
@@ -1221,7 +902,7 @@
 
 
 def putp(text):
-    text = _texttype(text)
+    text = _bytestype(text)
     return _check_ERR(lib.putp(text), "putp")
 
 
@@ -1347,23 +1028,17 @@
 
 def tigetflag(capname):
     _ensure_initialised_setupterm()
-    if isinstance(capname, str):
-        capname = capname.encode('utf-8')
-    return lib.tigetflag(capname)
+    return lib.tigetflag(capname.encode())
 
 
 def tigetnum(capname):
     _ensure_initialised_setupterm()
-    if isinstance(capname, str):
-        capname = capname.encode('utf-8')
-    return lib.tigetnum(capname)
+    return lib.tigetnum(capname.encode())
 
 
 def tigetstr(capname):
     _ensure_initialised_setupterm()
-    if isinstance(capname, str):
-        capname = capname.encode('utf-8')
-    val = lib.tigetstr(capname)
+    val = lib.tigetstr(capname.encode())
     if int(ffi.cast("intptr_t", val)) in (0, -1):
         return None
     return ffi.string(val)
diff --git a/lib_pypy/_curses_build.py b/lib_pypy/_curses_build.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_curses_build.py
@@ -0,0 +1,323 @@
+from cffi import FFI
+
+ffi = FFI()
+
+ffi.set_source("_curses_cffi", """
+#ifdef __APPLE__
+/* the following define is necessary for OS X 10.6+; without it, the
+   Apple-supplied ncurses.h sets NCURSES_OPAQUE to 1, and then Python
+   can't get at the WINDOW flags field. */
+#define NCURSES_OPAQUE 0
+#endif
+
+#include <ncurses.h>
+#include <panel.h>
+#include <term.h>
+
+#if defined STRICT_SYSV_CURSES
+#define _m_STRICT_SYSV_CURSES TRUE
+#else
+#define _m_STRICT_SYSV_CURSES FALSE
+#endif
+
+#if defined NCURSES_MOUSE_VERSION
+#define _m_NCURSES_MOUSE_VERSION TRUE
+#else
+#define _m_NCURSES_MOUSE_VERSION FALSE
+#endif
+
+#if defined __NetBSD__
+#define _m_NetBSD TRUE
+#else
+#define _m_NetBSD FALSE
+#endif
+
+int _m_ispad(WINDOW *win) {
+    // <curses.h> may not have _flags (and possibly _ISPAD),
+    // but for now let's assume that <ncurses.h> always has it
+    return (win->_flags & _ISPAD);
+}
+
+void _m_getsyx(int *yx) {
+    getsyx(yx[0], yx[1]);
+}
+""", libraries=['ncursesw', 'panel'])
+
+
+ffi.cdef("""
+typedef ... WINDOW;
+typedef ... SCREEN;
+typedef unsigned long... mmask_t;
+typedef unsigned char bool;
+typedef unsigned long... chtype;
+typedef chtype attr_t;
+
+typedef struct
+{
+    short id;           /* ID to distinguish multiple devices */
+    int x, y, z;        /* event coordinates (character-cell) */
+    mmask_t bstate;     /* button state bits */
+}
+MEVENT;
+
+static const int ERR, OK;
+static const int TRUE, FALSE;
+static const int KEY_MIN, KEY_MAX;
+
+static const int COLOR_BLACK;
+static const int COLOR_RED;
+static const int COLOR_GREEN;
+static const int COLOR_YELLOW;
+static const int COLOR_BLUE;
+static const int COLOR_MAGENTA;
+static const int COLOR_CYAN;
+static const int COLOR_WHITE;
+
+static const chtype A_ATTRIBUTES;
+static const chtype A_NORMAL;
+static const chtype A_STANDOUT;
+static const chtype A_UNDERLINE;
+static const chtype A_REVERSE;
+static const chtype A_BLINK;
+static const chtype A_DIM;
+static const chtype A_BOLD;
+static const chtype A_ALTCHARSET;
+static const chtype A_INVIS;
+static const chtype A_PROTECT;
+static const chtype A_CHARTEXT;
+static const chtype A_COLOR;
+
+static const int BUTTON1_RELEASED;
+static const int BUTTON1_PRESSED;
+static const int BUTTON1_CLICKED;
+static const int BUTTON1_DOUBLE_CLICKED;
+static const int BUTTON1_TRIPLE_CLICKED;
+static const int BUTTON2_RELEASED;
+static const int BUTTON2_PRESSED;
+static const int BUTTON2_CLICKED;
+static const int BUTTON2_DOUBLE_CLICKED;
+static const int BUTTON2_TRIPLE_CLICKED;
+static const int BUTTON3_RELEASED;
+static const int BUTTON3_PRESSED;
+static const int BUTTON3_CLICKED;
+static const int BUTTON3_DOUBLE_CLICKED;
+static const int BUTTON3_TRIPLE_CLICKED;
+static const int BUTTON4_RELEASED;
+static const int BUTTON4_PRESSED;
+static const int BUTTON4_CLICKED;
+static const int BUTTON4_DOUBLE_CLICKED;
+static const int BUTTON4_TRIPLE_CLICKED;
+static const int BUTTON_SHIFT;
+static const int BUTTON_CTRL;
+static const int BUTTON_ALT;
+static const int ALL_MOUSE_EVENTS;
+static const int REPORT_MOUSE_POSITION;
+
+int setupterm(char *, int, int *);
+
+WINDOW *stdscr;
+int COLORS;
+int COLOR_PAIRS;
+int COLS;
+int LINES;
+
+int baudrate(void);
+int beep(void);
+int box(WINDOW *, chtype, chtype);
+bool can_change_color(void);
+int cbreak(void);
+int clearok(WINDOW *, bool);
+int color_content(short, short*, short*, short*);
+int copywin(const WINDOW*, WINDOW*, int, int, int, int, int, int, int);
+int curs_set(int);
+int def_prog_mode(void);
+int def_shell_mode(void);
+int delay_output(int);
+int delwin(WINDOW *);
+WINDOW * derwin(WINDOW *, int, int, int, int);
+int doupdate(void);
+int echo(void);
+int endwin(void);
+char erasechar(void);
+void filter(void);
+int flash(void);
+int flushinp(void);
+chtype getbkgd(WINDOW *);
+WINDOW * getwin(FILE *);
+int halfdelay(int);
+bool has_colors(void);
+bool has_ic(void);
+bool has_il(void);
+void idcok(WINDOW *, bool);
+int idlok(WINDOW *, bool);
+void immedok(WINDOW *, bool);
+WINDOW * initscr(void);
+int init_color(short, short, short, short);
+int init_pair(short, short, short);
+int intrflush(WINDOW *, bool);
+bool isendwin(void);
+bool is_linetouched(WINDOW *, int);
+bool is_wintouched(WINDOW *);
+const char * keyname(int);
+int keypad(WINDOW *, bool);
+char killchar(void);
+int leaveok(WINDOW *, bool);
+char * longname(void);
+int meta(WINDOW *, bool);
+int mvderwin(WINDOW *, int, int);
+int mvwaddch(WINDOW *, int, int, const chtype);
+int mvwaddnstr(WINDOW *, int, int, const char *, int);
+int mvwaddstr(WINDOW *, int, int, const char *);
+int mvwchgat(WINDOW *, int, int, int, attr_t, short, const void *);
+int mvwdelch(WINDOW *, int, int);
+int mvwgetch(WINDOW *, int, int);
+int mvwgetnstr(WINDOW *, int, int, char *, int);
+int mvwin(WINDOW *, int, int);
+chtype mvwinch(WINDOW *, int, int);
+int mvwinnstr(WINDOW *, int, int, char *, int);
+int mvwinsch(WINDOW *, int, int, chtype);
+int mvwinsnstr(WINDOW *, int, int, const char *, int);
+int mvwinsstr(WINDOW *, int, int, const char *);
+int napms(int);
+WINDOW * newpad(int, int);
+WINDOW * newwin(int, int, int, int);
+int nl(void);
+int nocbreak(void);
+int nodelay(WINDOW *, bool);
+int noecho(void);
+int nonl(void);
+void noqiflush(void);
+int noraw(void);
+int notimeout(WINDOW *, bool);
+int overlay(const WINDOW*, WINDOW *);
+int overwrite(const WINDOW*, WINDOW *);
+int pair_content(short, short*, short*);
+int pechochar(WINDOW *, const chtype);
+int pnoutrefresh(WINDOW*, int, int, int, int, int, int);
+int prefresh(WINDOW *, int, int, int, int, int, int);
+int putwin(WINDOW *, FILE *);
+void qiflush(void);
+int raw(void);
+int redrawwin(WINDOW *);
+int resetty(void);
+int reset_prog_mode(void);
+int reset_shell_mode(void);
+int savetty(void);
+int scroll(WINDOW *);
+int scrollok(WINDOW *, bool);
+int start_color(void);
+WINDOW * subpad(WINDOW *, int, int, int, int);
+WINDOW * subwin(WINDOW *, int, int, int, int);
+int syncok(WINDOW *, bool);
+chtype termattrs(void);
+char * termname(void);
+int touchline(WINDOW *, int, int);
+int touchwin(WINDOW *);
+int typeahead(int);
+int ungetch(int);
+int untouchwin(WINDOW *);
+void use_env(bool);
+int waddch(WINDOW *, const chtype);
+int waddnstr(WINDOW *, const char *, int);
+int waddstr(WINDOW *, const char *);
+int wattron(WINDOW *, int);
+int wattroff(WINDOW *, int);
+int wattrset(WINDOW *, int);
+int wbkgd(WINDOW *, chtype);
+void wbkgdset(WINDOW *, chtype);
+int wborder(WINDOW *, chtype, chtype, chtype, chtype,
+            chtype, chtype, chtype, chtype);
+int wchgat(WINDOW *, int, attr_t, short, const void *);
+int wclear(WINDOW *);
+int wclrtobot(WINDOW *);
+int wclrtoeol(WINDOW *);
+void wcursyncup(WINDOW *);
+int wdelch(WINDOW *);
+int wdeleteln(WINDOW *);
+int wechochar(WINDOW *, const chtype);
+int werase(WINDOW *);
+int wgetch(WINDOW *);
+int wgetnstr(WINDOW *, char *, int);
+int whline(WINDOW *, chtype, int);
+chtype winch(WINDOW *);
+int winnstr(WINDOW *, char *, int);
+int winsch(WINDOW *, chtype);
+int winsdelln(WINDOW *, int);
+int winsertln(WINDOW *);
+int winsnstr(WINDOW *, const char *, int);
+int winsstr(WINDOW *, const char *);
+int wmove(WINDOW *, int, int);
+int wresize(WINDOW *, int, int);
+int wnoutrefresh(WINDOW *);
+int wredrawln(WINDOW *, int, int);
+int wrefresh(WINDOW *);
+int wscrl(WINDOW *, int);
+int wsetscrreg(WINDOW *, int, int);
+int wstandout(WINDOW *);
+int wstandend(WINDOW *);
+void wsyncdown(WINDOW *);
+void wsyncup(WINDOW *);
+void wtimeout(WINDOW *, int);
+int wtouchln(WINDOW *, int, int, int);
+int wvline(WINDOW *, chtype, int);
+int tigetflag(char *);
+int tigetnum(char *);
+char * tigetstr(char *);
+int putp(const char *);
+char * tparm(const char *, ...);
+int getattrs(const WINDOW *);
+int getcurx(const WINDOW *);
+int getcury(const WINDOW *);
+int getbegx(const WINDOW *);
+int getbegy(const WINDOW *);
+int getmaxx(const WINDOW *);
+int getmaxy(const WINDOW *);
+int getparx(const WINDOW *);
+int getpary(const WINDOW *);
+
+int getmouse(MEVENT *);
+int ungetmouse(MEVENT *);
+mmask_t mousemask(mmask_t, mmask_t *);
+bool wenclose(const WINDOW *, int, int);
+int mouseinterval(int);
+
+void setsyx(int y, int x);
+const char *unctrl(chtype);
+int use_default_colors(void);
+
+int has_key(int);
+bool is_term_resized(int, int);
+
+#define _m_STRICT_SYSV_CURSES ...
+#define _m_NCURSES_MOUSE_VERSION ...
+#define _m_NetBSD ...
+int _m_ispad(WINDOW *);
+
+chtype acs_map[];
+
+// For _curses_panel:
+
+typedef ... PANEL;
+
+WINDOW *panel_window(const PANEL *);
+void update_panels(void);
+int hide_panel(PANEL *);
+int show_panel(PANEL *);
+int del_panel(PANEL *);
+int top_panel(PANEL *);
+int bottom_panel(PANEL *);
+PANEL *new_panel(WINDOW *);
+PANEL *panel_above(const PANEL *);
+PANEL *panel_below(const PANEL *);
+int set_panel_userptr(PANEL *, void *);
+const void *panel_userptr(const PANEL *);
+int move_panel(PANEL *, int, int);
+int replace_panel(PANEL *,WINDOW *);
+int panel_hidden(const PANEL *);
+
+void _m_getsyx(int *yx);
+""")
+
+
+if __name__ == "__main__":
+    ffi.compile()
diff --git a/lib_pypy/_functools.py b/lib_pypy/_functools.py
--- a/lib_pypy/_functools.py
+++ b/lib_pypy/_functools.py
@@ -32,16 +32,16 @@
     partial(func, *args, **keywords) - new function with partial application
     of the given arguments and keywords.
     """
-
-    def __init__(self, *args, **keywords):
-        if not args:
-            raise TypeError('__init__() takes at least 2 arguments (1 given)')
-        func, args = args[0], args[1:]
+    def __init__(*args, **keywords):
+        if len(args) < 2:
+            raise TypeError('__init__() takes at least 2 arguments (%d given)'
+                            % len(args))
+        self, func, args = args[0], args[1], args[2:]
         if not callable(func):
             raise TypeError("the first argument must be callable")
         self._func = func
         self._args = args
-        self._keywords = keywords or None
+        self._keywords = keywords
 
     def __delattr__(self, key):
         if key == '__dict__':
@@ -61,9 +61,9 @@
         return self._keywords
 
     def __call__(self, *fargs, **fkeywords):
-        if self.keywords is not None:
-            fkeywords = dict(self.keywords, **fkeywords)
-        return self.func(*(self.args + fargs), **fkeywords)
+        if self._keywords:
+            fkeywords = dict(self._keywords, **fkeywords)
+        return self._func(*(self._args + fargs), **fkeywords)
 
     def __repr__(self):
         cls = type(self)
@@ -84,10 +84,13 @@
                 ('_func', '_args', '_keywords'))
         if len(d) == 0:
             d = None
-        return (type(self), (self.func,),
-                (self.func, self.args, self.keywords, d))
+        return (type(self), (self._func,),
+                (self._func, self._args, self._keywords, d))
 
     def __setstate__(self, state):
-        self._func, self._args, self._keywords, d = state
+        func, args, keywords, d = state
         if d is not None:
             self.__dict__.update(d)
+        self._func = func
+        self._args = args
+        self._keywords = keywords
diff --git a/lib_pypy/_gdbm.py b/lib_pypy/_gdbm.py
--- a/lib_pypy/_gdbm.py
+++ b/lib_pypy/_gdbm.py
@@ -1,72 +1,6 @@
-import cffi, os, sys
-
-ffi = cffi.FFI()
-ffi.cdef('''
-#define GDBM_READER ...
-#define GDBM_WRITER ...
-#define GDBM_WRCREAT ...
-#define GDBM_NEWDB ...
-#define GDBM_FAST ...
-#define GDBM_SYNC ...
-#define GDBM_NOLOCK ...
-#define GDBM_REPLACE ...
-
-void* gdbm_open(char *, int, int, int, void (*)());
-void gdbm_close(void*);
-
-typedef struct {
-    char *dptr;
-    int   dsize;
-} datum;
-
-datum gdbm_fetch(void*, datum);
-datum pygdbm_fetch(void*, char*, int);
-int gdbm_delete(void*, datum);
-int gdbm_store(void*, datum, datum, int);
-int gdbm_exists(void*, datum);
-int pygdbm_exists(void*, char*, int);
-
-int gdbm_reorganize(void*);
-
-datum gdbm_firstkey(void*);
-datum gdbm_nextkey(void*, datum);
-void gdbm_sync(void*);
-
-char* gdbm_strerror(int);
-int gdbm_errno;
-
-void free(void*);
-''')
-
-try:
-    verify_code = '''
-    #include "gdbm.h"
-
-    static datum pygdbm_fetch(GDBM_FILE gdbm_file, char *dptr, int dsize) {
-        datum key = {dptr, dsize};
-        return gdbm_fetch(gdbm_file, key);
-    }
-
-    static int pygdbm_exists(GDBM_FILE gdbm_file, char *dptr, int dsize) {
-        datum key = {dptr, dsize};
-        return gdbm_exists(gdbm_file, key);
-    }
-    
-    '''
-    if sys.platform.startswith('freebsd'):
-        import os.path
-        _localbase = os.environ.get('LOCALBASE', '/usr/local')
-        lib = ffi.verify(verify_code, libraries=['gdbm'],
-             include_dirs=[os.path.join(_localbase, 'include')],
-             library_dirs=[os.path.join(_localbase, 'lib')]
-        )
-    else:
-        lib = ffi.verify(verify_code, libraries=['gdbm'])
-except cffi.VerificationError as e:
-    # distutils does not preserve the actual message,
-    # but the verification is simple enough that the
-    # failure must be due to missing gdbm dev libs
-    raise ImportError('%s: %s' %(e.__class__.__name__, e))
+from _gdbm_cffi import ffi, lib    # generated by _gdbm_build.py
+import sys, os, threading
+_lock = threading.Lock()
 
 class error(IOError):
     pass
@@ -87,59 +21,71 @@
     return {'dptr': ffi.new("char[]", key), 'dsize': len(key)}
 
 class gdbm(object):
-    ll_dbm = None
+    __ll_dbm = None
+
+    # All public methods need to acquire the lock; all private methods
+    # assume the lock is already held.  Thus public methods cannot call
+    # other public methods.
 
     def __init__(self, filename, iflags, mode):
-        res = lib.gdbm_open(filename, 0, iflags, mode, ffi.NULL)
-        self.size = -1
-        if not res:
-            self._raise_from_errno()
-        self.ll_dbm = res
+        with _lock:
+            res = lib.gdbm_open(filename, 0, iflags, mode, ffi.NULL)
+            self.__size = -1
+            if not res:
+                self.__raise_from_errno()
+            self.__ll_dbm = res
 
     def close(self):
-        if self.ll_dbm:
-            lib.gdbm_close(self.ll_dbm)
-            self.ll_dbm = None
+        with _lock:
+            if self.__ll_dbm:
+                lib.gdbm_close(self.__ll_dbm)
+                self.__ll_dbm = None
 
-    def _raise_from_errno(self):
+    def __raise_from_errno(self):
         if ffi.errno:
             raise error(ffi.errno, os.strerror(ffi.errno))
         raise error(lib.gdbm_errno, lib.gdbm_strerror(lib.gdbm_errno))
 
     def __len__(self):
-        if self.size < 0:
-            self.size = len(self.keys())

From noreply at buildbot.pypy.org  Tue Jun  9 19:06:07 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Tue,  9 Jun 2015 19:06:07 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Remove 'utf-8' argument from calls to
 str.encode() in this file because it's already the default argument.
Message-ID: <20150609170607.CD3261C130E@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob <me at manueljacob.de>
Branch: py3k
Changeset: r77998:55d423041e00
Date: 2015-06-09 19:06 +0200
http://bitbucket.org/pypy/pypy/changeset/55d423041e00/

Log:	Remove 'utf-8' argument from calls to str.encode() in this file
	because it's already the default argument.

diff --git a/lib_pypy/_curses.py b/lib_pypy/_curses.py
--- a/lib_pypy/_curses.py
+++ b/lib_pypy/_curses.py
@@ -169,7 +169,7 @@
     if isinstance(text, bytes):
         return text
     elif isinstance(text, str):
-        return text.encode('utf-8')
+        return text.encode()
     else:
         raise TypeError("bytes or str expected, got a '%s' object"
                         % (type(text).__name__,))
@@ -809,7 +809,7 @@
     if term is None:
         term = ffi.NULL
     elif isinstance(term, str):
-        term = term.encode('utf-8')
+        term = term.encode()
     err = ffi.new("int *")
     if lib.setupterm(term, fd, err) == lib.ERR:
         err = err[0]

From noreply at buildbot.pypy.org  Tue Jun  9 20:41:38 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue,  9 Jun 2015 20:41:38 +0200 (CEST)
Subject: [pypy-commit] cffi default: Add some checks that will fail in PyPy
Message-ID: <20150609184138.786531C048F@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2181:1ba1aa9565ff
Date: 2015-06-09 20:41 +0200
http://bitbucket.org/cffi/cffi/changeset/1ba1aa9565ff/

Log:	Add some checks that will fail in PyPy

diff --git a/testing/cffi1/test_recompiler.py b/testing/cffi1/test_recompiler.py
--- a/testing/cffi1/test_recompiler.py
+++ b/testing/cffi1/test_recompiler.py
@@ -409,6 +409,10 @@
     # 'x' is another <built-in method> object on lib, made very indirectly
     x = type(lib).__dir__.__get__(lib)
     py.test.raises(TypeError, ffi.typeof, x)
+    #
+    assert hasattr(lib.sin, '__name__')     # present on built-in functions on
+    assert hasattr(lib.sin, '__module__')   # CPython; must be emulated on PyPy
+    assert hasattr(lib.sin, '__doc__')
 
 def test_verify_anonymous_struct_with_typedef():
     ffi = FFI()

From noreply at buildbot.pypy.org  Tue Jun  9 20:50:23 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue,  9 Jun 2015 20:50:23 +0200 (CEST)
Subject: [pypy-commit] cffi default: Make the test more precise
Message-ID: <20150609185023.5DD101C0695@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2182:713b2badd33b
Date: 2015-06-09 20:47 +0200
http://bitbucket.org/cffi/cffi/changeset/713b2badd33b/

Log:	Make the test more precise

diff --git a/testing/cffi1/test_recompiler.py b/testing/cffi1/test_recompiler.py
--- a/testing/cffi1/test_recompiler.py
+++ b/testing/cffi1/test_recompiler.py
@@ -410,9 +410,10 @@
     x = type(lib).__dir__.__get__(lib)
     py.test.raises(TypeError, ffi.typeof, x)
     #
-    assert hasattr(lib.sin, '__name__')     # present on built-in functions on
-    assert hasattr(lib.sin, '__module__')   # CPython; must be emulated on PyPy
-    assert hasattr(lib.sin, '__doc__')
+    # present on built-in functions on CPython; must be emulated on PyPy:
+    assert lib.sin.__name__ == 'sin'
+    assert lib.sin.__module__ == '_CFFI_test_math_sin_type'
+    assert lib.sin.__doc__ == 'direct call to the C function of the same name'
 
 def test_verify_anonymous_struct_with_typedef():
     ffi = FFI()

From noreply at buildbot.pypy.org  Tue Jun  9 20:51:10 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue,  9 Jun 2015 20:51:10 +0200 (CEST)
Subject: [pypy-commit] pypy default: Import cffi's 713b2badd33b and fix
Message-ID: <20150609185110.E6C011C0695@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r77999:7959ab6b0b35
Date: 2015-06-09 20:51 +0200
http://bitbucket.org/pypy/pypy/changeset/7959ab6b0b35/

Log:	Import cffi's 713b2badd33b and fix

diff --git a/pypy/module/_cffi_backend/lib_obj.py b/pypy/module/_cffi_backend/lib_obj.py
--- a/pypy/module/_cffi_backend/lib_obj.py
+++ b/pypy/module/_cffi_backend/lib_obj.py
@@ -65,7 +65,7 @@
         ptr = rffi.cast(rffi.CCHARP, g.c_address)
         assert ptr
         return W_FunctionWrapper(self.space, ptr, g.c_size_or_direct_fn, w_ct,
-                                 locs, rawfunctype, fnname)
+                                 locs, rawfunctype, fnname, self.libname)
 
     @jit.elidable_promote()
     def _get_attr_elidable(self, attr):
diff --git a/pypy/module/_cffi_backend/test/test_recompiler.py b/pypy/module/_cffi_backend/test/test_recompiler.py
--- a/pypy/module/_cffi_backend/test/test_recompiler.py
+++ b/pypy/module/_cffi_backend/test/test_recompiler.py
@@ -418,6 +418,11 @@
         # 'x' is another <built-in method> object on lib, made very indirectly
         x = type(lib).__dir__.__get__(lib)
         raises(TypeError, ffi.typeof, x)
+        #
+        # present on built-in functions on CPython; must be emulated on PyPy:
+        assert lib.sin.__name__ == 'sin'
+        assert lib.sin.__module__ == '_CFFI_test_math_sin_type'
+        assert lib.sin.__doc__=='direct call to the C function of the same name'
 
     def test_verify_anonymous_struct_with_typedef(self):
         ffi, lib = self.prepare(
diff --git a/pypy/module/_cffi_backend/wrapper.py b/pypy/module/_cffi_backend/wrapper.py
--- a/pypy/module/_cffi_backend/wrapper.py
+++ b/pypy/module/_cffi_backend/wrapper.py
@@ -1,6 +1,6 @@
 from pypy.interpreter.error import oefmt
 from pypy.interpreter.baseobjspace import W_Root
-from pypy.interpreter.typedef import TypeDef
+from pypy.interpreter.typedef import TypeDef, interp_attrproperty
 from pypy.interpreter.gateway import interp2app
 from rpython.rlib import jit
 
@@ -21,9 +21,10 @@
     also returns the original struct/union signature.
     """
     _immutable_ = True
+    common_doc_str = 'direct call to the C function of the same name'
 
     def __init__(self, space, fnptr, directfnptr, ctype,
-                 locs, rawfunctype, fnname):
+                 locs, rawfunctype, fnname, modulename):
         assert isinstance(ctype, W_CTypeFunc)
         assert ctype.cif_descr is not None     # not for '...' functions
         assert locs is None or len(ctype.fargs) == len(locs)
@@ -35,6 +36,7 @@
         self.locs = locs
         self.rawfunctype = rawfunctype
         self.fnname = fnname
+        self.modulename = modulename
         self.nargs_expected = len(ctype.fargs) - (locs is not None and
                                                   locs[0] == 'R')
 
@@ -111,5 +113,8 @@
         'FFIFunctionWrapper',
         __repr__ = interp2app(W_FunctionWrapper.descr_repr),
         __call__ = interp2app(W_FunctionWrapper.descr_call),
+        __name__ = interp_attrproperty('fnname', cls=W_FunctionWrapper),
+        __module__ = interp_attrproperty('modulename', cls=W_FunctionWrapper),
+        __doc__ = interp_attrproperty('common_doc_str', cls=W_FunctionWrapper),
         )
 W_FunctionWrapper.typedef.acceptable_as_base_class = False

From noreply at buildbot.pypy.org  Tue Jun  9 20:53:44 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Tue,  9 Jun 2015 20:53:44 +0200 (CEST)
Subject: [pypy-commit] pypy unicode-dtype: some progress
Message-ID: <20150609185344.CC5631C0695@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: unicode-dtype
Changeset: r78000:f68fb42cff20
Date: 2015-06-06 16:16 +0100
http://bitbucket.org/pypy/pypy/changeset/f68fb42cff20/

Log:	some progress

diff --git a/pypy/module/micronumpy/casting.py b/pypy/module/micronumpy/casting.py
--- a/pypy/module/micronumpy/casting.py
+++ b/pypy/module/micronumpy/casting.py
@@ -325,6 +325,8 @@
         return complex_dtype
     elif space.isinstance_w(w_obj, space.w_str):
         return variable_dtype(space, 'S%d' % space.len_w(w_obj))
+    elif space.isinstance_w(w_obj, space.w_unicode):
+        return new_unicode_dtype(space, space.len_w(w_obj))
     return object_dtype
 
 @signature(ann.instance(W_Dtype), ann.instance(W_Dtype), returns=ann.bool())
diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py
--- a/pypy/module/micronumpy/test/test_ndarray.py
+++ b/pypy/module/micronumpy/test/test_ndarray.py
@@ -1,3 +1,4 @@
+# -*- encoding: utf-8 -*-
 import py
 import sys
 
@@ -322,6 +323,13 @@
         assert b.flags['C']
         assert (b == a).all()
 
+    def test_unicode(self):
+        import numpy as np
+        a = np.array([u'A&#255;', u'abc'], dtype=np.dtype('U'))
+        assert a.shape == (2,)
+        assert a.dtype == np.dtype('U3')
+        assert a[0] == u'A&#255;'
+
     def test_dtype_attribute(self):
         import numpy as np
         a = np.array(40000, dtype='uint16')
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -11,7 +11,7 @@
     most_neg_value_of, LONG_BIT
 from rpython.rlib.rawstorage import (alloc_raw_storage,
     raw_storage_getitem_unaligned, raw_storage_setitem_unaligned)
-from rpython.rlib.rstring import StringBuilder
+from rpython.rlib.rstring import StringBuilder, UnicodeBuilder
 from rpython.rlib.rstruct.ieee import (float_pack, float_unpack, unpack_float,
                                        pack_float80, unpack_float80)
 from rpython.rlib.rstruct.nativefmttable import native_is_bigendian
@@ -2190,7 +2190,7 @@
             self._store(storage, i, offset, box, width)
 
 class UnicodeType(FlexibleType):
-    T = lltype.Char
+    T = lltype.UniChar
     num = NPY.UNICODE
     kind = NPY.UNICODELTR
     char = NPY.UNICODELTR
@@ -2202,58 +2202,75 @@
     def coerce(self, space, dtype, w_item):
         if isinstance(w_item, boxes.W_UnicodeBox):
             return w_item
-        raise OperationError(space.w_NotImplementedError, space.wrap(
-            "coerce (probably from set_item) not implemented for unicode type"))
+        value = space.unicode_w(w_item)
+        return boxes.W_UnicodeBox(value)
 
     def store(self, arr, i, offset, box):
         assert isinstance(box, boxes.W_UnicodeBox)
-        raise oefmt(self.space.w_NotImplementedError, "unicode type not completed")
+        value = box._value
+        for k in range(len(value)):
+            index = i + offset + 4*k
+            data = rffi.cast(Int32.T, ord(box._value[k]))
+            raw_storage_setitem_unaligned(arr.storage, index, data)
 
     def read(self, arr, i, offset, dtype=None):
-        raise oefmt(self.space.w_NotImplementedError, "unicode type not completed")
+        if dtype is None:
+            dtype = arr.dtype
+        size = dtype.elsize // 4
+        builder = UnicodeBuilder(size)
+        with arr as storage:
+            for k in range(size):
+                index = i + offset + 4*k
+                codepoint = raw_storage_getitem_unaligned(
+                    Int32.T, arr.storage, index)
+                char = unichr(codepoint)
+                if char == u'\0':
+                    break
+                builder.append(char)
+        return boxes.W_UnicodeBox(builder.build())
 
     def str_format(self, item, add_quotes=True):
-        raise oefmt(self.space.w_NotImplementedError, "unicode type not completed")
+        raise NotImplementedError
 
     def to_builtin_type(self, space, box):
-        raise oefmt(self.space.w_NotImplementedError, "unicode type not completed")
+        raise NotImplementedError
 
     def eq(self, v1, v2):
-        raise oefmt(self.space.w_NotImplementedError, "unicode type not completed")
+        raise NotImplementedError
 
     def ne(self, v1, v2):
-        raise oefmt(self.space.w_NotImplementedError, "unicode type not completed")
+        raise NotImplementedError
 
     def lt(self, v1, v2):
-        raise oefmt(self.space.w_NotImplementedError, "unicode type not completed")
+        raise NotImplementedError
 
     def le(self, v1, v2):
-        raise oefmt(self.space.w_NotImplementedError, "unicode type not completed")
+        raise NotImplementedError
 
     def gt(self, v1, v2):
-        raise oefmt(self.space.w_NotImplementedError, "unicode type not completed")
+        raise NotImplementedError
 
     def ge(self, v1, v2):
-        raise oefmt(self.space.w_NotImplementedError, "unicode type not completed")
+        raise NotImplementedError
 
     def logical_and(self, v1, v2):
-        raise oefmt(self.space.w_NotImplementedError, "unicode type not completed")
+        raise NotImplementedError
 
     def logical_or(self, v1, v2):
-        raise oefmt(self.space.w_NotImplementedError, "unicode type not completed")
+        raise NotImplementedError
 
     def logical_not(self, v):
-        raise oefmt(self.space.w_NotImplementedError, "unicode type not completed")
+        raise NotImplementedError
 
     @str_binary_op
     def logical_xor(self, v1, v2):
-        raise oefmt(self.space.w_NotImplementedError, "unicode type not completed")
+        raise NotImplementedError
 
     def bool(self, v):
-        raise oefmt(self.space.w_NotImplementedError, "unicode type not completed")
+        raise NotImplementedError
 
     def fill(self, storage, width, box, start, stop, offset, gcstruct):
-        raise oefmt(self.space.w_NotImplementedError, "unicode type not completed")
+        raise NotImplementedError
 
 
 class VoidType(FlexibleType):

From noreply at buildbot.pypy.org  Tue Jun  9 20:53:47 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Tue,  9 Jun 2015 20:53:47 +0200 (CEST)
Subject: [pypy-commit] pypy unicode-dtype: hg merge default
Message-ID: <20150609185347.69B0F1C0695@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: unicode-dtype
Changeset: r78001:32e9412990e1
Date: 2015-06-09 16:20 +0100
http://bitbucket.org/pypy/pypy/changeset/32e9412990e1/

Log:	hg merge default

diff too long, truncating to 2000 out of 2175 lines

diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: cffi
-Version: 1.1.0
+Version: 1.1.2
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI, CDefError, FFIError
 from .ffiplatform import VerificationError, VerificationMissing
 
-__version__ = "1.1.0"
-__version_info__ = (1, 1, 0)
+__version__ = "1.1.2"
+__version_info__ = (1, 1, 2)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/lib_pypy/cffi/gc_weakref.py b/lib_pypy/cffi/gc_weakref.py
--- a/lib_pypy/cffi/gc_weakref.py
+++ b/lib_pypy/cffi/gc_weakref.py
@@ -2,18 +2,23 @@
 
 
 class GcWeakrefs(object):
-    # code copied and adapted from WeakKeyDictionary.
-
     def __init__(self, ffi):
         self.ffi = ffi
-        self.data = data = {}
-        def remove(k):
-            destructor, cdata = data.pop(k)
-            destructor(cdata)
-        self.remove = remove
+        self.data = {}
+        self.nextindex = 0
 
     def build(self, cdata, destructor):
         # make a new cdata of the same type as the original one
         new_cdata = self.ffi.cast(self.ffi._backend.typeof(cdata), cdata)
-        self.data[ref(new_cdata, self.remove)] = destructor, cdata
+        #
+        def remove(key):
+            # careful, this function is not protected by any lock
+            old_key = self.data.pop(index)
+            assert old_key is key
+            destructor(cdata)
+        #
+        key = ref(new_cdata, remove)
+        index = self.nextindex
+        self.nextindex = index + 1     # we're protected by the lock here
+        self.data[index] = key
         return new_cdata
diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py
--- a/lib_pypy/cffi/recompiler.py
+++ b/lib_pypy/cffi/recompiler.py
@@ -775,7 +775,8 @@
             try:
                 if ftype.is_integer_type() or fbitsize >= 0:
                     # accept all integers, but complain on float or double
-                    prnt('  (void)((p->%s) << 1);' % fname)
+                    prnt("  (void)((p->%s) << 1);  /* check that '%s.%s' is "
+                         "an integer */" % (fname, cname, fname))
                     continue
                 # only accept exactly the type declared, except that '[]'
                 # is interpreted as a '*' and so will match any array length.
@@ -949,7 +950,7 @@
             prnt('{')
             prnt('  int n = (%s) <= 0;' % (name,))
             prnt('  *o = (unsigned long long)((%s) << 0);'
-                 '  /* check that we get an integer */' % (name,))
+                 '  /* check that %s is an integer */' % (name, name))
             if check_value is not None:
                 if check_value > 0:
                     check_value = '%dU' % (check_value,)
@@ -1088,8 +1089,9 @@
         self.cffi_types[index] = CffiOp(OP_PRIMITIVE, prim_index)
 
     def _emit_bytecode_UnknownIntegerType(self, tp, index):
-        s = '_cffi_prim_int(sizeof(%s), (((%s)-1) << 0) <= 0)' % (
-            tp.name, tp.name)
+        s = ('_cffi_prim_int(sizeof(%s), (\n'
+             '           ((%s)-1) << 0 /* check that %s is an integer type */\n'
+             '         ) <= 0)' % (tp.name, tp.name, tp.name))
         self.cffi_types[index] = CffiOp(OP_PRIMITIVE, s)
 
     def _emit_bytecode_RawFunctionType(self, tp, index):
diff --git a/lib_pypy/cffi/setuptools_ext.py b/lib_pypy/cffi/setuptools_ext.py
--- a/lib_pypy/cffi/setuptools_ext.py
+++ b/lib_pypy/cffi/setuptools_ext.py
@@ -18,7 +18,9 @@
     # __init__.py files may already try to import the file that
     # we are generating.
     with open(filename) as f:
-        code = compile(f.read(), filename, 'exec')
+        src = f.read()
+    src += '\n'      # Python 2.6 compatibility
+    code = compile(src, filename, 'exec')
     exec(code, glob, glob)
 
 
diff --git a/lib_pypy/cffi/vengine_gen.py b/lib_pypy/cffi/vengine_gen.py
--- a/lib_pypy/cffi/vengine_gen.py
+++ b/lib_pypy/cffi/vengine_gen.py
@@ -402,12 +402,16 @@
         else:
             assert tp is not None
             assert check_value is None
-            prnt(tp.get_c_name(' %s(void)' % funcname, name),)
-            prnt('{')
             if category == 'var':
                 ampersand = '&'
             else:
                 ampersand = ''
+            extra = ''
+            if category == 'const' and isinstance(tp, model.StructOrUnion):
+                extra = 'const *'
+                ampersand = '&'
+            prnt(tp.get_c_name(' %s%s(void)' % (extra, funcname), name))
+            prnt('{')
             prnt('  return (%s%s);' % (ampersand, name))
             prnt('}')
         prnt()
@@ -436,9 +440,14 @@
                 value += (1 << (8*self.ffi.sizeof(BLongLong)))
         else:
             assert check_value is None
-            BFunc = self.ffi._typeof_locked(tp.get_c_name('(*)(void)', name))[0]
+            fntypeextra = '(*)(void)'
+            if isinstance(tp, model.StructOrUnion):
+                fntypeextra = '*' + fntypeextra
+            BFunc = self.ffi._typeof_locked(tp.get_c_name(fntypeextra, name))[0]
             function = module.load_function(BFunc, funcname)
             value = function()
+            if isinstance(tp, model.StructOrUnion):
+                value = value[0]
         return value
 
     def _loaded_gen_constant(self, tp, name, module, library):
diff --git a/lib_pypy/datetime.py b/lib_pypy/datetime.py
--- a/lib_pypy/datetime.py
+++ b/lib_pypy/datetime.py
@@ -536,16 +536,17 @@
         return self
 
     def __repr__(self):
+        module = "datetime." if self.__class__ is timedelta else ""
         if self._microseconds:
-            return "%s(%d, %d, %d)" % ('datetime.' + self.__class__.__name__,
+            return "%s(%d, %d, %d)" % (module + self.__class__.__name__,
                                        self._days,
                                        self._seconds,
                                        self._microseconds)
         if self._seconds:
-            return "%s(%d, %d)" % ('datetime.' + self.__class__.__name__,
+            return "%s(%d, %d)" % (module + self.__class__.__name__,
                                    self._days,
                                    self._seconds)
-        return "%s(%d)" % ('datetime.' + self.__class__.__name__, self._days)
+        return "%s(%d)" % (module + self.__class__.__name__, self._days)
 
     def __str__(self):
         mm, ss = divmod(self._seconds, 60)
@@ -798,7 +799,8 @@
         >>> repr(dt)
         'datetime.datetime(2010, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)'
         """
-        return "%s(%d, %d, %d)" % ('datetime.' + self.__class__.__name__,
+        module = "datetime." if self.__class__ is date else ""
+        return "%s(%d, %d, %d)" % (module + self.__class__.__name__,
                                    self._year,
                                    self._month,
                                    self._day)
@@ -1286,7 +1288,8 @@
             s = ", %d" % self._second
         else:
             s = ""
-        s= "%s(%d, %d%s)" % ('datetime.' + self.__class__.__name__,
+        module = "datetime." if self.__class__ is time else ""
+        s= "%s(%d, %d%s)" % (module + self.__class__.__name__,
                              self._hour, self._minute, s)
         if self._tzinfo is not None:
             assert s[-1:] == ")"
@@ -1698,7 +1701,8 @@
         if L[-1] == 0:
             del L[-1]
         s = ", ".join(map(str, L))
-        s = "%s(%s)" % ('datetime.' + self.__class__.__name__, s)
+        module = "datetime." if self.__class__ is datetime else ""
+        s = "%s(%s)" % (module + self.__class__.__name__, s)
         if self._tzinfo is not None:
             assert s[-1:] == ")"
             s = s[:-1] + ", tzinfo=%r" % self._tzinfo + ")"
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -1,3 +1,4 @@
+import os
 import sys
 
 import py
@@ -38,7 +39,7 @@
     "_csv", "cppyy", "_pypyjson"
 ])
 
-if sys.platform.startswith('linux') and sys.maxint > 2147483647:
+if sys.platform.startswith('linux') and os.uname()[4] == 'x86_64':
     working_modules.add('_vmprof')
 
 translation_modules = default_modules.copy()
diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -21,7 +21,10 @@
     this_dir = os.path.dirname(sys.argv[0])
 
 def debug(msg):
-    os.write(2, "debug: " + msg + '\n')
+    try:
+        os.write(2, "debug: " + msg + '\n')
+    except OSError:
+        pass     # bah, no working stderr :-(
 
 # __________  Entry point  __________
 
diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -2,7 +2,7 @@
 from pypy.interpreter.mixedmodule import MixedModule
 from rpython.rlib import rdynload
 
-VERSION = "1.1.0"
+VERSION = "1.1.2"
 
 
 class Module(MixedModule):
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -3335,4 +3335,4 @@
 
 def test_version():
     # this test is here mostly for PyPy
-    assert __version__ == "1.1.0"
+    assert __version__ == "1.1.2"
diff --git a/pypy/module/_multiprocessing/interp_connection.py b/pypy/module/_multiprocessing/interp_connection.py
--- a/pypy/module/_multiprocessing/interp_connection.py
+++ b/pypy/module/_multiprocessing/interp_connection.py
@@ -28,6 +28,7 @@
 
 class W_BaseConnection(W_Root):
     BUFFER_SIZE = 1024
+    buffer = lltype.nullptr(rffi.CCHARP.TO)
 
     def __init__(self, flags):
         self.flags = flags
@@ -35,7 +36,8 @@
                                     flavor='raw')
 
     def __del__(self):
-        lltype.free(self.buffer, flavor='raw')
+        if self.buffer:
+            lltype.free(self.buffer, flavor='raw')
         try:
             self.do_close()
         except OSError:
@@ -204,6 +206,7 @@
 
 class W_FileConnection(W_BaseConnection):
     INVALID_HANDLE_VALUE = -1
+    fd = INVALID_HANDLE_VALUE
 
     if sys.platform == 'win32':
         def WRITE(self, data):
diff --git a/pypy/module/_multiprocessing/test/test_connection.py b/pypy/module/_multiprocessing/test/test_connection.py
--- a/pypy/module/_multiprocessing/test/test_connection.py
+++ b/pypy/module/_multiprocessing/test/test_connection.py
@@ -189,9 +189,11 @@
         assert data2 == '\x00\x00\x00\x04defg'
 
     def test_repr(self):
-        import _multiprocessing
-        c = _multiprocessing.Connection(1)
-        assert repr(c) == '<read-write Connection, handle 1>'
+        import _multiprocessing, os
+        fd = os.dup(1)     # closed by Connection.__del__
+        c = _multiprocessing.Connection(fd)
+        assert repr(c) == '<read-write Connection, handle %d>' % fd
         if hasattr(_multiprocessing, 'PipeConnection'):
-            c = _multiprocessing.PipeConnection(1)
-            assert repr(c) == '<read-write PipeConnection, handle 1>'
+            fd = os.dup(1)     # closed by PipeConnection.__del__
+            c = _multiprocessing.PipeConnection(fd)
+            assert repr(c) == '<read-write PipeConnection, handle %d>' % fd
diff --git a/pypy/module/cpyext/ndarrayobject.py b/pypy/module/cpyext/ndarrayobject.py
--- a/pypy/module/cpyext/ndarrayobject.py
+++ b/pypy/module/cpyext/ndarrayobject.py
@@ -65,15 +65,7 @@
 @cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL)
 def _PyArray_FLAGS(space, w_array):
     assert isinstance(w_array, W_NDimArray)
-    flags = NPY_BEHAVED_NS
-    if isinstance(w_array.implementation, ConcreteArray):
-        flags |= NPY_OWNDATA
-    if len(w_array.get_shape()) < 2:
-        flags |= NPY_CONTIGUOUS
-    elif w_array.implementation.order == 'C':
-        flags |= NPY_C_CONTIGUOUS
-    else:
-        flags |= NPY_F_CONTIGUOUS
+    flags = NPY_BEHAVED_NS | w_array.get_flags()
     return flags
 
 @cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL)
diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py
--- a/pypy/module/imp/importing.py
+++ b/pypy/module/imp/importing.py
@@ -30,7 +30,15 @@
 IMP_HOOK = 9
 
 SO = '.pyd' if _WIN32 else '.so'
-DEFAULT_SOABI = 'pypy-%d%d' % PYPY_VERSION[:2]
+
+# this used to change for every minor version, but no longer does: there
+# is little point any more, as the so's tend to be cross-version-
+# compatible, more so than between various versions of CPython.  Be
+# careful if we need to update it again: it is now used for both cpyext
+# and cffi so's.  If we do have to update it, we'd likely need a way to
+# split the two usages again.
+#DEFAULT_SOABI = 'pypy-%d%d' % PYPY_VERSION[:2]
+DEFAULT_SOABI = 'pypy-26'
 
 @specialize.memo()
 def get_so_extension(space):
diff --git a/pypy/module/micronumpy/boxes.py b/pypy/module/micronumpy/boxes.py
--- a/pypy/module/micronumpy/boxes.py
+++ b/pypy/module/micronumpy/boxes.py
@@ -10,6 +10,7 @@
 from rpython.rlib.rarithmetic import LONG_BIT
 from rpython.rlib.rstring import StringBuilder
 from rpython.rlib.objectmodel import specialize
+from rpython.rlib import jit
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.tool.sourcetools import func_with_new_name
 from pypy.module.micronumpy import constants as NPY
@@ -66,7 +67,8 @@
         assert isinstance(multiarray, MixedModule)
         scalar = multiarray.get("scalar")
 
-        ret = space.newtuple([scalar, space.newtuple([space.wrap(self._get_dtype(space)), space.wrap(self.raw_str())])])
+        ret = space.newtuple([scalar, space.newtuple(
+            [space.wrap(self._get_dtype(space)), space.wrap(self.raw_str())])])
         return ret
 
 
@@ -368,13 +370,11 @@
             if dtype.elsize != self.get_dtype(space).elsize:
                 raise OperationError(space.w_ValueError, space.wrap(
                     "new type not compatible with array."))
-        if dtype.is_str_or_unicode():
-            return dtype.coerce(space, space.wrap(self.raw_str()))
-        elif dtype.is_record():
+        if dtype.is_record():
             raise OperationError(space.w_NotImplementedError, space.wrap(
                 "viewing scalar as record not implemented"))
         else:
-            return dtype.itemtype.runpack_str(space, self.raw_str())
+            return dtype.runpack_str(space, self.raw_str())
 
     def descr_self(self, space):
         return self
@@ -536,8 +536,20 @@
     def get_dtype(self, space):
         return self.dtype
 
+    @jit.unroll_safe
     def raw_str(self):
-        return self.arr.dtype.itemtype.to_str(self)
+        builder = StringBuilder()
+        i = self.ofs
+        end = i + self.dtype.elsize
+        with self.arr as storage:
+            while i < end:
+                assert isinstance(storage[i], str)
+                if storage[i] == '\x00':
+                    break
+                builder.append(storage[i])
+                i += 1
+            return builder.build()
+
 
 class W_VoidBox(W_FlexibleBox):
     def descr_getitem(self, space, w_item):
@@ -562,7 +574,7 @@
         if isinstance(dtype.itemtype, VoidType):
             read_val = dtype.itemtype.readarray(self.arr, self.ofs, ofs, dtype)
         else:
-            read_val = dtype.itemtype.read(self.arr, self.ofs, ofs, dtype)
+            read_val = dtype.read(self.arr, self.ofs, ofs)
         if isinstance (read_val, W_StringBox):
             # StringType returns a str
             return space.wrap(dtype.itemtype.to_str(read_val))
@@ -582,7 +594,7 @@
             raise oefmt(space.w_IndexError, "222only integers, slices (`:`), "
                 "ellipsis (`...`), numpy.newaxis (`None`) and integer or "
                 "boolean arrays are valid indices")
-        dtype.itemtype.store(self.arr, self.ofs, ofs,
+        dtype.store(self.arr, self.ofs, ofs,
                              dtype.coerce(space, w_value))
 
     def convert_to(self, space, dtype):
diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py
--- a/pypy/module/micronumpy/concrete.py
+++ b/pypy/module/micronumpy/concrete.py
@@ -17,7 +17,6 @@
 from rpython.rtyper.annlowlevel import cast_gcref_to_instance
 from pypy.interpreter.baseobjspace import W_Root
 
-
 class BaseConcreteArray(object):
     _immutable_fields_ = ['dtype?', 'storage', 'start', 'size', 'shape[*]',
                           'strides[*]', 'backstrides[*]', 'order', 'gcstruct',
@@ -44,13 +43,13 @@
         return backstrides
 
     def getitem(self, index):
-        return self.dtype.itemtype.read(self, index, 0)
+        return self.dtype.read(self, index, 0)
 
     def getitem_bool(self, index):
-        return self.dtype.itemtype.read_bool(self, index, 0)
+        return self.dtype.read_bool(self, index, 0)
 
     def setitem(self, index, value):
-        self.dtype.itemtype.store(self, index, 0, value)
+        self.dtype.store(self, index, 0, value)
 
     @jit.unroll_safe
     def setslice(self, space, arr):
@@ -334,12 +333,19 @@
     def get_buffer(self, space, readonly):
         return ArrayBuffer(self, readonly)
 
-    def astype(self, space, dtype):
+    def astype(self, space, dtype, order):
         # copy the general pattern of the strides
         # but make the array storage contiguous in memory
         shape = self.get_shape()
         strides = self.get_strides()
-        if len(strides) > 0:
+        if order not in ('C', 'F'):
+            raise oefmt(space.w_ValueError, "Unknown order %s in astype", order)
+        if len(strides) == 0:
+            t_strides = []
+            backstrides = []
+        elif order != self.order:
+            t_strides, backstrides = calc_strides(shape, dtype, order)
+        else:
             mins = strides[0]
             t_elsize = dtype.elsize
             for s in strides:
@@ -347,10 +353,7 @@
                     mins = s
             t_strides = [s * t_elsize / mins for s in strides]
             backstrides = calc_backstrides(t_strides, shape)
-        else:
-            t_strides = []
-            backstrides = []
-        impl = ConcreteArray(shape, dtype, self.order, t_strides, backstrides)
+        impl = ConcreteArray(shape, dtype, order, t_strides, backstrides)
         loop.setslice(space, impl.get_shape(), impl, self)
         return impl
 
@@ -376,7 +379,7 @@
         gc._trace_callback(callback, arg, storage)
         storage += step
         i += 1
-    
+
 lambda_customtrace = lambda: customtrace
 
 def _setup():
@@ -409,8 +412,9 @@
         self.gcstruct = V_OBJECTSTORE
 
     def fill(self, space, box):
-        self.dtype.itemtype.fill(self.storage, self.dtype.elsize,
-                                 box, 0, self.size, 0, self.gcstruct)
+        self.dtype.itemtype.fill(
+            self.storage, self.dtype.elsize, self.dtype.is_native(),
+            box, 0, self.size, 0, self.gcstruct)
 
     def set_shape(self, space, orig_array, new_shape):
         strides, backstrides = calc_strides(new_shape, self.dtype,
@@ -440,7 +444,7 @@
         gcstruct = V_OBJECTSTORE
         flags = NPY.ARRAY_ALIGNED | NPY.ARRAY_WRITEABLE
         if storage == lltype.nullptr(RAW_STORAGE):
-            length = support.product(shape) 
+            length = support.product(shape)
             if dtype.num == NPY.OBJECT:
                 storage = dtype.itemtype.malloc(length * dtype.elsize, zero=True)
                 gcstruct = _create_objectstore(storage, length, dtype.elsize)
@@ -502,7 +506,7 @@
         ConcreteArray.__init__(self, shape, dtype, order, strides, backstrides,
                     storage, zero)
         self.flags &= ~ NPY.ARRAY_WRITEABLE
-        
+
     def descr_setitem(self, space, orig_array, w_index, w_value):
         raise OperationError(space.w_ValueError, space.wrap(
             "assignment destination is read-only"))
diff --git a/pypy/module/micronumpy/descriptor.py b/pypy/module/micronumpy/descriptor.py
--- a/pypy/module/micronumpy/descriptor.py
+++ b/pypy/module/micronumpy/descriptor.py
@@ -5,8 +5,10 @@
 from pypy.interpreter.gateway import interp2app, unwrap_spec
 from pypy.interpreter.typedef import (TypeDef, GetSetProperty,
                                       interp_attrproperty, interp_attrproperty_w)
+from rpython.annotator.model import SomeChar
 from rpython.rlib import jit
-from rpython.rlib.objectmodel import specialize, compute_hash, we_are_translated
+from rpython.rlib.objectmodel import (
+        specialize, compute_hash, we_are_translated, enforceargs)
 from rpython.rlib.rarithmetic import r_longlong, r_ulonglong
 from pypy.module.micronumpy import types, boxes, support, constants as NPY
 from .base import W_NDimArray
@@ -38,6 +40,15 @@
     out = W_NDimArray.from_shape(space, shape, dtype)
     return out
 
+def byteorder_w(space, w_str):
+    order = space.str_w(w_str)
+    if len(order) != 1:
+        raise oefmt(space.w_ValueError,
+                "endian is not 1-char string in Numpy dtype unpickling")
+    endian = order[0]
+    if endian not in (NPY.LITTLE, NPY.BIG, NPY.NATIVE, NPY.IGNORE):
+        raise oefmt(space.w_ValueError, "Invalid byteorder %s", endian)
+    return endian
 
 
 class W_Dtype(W_Root):
@@ -45,15 +56,13 @@
         "itemtype?", "w_box_type", "byteorder?", "names?", "fields?",
         "elsize?", "alignment?", "shape?", "subdtype?", "base?"]
 
-    def __init__(self, itemtype, w_box_type, byteorder=None, names=[],
+    @enforceargs(byteorder=SomeChar())
+    def __init__(self, itemtype, w_box_type, byteorder=NPY.NATIVE, names=[],
                  fields={}, elsize=None, shape=[], subdtype=None):
         self.itemtype = itemtype
         self.w_box_type = w_box_type
-        if byteorder is None:
-            if itemtype.get_element_size() == 1 or isinstance(itemtype, types.ObjectType):
-                byteorder = NPY.IGNORE
-            else:
-                byteorder = NPY.NATIVE
+        if itemtype.get_element_size() == 1 or isinstance(itemtype, types.ObjectType):
+            byteorder = NPY.IGNORE
         self.byteorder = byteorder
         self.names = names
         self.fields = fields
@@ -137,7 +146,8 @@
         return bool(self.fields)
 
     def is_native(self):
-        return self.byteorder in (NPY.NATIVE, NPY.NATBYTE)
+        # Use ord() to ensure that self.byteorder is a char and JITs properly
+        return ord(self.byteorder) in (ord(NPY.NATIVE), ord(NPY.NATBYTE))
 
     def as_signed(self, space):
         """Convert from an unsigned integer dtype to its signed partner"""
@@ -397,6 +407,20 @@
             return space.wrap(0)
         return space.wrap(len(self.fields))
 
+    def runpack_str(self, space, s):
+        if self.is_str_or_unicode():
+            return self.coerce(space, space.wrap(s))
+        return self.itemtype.runpack_str(space, s, self.is_native())
+
+    def store(self, arr, i, offset, value):
+        return self.itemtype.store(arr, i, offset, value, self.is_native())
+
+    def read(self, arr, i, offset):
+        return self.itemtype.read(arr, i, offset, self)
+
+    def read_bool(self, arr, i, offset):
+        return self.itemtype.read_bool(arr, i, offset, self)
+
     def descr_reduce(self, space):
         w_class = space.type(self)
         builder_args = space.newtuple([
@@ -432,7 +456,7 @@
                         "can't handle version %d of numpy.dtype pickle",
                         version)
 
-        endian = space.str_w(space.getitem(w_data, space.wrap(1)))
+        endian = byteorder_w(space, space.getitem(w_data, space.wrap(1)))
         if endian == NPY.NATBYTE:
             endian = NPY.NATIVE
 
@@ -492,11 +516,10 @@
                 endian = NPY.OPPBYTE if self.is_native() else NPY.NATBYTE
             elif newendian != NPY.IGNORE:
                 endian = newendian
-        itemtype = self.itemtype.__class__(space, endian in (NPY.NATIVE, NPY.NATBYTE))
         fields = self.fields
         if fields is None:
             fields = {}
-        return W_Dtype(itemtype,
+        return W_Dtype(self.itemtype,
                        self.w_box_type, byteorder=endian, elsize=self.elsize,
                        names=self.names, fields=fields,
                        shape=self.shape, subdtype=self.subdtype)
diff --git a/pypy/module/micronumpy/loop.py b/pypy/module/micronumpy/loop.py
--- a/pypy/module/micronumpy/loop.py
+++ b/pypy/module/micronumpy/loop.py
@@ -566,10 +566,7 @@
     while not ai.done(state):
         fromstring_driver.jit_merge_point(dtype=dtype, itemsize=itemsize)
         sub = s[i*itemsize:i*itemsize + itemsize]
-        if dtype.is_str_or_unicode():
-            val = dtype.coerce(space, space.wrap(sub))
-        else:
-            val = dtype.itemtype.runpack_str(space, sub)
+        val = dtype.runpack_str(space, sub)
         ai.setitem(state, val)
         state = ai.next(state)
         i += 1
diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py
--- a/pypy/module/micronumpy/ndarray.py
+++ b/pypy/module/micronumpy/ndarray.py
@@ -20,6 +20,7 @@
 from pypy.module.micronumpy.flagsobj import W_FlagsObject
 from pypy.module.micronumpy.strides import get_shape_from_iterable, \
     shape_agreement, shape_agreement_multiple, is_c_contiguous, is_f_contiguous
+from pypy.module.micronumpy.casting import can_cast_array
 
 
 def _match_dot_shapes(space, left, right):
@@ -43,7 +44,6 @@
         raise oefmt(space.w_ValueError, "objects are not aligned")
     return out_shape, right_critical_dim
 
-
 class __extend__(W_NDimArray):
     @jit.unroll_safe
     def descr_get_shape(self, space):
@@ -279,7 +279,7 @@
                 s.append(separator)
                 s.append(' ')
             if self.is_scalar() and dtype.is_str():
-                s.append(dtype.itemtype.to_str(i.getitem(state)))
+                s.append(i.getitem(state).raw_str())
             else:
                 s.append(dtype.itemtype.str_format(i.getitem(state), add_quotes=True))
             state = i.next(state)
@@ -592,10 +592,11 @@
         if self.is_scalar():
             return space.wrap(0)
         dtype = self.get_dtype().descr_newbyteorder(space, NPY.NATIVE)
-        contig = self.implementation.astype(space, dtype)
+        contig = self.implementation.astype(space, dtype, self.get_order())
         return contig.argsort(space, w_axis)
 
-    def descr_astype(self, space, w_dtype):
+    @unwrap_spec(order=str, casting=str, subok=bool, copy=bool)
+    def descr_astype(self, space, w_dtype, order='K', casting='unsafe', subok=True, copy=True):
         cur_dtype = self.get_dtype()
         new_dtype = space.interp_w(descriptor.W_Dtype, space.call_function(
             space.gettypefor(descriptor.W_Dtype), w_dtype))
@@ -603,13 +604,32 @@
             raise oefmt(space.w_NotImplementedError,
                         "astype(%s) not implemented yet",
                         new_dtype.get_name())
-        if new_dtype.num == NPY.STRING and new_dtype.elsize == 0:
-            if cur_dtype.num == NPY.STRING:
-                new_dtype = descriptor.variable_dtype(
-                    space, 'S' + str(cur_dtype.elsize))
+        if new_dtype.is_str() and new_dtype.elsize == 0:
+            elsize = 0
+            itype = cur_dtype.itemtype
+            for i in range(self.get_size()):
+                elsize = max(elsize, len(itype.str_format(self.implementation.getitem(i), add_quotes=False)))
+            new_dtype = descriptor.variable_dtype(
+                    space, 'S' + str(elsize))
+
+        if not can_cast_array(space, self, new_dtype, casting):
+            raise oefmt(space.w_TypeError, "Cannot cast array from %s to %s"
+                        "according to the rule %s",
+                        space.str_w(self.get_dtype().descr_repr(space)),
+                        space.str_w(new_dtype.descr_repr(space)), casting)
+        order  = support.get_order_as_CF(self.get_order(), order)
+        if (not copy and new_dtype == self.get_dtype() and order == self.get_order()
+                and (subok or type(self) is W_NDimArray)):
+            return self
         impl = self.implementation
-        new_impl = impl.astype(space, new_dtype)
-        return wrap_impl(space, space.type(self), self, new_impl)
+        new_impl = impl.astype(space, new_dtype, order)
+        if new_impl is None:
+            return self
+        if subok:
+            w_type = space.type(self)
+        else:
+            w_type = None
+        return wrap_impl(space, w_type, self, new_impl)
 
     def descr_get_base(self, space):
         impl = self.implementation
diff --git a/pypy/module/micronumpy/nditer.py b/pypy/module/micronumpy/nditer.py
--- a/pypy/module/micronumpy/nditer.py
+++ b/pypy/module/micronumpy/nditer.py
@@ -460,17 +460,18 @@
         # handle w_op_dtypes part 2: copy where needed if possible
         if len(self.dtypes) > 0:
             for i in range(len(self.seq)):
-                selfd = self.dtypes[i]
+                self_d = self.dtypes[i]
                 seq_d = self.seq[i].get_dtype()
-                if not selfd:
+                if not self_d:
                     self.dtypes[i] = seq_d
-                elif selfd != seq_d:
+                elif self_d != seq_d:
                     if not 'r' in self.op_flags[i].tmp_copy:
                         raise oefmt(space.w_TypeError,
                                     "Iterator operand required copying or "
                                     "buffering for operand %d", i)
                     impl = self.seq[i].implementation
-                    new_impl = impl.astype(space, selfd)
+                    order = support.get_order_as_CF(impl.order, self.order)
+                    new_impl = impl.astype(space, self_d, order)
                     self.seq[i] = W_NDimArray(new_impl)
         else:
             #copy them from seq
diff --git a/pypy/module/micronumpy/support.py b/pypy/module/micronumpy/support.py
--- a/pypy/module/micronumpy/support.py
+++ b/pypy/module/micronumpy/support.py
@@ -161,3 +161,14 @@
     w_priority_r = space.findattr(w_rhs, space.wrap('__array_priority__')) or w_zero
     # XXX what is better, unwrapping values or space.gt?
     return space.is_true(space.gt(w_priority_r, w_priority_l))
+
+def get_order_as_CF(proto_order, req_order):
+    if req_order == 'C':
+        return 'C'
+    elif req_order == 'F':
+        return 'F'
+    elif req_order == 'K':
+        return proto_order
+    elif req_order == 'A':
+        return proto_order
+
diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py
--- a/pypy/module/micronumpy/test/test_ndarray.py
+++ b/pypy/module/micronumpy/test/test_ndarray.py
@@ -2249,6 +2249,15 @@
         assert c.shape == b.shape
         assert c.strides == (8,)
 
+        exc = raises(TypeError, a.astype, 'i8', casting='safe')
+        assert exc.value.message.startswith(
+                "Cannot cast array from dtype('complex128') to dtype('int64')")
+        a = arange(6, dtype='f4').reshape(2, 3)
+        b = a.astype('f4', copy=False)
+        assert a is b
+        b = a.astype('f4', order='C', copy=False)
+        assert a is b
+
     def test_base(self):
         from numpy import array
         assert array(1).base is None
diff --git a/pypy/module/micronumpy/test/test_object_arrays.py b/pypy/module/micronumpy/test/test_object_arrays.py
--- a/pypy/module/micronumpy/test/test_object_arrays.py
+++ b/pypy/module/micronumpy/test/test_object_arrays.py
@@ -83,8 +83,8 @@
     def test_complex_op(self):
         import numpy as np
         import sys
-        a = np.array(['abc', 'def'], dtype=object) 
-        b = np.array([1, 2, 3], dtype=object) 
+        a = np.array(['abc', 'def'], dtype=object)
+        b = np.array([1, 2, 3], dtype=object)
         c = np.array([complex(1, 1), complex(1, -1)], dtype=object)
         for arg in (a,b,c):
             assert (arg == np.real(arg)).all()
@@ -164,3 +164,11 @@
         a = np.array([(1, 'object')], dt)
         # Wrong way - should complain about writing buffer to object dtype
         raises(ValueError, np.array, [1, 'object'], dt)
+
+    def test_astype(self):
+        import numpy as np
+        a = np.array([b'a' * 100], dtype='O')
+        assert 'a' * 100 in str(a)
+        b = a.astype('S')
+        assert 'a' * 100 in str(b)
+
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -133,12 +133,11 @@
     return dispatcher
 
 class BaseType(object):
-    _immutable_fields_ = ['native', 'space']
+    _immutable_fields_ = ['space']
     strlen = 0  # chars needed to print any possible value of the type
 
-    def __init__(self, space, native=True):
+    def __init__(self, space):
         assert isinstance(space, ObjSpace)
-        self.native = native
         self.space = space
 
     def __repr__(self):
@@ -199,37 +198,38 @@
     def default_fromstring(self, space):
         raise NotImplementedError
 
-    def _read(self, storage, i, offset):
+    def _read(self, storage, i, offset, native):
         res = raw_storage_getitem_unaligned(self.T, storage, i + offset)
-        if not self.native:
+        if not native:
             res = byteswap(res)
         return res
 
-    def _write(self, storage, i, offset, value):
-        if not self.native:
+    def _write(self, storage, i, offset, value, native):
+        if not native:
             value = byteswap(value)
         raw_storage_setitem_unaligned(storage, i + offset, value)
 
-    def read(self, arr, i, offset, dtype=None):
+    def read(self, arr, i, offset, dtype):
         with arr as storage:
-            return self.box(self._read(storage, i, offset))
-
-    def read_bool(self, arr, i, offset):
+            return self.box(self._read(storage, i, offset, dtype.is_native()))
+
+    def read_bool(self, arr, i, offset, dtype):
         with arr as storage:
-            return bool(self.for_computation(self._read(storage, i, offset)))
-
-    def store(self, arr, i, offset, box):
+            return bool(self.for_computation(
+                self._read(storage, i, offset, dtype.is_native())))
+
+    def store(self, arr, i, offset, box, native):
         with arr as storage:
-            self._write(storage, i, offset, self.unbox(box))
-
-    def fill(self, storage, width, box, start, stop, offset, gcstruct):
+            self._write(storage, i, offset, self.unbox(box), native)
+
+    def fill(self, storage, width, native, box, start, stop, offset, gcstruct):
         value = self.unbox(box)
         for i in xrange(start, stop, width):
-            self._write(storage, i, offset, value)
-
-    def runpack_str(self, space, s):
+            self._write(storage, i, offset, value, native)
+
+    def runpack_str(self, space, s, native):
         v = rffi.cast(self.T, runpack(self.format_code, s))
-        if not self.native:
+        if not native:
             v = byteswap(v)
         return self.box(v)
 
@@ -1058,10 +1058,10 @@
     def box(self, value):
         return self.BoxType(rffi.cast(rffi.DOUBLE, value))
 
-    def runpack_str(self, space, s):
+    def runpack_str(self, space, s, native):
         assert len(s) == 2
         fval = self.box(unpack_float(s, native_is_bigendian))
-        if not self.native:
+        if not native:
             fval = self.byteswap(fval)
         return fval
 
@@ -1074,19 +1074,19 @@
         swapped = byteswap(rffi.cast(self._STORAGE_T, hbits))
         return self.box(float_unpack(r_ulonglong(swapped), 2))
 
-    def _read(self, storage, i, offset):
+    def _read(self, storage, i, offset, native):
         hbits = raw_storage_getitem_unaligned(self._STORAGE_T, storage, i + offset)
-        if not self.native:
+        if not native:
             hbits = byteswap(hbits)
         return float_unpack(r_ulonglong(hbits), 2)
 
-    def _write(self, storage, i, offset, value):
+    def _write(self, storage, i, offset, value, native):
         try:
             hbits = float_pack(value, 2)
         except OverflowError:
             hbits = float_pack(rfloat.INFINITY, 2)
         hbits = rffi.cast(self._STORAGE_T, hbits)
-        if not self.native:
+        if not native:
             hbits = byteswap(hbits)
         raw_storage_setitem_unaligned(storage, i + offset, hbits)
 
@@ -1148,14 +1148,14 @@
         op = '+' if imag >= 0 or rfloat.isnan(imag) else ''
         return ''.join(['(', real_str, op, imag_str, ')'])
 
-    def runpack_str(self, space, s):
-        comp = self.ComponentBoxType._get_dtype(space).itemtype
+    def runpack_str(self, space, s, native):
+        comp = self.ComponentBoxType._get_dtype(space)
         l = len(s) // 2
         real = comp.runpack_str(space, s[:l])
         imag = comp.runpack_str(space, s[l:])
-        if not self.native:
-            real = comp.byteswap(real)
-            imag = comp.byteswap(imag)
+        if not native:
+            real = comp.itemtype.byteswap(real)
+            imag = comp.itemtype.byteswap(imag)
         return self.composite(real, imag)
 
     @staticmethod
@@ -1174,9 +1174,10 @@
         real, imag = self.for_computation(self.unbox(v))
         return bool(real) or bool(imag)
 
-    def read_bool(self, arr, i, offset):
+    def read_bool(self, arr, i, offset, dtype):
         with arr as storage:
-            v = self.for_computation(self._read(storage, i, offset))
+            v = self.for_computation(
+                self._read(storage, i, offset, dtype.is_native()))
             return bool(v[0]) or bool(v[1])
 
     def get_element_size(self):
@@ -1219,35 +1220,35 @@
         assert isinstance(box, self.BoxType)
         return box.real, box.imag
 
-    def _read(self, storage, i, offset):
+    def _read(self, storage, i, offset, native):
         real = raw_storage_getitem_unaligned(self.T, storage, i + offset)
         imag = raw_storage_getitem_unaligned(self.T, storage, i + offset + rffi.sizeof(self.T))
-        if not self.native:
+        if not native:
             real = byteswap(real)
             imag = byteswap(imag)
         return real, imag
 
-    def read(self, arr, i, offset, dtype=None):
+    def read(self, arr, i, offset, dtype):
         with arr as storage:
-            real, imag = self._read(storage, i, offset)
+            real, imag = self._read(storage, i, offset, dtype.is_native())
             return self.box_complex(real, imag)
 
-    def _write(self, storage, i, offset, value):
+    def _write(self, storage, i, offset, value, native):
         real, imag = value
-        if not self.native:
+        if not native:
             real = byteswap(real)
             imag = byteswap(imag)
         raw_storage_setitem_unaligned(storage, i + offset, real)
         raw_storage_setitem_unaligned(storage, i + offset + rffi.sizeof(self.T), imag)
 
-    def store(self, arr, i, offset, box):
+    def store(self, arr, i, offset, box, native):
         with arr as storage:
-            self._write(storage, i, offset, self.unbox(box))
-
-    def fill(self, storage, width, box, start, stop, offset, gcstruct):
+            self._write(storage, i, offset, self.unbox(box), native)
+
+    def fill(self, storage, width, native, box, start, stop, offset, gcstruct):
         value = self.unbox(box)
         for i in xrange(start, stop, width):
-            self._write(storage, i, offset, value)
+            self._write(storage, i, offset, value, native)
 
     @complex_binary_op
     def add(self, v1, v2):
@@ -1745,10 +1746,10 @@
         char = NPY.LONGDOUBLELTR
         BoxType = boxes.W_FloatLongBox
 
-        def runpack_str(self, space, s):
+        def runpack_str(self, space, s, native):
             assert len(s) == boxes.long_double_size
             fval = self.box(unpack_float80(s, native_is_bigendian))
-            if not self.native:
+            if not native:
                 fval = self.byteswap(fval)
             return fval
 
@@ -1788,14 +1789,14 @@
         # return the item itself
         return self.unbox(self.box(w_item))
 
-    def store(self, arr, i, offset, box):
+    def store(self, arr, i, offset, box, native):
         if arr.gcstruct is V_OBJECTSTORE:
             raise oefmt(self.space.w_NotImplementedError,
                 "cannot store object in array with no gc hook")
         self._write(arr.storage, i, offset, self.unbox(box),
                     arr.gcstruct)
 
-    def read(self, arr, i, offset, dtype=None):
+    def read(self, arr, i, offset, dtype):
         return self.box(self._read(arr.storage, i, offset))
 
     def byteswap(self, w_v):
@@ -1814,7 +1815,7 @@
         raw_storage_setitem_unaligned(storage, i + offset, value)
 
     @jit.dont_look_inside
-    def _read(self, storage, i, offset):
+    def _read(self, storage, i, offset, native=True):
         res = raw_storage_getitem_unaligned(self.T, storage, i + offset)
         if we_are_translated():
             gcref = rffi.cast(llmemory.GCREF, res)
@@ -1823,7 +1824,7 @@
             w_obj = _all_objs_for_tests[res]
         return w_obj
 
-    def fill(self, storage, width, box, start, stop, offset, gcstruct):
+    def fill(self, storage, width, native, box, start, stop, offset, gcstruct):
         value = self.unbox(box)
         for i in xrange(start, stop, width):
             self._write(storage, i, offset, value, gcstruct)
@@ -1866,7 +1867,7 @@
     def str_format(self, box, add_quotes=True):
         return self.space.str_w(self.space.repr(self.unbox(box)))
 
-    def runpack_str(self, space, s):
+    def runpack_str(self, space, s, native):
         raise oefmt(space.w_NotImplementedError,
                     "fromstring not implemented for object type")
 
@@ -2051,20 +2052,8 @@
     def get_element_size(self):
         return rffi.sizeof(self.T)
 
-    @jit.unroll_safe
     def to_str(self, item):
-        builder = StringBuilder()
-        assert isinstance(item, boxes.W_FlexibleBox)
-        i = item.ofs
-        end = i + item.dtype.elsize
-        with item.arr as storage:
-            while i < end:
-                assert isinstance(storage[i], str)
-                if storage[i] == '\x00':
-                    break
-                builder.append(storage[i])
-                i += 1
-            return builder.build()
+        return item.raw_str()
 
 def str_unary_op(func):
     specialize.argtype(1)(func)
@@ -2105,7 +2094,7 @@
                 storage[j] = '\x00'
             return boxes.W_StringBox(arr,  0, arr.dtype)
 
-    def store(self, arr, i, offset, box):
+    def store(self, arr, i, offset, box, native):
         assert isinstance(box, boxes.W_StringBox)
         size = min(arr.dtype.elsize - offset, box.arr.size - box.ofs)
         with arr as storage:
@@ -2118,9 +2107,7 @@
             for k in range(size):
                 storage[k + offset + i] = box_storage[k + box.ofs]
 
-    def read(self, arr, i, offset, dtype=None):
-        if dtype is None:
-            dtype = arr.dtype
+    def read(self, arr, i, offset, dtype):
         return boxes.W_StringBox(arr, i + offset, dtype)
 
     def str_format(self, item, add_quotes=True):
@@ -2185,7 +2172,7 @@
     def bool(self, v):
         return bool(self.to_str(v))
 
-    def fill(self, storage, width, box, start, stop, offset, gcstruct):
+    def fill(self, storage, width, native, box, start, stop, offset, gcstruct):
         for i in xrange(start, stop, width):
             self._store(storage, i, offset, box, width)
 
@@ -2205,7 +2192,7 @@
         value = space.unicode_w(w_item)
         return boxes.W_UnicodeBox(value)
 
-    def store(self, arr, i, offset, box):
+    def store(self, arr, i, offset, box, native):
         assert isinstance(box, boxes.W_UnicodeBox)
         value = box._value
         for k in range(len(value)):
@@ -2213,7 +2200,7 @@
             data = rffi.cast(Int32.T, ord(box._value[k]))
             raw_storage_setitem_unaligned(arr.storage, index, data)
 
-    def read(self, arr, i, offset, dtype=None):
+    def read(self, arr, i, offset, dtype):
         if dtype is None:
             dtype = arr.dtype
         size = dtype.elsize // 4
@@ -2269,7 +2256,7 @@
     def bool(self, v):
         raise NotImplementedError
 
-    def fill(self, storage, width, box, start, stop, offset, gcstruct):
+    def fill(self, storage, width, native, box, start, stop, offset, gcstruct):
         raise NotImplementedError
 
 
@@ -2293,8 +2280,8 @@
         itemtype = subdtype.itemtype
         if len(shape) <= 1:
             for i in range(len(items_w)):
-                w_box = itemtype.coerce(space, subdtype, items_w[i])
-                itemtype.store(arr, 0, ofs, w_box)
+                w_box = subdtype.coerce(space, items_w[i])
+                subdtype.store(arr, 0, ofs, w_box)
                 ofs += itemtype.get_element_size()
         else:
             for w_item in items_w:
@@ -2311,13 +2298,13 @@
         return boxes.W_VoidBox(arr, 0, dtype)
 
     @jit.unroll_safe
-    def store(self, arr, i, ofs, box):
+    def store(self, arr, i, offset, box, native):
         assert i == 0
         assert isinstance(box, boxes.W_VoidBox)
         assert box.dtype is box.arr.dtype
         with arr as arr_storage, box.arr as box_storage:
             for k in range(box.arr.dtype.elsize):
-                arr_storage[k + ofs] = box_storage[k + box.ofs]
+                arr_storage[k + offset] = box_storage[k + box.ofs]
 
     def readarray(self, arr, i, offset, dtype=None):
         from pypy.module.micronumpy.base import W_NDimArray
@@ -2330,9 +2317,7 @@
                                     dtype.subdtype)
         return W_NDimArray(implementation)
 
-    def read(self, arr, i, offset, dtype=None):
-        if dtype is None:
-            dtype = arr.dtype
+    def read(self, arr, i, offset, dtype):
         return boxes.W_VoidBox(arr, i + offset, dtype)
 
     @jit.unroll_safe
@@ -2351,10 +2336,11 @@
         ret_unwrapped = []
         for name in dt.names:
             ofs, dtype = dt.fields[name]
+            # XXX: code duplication with W_VoidBox.descr_getitem()
             if isinstance(dtype.itemtype, VoidType):
                 read_val = dtype.itemtype.readarray(item.arr, ofs, 0, dtype)
             else:
-                read_val = dtype.itemtype.read(item.arr, ofs, 0, dtype)
+                read_val = dtype.read(item.arr, ofs, 0)
             if isinstance (read_val, boxes.W_StringBox):
                 # StringType returns a str
                 read_val = space.wrap(dtype.itemtype.to_str(read_val))
@@ -2373,9 +2359,7 @@
     kind = NPY.VOIDLTR
     char = NPY.VOIDLTR
 
-    def read(self, arr, i, offset, dtype=None):
-        if dtype is None:
-            dtype = arr.dtype
+    def read(self, arr, i, offset, dtype):
         return boxes.W_VoidBox(arr, i + offset, dtype)
 
     @jit.unroll_safe
@@ -2410,22 +2394,21 @@
         arr = VoidBoxStorage(dtype.elsize, dtype)
         for i in range(len(dtype.fields)):
             ofs, subdtype = dtype.fields[dtype.names[i]]
-            itemtype = subdtype.itemtype
             try:
-                w_box = itemtype.coerce(space, subdtype, items_w[i])
+                w_box = subdtype.coerce(space, items_w[i])
             except IndexError:
-                w_box = itemtype.coerce(space, subdtype, None)
-            itemtype.store(arr, 0, ofs, w_box)
+                w_box = subdtype.coerce(space, None)
+            subdtype.store(arr, 0, ofs, w_box)
         return boxes.W_VoidBox(arr, 0, dtype)
 
-    def runpack_str(self, space, s):
+    def runpack_str(self, space, s, native):
         raise oefmt(space.w_NotImplementedError,
                     "fromstring not implemented for record types")
 
-    def store(self, arr, i, ofs, box):
+    def store(self, arr, i, offset, box, native):
         assert isinstance(box, boxes.W_VoidBox)
         with arr as storage:
-            self._store(storage, i, ofs, box, box.dtype.elsize)
+            self._store(storage, i, offset, box, box.dtype.elsize)
 
     @jit.unroll_safe
     def _store(self, storage, i, ofs, box, size):
@@ -2433,7 +2416,7 @@
             for k in range(size):
                 storage[k + i + ofs] = box_storage[k + box.ofs]
 
-    def fill(self, storage, width, box, start, stop, offset, gcstruct):
+    def fill(self, storage, width, native, box, start, stop, offset, gcstruct):
         assert isinstance(box, boxes.W_VoidBox)
         assert width == box.dtype.elsize
         for i in xrange(start, stop, width):
@@ -2449,9 +2432,8 @@
         dtype = box.dtype
         for name in dtype.names:
             ofs, subdtype = dtype.fields[name]
-            itemtype = subdtype.itemtype
-            subbox = itemtype.read(box.arr, box.ofs, ofs, subdtype)
-            items.append(itemtype.to_builtin_type(space, subbox))
+            subbox = subdtype.read(box.arr, box.ofs, ofs)
+            items.append(subdtype.itemtype.to_builtin_type(space, subbox))
         return space.newtuple(items)
 
     @jit.unroll_safe
@@ -2461,12 +2443,12 @@
         first = True
         for name in box.dtype.names:
             ofs, subdtype = box.dtype.fields[name]
-            tp = subdtype.itemtype
             if first:
                 first = False
             else:
                 pieces.append(", ")
-            val = tp.read(box.arr, box.ofs, ofs, subdtype)
+            val = subdtype.read(box.arr, box.ofs, ofs)
+            tp = subdtype.itemtype
             pieces.append(tp.str_format(val, add_quotes=add_quotes))
         pieces.append(")")
         return "".join(pieces)
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
@@ -1458,6 +1458,63 @@
         import gc; gc.collect(); gc.collect(); gc.collect()
         assert seen == [1]
 
+    def test_gc_2(self):
+        ffi = FFI(backend=self.Backend())
+        p = ffi.new("int *", 123)
+        seen = []
+        q1 = ffi.gc(p, lambda p: seen.append(1))
+        q2 = ffi.gc(q1, lambda p: seen.append(2))
+        import gc; gc.collect()
+        assert seen == []
+        del q1, q2
+        import gc; gc.collect(); gc.collect(); gc.collect(); gc.collect()
+        assert seen == [2, 1]
+
+    def test_gc_3(self):
+        ffi = FFI(backend=self.Backend())
+        p = ffi.new("int *", 123)
+        r = ffi.new("int *", 123)
+        seen = []
+        seen_r = []
+        q1 = ffi.gc(p, lambda p: seen.append(1))
+        s1 = ffi.gc(r, lambda r: seen_r.append(4))
+        q2 = ffi.gc(q1, lambda p: seen.append(2))
+        s2 = ffi.gc(s1, lambda r: seen_r.append(5))
+        q3 = ffi.gc(q2, lambda p: seen.append(3))
+        import gc; gc.collect()
+        assert seen == []
+        assert seen_r == []
+        del q1, q2, q3, s2, s1
+        import gc; gc.collect(); gc.collect(); gc.collect(); gc.collect()
+        assert seen == [3, 2, 1]
+        assert seen_r == [5, 4]
+
+    def test_gc_4(self):
+        ffi = FFI(backend=self.Backend())
+        p = ffi.new("int *", 123)
+        seen = []
+        q1 = ffi.gc(p, lambda p: seen.append(1))
+        q2 = ffi.gc(q1, lambda p: seen.append(2))
+        q3 = ffi.gc(q2, lambda p: seen.append(3))
+        import gc; gc.collect()
+        assert seen == []
+        del q1, q3     # q2 remains, and has a hard ref to q1
+        import gc; gc.collect(); gc.collect(); gc.collect()
+        assert seen == [3]
+
+    def test_gc_finite_list(self):
+        ffi = FFI(backend=self.Backend())
+        p = ffi.new("int *", 123)
+        keepalive = []
+        for i in range(10):
+            keepalive.append(ffi.gc(p, lambda p: None))
+            assert len(ffi.gc_weakrefs.data) == i + 1  #should be a private attr
+        del keepalive[:]
+        import gc; gc.collect(); gc.collect()
+        for i in range(10):
+            keepalive.append(ffi.gc(p, lambda p: None))
+        assert len(ffi.gc_weakrefs.data) == 10
+
     def test_CData_CType(self):
         ffi = FFI(backend=self.Backend())
         assert isinstance(ffi.cast("int", 0), ffi.CData)
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py
@@ -2228,3 +2228,11 @@
     ffi.cdef("static const int FOO = 123;")
     e = py.test.raises(VerificationError, ffi.verify, "#define FOO 124")
     assert str(e.value).endswith("FOO has the real value 124, not 123")
+
+def test_const_struct_global():
+    ffi = FFI()
+    ffi.cdef("typedef struct { int x; ...; } T; const T myglob;")
+    lib = ffi.verify("typedef struct { double y; int x; } T;"
+                     "const T myglob = { 0.1, 42 };")
+    assert ffi.typeof(lib.myglob) == ffi.typeof("T")
+    assert lib.myglob.x == 42
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zintegration.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zintegration.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zintegration.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zintegration.py
@@ -1,6 +1,5 @@
 # Generated by pypy/tool/import_cffi.py
 import py, os, sys, shutil
-import imp
 import subprocess
 from pypy.module.test_lib_pypy.cffi_tests.udir import udir
 
@@ -16,28 +15,12 @@
     except OSError as e:
         py.test.skip("Cannot execute virtualenv: %s" % (e,))
 
-    try:
-        deepcopy = os.symlink
-    except:
-        import shutil, errno
-        def deepcopy(src, dst):
-            try:
-                shutil.copytree(src, dst)
-            except OSError as e:
-                if e.errno in (errno.ENOTDIR, errno.EINVAL):
-                    shutil.copy(src, dst)
-                else:
-                    print('got errno')
-                    print(e.errno)
-                    print('not')
-                    print(errno.ENOTDIR)
-                    raise
-
     site_packages = None
     for dirpath, dirnames, filenames in os.walk(str(tmpdir)):
         if os.path.basename(dirpath) == 'site-packages':
             site_packages = dirpath
             break
+    paths = ""
     if site_packages:
         try:
             from cffi import _pycparser
@@ -50,15 +33,22 @@
                 pass
             else:
                 modules += ('ply',)   # needed for older versions of pycparser
+        paths = []
         for module in modules:
-            target = imp.find_module(module)[1]
-            deepcopy(target, os.path.join(site_packages,
-                                            os.path.basename(target)))
-    return tmpdir
+            target = __import__(module, None, None, [])
+            src = os.path.abspath(target.__file__)
+            for end in ['__init__.pyc', '__init__.pyo', '__init__.py']:
+                if src.lower().endswith(end):
+                    src = src[:-len(end)-1]
+                    break
+            paths.append(os.path.dirname(src))
+        paths = os.pathsep.join(paths)
+    return tmpdir, paths
 
 SNIPPET_DIR = py.path.local(__file__).join('..', 'snippets')
 
-def really_run_setup_and_program(dirname, venv_dir, python_snippet):
+def really_run_setup_and_program(dirname, venv_dir_and_paths, python_snippet):
+    venv_dir, paths = venv_dir_and_paths
     def remove(dir):
         dir = str(SNIPPET_DIR.join(dirname, dir))
         shutil.rmtree(dir, ignore_errors=True)
@@ -76,9 +66,11 @@
         else:
             bindir = 'bin'
         vp = str(venv_dir.join(bindir).join('python'))
-        subprocess.check_call((vp, 'setup.py', 'clean'))
-        subprocess.check_call((vp, 'setup.py', 'install'))
-        subprocess.check_call((vp, str(python_f)))
+        env = os.environ.copy()
+        env['PYTHONPATH'] = paths
+        subprocess.check_call((vp, 'setup.py', 'clean'), env=env)
+        subprocess.check_call((vp, 'setup.py', 'install'), env=env)
+        subprocess.check_call((vp, str(python_f)), env=env)
     finally:
         os.chdir(olddir)
 
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_ffi_obj.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_ffi_obj.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_ffi_obj.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_ffi_obj.py
@@ -1,5 +1,5 @@
 # Generated by pypy/tool/import_cffi.py
-import py
+import py, sys
 import _cffi_backend as _cffi1_backend
 
 
@@ -66,6 +66,7 @@
     ffi = _cffi1_backend.FFI()
     p = ffi.new("char[]", init=b"foobar\x00baz")
     assert ffi.string(p) == b"foobar"
+    assert ffi.string(cdata=p, maxlen=3) == b"foo"
 
 def test_ffi_errno():
     # xxx not really checking errno, just checking that we can read/write it
@@ -158,11 +159,19 @@
     assert str(e.value) == ("undefined struct/union name\n"
                             "struct never_heard_of_s\n"
                             "       ^")
+    e = py.test.raises(ffi.error, ffi.cast, "\t\n\x01\x1f~\x7f\x80\xff", 0)
+    marks = "?" if sys.version_info < (3,) else "??"
+    assert str(e.value) == ("identifier expected\n"
+                            "  ??~?%s%s\n"
+                            "  ^" % (marks, marks))
+    e = py.test.raises(ffi.error, ffi.cast, "X" * 600, 0)
+    assert str(e.value) == ("undefined type name")
 
 def test_ffi_buffer():
     ffi = _cffi1_backend.FFI()
     a = ffi.new("signed char[]", [5, 6, 7])
     assert ffi.buffer(a)[:] == b'\x05\x06\x07'
+    assert ffi.buffer(cdata=a, size=2)[:] == b'\x05\x06'
 
 def test_ffi_from_buffer():
     import array
@@ -179,3 +188,11 @@
     ffi = _cffi1_backend.FFI()
     assert isinstance(ffi.cast("int", 42), CData)
     assert isinstance(ffi.typeof("int"), CType)
+
+def test_ffi_getwinerror():
+    if sys.platform != "win32":
+        py.test.skip("for windows")
+    ffi = _cffi1_backend.FFI()
+    n = (1 << 29) + 42
+    code, message = ffi.getwinerror(code=n)
+    assert code == n
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py
@@ -33,7 +33,9 @@
         struct ab { int a, b; };
         struct abc { int a, b, c; };
 
-        enum foq { A0, B0, CC0, D0 };
+        /* don't use A0, B0, CC0, D0 because termios.h might be included
+           and it has its own #defines for these names */
+        enum foq { cffiA0, cffiB0, cffiCC0, cffiD0 };
         enum bar { A1, B1=-2, CC1, D1, E1 };
         enum baz { A2=0x1000, B2=0x2000 };
         enum foo2 { A3, B3, C3, D3 };
@@ -879,9 +881,9 @@
 
     def test_enum(self):
         # enum foq { A0, B0, CC0, D0 };
-        assert ffi.string(ffi.cast("enum foq", 0)) == "A0"
-        assert ffi.string(ffi.cast("enum foq", 2)) == "CC0"
-        assert ffi.string(ffi.cast("enum foq", 3)) == "D0"
+        assert ffi.string(ffi.cast("enum foq", 0)) == "cffiA0"
+        assert ffi.string(ffi.cast("enum foq", 2)) == "cffiCC0"
+        assert ffi.string(ffi.cast("enum foq", 3)) == "cffiD0"
         assert ffi.string(ffi.cast("enum foq", 4)) == "4"
         # enum bar { A1, B1=-2, CC1, D1, E1 };
         assert ffi.string(ffi.cast("enum bar", 0)) == "A1"
@@ -1408,6 +1410,47 @@
         import gc; gc.collect(); gc.collect(); gc.collect()
         assert seen == [1]
 
+    def test_gc_2(self):
+        p = ffi.new("int *", 123)
+        seen = []
+        q1 = ffi.gc(p, lambda p: seen.append(1))
+        q2 = ffi.gc(q1, lambda p: seen.append(2))
+        import gc; gc.collect()
+        assert seen == []
+        del q1, q2
+        import gc; gc.collect(); gc.collect(); gc.collect(); gc.collect()
+        assert seen == [2, 1]
+
+    def test_gc_3(self):
+        p = ffi.new("int *", 123)
+        r = ffi.new("int *", 123)
+        seen = []
+        seen_r = []
+        q1 = ffi.gc(p, lambda p: seen.append(1))
+        s1 = ffi.gc(r, lambda r: seen_r.append(4))
+        q2 = ffi.gc(q1, lambda p: seen.append(2))
+        s2 = ffi.gc(s1, lambda r: seen_r.append(5))
+        q3 = ffi.gc(q2, lambda p: seen.append(3))
+        import gc; gc.collect()
+        assert seen == []
+        assert seen_r == []
+        del q1, q2, q3, s2, s1
+        import gc; gc.collect(); gc.collect(); gc.collect(); gc.collect()
+        assert seen == [3, 2, 1]
+        assert seen_r == [5, 4]
+
+    def test_gc_4(self):
+        p = ffi.new("int *", 123)
+        seen = []
+        q1 = ffi.gc(p, lambda p: seen.append(1))
+        q2 = ffi.gc(q1, lambda p: seen.append(2))
+        q3 = ffi.gc(q2, lambda p: seen.append(3))
+        import gc; gc.collect()
+        assert seen == []
+        del q1, q3     # q2 remains, and has a hard ref to q1
+        import gc; gc.collect(); gc.collect(); gc.collect()
+        assert seen == [3]
+
     def test_CData_CType(self):
         assert isinstance(ffi.cast("int", 0), ffi.CData)
         assert isinstance(ffi.new("int *"), ffi.CData)
@@ -1534,8 +1577,8 @@
         assert p.a == -52525
         #
         p = ffi.cast("enum foq", 2)
-        assert ffi.string(p) == "CC0"
-        assert ffi2.sizeof("char[CC0]") == 2
+        assert ffi.string(p) == "cffiCC0"
+        assert ffi2.sizeof("char[cffiCC0]") == 2
         #
         p = ffi.new("anon_foo_t *", [-52526])
         assert p.a == -52526
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_re_python.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_re_python.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_re_python.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_re_python.py
@@ -8,6 +8,7 @@
 
 def setup_module(mod):
     SRC = """
+    #include <string.h>
     #define FOOBAR (-42)
     static const int FOOBAZ = -43;
     #define BIGPOS 420000000000L
@@ -54,6 +55,7 @@
     struct foo_s;
     typedef struct bar_s { int x; signed char a[]; } bar_t;
     enum foo_e { AA, BB, CC };
+    int strlen(const char *);
     """)
     ffi.set_source('re_python_pysrc', None)
     ffi.emit_python_code(str(tmpdir.join('re_python_pysrc.py')))
@@ -82,10 +84,20 @@
 def test_function_with_varargs():
     import _cffi_backend
     from re_python_pysrc import ffi
-    lib = ffi.dlopen(extmod)
+    lib = ffi.dlopen(extmod, 0)
     assert lib.add43(45, ffi.cast("int", -5)) == 45
     assert type(lib.add43) is _cffi_backend.FFI.CData
 
+def test_dlopen_none():
+    import _cffi_backend
+    from re_python_pysrc import ffi
+    name = None
+    if sys.platform == 'win32':
+        import ctypes.util
+        name = ctypes.util.find_msvcrt()
+    lib = ffi.dlopen(name)
+    assert lib.strlen(b"hello") == 5
+
 def test_dlclose():
     import _cffi_backend
     from re_python_pysrc import ffi
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
@@ -993,3 +993,13 @@
     ffi.typeof('function_t*')
     lib.function(ffi.NULL)
     # assert did not crash
+
+def test_alignment_of_longlong():
+    ffi = FFI()
+    x1 = ffi.alignof('unsigned long long')
+    assert x1 in [4, 8]
+    ffi.cdef("struct foo_s { unsigned long long x; };")
+    lib = verify(ffi, 'test_alignment_of_longlong',
+                 "struct foo_s { unsigned long long x; };")
+    assert ffi.alignof('unsigned long long') == x1
+    assert ffi.alignof('struct foo_s') == x1
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py
@@ -2118,25 +2118,19 @@
     try:
         ffi1 = FFI()
         ffi1.cdef("int foo_verify_dlopen_flags;")
-
-        sys.setdlopenflags(ffi1.RTLD_GLOBAL | ffi1.RTLD_LAZY)
+        sys.setdlopenflags(ffi1.RTLD_GLOBAL | ffi1.RTLD_NOW)
         lib1 = ffi1.verify("int foo_verify_dlopen_flags;")
-        lib2 = get_second_lib()
-
-        lib1.foo_verify_dlopen_flags = 42
-        assert lib2.foo_verify_dlopen_flags == 42
-        lib2.foo_verify_dlopen_flags += 1
-        assert lib1.foo_verify_dlopen_flags == 43
     finally:
         sys.setdlopenflags(old)
 
-def get_second_lib():
-    # Hack, using modulename makes the test fail
     ffi2 = FFI()
-    ffi2.cdef("int foo_verify_dlopen_flags;")
-    lib2 = ffi2.verify("int foo_verify_dlopen_flags;",
-                       flags=ffi2.RTLD_GLOBAL | ffi2.RTLD_LAZY)
-    return lib2
+    ffi2.cdef("int *getptr(void);")
+    lib2 = ffi2.verify("""
+        extern int foo_verify_dlopen_flags;
+        static int *getptr(void) { return &foo_verify_dlopen_flags; }
+    """)
+    p = lib2.getptr()
+    assert ffi1.addressof(lib1, 'foo_verify_dlopen_flags') == p
 
 def test_consider_not_implemented_function_type():
     ffi = FFI()
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_zdist.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_zdist.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_zdist.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_zdist.py
@@ -30,13 +30,17 @@
         if hasattr(self, 'saved_cwd'):
             os.chdir(self.saved_cwd)
 
-    def run(self, args):
+    def run(self, args, cwd=None):
         env = os.environ.copy()
-        newpath = self.rootdir
-        if 'PYTHONPATH' in env:
-            newpath += os.pathsep + env['PYTHONPATH']
-        env['PYTHONPATH'] = newpath
-        subprocess.check_call([self.executable] + args, env=env)
+        # a horrible hack to prevent distutils from finding ~/.pydistutils.cfg
+        # (there is the --no-user-cfg option, but not in Python 2.6...)
+        env['HOME'] = '/this/path/does/not/exist'
+        if cwd is None:
+            newpath = self.rootdir
+            if 'PYTHONPATH' in env:
+                newpath += os.pathsep + env['PYTHONPATH']
+            env['PYTHONPATH'] = newpath
+        subprocess.check_call([self.executable] + args, cwd=cwd, env=env)
 
     def _prepare_setuptools(self):
         if hasattr(TestDist, '_setuptools_ready'):
@@ -45,8 +49,7 @@
             import setuptools
         except ImportError:
             py.test.skip("setuptools not found")
-        subprocess.check_call([self.executable, 'setup.py', 'egg_info'],
-                              cwd=self.rootdir)
+        self.run(['setup.py', 'egg_info'], cwd=self.rootdir)
         TestDist._setuptools_ready = True
 
     def check_produced_files(self, content, curdir=None):
diff --git a/pypy/module/test_lib_pypy/test_datetime.py b/pypy/module/test_lib_pypy/test_datetime.py
--- a/pypy/module/test_lib_pypy/test_datetime.py
+++ b/pypy/module/test_lib_pypy/test_datetime.py
@@ -6,9 +6,40 @@
 
 class BaseTestDatetime:
     def test_repr(self):
-        print datetime
-        expected = "datetime.datetime(1, 2, 3, 0, 0)"
-        assert repr(datetime.datetime(1,2,3)) == expected
+        checks = (
+            (datetime.date(2015, 6, 8), "datetime.date(2015, 6, 8)"),
+            (datetime.datetime(2015, 6, 8, 12, 34, 56), "datetime.datetime(2015, 6, 8, 12, 34, 56)"),
+            (datetime.time(12, 34, 56), "datetime.time(12, 34, 56)"),
+            (datetime.timedelta(1), "datetime.timedelta(1)"),
+            (datetime.timedelta(1, 2), "datetime.timedelta(1, 2)"),
+            (datetime.timedelta(1, 2, 3), "datetime.timedelta(1, 2, 3)"),
+        )
+        for obj, expected in checks:
+            assert repr(obj) == expected
+
+    def test_repr_overridden(self):
+        class date_safe(datetime.date):
+            pass
+
+        class datetime_safe(datetime.datetime):
+            pass
+
+        class time_safe(datetime.time):
+            pass
+
+        class timedelta_safe(datetime.timedelta):
+            pass
+
+        checks = (
+            (date_safe(2015, 6, 8), "date_safe(2015, 6, 8)"),
+            (datetime_safe(2015, 6, 8, 12, 34, 56), "datetime_safe(2015, 6, 8, 12, 34, 56)"),
+            (time_safe(12, 34, 56), "time_safe(12, 34, 56)"),
+            (timedelta_safe(1), "timedelta_safe(1)"),
+            (timedelta_safe(1, 2), "timedelta_safe(1, 2)"),
+            (timedelta_safe(1, 2, 3), "timedelta_safe(1, 2, 3)"),
+        )
+        for obj, expected in checks:
+            assert repr(obj) == expected
 
     def test_attributes(self):
         for x in [datetime.date.today(),
diff --git a/rpython/annotator/bookkeeper.py b/rpython/annotator/bookkeeper.py
--- a/rpython/annotator/bookkeeper.py
+++ b/rpython/annotator/bookkeeper.py
@@ -21,6 +21,7 @@
 from rpython.annotator.argument import simple_args
 from rpython.rlib.objectmodel import r_dict, r_ordereddict, Symbolic
 from rpython.tool.algo.unionfind import UnionFind
+from rpython.tool.flattenrec import FlattenRecursion
 from rpython.rtyper import extregistry
 
 
@@ -425,6 +426,8 @@
             self.methoddescs[key] = result
             return result
 
+    _see_mutable_flattenrec = FlattenRecursion()
+
     def see_mutable(self, x):
         key = (x.__class__, x)
         if key in self.seen_mutable:
@@ -433,8 +436,11 @@
         self.seen_mutable[key] = True
         self.event('mutable', x)
         source = InstanceSource(self, x)
-        for attr in source.all_instance_attributes():
-            clsdef.add_source_for_attribute(attr, source) # can trigger reflowing
+        def delayed():
+            for attr in source.all_instance_attributes():
+                clsdef.add_source_for_attribute(attr, source)
+                # ^^^ can trigger reflowing
+        self._see_mutable_flattenrec(delayed)
 
     def valueoftype(self, t):
         return annotationoftype(t, self)
diff --git a/rpython/jit/backend/llsupport/assembler.py b/rpython/jit/backend/llsupport/assembler.py
--- a/rpython/jit/backend/llsupport/assembler.py
+++ b/rpython/jit/backend/llsupport/assembler.py
@@ -109,10 +109,13 @@
                 kind='unicode')
         else:
             self.malloc_slowpath_unicode = None
-        self.cond_call_slowpath = [self._build_cond_call_slowpath(False, False),
-                                   self._build_cond_call_slowpath(False, True),
-                                   self._build_cond_call_slowpath(True, False),
-                                   self._build_cond_call_slowpath(True, True)]
+        lst = [0, 0, 0, 0]
+        lst[0] = self._build_cond_call_slowpath(False, False)
+        lst[1] = self._build_cond_call_slowpath(False, True)
+        if self.cpu.supports_floats:
+            lst[2] = self._build_cond_call_slowpath(True, False)
+            lst[3] = self._build_cond_call_slowpath(True, True)
+        self.cond_call_slowpath = lst
 
         self._build_stack_check_slowpath()
         self._build_release_gil(gc_ll_descr.gcrootmap)
diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -382,7 +382,8 @@
             # we have one word to align
             mc.SUB_ri(esp.value, 7 * WORD) # align and reserve some space
             mc.MOV_sr(WORD, eax.value) # save for later
-            mc.MOVSD_sx(2 * WORD, xmm0.value)   # 32-bit: also 3 * WORD
+            if self.cpu.supports_floats:
+                mc.MOVSD_sx(2 * WORD, xmm0.value)   # 32-bit: also 3 * WORD
             if IS_X86_32:
                 mc.MOV_sr(4 * WORD, edx.value)
                 mc.MOV_sr(0, ebp.value)
@@ -423,7 +424,8 @@
         else:
             if IS_X86_32:
                 mc.MOV_rs(edx.value, 4 * WORD)
-            mc.MOVSD_xs(xmm0.value, 2 * WORD)
+            if self.cpu.supports_floats:
+                mc.MOVSD_xs(xmm0.value, 2 * WORD)
             mc.MOV_rs(eax.value, WORD) # restore
             self._restore_exception(mc, exc0, exc1)
             mc.MOV(exc0, RawEspLoc(WORD * 5, REF))
diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -103,6 +103,7 @@
 
 # ____________________________________________________________
 
+
 def compile_loop(metainterp, greenkey, start,
                  inputargs, jumpargs,
                  full_preamble_needed=True,
@@ -148,27 +149,28 @@
     if part.quasi_immutable_deps:
         loop.quasi_immutable_deps.update(part.quasi_immutable_deps)
     if part.operations[-1].getopnum() == rop.LABEL:
-        inliner = Inliner(inputargs, jumpargs)
-        part.quasi_immutable_deps = None
-        part.operations = [part.operations[-1]] + \
-                          [inliner.inline_op(h_ops[i]) for i in range(start, len(h_ops))] + \
-                          [ResOperation(rop.JUMP, [inliner.inline_arg(a) for a in jumpargs],
-                                        None, descr=jitcell_token)]
-        target_token = part.operations[0].getdescr()
-        assert isinstance(target_token, TargetToken)
-        all_target_tokens.append(target_token)
-        inputargs = jumpargs
-        jumpargs = part.operations[-1].getarglist()
+        if start_state is not None:
+            inliner = Inliner(inputargs, jumpargs)
+            part.quasi_immutable_deps = None
+            part.operations = [part.operations[-1]] + \
+                              [inliner.inline_op(h_ops[i]) for i in range(start, len(h_ops))] + \
+                              [ResOperation(rop.JUMP, [inliner.inline_arg(a) for a in jumpargs],
+                                            None, descr=jitcell_token)]
+            target_token = part.operations[0].getdescr()
+            assert isinstance(target_token, TargetToken)
+            all_target_tokens.append(target_token)
+            inputargs = jumpargs
+            jumpargs = part.operations[-1].getarglist()
 
-        try:
-            optimize_trace(metainterp_sd, jitdriver_sd, part, enable_opts,
-                           start_state=start_state, export_state=False)
-        except InvalidLoop:
-            return None
+            try:
+                optimize_trace(metainterp_sd, jitdriver_sd, part, enable_opts,
+                               start_state=start_state, export_state=False)
+            except InvalidLoop:
+                return None
 
-        loop.operations = loop.operations[:-1] + part.operations
-        if part.quasi_immutable_deps:
-            loop.quasi_immutable_deps.update(part.quasi_immutable_deps)
+            loop.operations = loop.operations[:-1] + part.operations
+            if part.quasi_immutable_deps:
+                loop.quasi_immutable_deps.update(part.quasi_immutable_deps)
     assert part.operations[-1].getopnum() != rop.LABEL
 
     if not loop.quasi_immutable_deps:
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -154,6 +154,22 @@
 
         loop.operations = self.optimizer.get_newoperations()
         if export_state:
+            jd_sd = self.optimizer.jitdriver_sd
+            try:
+                threshold = jd_sd.warmstate.disable_unrolling_threshold
+            except AttributeError:    # tests only
+                threshold = sys.maxint
+            if len(loop.operations) > threshold:
+                if loop.operations[0].getopnum() == rop.LABEL:
+                    # abandoning unrolling, too long
+                    new_descr = stop_label.getdescr()
+                    if loop.operations[0].getopnum() == rop.LABEL:
+                        new_descr = loop.operations[0].getdescr()
+                    stop_label = stop_label.copy_and_change(rop.JUMP,
+                                        descr=new_descr)
+                    self.optimizer.send_extra_operation(stop_label)
+                    loop.operations = self.optimizer.get_newoperations()
+                    return None
             final_state = self.export_state(stop_label)
         else:
             final_state = None
diff --git a/rpython/jit/metainterp/warmspot.py b/rpython/jit/metainterp/warmspot.py
--- a/rpython/jit/metainterp/warmspot.py
+++ b/rpython/jit/metainterp/warmspot.py
@@ -70,7 +70,7 @@
 def jittify_and_run(interp, graph, args, repeat=1, graph_and_interp_only=False,
                     backendopt=False, trace_limit=sys.maxint,
                     inline=False, loop_longevity=0, retrace_limit=5,
-                    function_threshold=4,
+                    function_threshold=4, disable_unrolling=sys.maxint,
                     enable_opts=ALL_OPTS_NAMES, max_retrace_guards=15, 
                     max_unroll_recursion=7, **kwds):
     from rpython.config.config import ConfigError
@@ -95,6 +95,7 @@
         jd.warmstate.set_param_max_retrace_guards(max_retrace_guards)
         jd.warmstate.set_param_enable_opts(enable_opts)
         jd.warmstate.set_param_max_unroll_recursion(max_unroll_recursion)
+        jd.warmstate.set_param_disable_unrolling(disable_unrolling)
     warmrunnerdesc.finish()
     if graph_and_interp_only:
         return interp, graph
diff --git a/rpython/jit/metainterp/warmstate.py b/rpython/jit/metainterp/warmstate.py
--- a/rpython/jit/metainterp/warmstate.py
+++ b/rpython/jit/metainterp/warmstate.py
@@ -256,6 +256,9 @@
     def set_param_inlining(self, value):
         self.inlining = value
 
+    def set_param_disable_unrolling(self, value):
+        self.disable_unrolling_threshold = value
+
     def set_param_enable_opts(self, value):
         from rpython.jit.metainterp.optimizeopt import ALL_OPTS_DICT, ALL_OPTS_NAMES
 
diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py
--- a/rpython/rlib/jit.py
+++ b/rpython/rlib/jit.py
@@ -549,6 +549,7 @@
     'retrace_limit': 'how many times we can try retracing before giving up',
     'max_retrace_guards': 'number of extra guards a retrace can cause',
     'max_unroll_loops': 'number of extra unrollings a loop can cause',
+    'disable_unrolling': 'after how many operations we should not unroll',
     'enable_opts': 'INTERNAL USE ONLY (MAY NOT WORK OR LEAD TO CRASHES): '
                    'optimizations to enable, or all = %s' % ENABLE_ALL_OPTS,
     'max_unroll_recursion': 'how many levels deep to unroll a recursive function'
@@ -564,6 +565,7 @@
               'retrace_limit': 5,
               'max_retrace_guards': 15,
               'max_unroll_loops': 0,
+              'disable_unrolling': 100,
               'enable_opts': 'all',
               'max_unroll_recursion': 7,
               }
diff --git a/rpython/rtyper/rclass.py b/rpython/rtyper/rclass.py
--- a/rpython/rtyper/rclass.py
+++ b/rpython/rtyper/rclass.py
@@ -7,6 +7,7 @@
 from rpython.rlib.objectmodel import UnboxedValue
 from rpython.tool.pairtype import pairtype, pair
 from rpython.tool.identity_dict import identity_dict
+from rpython.tool.flattenrec import FlattenRecursion
 from rpython.rtyper.extregistry import ExtRegistryEntry
 from rpython.rtyper.error import TyperError
 from rpython.rtyper.lltypesystem import lltype
@@ -767,11 +768,14 @@
             self.initialize_prebuilt_data(Ellipsis, self.classdef, result)
             return result
 
+    _initialize_data_flattenrec = FlattenRecursion()
+
     def initialize_prebuilt_instance(self, value, classdef, result):
         # must fill in the hash cache before the other ones
         # (see test_circular_hash_initialization)
         self.initialize_prebuilt_hash(value, result)
-        self.initialize_prebuilt_data(value, classdef, result)
+        self._initialize_data_flattenrec(self.initialize_prebuilt_data,
+                                         value, classdef, result)
 
     def get_ll_hash_function(self):
         return ll_inst_hash
diff --git a/rpython/rtyper/test/test_rclass.py b/rpython/rtyper/test/test_rclass.py
--- a/rpython/rtyper/test/test_rclass.py
+++ b/rpython/rtyper/test/test_rclass.py
@@ -1279,3 +1279,16 @@
             return cls[k](a, b).b
 
         assert self.interpret(f, [1, 4, 7]) == 7
+
+    def test_flatten_convert_const(self):
+        # check that we can convert_const() a chain of more than 1000
+        # instances
+        class A(object):
+            def __init__(self, next):
+                self.next = next
+        a = None

From noreply at buildbot.pypy.org  Tue Jun  9 20:53:48 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Tue,  9 Jun 2015 20:53:48 +0200 (CEST)
Subject: [pypy-commit] pypy unicode-dtype: Add comparison operations for
	unicode
Message-ID: <20150609185348.97BC41C0695@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: unicode-dtype
Changeset: r78002:771d426b475d
Date: 2015-06-09 18:03 +0100
http://bitbucket.org/pypy/pypy/changeset/771d426b475d/

Log:	Add comparison operations for unicode

diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -2223,22 +2223,34 @@
         raise NotImplementedError
 
     def eq(self, v1, v2):
-        raise NotImplementedError
+        assert isinstance(v1, boxes.W_UnicodeBox)
+        assert isinstance(v2, boxes.W_UnicodeBox)
+        return v1._value == v2._value
 
     def ne(self, v1, v2):
-        raise NotImplementedError
+        assert isinstance(v1, boxes.W_UnicodeBox)
+        assert isinstance(v2, boxes.W_UnicodeBox)
+        return v1._value != v2._value
 
     def lt(self, v1, v2):
-        raise NotImplementedError
+        assert isinstance(v1, boxes.W_UnicodeBox)
+        assert isinstance(v2, boxes.W_UnicodeBox)
+        return v1._value < v2._value
 
     def le(self, v1, v2):
-        raise NotImplementedError
+        assert isinstance(v1, boxes.W_UnicodeBox)
+        assert isinstance(v2, boxes.W_UnicodeBox)
+        return v1._value <= v2._value
 
     def gt(self, v1, v2):
-        raise NotImplementedError
+        assert isinstance(v1, boxes.W_UnicodeBox)
+        assert isinstance(v2, boxes.W_UnicodeBox)
+        return v1._value > v2._value
 
     def ge(self, v1, v2):
-        raise NotImplementedError
+        assert isinstance(v1, boxes.W_UnicodeBox)
+        assert isinstance(v2, boxes.W_UnicodeBox)
+        return v1._value >= v2._value
 
     def logical_and(self, v1, v2):
         raise NotImplementedError

From noreply at buildbot.pypy.org  Tue Jun  9 20:53:49 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Tue,  9 Jun 2015 20:53:49 +0200 (CEST)
Subject: [pypy-commit] pypy unicode-dtype: update test,
 since unicode is implemented but sorting still isn't
Message-ID: <20150609185349.B7A9E1C0695@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: unicode-dtype
Changeset: r78003:4cfe2a34d2ff
Date: 2015-06-09 18:25 +0100
http://bitbucket.org/pypy/pypy/changeset/4cfe2a34d2ff/

Log:	update test, since unicode is implemented but sorting still isn't

diff --git a/pypy/module/micronumpy/test/test_selection.py b/pypy/module/micronumpy/test/test_selection.py
--- a/pypy/module/micronumpy/test/test_selection.py
+++ b/pypy/module/micronumpy/test/test_selection.py
@@ -210,22 +210,28 @@
             assert (c == a).all(), msg
 
     def test_sort_unicode(self):
+        import sys
         from numpy import array
         # test unicode sorts.
         s = 'aaaaaaaa'
-        try:
-            a = array([s + chr(i) for i in range(101)], dtype=unicode)
-            b = a[::-1].copy()
-        except:
-            skip('unicode type not supported yet')
-        for kind in ['q', 'm', 'h'] :
+        a = array([s + chr(i) for i in range(101)], dtype=unicode)
+        b = a[::-1].copy()
+        for kind in ['q', 'm', 'h']:
             msg = "unicode sort, kind=%s" % kind
-            c = a.copy();
-            c.sort(kind=kind)
-            assert (c == a).all(), msg
-            c = b.copy();
-            c.sort(kind=kind)
-            assert (c == a).all(), msg
+            c = a.copy()
+            if '__pypy__' in sys.builtin_module_names:
+                exc = raises(NotImplementedError, "c.sort(kind=kind)")
+                assert 'non-numeric types' in exc.value.message
+            else:
+                c.sort(kind=kind)
+                assert (c == a).all(), msg
+            c = b.copy()
+            if '__pypy__' in sys.builtin_module_names:
+                exc = raises(NotImplementedError, "c.sort(kind=kind)")
+                assert 'non-numeric types' in exc.value.message
+            else:
+                c.sort(kind=kind)
+                assert (c == a).all(), msg
 
     def test_sort_objects(self):
         # test object array sorts.

From noreply at buildbot.pypy.org  Tue Jun  9 20:53:50 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Tue,  9 Jun 2015 20:53:50 +0200 (CEST)
Subject: [pypy-commit] pypy unicode-dtype: fix unicode array creation
Message-ID: <20150609185350.DC69A1C0695@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: unicode-dtype
Changeset: r78004:3591f1ef0fb0
Date: 2015-06-09 19:20 +0100
http://bitbucket.org/pypy/pypy/changeset/3591f1ef0fb0/

Log:	fix unicode array creation

diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py
--- a/pypy/module/micronumpy/test/test_ndarray.py
+++ b/pypy/module/micronumpy/test/test_ndarray.py
@@ -325,10 +325,11 @@
 
     def test_unicode(self):
         import numpy as np
-        a = np.array([u'A&#255;', u'abc'], dtype=np.dtype('U'))
-        assert a.shape == (2,)
+        a = np.array([3, u'A&#255;', ''], dtype='U3')
+        assert a.shape == (3,)
         assert a.dtype == np.dtype('U3')
-        assert a[0] == u'A&#255;'
+        assert a[0] == u'3'
+        assert a[1] == u'A&#255;'
 
     def test_dtype_attribute(self):
         import numpy as np
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -2189,7 +2189,7 @@
     def coerce(self, space, dtype, w_item):
         if isinstance(w_item, boxes.W_UnicodeBox):
             return w_item
-        value = space.unicode_w(w_item)
+        value = space.unicode_w(space.unicode_from_object(w_item))
         return boxes.W_UnicodeBox(value)
 
     def store(self, arr, i, offset, box, native):

From noreply at buildbot.pypy.org  Tue Jun  9 20:55:23 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue,  9 Jun 2015 20:55:23 +0200 (CEST)
Subject: [pypy-commit] cffi default: Whatsnew
Message-ID: <20150609185523.8DA541C0695@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2183:abc8ff5b2885
Date: 2015-06-09 20:56 +0200
http://bitbucket.org/cffi/cffi/changeset/abc8ff5b2885/

Log:	Whatsnew

diff --git a/doc/source/whatsnew.rst b/doc/source/whatsnew.rst
--- a/doc/source/whatsnew.rst
+++ b/doc/source/whatsnew.rst
@@ -14,6 +14,10 @@
   ``int a[5][...];`` is supported (but probably less useful: remember
   that in C it means ``int (a[5])[...];``).
 
+* PyPy: the ``lib.some_function`` objects were missing the attributes
+  ``__name__``, ``__module__`` and ``__doc__`` that are expected e.g. by
+  some decorators-management functions from ``functools``.
+
 
 1.1.2
 =====

From noreply at buildbot.pypy.org  Wed Jun 10 07:48:03 2015
From: noreply at buildbot.pypy.org (cfbolz)
Date: Wed, 10 Jun 2015 07:48:03 +0200 (CEST)
Subject: [pypy-commit] benchmarks default: ouch. sorry about that.
Message-ID: <20150610054803.3EF411C0579@cobra.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r330:29a5c9c83f8c
Date: 2015-06-10 07:48 +0200
http://bitbucket.org/pypy/benchmarks/changeset/29a5c9c83f8c/

Log:	ouch. sorry about that.

diff --git a/benchmarks.py b/benchmarks.py
--- a/benchmarks.py
+++ b/benchmarks.py
@@ -83,7 +83,7 @@
              'raytrace-simple', 'crypto_pyaes', 'bm_mako', 'bm_chameleon',
              'json_bench', 'pidigits', 'hexiom2', 'eparse', 'deltablue',
              'bm_dulwich_log', 'bm_krakatau', 'bm_mdp', 'pypy_interp',
-             'sqlitesynth]:
+             'sqlitesynth']:
     _register_new_bm(name, name, globals(), **opts.get(name, {}))
 
 for name in ['names', 'iteration', 'tcp', 'pb', ]:#'web']:#, 'accepts']:

From noreply at buildbot.pypy.org  Wed Jun 10 10:50:38 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Wed, 10 Jun 2015 10:50:38 +0200 (CEST)
Subject: [pypy-commit] jitviewer vmprof-address: adding vmprof address to
 the right side of a loop
Message-ID: <20150610085038.70A4B1C0845@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vmprof-address
Changeset: r272:0d1b70823758
Date: 2015-06-10 10:50 +0200
http://bitbucket.org/pypy/jitviewer/changeset/0d1b70823758/

Log:	adding vmprof address to the right side of a loop extended the
	search to be able to lookup hex addresses (must start with 0x...)

diff --git a/_jitviewer/app.py b/_jitviewer/app.py
--- a/_jitviewer/app.py
+++ b/_jitviewer/app.py
@@ -102,8 +102,10 @@
                                                     limit=1,
                                                     inputargs=loop.inputargs,
                                                     loopname=name)
+                func.start_ofs = loop.start_ofs
             except CannotFindFile:
                 func = DummyFunc()
+                func.start_ofs = -1
             func.count = getattr(loop, 'count', '?')
             func.descr = mangle_descr(loop.descr)
             loops.append(func)
@@ -177,7 +179,11 @@
                 source = CodeRepr(source, code, loop)
             except (IOError, OSError):
                 source = CodeReprNoFile(loop)
+        loop_addr = None
+        if hasattr(orig_loop, 'start_ofs'):
+            loop_addr = hex(orig_loop.start_ofs)[:-1]
         d = {'html': flask.render_template('loop.html',
+                                           loop_addr=loop_addr,
                                            source=source,
                                            current_loop=name,
                                            upper_path=up,
diff --git a/_jitviewer/static/app.js b/_jitviewer/static/app.js
--- a/_jitviewer/static/app.js
+++ b/_jitviewer/static/app.js
@@ -74,9 +74,18 @@
     },
     "#inp-bar keyup": function(el, ev){
         var v = el.val();
+        var number = 0;
+        if (v.indexOf("0x") === 0) {
+          // search for the start offset vmprof address
+          var number = parseInt(v.substring(2), 16);
+        }
         $(".loopitem").each(function (i, l) {
             var name = $(l).attr('name');
-            if(name.search(v) != -1){
+            var show = name.search(v) != -1;
+            if (number !== 0) {
+              show = parseInt($(l).data('start-ofs')) === number;
+            }
+            if(show){
                 $(l).show();
             } else {
                 $(l).hide();
diff --git a/_jitviewer/static/style.css b/_jitviewer/static/style.css
--- a/_jitviewer/static/style.css
+++ b/_jitviewer/static/style.css
@@ -249,5 +249,12 @@
     font-size: 25px;
 }
 
+.vmprof-address {
+    float: right;
+    font-size: 12px;
+    font-weight: bold;
+    padding-right: 10px;
+}
+
 /* End of Formatting 
 -----------------------------------------*/
diff --git a/_jitviewer/templates/index.html b/_jitviewer/templates/index.html
--- a/_jitviewer/templates/index.html
+++ b/_jitviewer/templates/index.html
@@ -38,7 +38,9 @@
   <div id="loops">
     <ul>
       {% for item in loops %}
-      <li class="loopitem" id="{{item.descr}}" name="{{item.repr()}}"><span><a href="#" data-name="{{ item.descr }}">{{item.repr()}}</a> run {{item.count}} times</span></li>
+      <li class="loopitem" id="{{item.descr}}" name="{{item.repr()}}" data-start-ofs="{{item.start_ofs}}">
+        <span><a href="#" data-name="{{ item.descr }}">{{item.repr()}}</a> run {{item.count}} times</span>
+      </li>
       {% endfor %}
     </ul>
     {% if extra_data %}
diff --git a/_jitviewer/templates/loop.html b/_jitviewer/templates/loop.html
--- a/_jitviewer/templates/loop.html
+++ b/_jitviewer/templates/loop.html
@@ -7,6 +7,9 @@
        <div id="line-{{loop.index + source.firstlineno - 1}}" class="source visible">{{sourceline.line}}</div>
        {% if sourceline.chunks %}
           <div class="operations">
+              {% if loop_addr %}
+              <div class="vmprof-address"> vmprof-address {{ loop_addr }}</div>
+              {% endif %}
               {% for chunk in sourceline.chunks %}
                  {% if chunk.is_bytecode %}
                    <span class="{{chunk.cssclass}}"><span class="bytecodepos">{{chunk.bytecode_no}} </span>{{chunk.html_repr()}}</span><br/>

From noreply at buildbot.pypy.org  Wed Jun 10 10:50:39 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Wed, 10 Jun 2015 10:50:39 +0200 (CEST)
Subject: [pypy-commit] jitviewer vmprof-address: merged defaut
Message-ID: <20150610085039.96CD61C0845@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vmprof-address
Changeset: r273:294cee8b9fda
Date: 2015-06-10 10:51 +0200
http://bitbucket.org/pypy/jitviewer/changeset/294cee8b9fda/

Log:	merged defaut

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -4,3 +4,4 @@
 62ad3e746dacc21c8e5dff2a37738659e1b61b7a pypy-2.4
 62ad3e746dacc21c8e5dff2a37738659e1b61b7a pypy-2.3
 ec561fb900e02df04e47b11c413f4a8449cbbb3a pypy-2.5
+3a0152b4ac6b8f930c493ef357fc5e9d8f4b91b7 pypy-2.6.0
diff --git a/_jitviewer/parser.py b/_jitviewer/parser.py
--- a/_jitviewer/parser.py
+++ b/_jitviewer/parser.py
@@ -100,7 +100,8 @@
             
         obj = self.getarg(0)
         return '%s = ((%s.%s)%s).%s' % (self.wrap_html(self.res),
-                                        namespace, classname, obj, field)
+                                        namespace, classname,
+                                        self.wrap_html(obj), field)
 
     def repr_getfield_gc_pure(self):
         return self.repr_getfield_gc() + " [pure]"

From noreply at buildbot.pypy.org  Wed Jun 10 11:41:54 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Wed, 10 Jun 2015 11:41:54 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: vector boxes need to be considered on
	guard exit
Message-ID: <20150610094154.294CF1C0845@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r78005:37e903d27286
Date: 2015-06-10 11:39 +0200
http://bitbucket.org/pypy/pypy/changeset/37e903d27286/

Log:	vector boxes need to be considered on guard exit

diff --git a/rpython/jit/backend/llsupport/assembler.py b/rpython/jit/backend/llsupport/assembler.py
--- a/rpython/jit/backend/llsupport/assembler.py
+++ b/rpython/jit/backend/llsupport/assembler.py
@@ -166,7 +166,8 @@
     def store_info_on_descr(self, startspos, guardtok):
         withfloats = False
         for box in guardtok.failargs:
-            if box is not None and box.type == FLOAT:
+            if box is not None and \
+               (box.type == FLOAT or box.type == VECTOR):
                 withfloats = True
                 break
         exc = guardtok.exc

From noreply at buildbot.pypy.org  Wed Jun 10 11:41:55 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Wed, 10 Jun 2015 11:41:55 +0200 (CEST)
Subject: [pypy-commit] pypy vmprof-address: changes to view the bootstrap
	address of vmprof
Message-ID: <20150610094155.6083C1C0845@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vmprof-address
Changeset: r78006:2ff1050d072a
Date: 2015-06-10 11:41 +0200
http://bitbucket.org/pypy/pypy/changeset/2ff1050d072a/

Log:	changes to view the bootstrap address of vmprof

diff --git a/rpython/jit/backend/tool/viewcode.py b/rpython/jit/backend/tool/viewcode.py
--- a/rpython/jit/backend/tool/viewcode.py
+++ b/rpython/jit/backend/tool/viewcode.py
@@ -57,6 +57,7 @@
         'x86_32': 'i386',
         'x86_64': 'i386:x86-64',
         'x86-64': 'i386:x86-64',
+        'x86-64-sse4': 'i386:x86-64',
         'i386': 'i386',
         'arm': 'arm',
         'arm_32': 'arm',
@@ -264,7 +265,7 @@
                 while j<len(self.ranges) and coderange.touches(self.ranges[j]):
                     coderange.update_from_old(self.ranges[j])
                     j += 1
-                self.ranges[i:j] = [coderange]
+                self.ranges.insert(i, coderange)
             elif line.startswith('LOG '):
                 pieces = line.split(None, 3)
                 assert pieces[1].startswith('@')
diff --git a/rpython/tool/jitlogparser/parser.py b/rpython/tool/jitlogparser/parser.py
--- a/rpython/tool/jitlogparser/parser.py
+++ b/rpython/tool/jitlogparser/parser.py
@@ -443,6 +443,9 @@
                               parser.postprocess(loop, backend_tp=bname,
                                                  backend_dump=dump,
                                                  dump_start=start_ofs))
+            loop.start_ofs = start_ofs
+        else:
+            loop.start_ofs = -1
         loops += split_trace(loop)
     return log, loops
 

From noreply at buildbot.pypy.org  Wed Jun 10 12:06:15 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Wed, 10 Jun 2015 12:06:15 +0200 (CEST)
Subject: [pypy-commit] pypy vmprof-address: manually loading address from
 log to get all addresses and be able to display all assembler dumps
Message-ID: <20150610100615.0AC9B1C1170@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vmprof-address
Changeset: r78007:c1dc41980f3b
Date: 2015-06-10 12:01 +0200
http://bitbucket.org/pypy/pypy/changeset/c1dc41980f3b/

Log:	manually loading address from log to get all addresses and be able
	to display all assembler dumps

diff --git a/rpython/jit/backend/tool/viewcode.py b/rpython/jit/backend/tool/viewcode.py
--- a/rpython/jit/backend/tool/viewcode.py
+++ b/rpython/jit/backend/tool/viewcode.py
@@ -265,7 +265,7 @@
                 while j<len(self.ranges) and coderange.touches(self.ranges[j]):
                     coderange.update_from_old(self.ranges[j])
                     j += 1
-                self.ranges.insert(i, coderange)
+                self.ranges[i:j] = [coderange]
             elif line.startswith('LOG '):
                 pieces = line.split(None, 3)
                 assert pieces[1].startswith('@')
diff --git a/rpython/tool/jitlogparser/parser.py b/rpython/tool/jitlogparser/parser.py
--- a/rpython/tool/jitlogparser/parser.py
+++ b/rpython/tool/jitlogparser/parser.py
@@ -407,16 +407,34 @@
 def import_log(logname, ParserCls=SimpleParser):
     log = parse_log_file(logname)
     addrs = parse_addresses(extract_category(log, 'jit-backend-addr'))
-    from rpython.jit.backend.tool.viewcode import World
-    world = World()
+    from rpython.jit.backend.tool.viewcode import CodeRange
+    ranges = {}
+    backend_name = None
     for entry in extract_category(log, 'jit-backend-dump'):
-        world.parse(entry.splitlines(True))
+        for line in entry.splitlines(True):
+            # copied from class World
+            if line.startswith('BACKEND '):
+                backend_name = line.split(' ')[1].strip()
+            if line.startswith('CODE_DUMP '):
+                pieces = line.split()
+                assert pieces[1].startswith('@')
+                assert pieces[2].startswith('+')
+                if len(pieces) == 3:
+                    continue     # empty line
+                baseaddr = long(pieces[1][1:], 16)
+                if baseaddr < 0:
+                    baseaddr += (2 * sys.maxint + 2)
+                offset = int(pieces[2][1:])
+                addr = baseaddr + offset
+                data = pieces[3].replace(':', '').decode('hex')
+                coderange = CodeRange(None, addr, data)
+                ranges[addr] = coderange
     dumps = {}
-    for r in world.ranges:
-        if r.addr in addrs and addrs[r.addr]:
-            name = addrs[r.addr].pop(0) # they should come in order
-            data = r.data.encode('hex')       # backward compatibility
-            dumps[name] = (world.backend_name, r.addr, data)
+    for addr, rang in ranges.items():
+        if addr in addrs and addrs[addr]:
+            name = addrs[addr].pop(0) # they should come in order
+            data = rang.data.encode('hex') # backward compatibility
+            dumps[name] = (backend_name, addr, data)
     loops = []
     cat = extract_category(log, 'jit-log-opt')
     if not cat:

From noreply at buildbot.pypy.org  Wed Jun 10 12:06:16 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Wed, 10 Jun 2015 12:06:16 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: loading all code regions instead of the
 merged ones from World
Message-ID: <20150610100616.462A51C1170@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r78008:09753b7b85af
Date: 2015-06-10 12:05 +0200
http://bitbucket.org/pypy/pypy/changeset/09753b7b85af/

Log:	loading all code regions instead of the merged ones from World

diff --git a/rpython/tool/jitlogparser/parser.py b/rpython/tool/jitlogparser/parser.py
--- a/rpython/tool/jitlogparser/parser.py
+++ b/rpython/tool/jitlogparser/parser.py
@@ -407,16 +407,35 @@
 def import_log(logname, ParserCls=SimpleParser):
     log = parse_log_file(logname)
     addrs = parse_addresses(extract_category(log, 'jit-backend-addr'))
-    from rpython.jit.backend.tool.viewcode import World
-    world = World()
+    from rpython.jit.backend.tool.viewcode import CodeRange
+    ranges = {}
+    backend_name = None
     for entry in extract_category(log, 'jit-backend-dump'):
-        world.parse(entry.splitlines(True))
+        for line in entry.splitlines(True):
+            # copied from class World
+            if line.startswith('BACKEND '):
+                backend_name = line.split(' ')[1].strip()
+            if line.startswith('CODE_DUMP '):
+                pieces = line.split()
+                assert pieces[1].startswith('@')
+                assert pieces[2].startswith('+')
+                if len(pieces) == 3:
+                    continue     # empty line
+                baseaddr = long(pieces[1][1:], 16)
+                if baseaddr < 0:
+                    baseaddr += (2 * sys.maxint + 2)
+                offset = int(pieces[2][1:])
+                addr = baseaddr + offset
+                data = pieces[3].replace(':', '').decode('hex')
+                coderange = CodeRange(None, addr, data)
+                ranges[addr] = coderange
     dumps = {}
-    for r in world.ranges:
-        if r.addr in addrs and addrs[r.addr]:
-            name = addrs[r.addr].pop(0) # they should come in order
-            data = r.data.encode('hex')       # backward compatibility
-            dumps[name] = (world.backend_name, r.addr, data)
+    for rang in sorted(ranges.values()):
+        addr = rang.addr
+        if addr in addrs and addrs[addr]:
+            name = addrs[addr].pop(0) # they should come in order
+            data = rang.data.encode('hex') # backward compatibility
+            dumps[name] = (backend_name, addr, data)
     loops = []
     cat = extract_category(log, 'jit-log-opt')
     if not cat:
@@ -443,6 +462,9 @@
                               parser.postprocess(loop, backend_tp=bname,
                                                  backend_dump=dump,
                                                  dump_start=start_ofs))
+            loop.start_ofs = start_ofs
+        else:
+            loop.start_ofs = -1
         loops += split_trace(loop)
     return log, loops
 

From noreply at buildbot.pypy.org  Wed Jun 10 12:06:17 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Wed, 10 Jun 2015 12:06:17 +0200 (CEST)
Subject: [pypy-commit] pypy vmprof-address: sorting the code ranges again
Message-ID: <20150610100617.65BC71C1170@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vmprof-address
Changeset: r78009:795973683fee
Date: 2015-06-10 12:06 +0200
http://bitbucket.org/pypy/pypy/changeset/795973683fee/

Log:	sorting the code ranges again

diff --git a/rpython/tool/jitlogparser/parser.py b/rpython/tool/jitlogparser/parser.py
--- a/rpython/tool/jitlogparser/parser.py
+++ b/rpython/tool/jitlogparser/parser.py
@@ -430,7 +430,8 @@
                 coderange = CodeRange(None, addr, data)
                 ranges[addr] = coderange
     dumps = {}
-    for addr, rang in ranges.items():
+    for rang in sorted(ranges.values()):
+        addr = rang.addr
         if addr in addrs and addrs[addr]:
             name = addrs[addr].pop(0) # they should come in order
             data = rang.data.encode('hex') # backward compatibility

From noreply at buildbot.pypy.org  Wed Jun 10 12:06:46 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 10 Jun 2015 12:06:46 +0200 (CEST)
Subject: [pypy-commit] pypy default: Don't put the "-g" option in the "gcc"
 invocation if we invoke it on a
Message-ID: <20150610100646.727561C1170@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78010:61996ba39135
Date: 2015-06-10 12:06 +0200
http://bitbucket.org/pypy/pypy/changeset/61996ba39135/

Log:	Don't put the "-g" option in the "gcc" invocation if we invoke it on
	a ".s" file. It seems that gcc propagates it to "as" in this case,
	but not in case were it is originally called on a ".c" file.

diff --git a/rpython/translator/c/genc.py b/rpython/translator/c/genc.py
--- a/rpython/translator/c/genc.py
+++ b/rpython/translator/c/genc.py
@@ -443,6 +443,12 @@
             mk.definition('OBJECTS1', '$(subst .asmgcc.s,.o,$(subst .c,.o,$(SOURCES)))')
             mk.definition('OBJECTS', '$(OBJECTS1) gcmaptable.s')
 
+            # the CFLAGS passed to gcc when invoked to assembler the .s file
+            # must not contain -g.  This confuses gcc 5.1.  (arigo) I failed
+            # to get any debugging symbols with gcc 5.1 and an older gdb
+            # 7.4.1-debian; I don't understand why at all because it works
+            # fine in manual examples.
+            mk.definition('CFLAGS_AS', '$(patsubst -g,,$(CFLAGS))')
 
             # the rule that transforms %.c into %.o, by compiling it to
             # %.s, then applying trackgcroot to get %.lbl.s and %.gcmap, and
@@ -452,7 +458,7 @@
                     '-o $*.s -S $< $(INCLUDEDIRS)',
                 '$(PYTHON) $(RPYDIR)/translator/c/gcc/trackgcroot.py '
                     '-t $*.s > $*.gctmp',
-                '$(CC) $(CFLAGS) -o $*.o -c $*.lbl.s',
+                '$(CC) $(CFLAGS_AS) -o $*.o -c $*.lbl.s',
                 'mv $*.gctmp $*.gcmap',
                 'rm $*.s $*.lbl.s'])
 

From noreply at buildbot.pypy.org  Wed Jun 10 12:29:24 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 10 Jun 2015 12:29:24 +0200 (CEST)
Subject: [pypy-commit] pypy default: Resolved the confusion with another
 confusion (but less so)
Message-ID: <20150610102924.238E31C03B2@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78011:a10c97822d2a
Date: 2015-06-10 12:29 +0200
http://bitbucket.org/pypy/pypy/changeset/a10c97822d2a/

Log:	Resolved the confusion with another confusion (but less so)

diff --git a/rpython/translator/c/genc.py b/rpython/translator/c/genc.py
--- a/rpython/translator/c/genc.py
+++ b/rpython/translator/c/genc.py
@@ -444,10 +444,9 @@
             mk.definition('OBJECTS', '$(OBJECTS1) gcmaptable.s')
 
             # the CFLAGS passed to gcc when invoked to assembler the .s file
-            # must not contain -g.  This confuses gcc 5.1.  (arigo) I failed
-            # to get any debugging symbols with gcc 5.1 and an older gdb
-            # 7.4.1-debian; I don't understand why at all because it works
-            # fine in manual examples.
+            # must not contain -g.  This confuses gcc 5.1.  (Note that it
+            # would seem that gcc 5.1 with "-g" does not produce debugging
+            # info in a format that gdb 4.7.1 can read.)
             mk.definition('CFLAGS_AS', '$(patsubst -g,,$(CFLAGS))')
 
             # the rule that transforms %.c into %.o, by compiling it to

From noreply at buildbot.pypy.org  Wed Jun 10 14:10:56 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Wed, 10 Jun 2015 14:10:56 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: added a new field to the resume guard
 descr to handle accumulation variables at guard exit
Message-ID: <20150610121056.835851C048F@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r78012:6497803fbba6
Date: 2015-06-10 14:10 +0200
http://bitbucket.org/pypy/pypy/changeset/6497803fbba6/

Log:	added a new field to the resume guard descr to handle accumulation
	variables at guard exit implemented the accumulation for float
	(64/32 bit) for x86 at guard exit (still need to fill the info at
	the new field of resume guard descrs)

diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -1809,6 +1809,12 @@
         """
         self.mc.force_frame_size(DEFAULT_FRAME_BYTES)
         startpos = self.mc.get_relative_pos()
+        # accumulation of a vectorized loop needs to patch
+        # some vector registers (e.g. sum).
+        if guardtok.faildescr.update_at_exit is not None:
+            for pae in guardtok.faildescr.update_at_exit:
+                self._update_at_exit(guardtok.fail_locs,pae)
+            guardtok.fail_descr.update_at_exit = None
         fail_descr, target = self.store_info_on_descr(startpos, guardtok)
         self.mc.PUSH(imm(fail_descr))
         self.push_gcmap(self.mc, guardtok.gcmap, push=True)
@@ -2471,6 +2477,41 @@
     # vector operations
     # ________________________________________
 
+    def _accum_update_at_exit(self, fail_locs, accum_descr):
+        """ If accumulation is done in this loop, at the guard exit
+        some vector registers must be adjusted to yield the correct value"""
+        pass
+        loc = fail_locs[accum_descr.position]
+        vector_var = accum_descr.vector_var
+        scalar_var = accum_descr.scalar_var
+        if accum_descr.operator == '+':
+            # reduction using plus
+            self._accum_reduce_float_sum(vector_var, scalar_var, loc)
+        else:
+            raise NotImplementedError("accum operator %s not implemented" %
+                                        (accum_descr.operator)) 
+
+    def _accum_reduce_sum(self, vector_var, scalar_var, regloc):
+        assert isinstance(vector_var, BoxVector)
+        assert isinstance(scalar_var, Box)
+        #
+        if vector_var.gettype() == FLOAT:
+            if vector_var.getsize() == 8:
+                # r = (r[0]+r[1],r[0]+r[1])
+                self.mc.HADDPD(regloc, regloc)
+                # upper bits (> 64) are dirty (but does not matter)
+                return
+            if vector_var.getsize() == 4:
+                # r = (r[0]+r[1],r[2]+r[3],r[0]+r[1],r[2]+r[3])
+                self.mc.HADDPS(regloc, regloc)
+                self.mc.HADDPS(regloc, regloc)
+                # invoking it a second time will gather the whole sum
+                # at the first element position
+                # the upper bits (>32) are dirty (but does not matter)
+                return
+
+        raise NotImplementedError("reduce sum for %s not impl." % vector_var)
+
     def genop_vec_getarrayitem_raw(self, op, arglocs, resloc):
         # considers item scale (raw_load does not)
         base_loc, ofs_loc, size_loc, ofs, integer_loc, aligned_loc = arglocs
diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -488,7 +488,8 @@
 
 class ResumeGuardDescr(ResumeDescr):
     _attrs_ = ('rd_numb', 'rd_count', 'rd_consts', 'rd_virtuals',
-               'rd_frame_info_list', 'rd_pendingfields', 'status')
+               'rd_frame_info_list', 'rd_pendingfields', 'status',
+               'update_at_exit')
     
     rd_numb = lltype.nullptr(NUMBERING)
     rd_count = 0
@@ -498,6 +499,7 @@
     rd_pendingfields = lltype.nullptr(PENDINGFIELDSP.TO)
 
     status = r_uint(0)
+    update_at_exit = None
 
     def copy_all_attributes_from(self, other):
         assert isinstance(other, ResumeGuardDescr)

From noreply at buildbot.pypy.org  Wed Jun 10 22:06:15 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Wed, 10 Jun 2015 22:06:15 +0200 (CEST)
Subject: [pypy-commit] pypy dtypes-compatability: add and modify tests for
 numpy compatibility, fix shape failures
Message-ID: <20150610200615.625F81C0845@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: dtypes-compatability
Changeset: r78016:1bb47ca4e3f7
Date: 2015-06-10 20:21 +0300
http://bitbucket.org/pypy/pypy/changeset/1bb47ca4e3f7/

Log:	add and modify tests for numpy compatibility, fix shape failures

diff --git a/pypy/module/micronumpy/descriptor.py b/pypy/module/micronumpy/descriptor.py
--- a/pypy/module/micronumpy/descriptor.py
+++ b/pypy/module/micronumpy/descriptor.py
@@ -58,7 +58,7 @@
 
     @enforceargs(byteorder=SomeChar())
     def __init__(self, itemtype, w_box_type, byteorder=NPY.NATIVE, names=[],
-                 fields={}, elsize=None, shape=[], subdtype=None):
+                 fields={}, elsize=-1, shape=[], subdtype=None):
         self.itemtype = itemtype
         self.w_box_type = w_box_type
         if itemtype.get_element_size() == 1 or isinstance(itemtype, types.ObjectType):
@@ -66,7 +66,7 @@
         self.byteorder = byteorder
         self.names = names
         self.fields = fields
-        if elsize is None:
+        if elsize < 0:
             elsize = itemtype.get_element_size()
         self.elsize = elsize
         self.alignment = itemtype.alignment
@@ -277,8 +277,8 @@
         self.names = names
 
     def descr_del_names(self, space):
-        raise OperationError(space.w_AttributeError, space.wrap(
-            "Cannot delete dtype names attribute"))
+        raise oefmt(space.w_AttributeError, 
+            "Cannot delete dtype names attribute")
 
     def eq(self, space, w_other):
         w_other = space.call_function(space.gettypefor(W_Dtype), w_other)
@@ -429,6 +429,7 @@
 
         version = space.wrap(3)
         endian = self.byteorder
+        flags = 0
         if endian == NPY.NATIVE:
             endian = NPY.NATBYTE
         subdescr = self.descr_get_subdtype(space)
@@ -436,14 +437,14 @@
         values = self.descr_get_fields(space)
         if self.is_flexible():
             w_size = space.wrap(self.elsize)
-            alignment = space.wrap(self.alignment)
+            w_alignment = space.wrap(self.alignment)
         else:
             w_size = space.wrap(-1)
-            alignment = space.wrap(-1)
-        flags = space.wrap(0)
+            w_alignment = space.wrap(-1)
+        w_flags = space.wrap(flags)
 
         data = space.newtuple([version, space.wrap(endian), subdescr,
-                               names, values, w_size, alignment, flags])
+                               names, values, w_size, w_alignment, w_flags])
         return space.newtuple([w_class, builder_args, data])
 
     def descr_setstate(self, space, w_data):
@@ -564,8 +565,7 @@
 
 
 def dtype_from_dict(space, w_dict):
-    raise OperationError(space.w_NotImplementedError, space.wrap(
-        "dtype from dict"))
+    raise oefmt(space.w_NotImplementedError, "dtype from dict")
 
 
 def dtype_from_spec(space, w_spec):
@@ -612,22 +612,37 @@
 def descr__new__(space, w_subtype, w_dtype, align=False, w_copy=None, w_shape=None):
     # align and w_copy are necessary for pickling
     cache = get_dtype_cache(space)
-
     if w_shape is not None and (space.isinstance_w(w_shape, space.w_int) or
                                 space.len_w(w_shape) > 0):
         subdtype = descr__new__(space, w_subtype, w_dtype, align, w_copy)
         assert isinstance(subdtype, W_Dtype)
         size = 1
         if space.isinstance_w(w_shape, space.w_int):
+            dim = space.int_w(w_shape)
+            if dim == 1:
+                return subdtype
             w_shape = space.newtuple([w_shape])
         shape = []
         for w_dim in space.fixedview(w_shape):
-            dim = space.int_w(w_dim)
+            try:
+                dim = space.int_w(w_dim)
+            except OperationError as e:
+                if e.match(space, space.w_OverflowError):
+                    raise oefmt(space.w_ValueError, "invalid shape in fixed-type tuple.")
+                else:
+                    raise
+            if dim > 2 ** 32 -1:
+                raise oefmt(space.w_ValueError, "invalid shape in fixed-type tuple: "
+                      "dimension does not fit into a C int.")
+            elif dim < 0:
+                raise oefmt(space.w_ValueError, "invalid shape in fixed-type tuple: "
+                      "dimension smaller than zero.")
             shape.append(dim)
             size *= dim
-        if size == 1:
-            return subdtype
         size *= subdtype.elsize
+        if size >= 2 ** 31:
+            raise oefmt(space.w_ValueError, "invalid shape in fixed-type tuple: "
+                  "dtype size in bytes must fit into a C int.")
         return W_Dtype(types.VoidType(space),
                        space.gettypefor(boxes.W_VoidBox),
                        shape=shape, subdtype=subdtype, elsize=size)
@@ -673,6 +688,10 @@
             return dtype
         if w_dtype is dtype.w_box_type:
             return dtype
+        if space.isinstance_w(w_dtype, space.w_type) and \
+           space.is_true(space.issubtype(w_dtype, dtype.w_box_type)):
+            return cache.w_objectdtype
+            #return W_Dtype(types.VoidType(space), w_box_type=dtype.w_box_type)
     if space.isinstance_w(w_dtype, space.w_type):
         return cache.w_objectdtype
     raise oefmt(space.w_TypeError, "data type not understood")
diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py
--- a/pypy/module/micronumpy/test/test_dtypes.py
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -92,6 +92,7 @@
         assert d == np.dtype('i8')
         assert d.shape == ()
         d = np.dtype((np.int64, 1,))
+        assert d.shape == ()
         assert d == np.dtype('i8')
         assert d.shape == ()
         d = np.dtype((np.int64, 4))
@@ -111,6 +112,7 @@
         assert "int8" == dtype("int8")
         raises(TypeError, lambda: dtype("int8") == 3)
         assert dtype(bool) == bool
+        assert dtype('f8') != dtype(('f8', (1,)))
 
     def test_dtype_cmp(self):
         from numpy import dtype
@@ -342,10 +344,10 @@
         raises(TypeError, type, "Foo", (dtype,), {})
 
     def test_can_subclass(self):
-        import numpy
-        class xyz(numpy.void):
+        import numpy as np
+        class xyz(np.void):
             pass
-        assert True
+        assert np.dtype(xyz).name == 'xyz'
 
     def test_index(self):
         import numpy as np
@@ -413,7 +415,7 @@
         assert loads(dumps(a.dtype)) == a.dtype
         assert np.dtype('bool').__reduce__() == (dtype, ('b1', 0, 1), (3, '|', None, None, None, -1, -1, 0))
         assert np.dtype('|V16').__reduce__() == (dtype, ('V16', 0, 1), (3, '|', None, None, None, 16, 1, 0))
-        assert np.dtype(('<f8', 2)).__reduce__() == (dtype, ('V16', 0, 1), (3, '|', (dtype('float64'), (2,)), None, None, 16, 1, 0))
+        assert np.dtype(('<f8', 2)).__reduce__() == (dtype, ('V16', 0, 1), (3, '|', (dtype('float64'), (2,)), None, None, 16, 8, 0))
 
     def test_newbyteorder(self):
         import numpy as np
@@ -1286,7 +1288,7 @@
                      ('x', 'y', 'z', 'value'),
                      {'y': (dtype('int32'), 4), 'x': (dtype('int32'), 0),
                       'z': (dtype('int32'), 8), 'value': (dtype('float64'), 12),
-                      }, 20, 1, 0))
+                      }, 20, 1, 16))
 
         new_d = loads(dumps(d))
 
@@ -1305,6 +1307,22 @@
 
         assert new_d.itemsize == d.itemsize == 76
 
+    def test_shape_invalid(self):
+        import numpy as np
+        # Check that the shape is valid.
+        max_int = 2 ** (8 * 4 - 1)
+        max_intp = 2 ** (8 * np.dtype('intp').itemsize - 1) - 1
+        # Too large values (the datatype is part of this)
+        raises(ValueError, np.dtype, [('a', 'f4', max_int // 4 + 1)])
+        raises(ValueError, np.dtype, [('a', 'f4', max_int + 1)])
+        raises(ValueError, np.dtype, [('a', 'f4', (max_int, 2))])
+        # Takes a different code path (fails earlier:
+        raises(ValueError, np.dtype, [('a', 'f4', max_intp + 1)])
+        # Negative values
+        raises(ValueError, np.dtype, [('a', 'f4', -1)])
+        raises(ValueError, np.dtype, [('a', 'f4', (-1, -1))])
+
+
 class AppTestNotDirect(BaseNumpyAppTest):
     def setup_class(cls):
         BaseNumpyAppTest.setup_class.im_func(cls)

From noreply at buildbot.pypy.org  Wed Jun 10 22:06:16 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Wed, 10 Jun 2015 22:06:16 +0200 (CEST)
Subject: [pypy-commit] pypy dtypes-compatability: fix subclass test,
	break translation
Message-ID: <20150610200616.88F711C0845@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: dtypes-compatability
Changeset: r78017:b65ab68b4506
Date: 2015-06-10 21:10 +0300
http://bitbucket.org/pypy/pypy/changeset/b65ab68b4506/

Log:	fix subclass test, break translation

diff --git a/pypy/module/micronumpy/descriptor.py b/pypy/module/micronumpy/descriptor.py
--- a/pypy/module/micronumpy/descriptor.py
+++ b/pypy/module/micronumpy/descriptor.py
@@ -690,8 +690,7 @@
             return dtype
         if space.isinstance_w(w_dtype, space.w_type) and \
            space.is_true(space.issubtype(w_dtype, dtype.w_box_type)):
-            return cache.w_objectdtype
-            #return W_Dtype(types.VoidType(space), w_box_type=dtype.w_box_type)
+            return W_Dtype(dtype.itemtype, w_box_type=w_dtype, elsize=0)
     if space.isinstance_w(w_dtype, space.w_type):
         return cache.w_objectdtype
     raise oefmt(space.w_TypeError, "data type not understood")

From noreply at buildbot.pypy.org  Wed Jun 10 22:06:17 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Wed, 10 Jun 2015 22:06:17 +0200 (CEST)
Subject: [pypy-commit] pypy dtypes-compatability: add, implement flags,
	fix alignment
Message-ID: <20150610200617.C4A9A1C0845@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: dtypes-compatability
Changeset: r78018:37b77ed73855
Date: 2015-06-10 23:04 +0300
http://bitbucket.org/pypy/pypy/changeset/37b77ed73855/

Log:	add, implement flags, fix alignment

diff --git a/pypy/module/micronumpy/constants.py b/pypy/module/micronumpy/constants.py
--- a/pypy/module/micronumpy/constants.py
+++ b/pypy/module/micronumpy/constants.py
@@ -92,6 +92,21 @@
 ARRAY_ELEMENTSTRIDES = 0x0080 # strides  are units of the dtype element size
 ARRAY_NOTSWAPPED  = 0x0200 #native byte order
 
+#dtype flags
+ITEM_REFCOUNT   = 0x01
+ITEM_HASOBJECT  = 0x01
+LIST_PICKLE     = 0x02
+ITEM_IS_POINTER = 0x04
+NEEDS_INIT      = 0x08
+NEEDS_PYAPI     = 0x10
+USE_GETITEM     = 0x20
+USE_SETITEM     = 0x40
+ALIGNED_STRUCT  = 0x80
+FROM_FIELDS     = NEEDS_INIT | LIST_PICKLE | ITEM_REFCOUNT | NEEDS_PYAPI
+OBJECT_DTYPE_FLAGS = (LIST_PICKLE | USE_GETITEM | ITEM_IS_POINTER |
+                        ITEM_REFCOUNT | NEEDS_INIT | NEEDS_PYAPI)
+
+
 LITTLE = '<'
 BIG = '>'
 NATIVE = '='
diff --git a/pypy/module/micronumpy/descriptor.py b/pypy/module/micronumpy/descriptor.py
--- a/pypy/module/micronumpy/descriptor.py
+++ b/pypy/module/micronumpy/descriptor.py
@@ -54,7 +54,7 @@
 class W_Dtype(W_Root):
     _immutable_fields_ = [
         "itemtype?", "w_box_type", "byteorder?", "names?", "fields?",
-        "elsize?", "alignment?", "shape?", "subdtype?", "base?"]
+        "elsize?", "alignment?", "shape?", "subdtype?", "base?", "flags?"]
 
     @enforceargs(byteorder=SomeChar())
     def __init__(self, itemtype, w_box_type, byteorder=NPY.NATIVE, names=[],
@@ -69,13 +69,17 @@
         if elsize < 0:
             elsize = itemtype.get_element_size()
         self.elsize = elsize
-        self.alignment = itemtype.alignment
         self.shape = shape
         self.subdtype = subdtype
+        self.flags = 0
+        if isinstance(itemtype, types.ObjectType):
+            self.flags = NPY.OBJECT_DTYPE_FLAGS
         if not subdtype:
             self.base = self
+            self.alignment = itemtype.get_element_size()
         else:
             self.base = subdtype.base
+            self.alignment = subdtype.itemtype.get_element_size()
 
     @property
     def num(self):
@@ -216,7 +220,7 @@
             return space.newlist(descr)
 
     def descr_get_hasobject(self, space):
-        return space.w_False
+        return space.wrap(self.is_object())
 
     def descr_get_isbuiltin(self, space):
         if self.fields is None:
@@ -238,6 +242,9 @@
     def descr_get_shape(self, space):
         return space.newtuple([space.wrap(dim) for dim in self.shape])
 
+    def descr_get_flags(self, space):
+        return space.wrap(self.flags)
+
     def descr_get_fields(self, space):
         if not self.fields:
             return space.w_None
@@ -429,7 +436,6 @@
 
         version = space.wrap(3)
         endian = self.byteorder
-        flags = 0
         if endian == NPY.NATIVE:
             endian = NPY.NATBYTE
         subdescr = self.descr_get_subdtype(space)
@@ -441,7 +447,7 @@
         else:
             w_size = space.wrap(-1)
             w_alignment = space.wrap(-1)
-        w_flags = space.wrap(flags)
+        w_flags = space.wrap(self.flags)
 
         data = space.newtuple([version, space.wrap(endian), subdescr,
                                names, values, w_size, w_alignment, w_flags])
@@ -466,6 +472,7 @@
         w_fields = space.getitem(w_data, space.wrap(4))
         size = space.int_w(space.getitem(w_data, space.wrap(5)))
         alignment = space.int_w(space.getitem(w_data, space.wrap(6)))
+        flags = space.int_w(space.getitem(w_data, space.wrap(7)))
 
         if (w_names == space.w_None) != (w_fields == space.w_None):
             raise oefmt(space.w_ValueError, "inconsistent fields and names in Numpy dtype unpickling")
@@ -507,6 +514,7 @@
         if self.is_flexible():
             self.elsize = size
             self.alignment = alignment
+        self.flags = flags
 
     @unwrap_spec(new_order=str)
     def descr_newbyteorder(self, space, new_order=NPY.SWAP):
@@ -560,12 +568,14 @@
     if align:
         # Set offset to the next power-of-two above offset
         offset = (offset + maxalign -1) & (-maxalign)
-    return W_Dtype(types.RecordType(space), space.gettypefor(boxes.W_VoidBox),
+    retval = W_Dtype(types.RecordType(space), space.gettypefor(boxes.W_VoidBox),
                    names=names, fields=fields, elsize=offset)
-
+    retval.flags |= NPY.NEEDS_PYAPI
+    return retval
 
 def dtype_from_dict(space, w_dict):
     raise oefmt(space.w_NotImplementedError, "dtype from dict")
+    retval.flags |= NPY.NEEDS_PYAPI
 
 
 def dtype_from_spec(space, w_spec):
@@ -690,7 +700,7 @@
             return dtype
         if space.isinstance_w(w_dtype, space.w_type) and \
            space.is_true(space.issubtype(w_dtype, dtype.w_box_type)):
-            return W_Dtype(dtype.itemtype, w_box_type=w_dtype, elsize=0)
+            return W_Dtype(dtype.itemtype, w_dtype, elsize=0)
     if space.isinstance_w(w_dtype, space.w_type):
         return cache.w_objectdtype
     raise oefmt(space.w_TypeError, "data type not understood")
@@ -720,6 +730,7 @@
     names = GetSetProperty(W_Dtype.descr_get_names,
                            W_Dtype.descr_set_names,
                            W_Dtype.descr_del_names),
+    flags = GetSetProperty(W_Dtype.descr_get_flags),
 
     __eq__ = interp2app(W_Dtype.descr_eq),
     __ne__ = interp2app(W_Dtype.descr_ne),

From noreply at buildbot.pypy.org  Wed Jun 10 22:24:59 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Wed, 10 Jun 2015 22:24:59 +0200 (CEST)
Subject: [pypy-commit] pypy unicode-dtype: Implement UnicodeType.fill()
Message-ID: <20150610202459.507DD1C048F@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: unicode-dtype
Changeset: r78019:f3f7e88f6b92
Date: 2015-06-10 05:12 +0100
http://bitbucket.org/pypy/pypy/changeset/f3f7e88f6b92/

Log:	Implement UnicodeType.fill()

diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py
--- a/pypy/module/micronumpy/test/test_ndarray.py
+++ b/pypy/module/micronumpy/test/test_ndarray.py
@@ -389,6 +389,9 @@
         assert zeros((), dtype='S') == ''
         assert zeros((), dtype='S').shape == ()
         assert zeros((), dtype='S').dtype == '|S1'
+        assert zeros(5, dtype='U')[4] == u''
+        assert zeros(5, dtype='U').shape == (5,)
+        assert zeros(5, dtype='U').dtype == '<U1'
 
     def test_check_shape(self):
         import numpy as np
@@ -2432,6 +2435,12 @@
         a.fill(12)
         assert (a == '1').all()
 
+    def test_unicode_filling(self):
+        import numpy as np
+        a = np.empty((10,10), dtype='U1')
+        a.fill(12)
+        assert (a == u'1').all()
+
     def test_boolean_indexing(self):
         import numpy as np
         a = np.zeros((1, 3))
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -50,6 +50,7 @@
             pass
         return _raw_storage_getitem_unaligned(T, storage, offset)
 '''
+
 def simple_unary_op(func):
     specialize.argtype(1)(func)
     @functools.wraps(func)
@@ -2195,10 +2196,20 @@
     def store(self, arr, i, offset, box, native):
         assert isinstance(box, boxes.W_UnicodeBox)
         value = box._value
-        for k in range(len(value)):
+        with arr as storage:
+            self._store(storage, i, offset, box, arr.dtype.elsize)
+
+    @jit.unroll_safe
+    def _store(self, storage, i, offset, box, width):
+        size = min(width // 4, len(box._value))
+        for k in range(size):
             index = i + offset + 4*k
             data = rffi.cast(Int32.T, ord(box._value[k]))
-            raw_storage_setitem_unaligned(arr.storage, index, data)
+            raw_storage_setitem_unaligned(storage, index, data)
+        for k in range(size, width // 4):
+            index = i + offset + 4*k
+            data = rffi.cast(Int32.T, 0)
+            raw_storage_setitem_unaligned(storage, index, data)
 
     def read(self, arr, i, offset, dtype):
         if dtype is None:
@@ -2269,7 +2280,9 @@
         raise NotImplementedError
 
     def fill(self, storage, width, native, box, start, stop, offset, gcstruct):
-        raise NotImplementedError
+        assert isinstance(box, boxes.W_UnicodeBox)
+        for i in xrange(start, stop, width):
+            self._store(storage, i, offset, box, width)
 
 
 class VoidType(FlexibleType):

From noreply at buildbot.pypy.org  Wed Jun 10 22:25:00 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Wed, 10 Jun 2015 22:25:00 +0200 (CEST)
Subject: [pypy-commit] pypy unicode-dtype: Handle str() and repr() on
	unicode scalars
Message-ID: <20150610202500.94DAA1C048F@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: unicode-dtype
Changeset: r78020:41c8a260855b
Date: 2015-06-10 19:48 +0100
http://bitbucket.org/pypy/pypy/changeset/41c8a260855b/

Log:	Handle str() and repr() on unicode scalars

diff --git a/pypy/module/micronumpy/test/test_object_arrays.py b/pypy/module/micronumpy/test/test_object_arrays.py
--- a/pypy/module/micronumpy/test/test_object_arrays.py
+++ b/pypy/module/micronumpy/test/test_object_arrays.py
@@ -171,4 +171,8 @@
         assert 'a' * 100 in str(a)
         b = a.astype('S')
         assert 'a' * 100 in str(b)
-
+        a = np.array([123], dtype='U')
+        assert a[0] == u'123'
+        b = a.astype('O')
+        assert b[0] == u'123'
+        assert type(b[0]) is unicode
diff --git a/pypy/module/micronumpy/test/test_scalar.py b/pypy/module/micronumpy/test/test_scalar.py
--- a/pypy/module/micronumpy/test/test_scalar.py
+++ b/pypy/module/micronumpy/test/test_scalar.py
@@ -1,3 +1,4 @@
+# -*- encoding:utf-8 -*-
 from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest
 
 class AppTestScalar(BaseNumpyAppTest):
@@ -462,9 +463,20 @@
         from numpy import str_
         assert isinstance(str_(3), str_)
         assert str_(3) == '3'
+        assert str(str_(3)) == '3'
+        assert repr(str_(3)) == "'3'"
 
     def test_unicode_boxes(self):
         from numpy import unicode_
         u = unicode_(3)
         assert isinstance(u, unicode)
         assert u == u'3'
+
+    def test_unicode_repr(self):
+        from numpy import unicode_
+        u = unicode_(3)
+        assert str(u) == '3'
+        assert repr(u) == "u'3'"
+        u = unicode_(u'A&#255;')
+        raises(UnicodeEncodeError, "str(u)")
+        assert repr(u) == repr(u'A&#255;')
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -2228,10 +2228,19 @@
         return boxes.W_UnicodeBox(builder.build())
 
     def str_format(self, item, add_quotes=True):
-        raise NotImplementedError
+        assert isinstance(item, boxes.W_UnicodeBox)
+        if add_quotes:
+            w_unicode = self.to_builtin_type(self.space, item)
+            return self.space.str_w(self.space.repr(w_unicode))
+        else:
+            # Same as W_UnicodeBox.descr_repr() but without quotes and prefix
+            from rpython.rlib.runicode import unicode_encode_unicode_escape
+            return unicode_encode_unicode_escape(item._value,
+                                                 len(item._value), 'strict')
 
     def to_builtin_type(self, space, box):
-        raise NotImplementedError
+        assert isinstance(box, boxes.W_UnicodeBox)
+        return space.wrap(box._value)
 
     def eq(self, v1, v2):
         assert isinstance(v1, boxes.W_UnicodeBox)

From noreply at buildbot.pypy.org  Wed Jun 10 22:25:01 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Wed, 10 Jun 2015 22:25:01 +0200 (CEST)
Subject: [pypy-commit] pypy unicode-dtype: Fix repr() of str and unicode
	scalars
Message-ID: <20150610202501.CB5DE1C048F@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: unicode-dtype
Changeset: r78021:b74fb5aa4741
Date: 2015-06-10 21:22 +0100
http://bitbucket.org/pypy/pypy/changeset/b74fb5aa4741/

Log:	Fix repr() of str and unicode scalars

diff --git a/pypy/module/micronumpy/boxes.py b/pypy/module/micronumpy/boxes.py
--- a/pypy/module/micronumpy/boxes.py
+++ b/pypy/module/micronumpy/boxes.py
@@ -196,7 +196,12 @@
                     "'%T' object is not iterable", self)
 
     def descr_str(self, space):
-        return space.wrap(self.get_dtype(space).itemtype.str_format(self, add_quotes=False))
+        tp = self.get_dtype(space).itemtype
+        return space.wrap(tp.str_format(self, add_quotes=False))
+
+    def descr_repr(self, space):
+        tp = self.get_dtype(space).itemtype
+        return space.wrap(tp.str_format(self, add_quotes=True))
 
     def descr_format(self, space, w_spec):
         return space.format(self.item(space), w_spec)
@@ -658,7 +663,7 @@
     __getitem__ = interp2app(W_GenericBox.descr_getitem),
     __iter__ = interp2app(W_GenericBox.descr_iter),
     __str__ = interp2app(W_GenericBox.descr_str),
-    __repr__ = interp2app(W_GenericBox.descr_str),
+    __repr__ = interp2app(W_GenericBox.descr_repr),
     __format__ = interp2app(W_GenericBox.descr_format),
     __int__ = interp2app(W_GenericBox.descr_int),
     __long__ = interp2app(W_GenericBox.descr_long),
diff --git a/pypy/module/micronumpy/test/test_scalar.py b/pypy/module/micronumpy/test/test_scalar.py
--- a/pypy/module/micronumpy/test/test_scalar.py
+++ b/pypy/module/micronumpy/test/test_scalar.py
@@ -478,5 +478,5 @@
         assert str(u) == '3'
         assert repr(u) == "u'3'"
         u = unicode_(u'A&#255;')
-        raises(UnicodeEncodeError, "str(u)")
+        # raises(UnicodeEncodeError, "str(u)")  # XXX
         assert repr(u) == repr(u'A&#255;')

From noreply at buildbot.pypy.org  Wed Jun 10 22:50:14 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 10 Jun 2015 22:50:14 +0200 (CEST)
Subject: [pypy-commit] stmgc c8-gil-like: fix fix fix
Message-ID: <20150610205014.81D751C11BD@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: c8-gil-like
Changeset: r1797:152b52431340
Date: 2015-06-10 16:51 +0200
http://bitbucket.org/pypy/stmgc/changeset/152b52431340/

Log:	fix fix fix

diff --git a/c8/demo/demo_simple.c b/c8/demo/demo_simple.c
--- a/c8/demo/demo_simple.c
+++ b/c8/demo/demo_simple.c
@@ -70,18 +70,20 @@
 
     object_t *tmp;
     int i = 0;
+
+    stm_enter_transactional_zone(&stm_thread_local);
     while (i < ITERS) {
-        stm_start_transaction(&stm_thread_local);
         tl_counter++;
         if (i % 500 < 250)
             STM_PUSH_ROOT(stm_thread_local, stm_allocate(16));//gl_counter++;
         else
             STM_POP_ROOT(stm_thread_local, tmp);
-        stm_commit_transaction();
+        stm_force_transaction_break(&stm_thread_local);
         i++;
     }
 
     OPT_ASSERT(org == (char *)stm_thread_local.shadowstack);
+    stm_leave_transactional_zone(&stm_thread_local);
 
     stm_rewind_jmp_leaveframe(&stm_thread_local, &rjbuf);
     stm_unregister_thread_local(&stm_thread_local);
diff --git a/c8/demo/test_shadowstack.c b/c8/demo/test_shadowstack.c
--- a/c8/demo/test_shadowstack.c
+++ b/c8/demo/test_shadowstack.c
@@ -43,17 +43,16 @@
     stm_register_thread_local(&stm_thread_local);
     stm_rewind_jmp_enterframe(&stm_thread_local, &rjbuf);
 
-    stm_start_transaction(&stm_thread_local);
+    stm_enter_transactional_zone(&stm_thread_local);
     node_t *node = (node_t *)stm_allocate(sizeof(struct node_s));
     node->value = 129821;
     STM_PUSH_ROOT(stm_thread_local, node);
     STM_PUSH_ROOT(stm_thread_local, 333);  /* odd value */
-    stm_commit_transaction();
 
     /* now in a new transaction, pop the node off the shadowstack, but
        then do a major collection.  It should still be found by the
        tracing logic. */
-    stm_start_transaction(&stm_thread_local);
+    stm_force_transaction_break(&stm_thread_local);
     STM_POP_ROOT_RET(stm_thread_local);
     STM_POP_ROOT(stm_thread_local, node);
     assert(node->value == 129821);
diff --git a/c8/stm/detach.c b/c8/stm/detach.c
--- a/c8/stm/detach.c
+++ b/c8/stm/detach.c
@@ -40,6 +40,12 @@
 
 void _stm_reattach_transaction(uintptr_t old, stm_thread_local_t *tl)
 {
+    if (old == 0) {
+        /* there was no detached inevitable transaction */
+        _stm_start_transaction(tl);
+        return;
+    }
+
     if (old & 1) {
         /* The detached transaction was fetched; wait until the s_mutex_lock
            is free. 
@@ -56,21 +62,15 @@
         s_mutex_unlock();
     }
 
-    if (old != 0) {
-        /* We took over the inevitable transaction originally detached
-           from a different thread.  We have to fix the %gs register if
-           it is incorrect.
-        */
-        ensure_gs_register(tl->last_associated_segment_num);
-        assert(STM_SEGMENT->running_thread == (stm_thread_local_t *)old);
-        STM_SEGMENT->running_thread = tl;
+    /* We took over the inevitable transaction originally detached
+       from a different thread.  We have to fix the %gs register if
+       it is incorrect.
+    */
+    ensure_gs_register(tl->last_associated_segment_num);
+    assert(STM_SEGMENT->running_thread == (stm_thread_local_t *)old);
+    STM_SEGMENT->running_thread = tl;
 
-        stm_safe_point();
-    }
-    else {
-        /* there was no detached inevitable transaction */
-        _stm_start_transaction(tl);
-    }
+    stm_safe_point();
 }
 
 static bool fetch_detached_transaction(void)
@@ -108,3 +108,10 @@
     pseg->safe_point = SP_RUNNING_DETACHED_FETCHED;
     return true;
 }
+
+void stm_force_transaction_break(stm_thread_local_t *tl)
+{
+    assert(STM_SEGMENT->running_thread == tl);
+    _stm_commit_transaction();
+    _stm_start_transaction(tl);
+}
diff --git a/c8/stmgc.h b/c8/stmgc.h
--- a/c8/stmgc.h
+++ b/c8/stmgc.h
@@ -433,13 +433,13 @@
         _stm_leave_noninevitable_transactional_zone();
 }
 
-/* stm_break_transaction() is in theory equivalent to
+/* stm_force_transaction_break() is in theory equivalent to
    stm_leave_transactional_zone() immediately followed by
    stm_enter_transactional_zone(); however, it is supposed to be
    called in CPU-heavy threads that had a transaction run for a while,
    and so it *always* forces a commit and starts the next transaction.
    The new transaction is never inevitable. */
-void stm_break_transaction(stm_thread_local_t *tl);
+void stm_force_transaction_break(stm_thread_local_t *tl);
 
 /* Abort the currently running transaction.  This function never
    returns: it jumps back to the start of the transaction (which must

From noreply at buildbot.pypy.org  Wed Jun 10 22:50:15 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 10 Jun 2015 22:50:15 +0200 (CEST)
Subject: [pypy-commit] stmgc c8-gil-like: intermediate check-in,
 will probably be changed again
Message-ID: <20150610205015.A43881C11BD@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: c8-gil-like
Changeset: r1798:c3d3a85bd978
Date: 2015-06-10 21:57 +0200
http://bitbucket.org/pypy/stmgc/changeset/c3d3a85bd978/

Log:	intermediate check-in, will probably be changed again

diff --git a/c8/stm/detach.c b/c8/stm/detach.c
--- a/c8/stm/detach.c
+++ b/c8/stm/detach.c
@@ -5,17 +5,18 @@
 
 /* _stm_detached_inevitable_from_thread is:
 
-   - NULL: there is no inevitable transaction, or it is not detached
+   - 0: there is no inevitable transaction, or it is not detached
 
    - a stm_thread_local_t pointer: this thread-local has detached its
      own inevitable transaction, and might concurrently reattach to it
      at any time
 
-   - a stm_thread_local_t pointer with the last bit set to 1: another
-     thread ran synchronize_all_threads(), so in order to reattach,
-     the detaching thread must first go through
-     s_mutex_lock()/s_mutex_unlock().
+   - DETACHED_AND_FETCHED: another thread ran
+     synchronize_all_threads(), so in order to reattach, the detaching
+     thread must first go through s_mutex_lock()/s_mutex_unlock().
 */
+#define DETACHED_AND_FETCHED  1
+
 volatile uintptr_t _stm_detached_inevitable_from_thread;
 
 
@@ -38,6 +39,20 @@
     }
 }
 
+static struct stm_priv_segment_info_s *detached_and_fetched(void)
+{
+    long i;
+    struct stm_priv_segment_info_s *result = NULL;
+    for (i = 1; i < NB_SEGMENTS; i++) {
+        if (get_priv_segment(i)->safe_point == SP_RUNNING_DETACHED_FETCHED) {
+            assert(result == NULL);
+            result = get_priv_segment(i);
+        }
+    }
+    assert(result != NULL);
+    return result;
+}
+
 void _stm_reattach_transaction(uintptr_t old, stm_thread_local_t *tl)
 {
     if (old == 0) {
@@ -46,19 +61,17 @@
         return;
     }
 
-    if (old & 1) {
+    if (old == DETACHED_AND_FETCHED) {
         /* The detached transaction was fetched; wait until the s_mutex_lock
-           is free. 
+           is free.  The fetched transaction can only be reattached by the
+           code here; there should be no risk of its state changing while
+           we wait.
          */
-        stm_thread_local_t *old_tl;
         struct stm_priv_segment_info_s *pseg;
-
-        old_tl = (stm_thread_local_t *)(--old);
-        pseg = get_priv_segment(old_tl->last_associated_segment_num);
-        assert(pseg->safe_point = SP_RUNNING_DETACHED_FETCHED);
-
         s_mutex_lock();
+        pseg = detached_and_fetched();
         pseg->safe_point = SP_RUNNING;
+        old = (uintptr_t)pseg->running_thread;
         s_mutex_unlock();
     }
 
@@ -88,13 +101,11 @@
     old = _stm_detached_inevitable_from_thread;
     if (old == 0)
         return false;
-    if (old & 1) {
-        /* we have the mutex here, so this detached transaction with the
-           last bit set cannot reattach in parallel */
-        tl = (stm_thread_local_t *)(old - 1);
-        pseg = get_priv_segment(tl->last_associated_segment_num);
-        assert(pseg->safe_point == SP_RUNNING_DETACHED_FETCHED);
-        (void)pseg;
+    if (old < NB_SEGMENTS) {
+        /* we have the mutex here, so this fetched detached transaction
+           cannot get reattached in parallel */
+        assert(get_priv_segment(old)->safe_point ==
+               SP_RUNNING_DETACHED_FETCHED);
         return true;
     }
 
@@ -115,3 +126,17 @@
     _stm_commit_transaction();
     _stm_start_transaction(tl);
 }
+
+static void commit_own_inevitable_detached_transaction(stm_thread_local_t *tl)
+{
+    uintptr_t cur = _stm_detached_inevitable_from_thread;
+    if ((cur & ~1) == (uintptr_t)tl) {
+        stm_enter_transactional_zone(tl);
+        _stm_commit_transaction();
+    }
+}
+
+REWRITE:. we need a function to grab and commit the detached inev transaction
+anyway.  So kill the special values of _stm_detached_inevitable_from_thread.
+And call that function from core.c when we wait for the inev transaction to
+finish
diff --git a/c8/stm/detach.h b/c8/stm/detach.h
--- a/c8/stm/detach.h
+++ b/c8/stm/detach.h
@@ -1,3 +1,4 @@
 
 static void setup_detach(void);
 static bool fetch_detached_transaction(void);
+static void commit_own_inevitable_detached_transaction(stm_thread_local_t *tl);
diff --git a/c8/stm/setup.c b/c8/stm/setup.c
--- a/c8/stm/setup.c
+++ b/c8/stm/setup.c
@@ -264,6 +264,9 @@
 
 void stm_unregister_thread_local(stm_thread_local_t *tl)
 {
+    /* If we own the detached inevitable transaction, commit it now */
+    commit_own_inevitable_detached_transaction(tl);
+
     s_mutex_lock();
     assert(tl->prev != NULL);
     assert(tl->next != NULL);

From noreply at buildbot.pypy.org  Wed Jun 10 22:50:16 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 10 Jun 2015 22:50:16 +0200 (CEST)
Subject: [pypy-commit] stmgc c8-gil-like: progress
Message-ID: <20150610205016.B940C1C11BD@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: c8-gil-like
Changeset: r1799:370c663b47f6
Date: 2015-06-10 22:50 +0200
http://bitbucket.org/pypy/stmgc/changeset/370c663b47f6/

Log:	progress

diff --git a/c8/stm/detach.c b/c8/stm/detach.c
--- a/c8/stm/detach.c
+++ b/c8/stm/detach.c
@@ -3,26 +3,20 @@
 #endif
 
 
-/* _stm_detached_inevitable_from_thread is:
+/* _stm_detached_inevitable_segnum is:
 
-   - 0: there is no inevitable transaction, or it is not detached
+   - -1: there is no inevitable transaction, or it is not detached
 
-   - a stm_thread_local_t pointer: this thread-local has detached its
-     own inevitable transaction, and might concurrently reattach to it
-     at any time
-
-   - DETACHED_AND_FETCHED: another thread ran
-     synchronize_all_threads(), so in order to reattach, the detaching
-     thread must first go through s_mutex_lock()/s_mutex_unlock().
+   - in range(1, NB_SEGMENTS): an inevitable transaction belongs to
+     the segment and was detached.  It might concurrently be
+     reattached at any time, with an XCHG (__sync_lock_test_and_set).
 */
-#define DETACHED_AND_FETCHED  1
-
-volatile uintptr_t _stm_detached_inevitable_from_thread;
+volatile int _stm_detached_inevitable_seg_num;
 
 
 static void setup_detach(void)
 {
-    _stm_detached_inevitable_from_thread = 0;
+    _stm_detached_inevitable_seg_num = -1;
 }
 
 
@@ -39,85 +33,24 @@
     }
 }
 
-static struct stm_priv_segment_info_s *detached_and_fetched(void)
+void _stm_reattach_transaction(int old, stm_thread_local_t *tl)
 {
-    long i;
-    struct stm_priv_segment_info_s *result = NULL;
-    for (i = 1; i < NB_SEGMENTS; i++) {
-        if (get_priv_segment(i)->safe_point == SP_RUNNING_DETACHED_FETCHED) {
-            assert(result == NULL);
-            result = get_priv_segment(i);
-        }
-    }
-    assert(result != NULL);
-    return result;
-}
-
-void _stm_reattach_transaction(uintptr_t old, stm_thread_local_t *tl)
-{
-    if (old == 0) {
+    if (old == -1) {
         /* there was no detached inevitable transaction */
         _stm_start_transaction(tl);
-        return;
     }
+    else {
+        /* We took over the inevitable transaction originally detached
+           from a different segment.  We have to fix the %gs register if
+           it is incorrect.
+        */
+        tl->last_associated_segment_num = old;
+        ensure_gs_register(old);
+        assert(STM_PSEGMENT->transaction_state == TS_INEVITABLE);
+        STM_SEGMENT->running_thread = tl;
 
-    if (old == DETACHED_AND_FETCHED) {
-        /* The detached transaction was fetched; wait until the s_mutex_lock
-           is free.  The fetched transaction can only be reattached by the
-           code here; there should be no risk of its state changing while
-           we wait.
-         */
-        struct stm_priv_segment_info_s *pseg;
-        s_mutex_lock();
-        pseg = detached_and_fetched();
-        pseg->safe_point = SP_RUNNING;
-        old = (uintptr_t)pseg->running_thread;
-        s_mutex_unlock();
+        stm_safe_point();
     }
-
-    /* We took over the inevitable transaction originally detached
-       from a different thread.  We have to fix the %gs register if
-       it is incorrect.
-    */
-    ensure_gs_register(tl->last_associated_segment_num);
-    assert(STM_SEGMENT->running_thread == (stm_thread_local_t *)old);
-    STM_SEGMENT->running_thread = tl;
-
-    stm_safe_point();
-}
-
-static bool fetch_detached_transaction(void)
-{
-    /* returns True if there is a detached transaction; afterwards, it
-       is not necessary to call fetch_detached_transaction() again
-       regularly.
-    */
-    uintptr_t old;
-    stm_thread_local_t *tl;
-    struct stm_priv_segment_info_s *pseg;
-    assert(_has_mutex_here);
-
- restart:
-    old = _stm_detached_inevitable_from_thread;
-    if (old == 0)
-        return false;
-    if (old < NB_SEGMENTS) {
-        /* we have the mutex here, so this fetched detached transaction
-           cannot get reattached in parallel */
-        assert(get_priv_segment(old)->safe_point ==
-               SP_RUNNING_DETACHED_FETCHED);
-        return true;
-    }
-
-    if (!__sync_bool_compare_and_swap(&_stm_detached_inevitable_from_thread,
-                                      old, old + 1))
-        goto restart;
-
-    tl = (stm_thread_local_t *)old;
-    pseg = get_priv_segment(tl->last_associated_segment_num);
-    assert(pseg->safe_point == SP_RUNNING);
-    pseg->safe_point = SP_RUNNING_DETACHED_FETCHED;
-    return true;
 }
 
 void stm_force_transaction_break(stm_thread_local_t *tl)
@@ -127,16 +60,32 @@
     _stm_start_transaction(tl);
 }
 
-static void commit_own_inevitable_detached_transaction(stm_thread_local_t *tl)
+static int fetch_detached_transaction(void)
 {
-    uintptr_t cur = _stm_detached_inevitable_from_thread;
-    if ((cur & ~1) == (uintptr_t)tl) {
-        stm_enter_transactional_zone(tl);
-        _stm_commit_transaction();
-    }
+    int cur = _stm_detached_inevitable_seg_num;
+    if (cur != -1)
+        cur = __sync_lock_test_and_set(    /* XCHG */
+            &_stm_detached_inevitable_seg_num, -1);
+    return cur;
 }
 
-REWRITE:. we need a function to grab and commit the detached inev transaction
-anyway.  So kill the special values of _stm_detached_inevitable_from_thread.
-And call that function from core.c when we wait for the inev transaction to
-finish
+static void commit_fetched_detached_transaction(int segnum)
+{
+    /* Here, 'seg_num' is the segment that contains the detached
+       inevitable transaction from fetch_detached_transaction(),
+       probably belonging to an unrelated thread.  We fetched it,
+       which means that nobody else can concurrently fetch it now, but
+       everybody will see that there is still a concurrent inevitable
+       transaction.  This should guarantee there are not race
+       conditions.
+    */
+    assert(segnum > 0);
+
+    int mysegnum = STM_SEGMENT->segment_num;
+    ensure_gs_register(segnum);
+
+    assert(STM_PSEGMENT->transaction_state == TS_INEVITABLE);
+    _stm_commit_transaction();   /* can't abort */
+
+    ensure_gs_register(mysegnum);
+}
diff --git a/c8/stm/detach.h b/c8/stm/detach.h
--- a/c8/stm/detach.h
+++ b/c8/stm/detach.h
@@ -1,4 +1,4 @@
 
 static void setup_detach(void);
-static bool fetch_detached_transaction(void);
-static void commit_own_inevitable_detached_transaction(stm_thread_local_t *tl);
+static int fetch_detached_transaction(void);
+static void commit_fetched_detached_transaction(int segnum);
diff --git a/c8/stm/setup.c b/c8/stm/setup.c
--- a/c8/stm/setup.c
+++ b/c8/stm/setup.c
@@ -264,9 +264,6 @@
 
 void stm_unregister_thread_local(stm_thread_local_t *tl)
 {
-    /* If we own the detached inevitable transaction, commit it now */
-    commit_own_inevitable_detached_transaction(tl);
-
     s_mutex_lock();
     assert(tl->prev != NULL);
     assert(tl->next != NULL);
diff --git a/c8/stm/sync.c b/c8/stm/sync.c
--- a/c8/stm/sync.c
+++ b/c8/stm/sync.c
@@ -66,6 +66,7 @@
 
 static void ensure_gs_register(long segnum)
 {
+    /* XXX use this instead of set_gs_register() in many places */
     if (STM_SEGMENT->segment_num != segnum) {
         set_gs_register(get_segment_base(segnum));
         assert(STM_SEGMENT->segment_num == segnum);
@@ -392,18 +393,7 @@
 
 static void synchronize_all_threads(enum sync_type_e sync_type)
 {
-    /* Regularly, we try fetch_detached_transaction(), which, if there
-       is a detached inevitable transaction, will take it out of the
-       global variable that reattaching tries to read, and put it in
-       our local 'detached_tl' variable.  The status of such a fetched
-       detached transaction is temporarily set from SP_RUNNING to
-       SP_RUNNING_DETACHED_FETCHED, which is not counted any more by
-       count_other_threads_sp_running().  'detached_tl' will be copied
-       back into '_stm_detached_inevitable_from_thread' by the other
-       thread running _stm_reattach_transaction(), later.
-    */
-    bool detached = fetch_detached_transaction();
-
+ restart:
     assert(_has_mutex());
     enter_safe_point_if_requested();
 
@@ -414,7 +404,6 @@
        enter_safe_point_if_requested() above.
     */
     if (UNLIKELY(globally_unique_transaction)) {
-        assert(!detached);
         assert(count_other_threads_sp_running() == 0);
         return;
     }
@@ -424,18 +413,17 @@
     /* If some other threads are SP_RUNNING, we cannot proceed now.
        Wait until all other threads are suspended. */
     while (count_other_threads_sp_running() > 0) {
+
+        int detached = fetch_detached_transaction();
+        if (detached >= 0) {
+            remove_requests_for_safe_point();    /* => C_REQUEST_REMOVED */
+            commit_fetched_detached_transaction(detached);
+            goto restart;
+        }
+
         STM_PSEGMENT->safe_point = SP_WAIT_FOR_C_AT_SAFE_POINT;
-        if (!detached) {
-            do {
-                detached = fetch_detached_transaction();
-                if (detached)
-                    break;
-            } while (!cond_wait_timeout(C_AT_SAFE_POINT, 0.00001));
-            /* every 10 microsec, try again fetch_detached_transaction() */
-        }
-        else {
-            cond_wait(C_AT_SAFE_POINT);
-        }
+        cond_wait_timeout(C_AT_SAFE_POINT, 0.00001);
+        /* every 10 microsec, try again fetch_detached_transaction() */
         STM_PSEGMENT->safe_point = SP_RUNNING;
 
         if (must_abort()) {
diff --git a/c8/stmgc.h b/c8/stmgc.h
--- a/c8/stmgc.h
+++ b/c8/stmgc.h
@@ -84,15 +84,16 @@
 object_t *_stm_allocate_slowpath(ssize_t);
 object_t *_stm_allocate_external(ssize_t);
 
-extern volatile uintptr_t _stm_detached_inevitable_from_thread;
+extern volatile int _stm_detached_inevitable_seg_num;
 long _stm_start_transaction(stm_thread_local_t *tl);
 void _stm_commit_transaction(void);
 void _stm_leave_noninevitable_transactional_zone(void);
-#define _stm_detach_inevitable_transaction(tl)  do {            \
-    write_fence();                                              \
-    _stm_detached_inevitable_from_thread = (uintptr_t)(tl);     \
+#define _stm_detach_inevitable_transaction(tl)  do {                    \
+    write_fence();                                                      \
+    assert((tl)->last_associated_segment_num == STM_SEGMENT->segment_num); \
+    _stm_detached_inevitable_seg_num = STM_SEGMENT->segment_num;        \
 } while (0)
-void _stm_reattach_transaction(uintptr_t old, stm_thread_local_t *tl);
+void _stm_reattach_transaction(int old, stm_thread_local_t *tl);
 void _stm_become_inevitable(const char*);
 void _stm_collectable_safe_point(void);
 
@@ -420,9 +421,11 @@
    transactions.
 */
 static inline void stm_enter_transactional_zone(stm_thread_local_t *tl) {
-    uintptr_t old = __sync_lock_test_and_set(    /* XCHG */
-        &_stm_detached_inevitable_from_thread, 0);
-    if (old != (uintptr_t)(tl))
+    int old = __sync_lock_test_and_set(    /* XCHG */
+        &_stm_detached_inevitable_seg_num, -1);
+    if (old == tl->last_associated_segment_num)
+        STM_SEGMENT->running_thread = tl;
+    else
         _stm_reattach_transaction(old, tl);
 }
 static inline void stm_leave_transactional_zone(stm_thread_local_t *tl) {
@@ -455,7 +458,7 @@
     if (!stm_is_inevitable())
         _stm_become_inevitable(msg);
     /* now, we're running the inevitable transaction, so: */
-    assert(_stm_detached_inevitable_from_thread == 0);
+    assert(_stm_detached_inevitable_seg_num == -1);
 }
 
 /* Forces a safe-point if needed.  Normally not needed: this is

From noreply at buildbot.pypy.org  Wed Jun 10 22:57:10 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 10 Jun 2015 22:57:10 +0200 (CEST)
Subject: [pypy-commit] stmgc c8-gil-like: Essential fix here: while waiting
 for the inevitable transaction to
Message-ID: <20150610205710.51B521C11BD@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: c8-gil-like
Changeset: r1800:9d96ac8b82e0
Date: 2015-06-10 22:57 +0200
http://bitbucket.org/pypy/stmgc/changeset/9d96ac8b82e0/

Log:	Essential fix here: while waiting for the inevitable transaction to
	commit, look if it is detached and commit it ourselves

diff --git a/c8/stm/core.c b/c8/stm/core.c
--- a/c8/stm/core.c
+++ b/c8/stm/core.c
@@ -496,11 +496,23 @@
 
 static void wait_for_other_inevitable(struct stm_commit_log_entry_s *old)
 {
+    int detached = fetch_detached_transaction();
+    if (detached >= 0) {
+        commit_fetched_detached_transaction(detached);
+        return;
+    }
+
     timing_event(STM_SEGMENT->running_thread, STM_WAIT_OTHER_INEVITABLE);
 
     while (old->next == INEV_RUNNING && !safe_point_requested()) {
         spin_loop();
         usleep(10);    /* XXXXXX */
+
+        detached = fetch_detached_transaction();
+        if (detached >= 0) {
+            commit_fetched_detached_transaction(detached);
+            break;
+        }
     }
     timing_event(STM_SEGMENT->running_thread, STM_WAIT_DONE);
 }
@@ -1278,7 +1290,8 @@
 
     assert(!_has_mutex());
     assert(STM_PSEGMENT->safe_point == SP_RUNNING);
-    assert(STM_PSEGMENT->running_pthread == pthread_self());
+    //assert(STM_PSEGMENT->running_pthread == pthread_self());
+    // ^^^ fails if detach.c commits a detached inevitable transaction
 
     dprintf(("> stm_commit_transaction()\n"));
     minor_collection(1);

From noreply at buildbot.pypy.org  Thu Jun 11 01:47:02 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Thu, 11 Jun 2015 01:47:02 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Add a fallback to the ncurses library if
 ncursesw is not available.
Message-ID: <20150610234702.E6DAD1C048F@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob <me at manueljacob.de>
Branch: py3k
Changeset: r78022:2705fe4f9105
Date: 2015-06-11 01:47 +0200
http://bitbucket.org/pypy/pypy/changeset/2705fe4f9105/

Log:	Add a fallback to the ncurses library if ncursesw is not available.

diff --git a/lib_pypy/_curses_build.py b/lib_pypy/_curses_build.py
--- a/lib_pypy/_curses_build.py
+++ b/lib_pypy/_curses_build.py
@@ -1,4 +1,22 @@
-from cffi import FFI
+from cffi import FFI, VerificationError
+
+
+def find_curses_library():
+    for curses_library in ['ncursesw', 'ncurses']:
+        ffi = FFI()
+        ffi.set_source("_curses_cffi_check", "", libraries=[curses_library])
+        try:
+            ffi.compile()
+        except VerificationError as e:
+            e_last = e
+            continue
+        else:
+            return curses_library
+
+    # If none of the libraries is available, present the user a meaningful
+    # error message
+    raise e_last
+
 
 ffi = FFI()
 
@@ -41,7 +59,7 @@
 void _m_getsyx(int *yx) {
     getsyx(yx[0], yx[1]);
 }
-""", libraries=['ncursesw', 'panel'])
+""", libraries=[find_curses_library(), 'panel'])
 
 
 ffi.cdef("""

From noreply at buildbot.pypy.org  Thu Jun 11 03:46:54 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Thu, 11 Jun 2015 03:46:54 +0200 (CEST)
Subject: [pypy-commit] pypy unicode-dtype: Add missing UnicodeType methods
Message-ID: <20150611014654.D9ED51C1016@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: unicode-dtype
Changeset: r78023:f9808839b92c
Date: 2015-06-10 21:51 +0100
http://bitbucket.org/pypy/pypy/changeset/f9808839b92c/

Log:	Add missing UnicodeType methods

diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -2273,20 +2273,32 @@
         return v1._value >= v2._value
 
     def logical_and(self, v1, v2):
-        raise NotImplementedError
+        assert isinstance(v1, boxes.W_UnicodeBox)
+        assert isinstance(v2, boxes.W_UnicodeBox)
+        if bool(v1) and bool(v2):
+            return Bool._True
+        return Bool._False
 
     def logical_or(self, v1, v2):
-        raise NotImplementedError
+        assert isinstance(v1, boxes.W_UnicodeBox)
+        assert isinstance(v2, boxes.W_UnicodeBox)
+        if bool(v1) or bool(v2):
+            return Bool._True
+        return Bool._False
 
     def logical_not(self, v):
-        raise NotImplementedError
-
-    @str_binary_op
+        assert isinstance(v, boxes.W_UnicodeBox)
+        return not bool(v)
+
     def logical_xor(self, v1, v2):
-        raise NotImplementedError
+        assert isinstance(v1, boxes.W_UnicodeBox)
+        assert isinstance(v2, boxes.W_UnicodeBox)
+        a = bool(v1)
+        b = bool(v2)
+        return (not b and a) or (not a and b)
 
     def bool(self, v):
-        raise NotImplementedError
+        return bool(v._value)
 
     def fill(self, storage, width, native, box, start, stop, offset, gcstruct):
         assert isinstance(box, boxes.W_UnicodeBox)

From noreply at buildbot.pypy.org  Thu Jun 11 03:46:56 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Thu, 11 Jun 2015 03:46:56 +0200 (CEST)
Subject: [pypy-commit] pypy unicode-dtype: fix translation
Message-ID: <20150611014656.2C6811C1016@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: unicode-dtype
Changeset: r78024:e0542894acbb
Date: 2015-06-11 02:46 +0100
http://bitbucket.org/pypy/pypy/changeset/e0542894acbb/

Log:	fix translation

diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -2298,6 +2298,7 @@
         return (not b and a) or (not a and b)
 
     def bool(self, v):
+        assert isinstance(v, boxes.W_UnicodeBox)
         return bool(v._value)
 
     def fill(self, storage, width, native, box, start, stop, offset, gcstruct):

From noreply at buildbot.pypy.org  Thu Jun 11 10:08:27 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Thu, 11 Jun 2015 10:08:27 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: signext (due to its missing x86 opcodes)
 is weigthed as "not a profitable operation"
Message-ID: <20150611080827.9E15F1C0E5B@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r78025:eaef430ca49d
Date: 2015-06-11 10:08 +0200
http://bitbucket.org/pypy/pypy/changeset/eaef430ca49d/

Log:	signext (due to its missing x86 opcodes) is weigthed as "not a
	profitable operation"

diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py
--- a/pypy/module/micronumpy/compile.py
+++ b/pypy/module/micronumpy/compile.py
@@ -724,8 +724,13 @@
                 raise ArgumentMismatch
             if self.name == "sum":
                 if len(self.args)>1:
-                    w_res = arr.descr_sum(interp.space,
+                    var = self.args[1]
+                    if isinstance(var, DtypeClass):
+                        w_res = arr.descr_sum(interp.space, None, var.execute(interp))
+                    else:
+                        w_res = arr.descr_sum(interp.space,
                                           self.args[1].execute(interp))
+
                 else:
                     w_res = arr.descr_sum(interp.space)
             elif self.name == "prod":
diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -332,6 +332,17 @@
         assert result == sum(range(30))
         self.check_vectorized(1, 1)
 
+    def define_sum_float_to_int16():
+        return """
+        a = |30|
+        sum(a,int16)
+        """
+
+    def test_sum_float_to_int16(self):
+        result = self.run("sum_float_to_int16")
+        assert result == sum(range(30))
+        self.check_vectorized(1, 1)
+
     def define_cumsum():
         return """
         a = |30|
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_costmodel.py b/rpython/jit/metainterp/optimizeopt/test/test_costmodel.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_costmodel.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_costmodel.py
@@ -119,7 +119,7 @@
         raw_store(p0, i4, i31, descr=int)
         """)
         savings = self.savings(loop1)
-        assert savings == 1
+        assert savings >= 0
 
     def test_sum(self):
         loop1 = self.parse("""
@@ -131,5 +131,26 @@
         savings = self.savings(loop1)
         assert savings == 2
 
+    def test_sum_float_to_int16(self):
+        loop1 = self.parse("""
+        f10 = raw_load(p0, i0, descr=double)
+        f11 = raw_load(p0, i1, descr=double)
+        i10 = cast_float_to_int(f10)
+        i11 = cast_float_to_int(f11)
+        i12 = int_signext(i10, 2)
+        i13 = int_signext(i11, 2)
+        i14 = int_add(i1, i12)
+        i16 = int_signext(i14, 2)
+        i15 = int_add(i16, i13)
+        i17 = int_signext(i15, 2)
+        """)
+        savings = self.savings(loop1)
+        # it does not benefit because signext has
+        # a very inefficient implementation (x86
+        # does not provide nice instr to convert
+        # integer sizes)
+        # signext -> no benefit, + 2x unpack
+        assert savings < 0
+
 class Test(CostModelBaseTest, LLtypeMixin):
     pass
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py b/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
@@ -12,7 +12,7 @@
 from rpython.jit.metainterp.optimizeopt.dependency import DependencyGraph
 from rpython.jit.metainterp.optimizeopt.unroll import Inliner
 from rpython.jit.metainterp.optimizeopt.vectorize import (VectorizingOptimizer, MemoryRef,
-        isomorphic, Pair, NotAVectorizeableLoop, NotAVectorizeableLoop, GuardStrengthenOpt)
+        isomorphic, Pair, NotAVectorizeableLoop, NotAProfitableLoop, GuardStrengthenOpt)
 from rpython.jit.metainterp.optimize import InvalidLoop
 from rpython.jit.metainterp.history import ConstInt, BoxInt, get_const_ptr_for_string
 from rpython.jit.metainterp import executor, compile, resume
@@ -109,6 +109,8 @@
         opt.find_adjacent_memory_refs()
         opt.extend_packset()
         opt.combine_packset()
+        if not opt.costmodel.profitable(opt.packset):
+            raise NotAProfitableLoop()
         opt.schedule(True)
         gso = GuardStrengthenOpt(opt.dependency_graph.index_vars)
         gso.propagate_all_forward(opt.loop)
diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/vectorize.py
@@ -361,11 +361,16 @@
         if not we_are_translated():
             # some test cases check the accumulation variables
             self.packset.accum_vars = {}
+            print "packs:"
             for pack in self.packset.packs:
                 accum = pack.accum
                 if accum:
                     self.packset.accum_vars[accum.var] = accum.pos
 
+                print " %dx %s (accum? %d) " % (len(pack.operations),
+                                                pack.operations[0].op.getopname(),
+                                                accum is not None)
+
     def schedule(self, vector=False):
         self.guard_early_exit = -1
         self.clear_newoperations()
@@ -394,6 +399,8 @@
     def unpack_from_vector(self, op, sched_data, renamer):
         renamer.rename(op)
         args = op.getarglist()
+        if op.getopnum() == rop.INT_SIGNEXT:
+            py.test.set_trace()
         for i, arg in enumerate(op.getarglist()):
             if isinstance(arg, Box):
                 argument = self._unpack_from_vector(i, arg, sched_data, renamer)
@@ -481,7 +488,7 @@
     def unpack_cost(self, index, op):
         raise NotImplementedError
 
-    def savings_for_pack(self, opnum, times):
+    def savings_for_pack(self, pack, times):
         raise NotImplementedError
 
     def savings_for_unpacking(self, node, index):
@@ -495,8 +502,8 @@
     def calculate_savings(self, packset):
         savings = 0
         for pack in packset.packs:
-            savings += self.savings_for_pack(pack.opnum, pack.opcount())
             op0 = pack.operations[0].getoperation()
+            savings += self.savings_for_pack(pack, pack.opcount())
             if op0.result:
                 for i,node in enumerate(pack.operations):
                     savings += self.savings_for_unpacking(node, i)
@@ -507,10 +514,22 @@
 
 class X86_CostModel(CostModel):
 
-    def savings_for_pack(self, opnum, times):
-        cost, benefit_factor = (1,1) # TODO custom values for different ops
+    def savings_for_pack(self, pack, times):
+        cost, benefit_factor = (1,1)
+        op = pack.operations[0].getoperation()
+        if op.getopnum() == rop.INT_SIGNEXT:
+            cost, benefit_factor = self.cb_signext(pack)
         return benefit_factor * times - cost
 
+    def cb_signext(self, pack):
+        op0 = pack.operations[0].getoperation()
+        size = op0.getarg(1).getint()
+        orig_size = pack.output_type.getsize()
+        if size == orig_size:
+            return 0,0
+        # no benefit for this operation! needs many x86 instr 
+        return 1,0
+
     def unpack_cost(self, index, op):
         if op.getdescr():
             if op.getdescr().is_array_of_floats():

From noreply at buildbot.pypy.org  Thu Jun 11 11:02:02 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Thu, 11 Jun 2015 11:02:02 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: added tests and parameterized one
Message-ID: <20150611090202.307911C1016@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r78026:645362226e3e
Date: 2015-06-11 10:55 +0200
http://bitbucket.org/pypy/pypy/changeset/645362226e3e/

Log:	added tests and parameterized one

diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py
--- a/pypy/module/micronumpy/compile.py
+++ b/pypy/module/micronumpy/compile.py
@@ -685,6 +685,14 @@
             dtype = get_dtype_cache(interp.space).w_int16dtype
         elif self.v == 'int32':
             dtype = get_dtype_cache(interp.space).w_int32dtype
+        elif self.v == 'uint':
+            dtype = get_dtype_cache(interp.space).w_uint64dtype
+        elif self.v == 'uint8':
+            dtype = get_dtype_cache(interp.space).w_uint8dtype
+        elif self.v == 'uint16':
+            dtype = get_dtype_cache(interp.space).w_uint16dtype
+        elif self.v == 'uint32':
+            dtype = get_dtype_cache(interp.space).w_uint32dtype
         elif self.v == 'float':
             dtype = get_dtype_cache(interp.space).w_float64dtype
         elif self.v == 'float32':
@@ -932,6 +940,16 @@
                     stack.append(DtypeClass('int32'))
                 elif token.v.strip(' ') == 'int64':
                     stack.append(DtypeClass('int'))
+                elif token.v.strip(' ') == 'uint':
+                    stack.append(DtypeClass('uint'))
+                elif token.v.strip(' ') == 'uint8':
+                    stack.append(DtypeClass('uint8'))
+                elif token.v.strip(' ') == 'uint16':
+                    stack.append(DtypeClass('uint16'))
+                elif token.v.strip(' ') == 'uint32':
+                    stack.append(DtypeClass('uint32'))
+                elif token.v.strip(' ') == 'uint64':
+                    stack.append(DtypeClass('uint'))
                 elif token.v.strip(' ') == 'float':
                     stack.append(DtypeClass('float'))
                 elif token.v.strip(' ') == 'float32':
diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -8,6 +8,7 @@
 from rpython.jit.metainterp.warmspot import reset_jit, get_stats
 from rpython.jit.metainterp.jitprof import Profiler
 from rpython.rlib.jit import Counters
+from rpython.rlib.rarithmetic import intmask
 from pypy.module.micronumpy import boxes
 from pypy.module.micronumpy.compile import FakeSpace, Parser, InterpreterState
 from pypy.module.micronumpy.base import W_NDimArray
@@ -72,6 +73,16 @@
                 return float(int(w_res.value))
             elif isinstance(w_res, boxes.W_Int16Box):
                 return float(int(w_res.value))
+            elif isinstance(w_res, boxes.W_Int8Box):
+                return float(int(w_res.value))
+            elif isinstance(w_res, boxes.W_UInt64Box):
+                return float(intmask(w_res.value))
+            elif isinstance(w_res, boxes.W_UInt32Box):
+                return float(intmask(w_res.value))
+            elif isinstance(w_res, boxes.W_UInt16Box):
+                return float(intmask(w_res.value))
+            elif isinstance(w_res, boxes.W_UInt8Box):
+                return float(intmask(w_res.value))
             elif isinstance(w_res, boxes.W_LongBox):
                 return float(w_res.value)
             elif isinstance(w_res, boxes.W_BoolBox):
@@ -337,12 +348,40 @@
         a = |30|
         sum(a,int16)
         """
-
     def test_sum_float_to_int16(self):
         result = self.run("sum_float_to_int16")
         assert result == sum(range(30))
+        self.check_vectorized(1, 0)
+    def define_sum_float_to_int32():
+        return """
+        a = |30|
+        sum(a,int32)
+        """
+    def test_sum_float_to_int32(self):
+        result = self.run("sum_float_to_int32")
+        assert result == sum(range(30))
         self.check_vectorized(1, 1)
 
+    def define_sum_float_to_float32():
+        return """
+        a = |30|
+        sum(a,float32)
+        """
+    def test_sum_float_to_float32(self):
+        result = self.run("sum_float_to_float32")
+        assert result == sum(range(30))
+        self.check_vectorized(1, 1)
+
+    def define_sum_float_to_uint64():
+        return """
+        a = |30|
+        sum(a,uint64)
+        """
+    def test_sum_float_to_uint64(self):
+        result = self.run("sum_float_to_uint64")
+        assert result == sum(range(30))
+        self.check_vectorized(1, 0) # unsigned
+
     def define_cumsum():
         return """
         a = |30|
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_costmodel.py b/rpython/jit/metainterp/optimizeopt/test/test_costmodel.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_costmodel.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_costmodel.py
@@ -131,26 +131,27 @@
         savings = self.savings(loop1)
         assert savings == 2
 
-    def test_sum_float_to_int16(self):
+    @py.test.mark.parametrize("bytes,s", [(1,-1),(2,-1),(4,0),(8,-1)])
+    def test_sum_float_to_int(self, bytes, s):
         loop1 = self.parse("""
         f10 = raw_load(p0, i0, descr=double)
         f11 = raw_load(p0, i1, descr=double)
         i10 = cast_float_to_int(f10)
         i11 = cast_float_to_int(f11)
-        i12 = int_signext(i10, 2)
-        i13 = int_signext(i11, 2)
+        i12 = int_signext(i10, {c})
+        i13 = int_signext(i11, {c})
         i14 = int_add(i1, i12)
-        i16 = int_signext(i14, 2)
+        i16 = int_signext(i14, {c})
         i15 = int_add(i16, i13)
-        i17 = int_signext(i15, 2)
-        """)
+        i17 = int_signext(i15, {c})
+        """.format(c=bytes))
         savings = self.savings(loop1)
         # it does not benefit because signext has
         # a very inefficient implementation (x86
         # does not provide nice instr to convert
         # integer sizes)
         # signext -> no benefit, + 2x unpack
-        assert savings < 0
+        assert savings <= s
 
 class Test(CostModelBaseTest, LLtypeMixin):
     pass

From noreply at buildbot.pypy.org  Thu Jun 11 11:16:04 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Thu, 11 Jun 2015 11:16:04 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: added uint vector addition test
Message-ID: <20150611091604.770DC1C0186@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r78027:685a9ce976c8
Date: 2015-06-11 11:16 +0200
http://bitbucket.org/pypy/pypy/changeset/685a9ce976c8/

Log:	added uint vector addition test

diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -169,6 +169,17 @@
         self.assert_float_equal(result, 17.0 + 17.0)
         self.check_vectorized(1, 1)
 
+    def define_uint_add():
+        return """
+        a = astype(|30|, uint64)
+        b = a + a
+        b -> 17
+        """
+    def test_uint_add(self):
+        result = self.run("uint_add")
+        assert int(result) == 17+17
+        self.check_vectorized(2, 1)
+
     def define_float32_add_const():
         return """
         a = astype(|30|, float32)

From noreply at buildbot.pypy.org  Thu Jun 11 11:26:57 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 11 Jun 2015 11:26:57 +0200 (CEST)
Subject: [pypy-commit] stmgc c8-gil-like: in-progress
Message-ID: <20150611092657.AB2EF1C0579@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: c8-gil-like
Changeset: r1801:f344eac5014c
Date: 2015-06-11 11:27 +0200
http://bitbucket.org/pypy/stmgc/changeset/f344eac5014c/

Log:	in-progress

diff --git a/c8/demo/demo_random.c b/c8/demo/demo_random.c
--- a/c8/demo/demo_random.c
+++ b/c8/demo/demo_random.c
@@ -347,7 +347,7 @@
 
     objptr_t p;
 
-    stm_start_transaction(&stm_thread_local);
+    stm_enter_transactional_zone(&stm_thread_local);
     assert(td.num_roots >= td.num_roots_at_transaction_start);
     td.num_roots = td.num_roots_at_transaction_start;
     p = NULL;
@@ -367,12 +367,19 @@
 
             long call_fork = (arg != NULL && *(long *)arg);
             if (call_fork == 0) {   /* common case */
-                stm_commit_transaction();
                 td.num_roots_at_transaction_start = td.num_roots;
-                if (get_rand(100) < 98) {
-                    stm_start_transaction(&stm_thread_local);
-                } else {
-                    stm_start_inevitable_transaction(&stm_thread_local);
+                if (get_rand(100) < 50) {
+                    stm_leave_transactional_zone(&stm_thread_local);
+                    /* Nothing here; it's unlikely that a different thread
+                       manages to steal the detached inev transaction.
+                       Give them a little chance with a usleep(). */
+                    fprintf(stderr, "sleep...\n");
+                    usleep(1);
+                    fprintf(stderr, "sleep done\n");
+                    stm_enter_transactional_zone(&stm_thread_local);
+                }
+                else {
+                    stm_force_transaction_break(&stm_thread_local);
                 }
                 td.num_roots = td.num_roots_at_transaction_start;
                 p = NULL;
@@ -401,16 +408,16 @@
         }
     }
     push_roots();
-    stm_commit_transaction();
+    stm_force_transaction_break(&stm_thread_local);
 
     /* even out the shadow stack before leaveframe: */
-    stm_start_inevitable_transaction(&stm_thread_local);
+    stm_become_inevitable(&stm_thread_local, "before leaveframe");
     while (td.num_roots > 0) {
         td.num_roots--;
         objptr_t t;
         STM_POP_ROOT(stm_thread_local, t);
     }
-    stm_commit_transaction();
+    stm_leave_transactional_zone(&stm_thread_local);
 
     stm_rewind_jmp_leaveframe(&stm_thread_local, &rjbuf);
     stm_unregister_thread_local(&stm_thread_local);
diff --git a/c8/stm/atomic.h b/c8/stm/atomic.h
--- a/c8/stm/atomic.h
+++ b/c8/stm/atomic.h
@@ -24,15 +24,21 @@
 
 #if defined(__i386__) || defined(__amd64__)
 
-# define HAVE_FULL_EXCHANGE_INSN
   static inline void spin_loop(void) { asm("pause" : : : "memory"); }
   static inline void write_fence(void) { asm("" : : : "memory"); }
+# define atomic_exchange(ptr, old, new)  do {           \
+        (old) = __sync_lock_test_and_set(ptr, new);     \
+    } while (0)
 
 #else
 
   static inline void spin_loop(void) { asm("" : : : "memory"); }
   static inline void write_fence(void) { __sync_synchronize(); }
 
+# define atomic_exchange(ptr, old, new)  do {           \
+        (old) = *(ptr);                                 \
+    } while (UNLIKELY(!__sync_bool_compare_and_swap(ptr, old, new)));
+
 #endif
 
 
diff --git a/c8/stm/core.c b/c8/stm/core.c
--- a/c8/stm/core.c
+++ b/c8/stm/core.c
@@ -496,8 +496,8 @@
 
 static void wait_for_other_inevitable(struct stm_commit_log_entry_s *old)
 {
-    int detached = fetch_detached_transaction();
-    if (detached >= 0) {
+    intptr_t detached = fetch_detached_transaction();
+    if (detached != 0) {
         commit_fetched_detached_transaction(detached);
         return;
     }
@@ -509,7 +509,7 @@
         usleep(10);    /* XXXXXX */
 
         detached = fetch_detached_transaction();
-        if (detached >= 0) {
+        if (detached != 0) {
             commit_fetched_detached_transaction(detached);
             break;
         }
@@ -1235,7 +1235,8 @@
     list_clear(STM_PSEGMENT->objects_pointing_to_nursery);
     list_clear(STM_PSEGMENT->old_objects_with_cards_set);
     list_clear(STM_PSEGMENT->large_overflow_objects);
-    timing_event(tl, event);
+    if (tl != NULL)
+        timing_event(tl, event);
 
     release_thread_segment(tl);
     /* cannot access STM_SEGMENT or STM_PSEGMENT from here ! */
@@ -1290,19 +1291,24 @@
 
     assert(!_has_mutex());
     assert(STM_PSEGMENT->safe_point == SP_RUNNING);
-    //assert(STM_PSEGMENT->running_pthread == pthread_self());
-    // ^^^ fails if detach.c commits a detached inevitable transaction
+    assert(STM_PSEGMENT->running_pthread == pthread_self());
 
     dprintf(("> stm_commit_transaction()\n"));
     minor_collection(1);
 
+    _core_commit_transaction();
+}
+
+static void _core_commit_transaction(void)
+{
     push_large_overflow_objects_to_other_segments();
     /* push before validate. otherwise they are reachable too early */
 
     bool was_inev = STM_PSEGMENT->transaction_state == TS_INEVITABLE;
     _validate_and_add_to_commit_log();
 
-    stm_rewind_jmp_forget(STM_SEGMENT->running_thread);
+    if (!was_inev)
+        stm_rewind_jmp_forget(STM_SEGMENT->running_thread);
 
     /* XXX do we still need a s_mutex_lock() section here? */
     s_mutex_lock();
@@ -1343,7 +1349,8 @@
 
     /* between transactions, call finalizers. this will execute
        a transaction itself */
-    invoke_general_finalizers(tl);
+    if (tl != NULL)
+        invoke_general_finalizers(tl);
 }
 
 static void reset_modified_from_backup_copies(int segment_num)
diff --git a/c8/stm/core.h b/c8/stm/core.h
--- a/c8/stm/core.h
+++ b/c8/stm/core.h
@@ -300,6 +300,7 @@
 
 static void _signal_handler(int sig, siginfo_t *siginfo, void *context);
 static bool _stm_validate(void);
+static void _core_commit_transaction(void);
 
 static inline bool was_read_remote(char *base, object_t *obj)
 {
diff --git a/c8/stm/detach.c b/c8/stm/detach.c
--- a/c8/stm/detach.c
+++ b/c8/stm/detach.c
@@ -2,21 +2,31 @@
 # error "must be compiled via stmgc.c"
 #endif
 
+/* Idea: if stm_leave_transactional_zone() is quickly followed by
+   stm_enter_transactional_zone() in the same thread, then we should
+   simply try to have one inevitable transaction that does both sides.
+   This is useful if there are many such small interruptions.
 
-/* _stm_detached_inevitable_segnum is:
+   stm_leave_transactional_zone() tries to make sure the transaction
+   is inevitable, and then sticks the current 'stm_thread_local_t *'
+   into _stm_detached_inevitable_from_thread.
+   stm_enter_transactional_zone() has a fast-path if the same
+   'stm_thread_local_t *' is still there.
 
-   - -1: there is no inevitable transaction, or it is not detached
+   If a different thread grabs it, it atomically replaces the value in
+   _stm_detached_inevitable_from_thread with -1, commits it (this part
+   involves reading for example the shadowstack of the thread that
+   originally detached), and at the point where we know the original
+   stm_thread_local_t is no longer relevant, we reset
+   _stm_detached_inevitable_from_thread to 0.
+*/
 
-   - in range(1, NB_SEGMENTS): an inevitable transaction belongs to
-     the segment and was detached.  It might concurrently be
-     reattached at any time, with an XCHG (__sync_lock_test_and_set).
-*/
-volatile int _stm_detached_inevitable_seg_num;
+volatile intptr_t _stm_detached_inevitable_from_thread;
 
 
 static void setup_detach(void)
 {
-    _stm_detached_inevitable_seg_num = -1;
+    _stm_detached_inevitable_from_thread = 0;
 }
 
 
@@ -26,50 +36,108 @@
 
     /* did it work? */
     if (STM_PSEGMENT->transaction_state == TS_INEVITABLE) {   /* yes */
-        _stm_detach_inevitable_transaction(STM_SEGMENT->running_thread);
+        dprintf(("leave_noninevitable_transactional_zone: now inevitable\n"));
+        stm_thread_local_t *tl = STM_SEGMENT->running_thread;
+        _stm_detach_inevitable_transaction(tl);
     }
     else {   /* no */
+        dprintf(("leave_noninevitable_transactional_zone: commit\n"));
         _stm_commit_transaction();
     }
 }
 
-void _stm_reattach_transaction(int old, stm_thread_local_t *tl)
+static void commit_external_inevitable_transaction(void)
 {
-    if (old == -1) {
-        /* there was no detached inevitable transaction */
-        _stm_start_transaction(tl);
+    assert(!_has_mutex());
+    assert(STM_PSEGMENT->safe_point == SP_RUNNING);
+    assert(STM_PSEGMENT->transaction_state == TS_INEVITABLE); /* can't abort */
+
+    exec_local_finalizers();
+    minor_collection(1);
+
+    /* from this point on, unlink the original 'stm_thread_local_t *'
+       from its segment.  Better do it as soon as possible, because
+       other threads might be spin-looping, waiting for the -1 to
+       disappear.  XXX could be done even earlier, as soon as we have
+       read the shadowstack inside the minor collection. */
+    STM_SEGMENT->running_thread = NULL;
+    write_fence();
+    assert(_stm_detached_inevitable_from_thread == -1);
+    _stm_detached_inevitable_from_thread = 0;
+
+    _core_commit_transaction();
+}
+
+void _stm_reattach_transaction(intptr_t old, stm_thread_local_t *tl)
+{
+ restart:
+    if (old != 0) {
+        if (old == -1) {
+            /* busy-loop: wait until _stm_detached_inevitable_from_thread
+               is reset to a value different from -1 */
+            while (_stm_detached_inevitable_from_thread == -1)
+                spin_loop();
+
+            /* then retry */
+            atomic_exchange(&_stm_detached_inevitable_from_thread, old, -1);
+            goto restart;
+        }
+
+        stm_thread_local_t *old_tl = (stm_thread_local_t *)old;
+        int remote_seg_num = old_tl->last_associated_segment_num;
+        dprintf(("reattach_transaction: commit detached from seg %d\n",
+                 remote_seg_num));
+
+        ensure_gs_register(remote_seg_num);
+        commit_external_inevitable_transaction();
     }
     else {
-        /* We took over the inevitable transaction originally detached
-           from a different segment.  We have to fix the %gs register if
-           it is incorrect.
-        */
-        tl->last_associated_segment_num = old;
-        ensure_gs_register(old);
-        assert(STM_PSEGMENT->transaction_state == TS_INEVITABLE);
-        STM_SEGMENT->running_thread = tl;
-
-        stm_safe_point();
+        assert(_stm_detached_inevitable_from_thread == -1);
+        _stm_detached_inevitable_from_thread = 0;
     }
+    dprintf(("reattach_transaction: start a new transaction\n"));
+    _stm_start_transaction(tl);
 }
 
 void stm_force_transaction_break(stm_thread_local_t *tl)
 {
+    dprintf(("> stm_force_transaction_break()\n"));
     assert(STM_SEGMENT->running_thread == tl);
     _stm_commit_transaction();
     _stm_start_transaction(tl);
 }
 
-static int fetch_detached_transaction(void)
+static intptr_t fetch_detached_transaction(void)
 {
-    int cur = _stm_detached_inevitable_seg_num;
-    if (cur != -1)
-        cur = __sync_lock_test_and_set(    /* XCHG */
-            &_stm_detached_inevitable_seg_num, -1);
+    intptr_t cur;
+ restart:
+    cur = _stm_detached_inevitable_from_thread;
+    if (cur == 0) {    /* fast-path */
+        return 0;   /* _stm_detached_inevitable_from_thread not changed */
+    }
+    if (cur != -1) {
+        atomic_exchange(&_stm_detached_inevitable_from_thread, cur, -1);
+        if (cur == 0) {
+            /* found 0, so change from -1 to 0 again and return */
+            _stm_detached_inevitable_from_thread = 0;
+            return 0;
+        }
+    }
+    if (cur == -1) {
+        /* busy-loop: wait until _stm_detached_inevitable_from_thread
+           is reset to a value different from -1 */
+        while (_stm_detached_inevitable_from_thread == -1)
+            spin_loop();
+        goto restart;
+    }
+    /* this is the only case where we grabbed a detached transaction.
+       _stm_detached_inevitable_from_thread is still -1, until
+       commit_fetched_detached_transaction() is called. */
+    assert(_stm_detached_inevitable_from_thread == -1);
     return cur;
 }
 
-static void commit_fetched_detached_transaction(int segnum)
+static void commit_fetched_detached_transaction(intptr_t old)
 {
     /* Here, 'seg_num' is the segment that contains the detached
        inevitable transaction from fetch_detached_transaction(),
@@ -79,13 +147,33 @@
        transaction.  This should guarantee there are not race
        conditions.
     */
+    int segnum = ((stm_thread_local_t *)old)->last_associated_segment_num;
+    dprintf(("commit_fetched_detached_transaction from seg %d\n", segnum));
     assert(segnum > 0);
 
     int mysegnum = STM_SEGMENT->segment_num;
     ensure_gs_register(segnum);
-
-    assert(STM_PSEGMENT->transaction_state == TS_INEVITABLE);
-    _stm_commit_transaction();   /* can't abort */
-
+    commit_external_inevitable_transaction();
     ensure_gs_register(mysegnum);
 }
+
+static void commit_detached_transaction_if_from(stm_thread_local_t *tl)
+{
+    intptr_t old;
+ restart:
+    old = _stm_detached_inevitable_from_thread;
+    if (old == (intptr_t)tl) {
+        if (!__sync_bool_compare_and_swap(&_stm_detached_inevitable_from_thread,
+                                          old, -1))
+            goto restart;
+        commit_fetched_detached_transaction(old);
+        return;
+    }
+    if (old == -1) {
+        /* busy-loop: wait until _stm_detached_inevitable_from_thread
+           is reset to a value different from -1 */
+        while (_stm_detached_inevitable_from_thread == -1)
+            spin_loop();
+        goto restart;
+    }
+}
diff --git a/c8/stm/detach.h b/c8/stm/detach.h
--- a/c8/stm/detach.h
+++ b/c8/stm/detach.h
@@ -1,4 +1,5 @@
 
 static void setup_detach(void);
-static int fetch_detached_transaction(void);
-static void commit_fetched_detached_transaction(int segnum);
+static intptr_t fetch_detached_transaction(void);
+static void commit_fetched_detached_transaction(intptr_t old);
+static void commit_detached_transaction_if_from(stm_thread_local_t *tl);
diff --git a/c8/stm/setup.c b/c8/stm/setup.c
--- a/c8/stm/setup.c
+++ b/c8/stm/setup.c
@@ -264,6 +264,8 @@
 
 void stm_unregister_thread_local(stm_thread_local_t *tl)
 {
+    commit_detached_transaction_if_from(tl);
+
     s_mutex_lock();
     assert(tl->prev != NULL);
     assert(tl->next != NULL);
diff --git a/c8/stm/sync.c b/c8/stm/sync.c
--- a/c8/stm/sync.c
+++ b/c8/stm/sync.c
@@ -215,10 +215,12 @@
         num = (num+1) % (NB_SEGMENTS-1);
         if (sync_ctl.in_use1[num+1] == 0) {
             /* we're getting 'num', a different number. */
-            dprintf(("acquired different segment: %d->%d\n",
-                     tl->last_associated_segment_num, num+1));
+            int old_num = tl->last_associated_segment_num;
+            dprintf(("acquired different segment: %d->%d\n", old_num, num+1));
             tl->last_associated_segment_num = num+1;
             set_gs_register(get_segment_base(num+1));
+            dprintf(("                            %d->%d\n", old_num, num+1));
+            (void)old_num;
             goto got_num;
         }
     }
@@ -245,18 +247,22 @@
 
 static void release_thread_segment(stm_thread_local_t *tl)
 {
+    int segnum;
     assert(_has_mutex());
 
     cond_signal(C_SEGMENT_FREE);
 
     assert(STM_SEGMENT->running_thread == tl);
-    assert(tl->last_associated_segment_num == STM_SEGMENT->segment_num);
-    assert(in_transaction(tl));
-    STM_SEGMENT->running_thread = NULL;
-    assert(!in_transaction(tl));
+    segnum = STM_SEGMENT->segment_num;
+    if (tl != NULL) {
+        assert(tl->last_associated_segment_num == segnum);
+        assert(in_transaction(tl));
+        STM_SEGMENT->running_thread = NULL;
+        assert(!in_transaction(tl));
+    }
 
-    assert(sync_ctl.in_use1[tl->last_associated_segment_num] == 1);
-    sync_ctl.in_use1[tl->last_associated_segment_num] = 0;
+    assert(sync_ctl.in_use1[segnum] == 1);
+    sync_ctl.in_use1[segnum] = 0;
 }
 
 __attribute__((unused))
@@ -414,8 +420,8 @@
        Wait until all other threads are suspended. */
     while (count_other_threads_sp_running() > 0) {
 
-        int detached = fetch_detached_transaction();
-        if (detached >= 0) {
+        intptr_t detached = fetch_detached_transaction();
+        if (detached != 0) {
             remove_requests_for_safe_point();    /* => C_REQUEST_REMOVED */
             commit_fetched_detached_transaction(detached);
             goto restart;
diff --git a/c8/stmgc.h b/c8/stmgc.h
--- a/c8/stmgc.h
+++ b/c8/stmgc.h
@@ -84,16 +84,16 @@
 object_t *_stm_allocate_slowpath(ssize_t);
 object_t *_stm_allocate_external(ssize_t);
 
-extern volatile int _stm_detached_inevitable_seg_num;
+extern volatile intptr_t _stm_detached_inevitable_from_thread;
 long _stm_start_transaction(stm_thread_local_t *tl);
 void _stm_commit_transaction(void);
 void _stm_leave_noninevitable_transactional_zone(void);
-#define _stm_detach_inevitable_transaction(tl)  do {                    \
-    write_fence();                                                      \
-    assert((tl)->last_associated_segment_num == STM_SEGMENT->segment_num); \
-    _stm_detached_inevitable_seg_num = STM_SEGMENT->segment_num;        \
+#define _stm_detach_inevitable_transaction(tl)  do {            \
+    write_fence();                                              \
+    assert(_stm_detached_inevitable_from_thread == 0);          \
+    _stm_detached_inevitable_from_thread = (intptr_t)(tl);      \
 } while (0)
-void _stm_reattach_transaction(int old, stm_thread_local_t *tl);
+void _stm_reattach_transaction(intptr_t old, stm_thread_local_t *tl);
 void _stm_become_inevitable(const char*);
 void _stm_collectable_safe_point(void);
 
@@ -421,12 +421,15 @@
    transactions.
 */
 static inline void stm_enter_transactional_zone(stm_thread_local_t *tl) {
-    int old = __sync_lock_test_and_set(    /* XCHG */
-        &_stm_detached_inevitable_seg_num, -1);
-    if (old == tl->last_associated_segment_num)
-        STM_SEGMENT->running_thread = tl;
-    else
+    intptr_t old;
+    atomic_exchange(&_stm_detached_inevitable_from_thread, old, -1);
+    if (old == (intptr_t)tl) {
+        _stm_detached_inevitable_from_thread = 0;
+    }
+    else {
         _stm_reattach_transaction(old, tl);
+        assert(_stm_detached_inevitable_from_thread == 0);
+    }
 }
 static inline void stm_leave_transactional_zone(stm_thread_local_t *tl) {
     assert(STM_SEGMENT->running_thread == tl);
@@ -458,7 +461,7 @@
     if (!stm_is_inevitable())
         _stm_become_inevitable(msg);
     /* now, we're running the inevitable transaction, so: */
-    assert(_stm_detached_inevitable_seg_num == -1);
+    assert(_stm_detached_inevitable_from_thread == 0);
 }
 
 /* Forces a safe-point if needed.  Normally not needed: this is

From noreply at buildbot.pypy.org  Thu Jun 11 11:27:26 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Thu, 11 Jun 2015 11:27:26 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: removed set_trace call
Message-ID: <20150611092726.917E51C0579@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r78028:2c6a43445680
Date: 2015-06-11 11:27 +0200
http://bitbucket.org/pypy/pypy/changeset/2c6a43445680/

Log:	removed set_trace call

diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/vectorize.py
@@ -399,8 +399,6 @@
     def unpack_from_vector(self, op, sched_data, renamer):
         renamer.rename(op)
         args = op.getarglist()
-        if op.getopnum() == rop.INT_SIGNEXT:
-            py.test.set_trace()
         for i, arg in enumerate(op.getarglist()):
             if isinstance(arg, Box):
                 argument = self._unpack_from_vector(i, arg, sched_data, renamer)

From noreply at buildbot.pypy.org  Thu Jun 11 11:32:55 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 11 Jun 2015 11:32:55 +0200 (CEST)
Subject: [pypy-commit] stmgc c8-gil-like: fix
Message-ID: <20150611093255.9D7091C0EB1@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: c8-gil-like
Changeset: r1802:cc87f515da59
Date: 2015-06-11 11:33 +0200
http://bitbucket.org/pypy/stmgc/changeset/cc87f515da59/

Log:	fix

diff --git a/c8/stm/core.h b/c8/stm/core.h
--- a/c8/stm/core.h
+++ b/c8/stm/core.h
@@ -162,7 +162,6 @@
 #ifdef STM_TESTS
     SP_WAIT_FOR_OTHER_THREAD,
 #endif
-    SP_RUNNING_DETACHED_FETCHED,
 };
 
 enum /* transaction_state */ {
diff --git a/c8/stm/sync.c b/c8/stm/sync.c
--- a/c8/stm/sync.c
+++ b/c8/stm/sync.c
@@ -423,7 +423,9 @@
         intptr_t detached = fetch_detached_transaction();
         if (detached != 0) {
             remove_requests_for_safe_point();    /* => C_REQUEST_REMOVED */
+            s_mutex_unlock();
             commit_fetched_detached_transaction(detached);
+            s_mutex_lock();
             goto restart;
         }
 

From noreply at buildbot.pypy.org  Thu Jun 11 11:37:04 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Thu, 11 Jun 2015 11:37:04 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: added llgraph impl for reduce (+,-,*)n
Message-ID: <20150611093704.179031C0F4B@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r78029:fa11cb002cc1
Date: 2015-06-11 11:35 +0200
http://bitbucket.org/pypy/pypy/changeset/fa11cb002cc1/

Log:	added llgraph impl for reduce (+,-,*)n

diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -3,7 +3,7 @@
 from rpython.jit.backend.llgraph import support
 from rpython.jit.backend.llsupport import symbolic
 from rpython.jit.metainterp.history import AbstractDescr
-from rpython.jit.metainterp.history import Const, getkind
+from rpython.jit.metainterp.history import Const, getkind, BoxVectorAccum
 from rpython.jit.metainterp.history import INT, REF, FLOAT, VOID, VECTOR
 from rpython.jit.metainterp.resoperation import rop
 from rpython.jit.metainterp.optimizeopt import intbounds
@@ -862,6 +862,17 @@
                 value = self.env[box]
             else:
                 value = None
+            if isinstance(box, BoxVectorAccum):
+                if box.operator == '+':
+                    value = sum(value)
+                elif box.operator == '-':
+                    def sub(acc, x): return acc - x
+                    value = reduce(sub, value, 0)
+                elif box.operator == '*':
+                    def prod(acc, x): return acc * x
+                    value = reduce(prod, value, 1)
+                else:
+                    raise NotImplementedError
             values.append(value)
         if hasattr(descr, '_llgraph_bridge'):
             target = (descr._llgraph_bridge, -1)

From noreply at buildbot.pypy.org  Thu Jun 11 12:04:56 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 11 Jun 2015 12:04:56 +0200 (CEST)
Subject: [pypy-commit] stmgc c8-gil-like: fix test
Message-ID: <20150611100456.A2D3B1C1570@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: c8-gil-like
Changeset: r1803:e414755a0ddc
Date: 2015-06-11 12:05 +0200
http://bitbucket.org/pypy/stmgc/changeset/e414755a0ddc/

Log:	fix test

diff --git a/c8/demo/demo_random.c b/c8/demo/demo_random.c
--- a/c8/demo/demo_random.c
+++ b/c8/demo/demo_random.c
@@ -367,7 +367,6 @@
 
             long call_fork = (arg != NULL && *(long *)arg);
             if (call_fork == 0) {   /* common case */
-                td.num_roots_at_transaction_start = td.num_roots;
                 if (get_rand(100) < 50) {
                     stm_leave_transactional_zone(&stm_thread_local);
                     /* Nothing here; it's unlikely that a different thread
@@ -376,10 +375,13 @@
                     fprintf(stderr, "sleep...\n");
                     usleep(1);
                     fprintf(stderr, "sleep done\n");
+                    td.num_roots_at_transaction_start = td.num_roots;
                     stm_enter_transactional_zone(&stm_thread_local);
                 }
                 else {
-                    stm_force_transaction_break(&stm_thread_local);
+                    _stm_commit_transaction();
+                    td.num_roots_at_transaction_start = td.num_roots;
+                    _stm_start_transaction(&stm_thread_local);
                 }
                 td.num_roots = td.num_roots_at_transaction_start;
                 p = NULL;

From noreply at buildbot.pypy.org  Thu Jun 11 12:42:01 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 11 Jun 2015 12:42:01 +0200 (CEST)
Subject: [pypy-commit] stmgc c8-gil-like: fixfix
Message-ID: <20150611104201.B5FB41C06D1@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: c8-gil-like
Changeset: r1804:ff025e03931c
Date: 2015-06-11 12:42 +0200
http://bitbucket.org/pypy/stmgc/changeset/ff025e03931c/

Log:	fixfix

diff --git a/c8/stm/core.h b/c8/stm/core.h
--- a/c8/stm/core.h
+++ b/c8/stm/core.h
@@ -162,6 +162,7 @@
 #ifdef STM_TESTS
     SP_WAIT_FOR_OTHER_THREAD,
 #endif
+    SP_COMMIT_OTHER_DETACHED,
 };
 
 enum /* transaction_state */ {
diff --git a/c8/stm/detach.c b/c8/stm/detach.c
--- a/c8/stm/detach.c
+++ b/c8/stm/detach.c
@@ -144,17 +144,32 @@
        probably belonging to an unrelated thread.  We fetched it,
        which means that nobody else can concurrently fetch it now, but
        everybody will see that there is still a concurrent inevitable
-       transaction.  This should guarantee there are not race
+       transaction.  This should guarantee there are no race
        conditions.
     */
+    int mysegnum = STM_SEGMENT->segment_num;
     int segnum = ((stm_thread_local_t *)old)->last_associated_segment_num;
     dprintf(("commit_fetched_detached_transaction from seg %d\n", segnum));
     assert(segnum > 0);
 
-    int mysegnum = STM_SEGMENT->segment_num;
-    ensure_gs_register(segnum);
+    if (segnum != mysegnum) {
+        s_mutex_lock();
+        assert(STM_PSEGMENT->safe_point == SP_RUNNING);
+        STM_PSEGMENT->safe_point = SP_COMMIT_OTHER_DETACHED;
+        s_mutex_unlock();
+
+        set_gs_register(get_segment_base(segnum));
+    }
     commit_external_inevitable_transaction();
-    ensure_gs_register(mysegnum);
+
+    if (segnum != mysegnum) {
+        set_gs_register(get_segment_base(mysegnum));
+
+        s_mutex_lock();
+        assert(STM_PSEGMENT->safe_point == SP_COMMIT_OTHER_DETACHED);
+        STM_PSEGMENT->safe_point = SP_RUNNING;
+        s_mutex_unlock();
+    }
 }
 
 static void commit_detached_transaction_if_from(stm_thread_local_t *tl)
diff --git a/c8/stm/sync.c b/c8/stm/sync.c
--- a/c8/stm/sync.c
+++ b/c8/stm/sync.c
@@ -329,6 +329,7 @@
     }
     assert(!pause_signalled);
     pause_signalled = true;
+    dprintf(("request to pause\n"));
 }
 
 static inline long count_other_threads_sp_running(void)
@@ -363,6 +364,7 @@
         if (get_segment(i)->nursery_end == NSE_SIGPAUSE)
             get_segment(i)->nursery_end = NURSERY_END;
     }
+    dprintf(("request removed\n"));
     cond_broadcast(C_REQUEST_REMOVED);
 }
 
@@ -380,6 +382,7 @@
         if (STM_SEGMENT->nursery_end == NURSERY_END)
             break;    /* no safe point requested */
 
+        dprintf(("enter safe point\n"));
         assert(STM_SEGMENT->nursery_end == NSE_SIGPAUSE);
         assert(pause_signalled);
 
@@ -394,6 +397,7 @@
         cond_wait(C_REQUEST_REMOVED);
         STM_PSEGMENT->safe_point = SP_RUNNING;
         timing_event(STM_SEGMENT->running_thread, STM_WAIT_DONE);
+        dprintf(("left safe point\n"));
     }
 }
 
diff --git a/c8/stmgc.h b/c8/stmgc.h
--- a/c8/stmgc.h
+++ b/c8/stmgc.h
@@ -428,7 +428,8 @@
     }
     else {
         _stm_reattach_transaction(old, tl);
-        assert(_stm_detached_inevitable_from_thread == 0);
+        /* _stm_detached_inevitable_from_thread should be 0 here, but
+           it can already have been changed from a parallel thread */
     }
 }
 static inline void stm_leave_transactional_zone(stm_thread_local_t *tl) {

From noreply at buildbot.pypy.org  Thu Jun 11 17:15:17 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Thu, 11 Jun 2015 17:15:17 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: some test changes,
 removed an error when the output type is none
Message-ID: <20150611151517.B7EEA1C0186@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r78030:c291ce811739
Date: 2015-06-11 17:15 +0200
http://bitbucket.org/pypy/pypy/changeset/c291ce811739/

Log:	some test changes, removed an error when the output type is none

diff --git a/rpython/jit/metainterp/optimizeopt/test/test_schedule.py b/rpython/jit/metainterp/optimizeopt/test/test_schedule.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_schedule.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_schedule.py
@@ -53,7 +53,7 @@
             if len(pack) == 1:
                 ops.append(pack[0].getoperation())
             else:
-                for op in vsd.as_vector_operation(Pack(pack), renamer):
+                for op in vsd.as_vector_operation(Pack(pack, None, None), renamer):
                     ops.append(op)
         loop.operations = ops
         if prepend_invariant:
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py b/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
@@ -53,7 +53,7 @@
     def vectoroptimizer(self, loop):
         metainterp_sd = FakeMetaInterpStaticData(self.cpu)
         jitdriver_sd = FakeJitDriverStaticData()
-        opt = VectorizingOptimizer(metainterp_sd, jitdriver_sd, loop, [])
+        opt = VectorizingOptimizer(metainterp_sd, jitdriver_sd, loop, 0)
         return opt
 
     def vectoroptimizer_unrolled(self, loop, unroll_factor = -1):
@@ -96,12 +96,15 @@
         opt.combine_packset()
         return opt
 
-    def schedule(self, loop, unroll_factor = -1):
+    def schedule(self, loop, unroll_factor = -1, with_guard_opt=False):
         opt = self.vectoroptimizer_unrolled(loop, unroll_factor)
         opt.find_adjacent_memory_refs()
         opt.extend_packset()
         opt.combine_packset()
         opt.schedule(True)
+        if with_guard_opt:
+            gso = GuardStrengthenOpt(opt.dependency_graph.index_vars)
+            gso.propagate_all_forward(opt.loop)
         return opt
 
     def vectorize(self, loop, unroll_factor = -1):
@@ -962,7 +965,7 @@
         i1 = vec_getarrayitem_raw(p0, i0, 16, descr=chararraydescr)
         jump(p0,i2)
         """.format(dead_code=dead_code)
-        vopt = self.vectorize(self.parse_loop(ops),15)
+        vopt = self.schedule(self.parse_loop(ops),15,with_guard_opt=True)
         self.assert_equal(vopt.loop, self.parse_loop(opt))
 
     def test_too_small_vector(self):
@@ -1304,8 +1307,11 @@
         guard_false(i39) [i1, p9, p8, p6, p4, p3, i33, i38, None, None, i26, i11, None, p13, None, None, p10]
         jump(i1, p10, i11, p8, i26, p3, p4, p13, i33, i38, p6, p9, i16, i17, i18, i19, i20, i21, i22, i23)
         """
-        opt = self.vectorize(self.parse_loop(trace))
-        self.debug_print_operations(opt.loop)
+        try:
+            self.vectorize(self.parse_loop(trace))
+            py.test.fail("axis sum is not profitable")
+        except NotAProfitableLoop:
+            pass
 
     def test_cast_1(self):
         trace = """
@@ -1364,7 +1370,7 @@
         guard_false(i17) [p2, i16, f9, i14, None, None, None, p3]
         jump(p3, i14, p2, i16, f9, i7, i8)
         """
-        opt = self.vectorize(self.parse_loop(trace))
+        opt = self.schedule(self.parse_loop(trace), with_guard_opt=True)
         self.debug_print_operations(opt.loop)
 
 
diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/vectorize.py
@@ -522,6 +522,8 @@
     def cb_signext(self, pack):
         op0 = pack.operations[0].getoperation()
         size = op0.getarg(1).getint()
+        if pack.output_type is None:
+            return 1,0
         orig_size = pack.output_type.getsize()
         if size == orig_size:
             return 0,0

From noreply at buildbot.pypy.org  Thu Jun 11 17:59:06 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 11 Jun 2015 17:59:06 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: a hack to make sure that we have a
 correct type in fielddescr
Message-ID: <20150611155906.1151E1C12D1@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r78031:12b6a3d6cfcf
Date: 2015-06-11 17:59 +0200
http://bitbucket.org/pypy/pypy/changeset/12b6a3d6cfcf/

Log:	a hack to make sure that we have a correct type in fielddescr

diff --git a/rpython/jit/backend/llsupport/descr.py b/rpython/jit/backend/llsupport/descr.py
--- a/rpython/jit/backend/llsupport/descr.py
+++ b/rpython/jit/backend/llsupport/descr.py
@@ -126,6 +126,16 @@
     def __repr__(self):
         return 'FieldDescr<%s>' % (self.name,)
 
+    def check_correct_type(self, struct):
+        if isinstance(self.parent_descr, SizeDescrWithVTable):
+            cls = llmemory.cast_adr_to_ptr(
+                heaptracker.int2adr(self.parent_descr.get_vtable()),
+                lltype.Ptr(rclass.OBJECT_VTABLE))
+            assert rclass.ll_isinstance(lltype.cast_opaque_ptr(
+                rclass.OBJECTPTR, struct), cls)
+        else:
+            pass
+
     def is_pointer_field(self):
         return self.flag == FLAG_POINTER
 
diff --git a/rpython/jit/backend/llsupport/llmodel.py b/rpython/jit/backend/llsupport/llmodel.py
--- a/rpython/jit/backend/llsupport/llmodel.py
+++ b/rpython/jit/backend/llsupport/llmodel.py
@@ -577,15 +577,18 @@
     @specialize.argtype(1)
     def bh_setfield_gc_i(self, struct, newvalue, fielddescr):
         ofs, size, _ = self.unpack_fielddescr_size(fielddescr)
+        fielddescr.check_correct_type(struct)
         self.write_int_at_mem(struct, ofs, size, newvalue)
 
     def bh_setfield_gc_r(self, struct, newvalue, fielddescr):
         ofs = self.unpack_fielddescr(fielddescr)
+        fielddescr.check_correct_type(struct)
         self.write_ref_at_mem(struct, ofs, newvalue)
 
     @specialize.argtype(1)
     def bh_setfield_gc_f(self, struct, newvalue, fielddescr):
         ofs = self.unpack_fielddescr(fielddescr)
+        fielddescr.check_correct_type(struct)
         self.write_float_at_mem(struct, ofs, newvalue)
 
     bh_setfield_raw_i = bh_setfield_gc_i
diff --git a/rpython/memory/gc/base.py b/rpython/memory/gc/base.py
--- a/rpython/memory/gc/base.py
+++ b/rpython/memory/gc/base.py
@@ -124,6 +124,9 @@
     def get_size(self, obj):
         return self._get_size_for_typeid(obj, self.get_type_id(obj))
 
+    def get_type_id_cast(self, obj):
+        return rffi.cast(lltype.Signed, self.get_type_id(obj))
+
     def get_size_incl_hash(self, obj):
         return self.get_size(obj)
 
diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py
--- a/rpython/memory/gctransform/framework.py
+++ b/rpython/memory/gctransform/framework.py
@@ -335,6 +335,10 @@
                 [s_gc, annmodel.SomeInteger(knowntype=llgroup.r_halfword)],
                 annmodel.SomeInteger())
 
+        self.gc_gettypeid_ptr = getfn(GCClass.get_type_id_cast,
+                                       [s_gc, SomeAddress()],
+                                       annmodel.SomeInteger())
+
         if hasattr(GCClass, 'writebarrier_before_copy'):
             self.wb_before_copy_ptr = \
                     getfn(GCClass.writebarrier_before_copy.im_func,
@@ -780,6 +784,16 @@
                                   v_addr, v_length],
                   resultvar=op.result)
 
+    def gct_gc_gettypeid(self, hop):
+        op = hop.spaceop
+        v_addr = op.args[0]
+        if v_addr.concretetype != llmemory.Address:
+            v_addr = hop.genop("cast_ptr_to_adr", [v_addr],
+                               resulttype=llmemory.Address)
+        hop.genop("direct_call", [self.gc_gettypeid_ptr, self.c_const_gc,
+                                  v_addr],
+                         resultvar=op.result)
+
     def gct_gc_writebarrier(self, hop):
         if self.write_barrier_ptr is None:
             return
diff --git a/rpython/memory/gcwrapper.py b/rpython/memory/gcwrapper.py
--- a/rpython/memory/gcwrapper.py
+++ b/rpython/memory/gcwrapper.py
@@ -74,6 +74,9 @@
             return lltype.malloc(TYPE, n, flavor=flavor, zero=zero,
                                  track_allocation=track_allocation)
 
+    def gettypeid(self, obj):
+        return self.get_type_id(lltype.typeOf(obj).TO)
+
     def add_memory_pressure(self, size):
         if hasattr(self.gc, 'raw_malloc_memory_pressure'):
             self.gc.raw_malloc_memory_pressure(size)
diff --git a/rpython/memory/test/gc_test_base.py b/rpython/memory/test/gc_test_base.py
--- a/rpython/memory/test/gc_test_base.py
+++ b/rpython/memory/test/gc_test_base.py
@@ -864,6 +864,16 @@
         else:
             assert res == 0 or res == 13
 
+    def test_gettypeid(self):
+        class A(object):
+            pass
+        
+        def fn():
+            a = A()
+            return rgc.get_typeid(a)
+
+        self.interpret(fn, [])
+
 
 from rpython.rlib.objectmodel import UnboxedValue
 
diff --git a/rpython/memory/test/test_transformed_gc.py b/rpython/memory/test/test_transformed_gc.py
--- a/rpython/memory/test/test_transformed_gc.py
+++ b/rpython/memory/test/test_transformed_gc.py
@@ -1353,6 +1353,22 @@
         res = func([])
         assert res == -1999
 
+    def define_gettypeid(cls):
+        class A(object):
+            pass
+        
+        def fn():
+            a = A()
+            return rgc.get_typeid(a)
+
+        return fn
+
+    def test_gettypeid(self):
+        func = self.runner("gettypeid")
+        res = func([])
+        print res
+
+
 from rpython.rlib.objectmodel import UnboxedValue
 
 class TaggedBase(object):
diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py
--- a/rpython/rlib/rgc.py
+++ b/rpython/rlib/rgc.py
@@ -715,3 +715,19 @@
         funcptr = hop.rtyper.annotate_helper_fn(ll_func, args_s)
         hop.exception_cannot_occur()
         lltype.attachRuntimeTypeInfo(TP, destrptr=funcptr)
+
+all_typeids = {}
+        
+def get_typeid(obj):
+    raise Exception("does not work untranslated")
+
+class GetTypeidEntry(ExtRegistryEntry):
+    _about_ = get_typeid
+
+    def compute_result_annotation(self, s_obj):
+        from rpython.annotator import model as annmodel
+        return annmodel.SomeInteger()
+
+    def specialize_call(self, hop):
+        hop.exception_cannot_occur()
+        return hop.genop('gc_gettypeid', hop.args_v, resulttype=hop.r_result)
diff --git a/rpython/rtyper/llinterp.py b/rpython/rtyper/llinterp.py
--- a/rpython/rtyper/llinterp.py
+++ b/rpython/rtyper/llinterp.py
@@ -708,6 +708,9 @@
     def op_gc_add_memory_pressure(self, size):
         self.heap.add_memory_pressure(size)
 
+    def op_gc_gettypeid(self, obj):
+        return lltype.cast_primitive(lltype.Signed, self.heap.gettypeid(obj))
+
     def op_shrink_array(self, obj, smallersize):
         return self.heap.shrink_array(obj, smallersize)
 
diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -498,6 +498,7 @@
     'gc_dump_rpy_heap'    : LLOp(),
     'gc_typeids_z'        : LLOp(),
     'gc_typeids_list'     : LLOp(),
+    'gc_gettypeid'        : LLOp(),
     'gc_gcflag_extra'     : LLOp(),
     'gc_add_memory_pressure': LLOp(),
 

From noreply at buildbot.pypy.org  Thu Jun 11 18:35:29 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Thu, 11 Jun 2015 18:35:29 +0200 (CEST)
Subject: [pypy-commit] pypy unicode-dtype: hg merge default
Message-ID: <20150611163529.25F7D1C12D1@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: unicode-dtype
Changeset: r78032:5b192b71776d
Date: 2015-06-11 17:35 +0100
http://bitbucket.org/pypy/pypy/changeset/5b192b71776d/

Log:	hg merge default

diff --git a/pypy/module/_cffi_backend/lib_obj.py b/pypy/module/_cffi_backend/lib_obj.py
--- a/pypy/module/_cffi_backend/lib_obj.py
+++ b/pypy/module/_cffi_backend/lib_obj.py
@@ -65,7 +65,7 @@
         ptr = rffi.cast(rffi.CCHARP, g.c_address)
         assert ptr
         return W_FunctionWrapper(self.space, ptr, g.c_size_or_direct_fn, w_ct,
-                                 locs, rawfunctype, fnname)
+                                 locs, rawfunctype, fnname, self.libname)
 
     @jit.elidable_promote()
     def _get_attr_elidable(self, attr):
diff --git a/pypy/module/_cffi_backend/test/test_recompiler.py b/pypy/module/_cffi_backend/test/test_recompiler.py
--- a/pypy/module/_cffi_backend/test/test_recompiler.py
+++ b/pypy/module/_cffi_backend/test/test_recompiler.py
@@ -418,6 +418,11 @@
         # 'x' is another <built-in method> object on lib, made very indirectly
         x = type(lib).__dir__.__get__(lib)
         raises(TypeError, ffi.typeof, x)
+        #
+        # present on built-in functions on CPython; must be emulated on PyPy:
+        assert lib.sin.__name__ == 'sin'
+        assert lib.sin.__module__ == '_CFFI_test_math_sin_type'
+        assert lib.sin.__doc__=='direct call to the C function of the same name'
 
     def test_verify_anonymous_struct_with_typedef(self):
         ffi, lib = self.prepare(
diff --git a/pypy/module/_cffi_backend/wrapper.py b/pypy/module/_cffi_backend/wrapper.py
--- a/pypy/module/_cffi_backend/wrapper.py
+++ b/pypy/module/_cffi_backend/wrapper.py
@@ -1,6 +1,6 @@
 from pypy.interpreter.error import oefmt
 from pypy.interpreter.baseobjspace import W_Root
-from pypy.interpreter.typedef import TypeDef
+from pypy.interpreter.typedef import TypeDef, interp_attrproperty
 from pypy.interpreter.gateway import interp2app
 from rpython.rlib import jit
 
@@ -21,9 +21,10 @@
     also returns the original struct/union signature.
     """
     _immutable_ = True
+    common_doc_str = 'direct call to the C function of the same name'
 
     def __init__(self, space, fnptr, directfnptr, ctype,
-                 locs, rawfunctype, fnname):
+                 locs, rawfunctype, fnname, modulename):
         assert isinstance(ctype, W_CTypeFunc)
         assert ctype.cif_descr is not None     # not for '...' functions
         assert locs is None or len(ctype.fargs) == len(locs)
@@ -35,6 +36,7 @@
         self.locs = locs
         self.rawfunctype = rawfunctype
         self.fnname = fnname
+        self.modulename = modulename
         self.nargs_expected = len(ctype.fargs) - (locs is not None and
                                                   locs[0] == 'R')
 
@@ -111,5 +113,8 @@
         'FFIFunctionWrapper',
         __repr__ = interp2app(W_FunctionWrapper.descr_repr),
         __call__ = interp2app(W_FunctionWrapper.descr_call),
+        __name__ = interp_attrproperty('fnname', cls=W_FunctionWrapper),
+        __module__ = interp_attrproperty('modulename', cls=W_FunctionWrapper),
+        __doc__ = interp_attrproperty('common_doc_str', cls=W_FunctionWrapper),
         )
 W_FunctionWrapper.typedef.acceptable_as_base_class = False
diff --git a/rpython/translator/c/genc.py b/rpython/translator/c/genc.py
--- a/rpython/translator/c/genc.py
+++ b/rpython/translator/c/genc.py
@@ -443,6 +443,11 @@
             mk.definition('OBJECTS1', '$(subst .asmgcc.s,.o,$(subst .c,.o,$(SOURCES)))')
             mk.definition('OBJECTS', '$(OBJECTS1) gcmaptable.s')
 
+            # the CFLAGS passed to gcc when invoked to assembler the .s file
+            # must not contain -g.  This confuses gcc 5.1.  (Note that it
+            # would seem that gcc 5.1 with "-g" does not produce debugging
+            # info in a format that gdb 4.7.1 can read.)
+            mk.definition('CFLAGS_AS', '$(patsubst -g,,$(CFLAGS))')
 
             # the rule that transforms %.c into %.o, by compiling it to
             # %.s, then applying trackgcroot to get %.lbl.s and %.gcmap, and
@@ -452,7 +457,7 @@
                     '-o $*.s -S $< $(INCLUDEDIRS)',
                 '$(PYTHON) $(RPYDIR)/translator/c/gcc/trackgcroot.py '
                     '-t $*.s > $*.gctmp',
-                '$(CC) $(CFLAGS) -o $*.o -c $*.lbl.s',
+                '$(CC) $(CFLAGS_AS) -o $*.o -c $*.lbl.s',
                 'mv $*.gctmp $*.gcmap',
                 'rm $*.s $*.lbl.s'])
 

From noreply at buildbot.pypy.org  Thu Jun 11 22:32:40 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 11 Jun 2015 22:32:40 +0200 (CEST)
Subject: [pypy-commit] stmgc c8-gil-like: in-progress
Message-ID: <20150611203240.A52D91C0EB1@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: c8-gil-like
Changeset: r1805:ea72e4a504fd
Date: 2015-06-11 16:22 +0200
http://bitbucket.org/pypy/stmgc/changeset/ea72e4a504fd/

Log:	in-progress

diff --git a/c8/demo/demo_random.c b/c8/demo/demo_random.c
--- a/c8/demo/demo_random.c
+++ b/c8/demo/demo_random.c
@@ -57,9 +57,16 @@
 }
 
 
+long check_size(long size)
+{
+    assert(size >= sizeof(struct node_s));
+    assert(size <= sizeof(struct node_s) + 4096*70);
+    return size;
+}
+
 ssize_t stmcb_size_rounded_up(struct object_s *ob)
 {
-    return ((struct node_s*)ob)->my_size;
+    return check_size(((struct node_s*)ob)->my_size);
 }
 
 void stmcb_trace(struct object_s *obj, void visit(object_t **))
@@ -69,7 +76,8 @@
 
     /* and the same value at the end: */
     /* note, ->next may be the same as last_next */
-    nodeptr_t *last_next = (nodeptr_t*)((char*)n + n->my_size - sizeof(void*));
+    nodeptr_t *last_next = (nodeptr_t*)((char*)n + check_size(n->my_size)
+                                        - sizeof(void*));
 
     assert(n->next == *last_next);
 
@@ -113,36 +121,36 @@
     }
 }
 
-void reload_roots()
-{
-    int i;
-    assert(td.num_roots == td.num_roots_at_transaction_start);
-    for (i = td.num_roots_at_transaction_start - 1; i >= 0; i--) {
-        if (td.roots[i])
-            STM_POP_ROOT(stm_thread_local, td.roots[i]);
-    }
-
-    for (i = 0; i < td.num_roots_at_transaction_start; i++) {
-        if (td.roots[i])
-            STM_PUSH_ROOT(stm_thread_local, td.roots[i]);
-    }
-}
-
 void push_roots()
 {
     int i;
+    assert(td.num_roots_at_transaction_start <= td.num_roots);
     for (i = td.num_roots_at_transaction_start; i < td.num_roots; i++) {
         if (td.roots[i])
             STM_PUSH_ROOT(stm_thread_local, td.roots[i]);
     }
+    STM_SEGMENT->no_safe_point_here = 0;
 }
 
 void pop_roots()
 {
     int i;
-    for (i = td.num_roots - 1; i >= td.num_roots_at_transaction_start; i--) {
-        if (td.roots[i])
+    STM_SEGMENT->no_safe_point_here = 1;
+
+    assert(td.num_roots_at_transaction_start <= td.num_roots);
+    for (i = td.num_roots - 1; i >= 0; i--) {
+        if (td.roots[i]) {
             STM_POP_ROOT(stm_thread_local, td.roots[i]);
+            assert(td.roots[i]);
+        }
+    }
+
+    fprintf(stderr, "stm_is_inevitable() = %d\n", (int)stm_is_inevitable());
+    for (i = 0; i < td.num_roots_at_transaction_start; i++) {
+        if (td.roots[i]) {
+            fprintf(stderr, "root %d: %p\n", i, td.roots[i]);
+            STM_PUSH_ROOT(stm_thread_local, td.roots[i]);
+        }
     }
 }
 
@@ -150,6 +158,7 @@
 {
     int i;
     assert(idx >= td.num_roots_at_transaction_start);
+    assert(idx < td.num_roots);
 
     for (i = idx; i < td.num_roots - 1; i++)
         td.roots[i] = td.roots[i + 1];
@@ -158,6 +167,7 @@
 
 void add_root(objptr_t r)
 {
+    assert(td.num_roots_at_transaction_start <= td.num_roots);
     if (r && td.num_roots < MAXROOTS) {
         td.roots[td.num_roots++] = r;
     }
@@ -184,7 +194,8 @@
         nodeptr_t n = (nodeptr_t)p;
 
         /* and the same value at the end: */
-        nodeptr_t TLPREFIX *last_next = (nodeptr_t TLPREFIX *)((stm_char*)n + n->my_size - sizeof(void*));
+        nodeptr_t TLPREFIX *last_next = (nodeptr_t TLPREFIX *)((stm_char*)n +
+                                       check_size(n->my_size) - sizeof(void*));
         assert(n->next == *last_next);
         n->next = (nodeptr_t)v;
         *last_next = (nodeptr_t)v;
@@ -196,7 +207,8 @@
     nodeptr_t n = (nodeptr_t)p;
 
     /* and the same value at the end: */
-    nodeptr_t TLPREFIX *last_next = (nodeptr_t TLPREFIX *)((stm_char*)n + n->my_size - sizeof(void*));
+    nodeptr_t TLPREFIX *last_next = (nodeptr_t TLPREFIX *)((stm_char*)n +
+                                       check_size(n->my_size) - sizeof(void*));
     OPT_ASSERT(n->next == *last_next);
 
     return n->next;
@@ -229,7 +241,7 @@
                            sizeof(struct node_s) + (get_rand(100000) & ~15),
                            sizeof(struct node_s) + 4096,
                            sizeof(struct node_s) + 4096*70};
-        size_t size = sizes[get_rand(4)];
+        size_t size = check_size(sizes[get_rand(4)]);
         p = stm_allocate(size);
         nodeptr_t n = (nodeptr_t)p;
         n->sig = SIGNATURE;
@@ -240,7 +252,6 @@
         n->next = NULL;
         *last_next = NULL;
         pop_roots();
-        /* reload_roots not necessary, all are old after start_transaction */
         break;
     case 4:  // read and validate 'p'
         read_barrier(p);
@@ -306,7 +317,9 @@
         stm_become_inevitable(&stm_thread_local, "please");
         pop_roots();
         return NULL;
-    } else if (get_rand(240) == 1) {
+    } else if (0 &&            // XXXXXXXXXXXXXXXXXXXXX
+                                                       
+               get_rand(240) == 1) {
         push_roots();
         stm_become_globally_unique_transaction(&stm_thread_local, "really");
         fprintf(stderr, "[GUT/%d]", (int)STM_SEGMENT->segment_num);
@@ -352,13 +365,16 @@
     td.num_roots = td.num_roots_at_transaction_start;
     p = NULL;
     pop_roots();                /* does nothing.. */
-    reload_roots();
 
     while (td.steps_left-->0) {
         if (td.steps_left % 8 == 0)
             fprintf(stdout, "#");
 
-        assert(p == NULL || ((nodeptr_t)p)->sig == SIGNATURE);
+        int local_seg = STM_SEGMENT->segment_num;
+        int p_sig = p == NULL ? 0 : ((nodeptr_t)p)->sig;
+
+        assert(p == NULL || p_sig == SIGNATURE);
+        (void)local_seg;
 
         p = do_step(p);
 
@@ -366,7 +382,9 @@
             push_roots();
 
             long call_fork = (arg != NULL && *(long *)arg);
-            if (call_fork == 0) {   /* common case */
+            if (1 ||   // XXXXXXXXXXXXXXXX
+                                          
+                call_fork == 0) {   /* common case */
                 if (get_rand(100) < 50) {
                     stm_leave_transactional_zone(&stm_thread_local);
                     /* Nothing here; it's unlikely that a different thread
@@ -386,7 +404,6 @@
                 td.num_roots = td.num_roots_at_transaction_start;
                 p = NULL;
                 pop_roots();
-                reload_roots();
             }
             else {
                 /* run a fork() inside the transaction */
diff --git a/c8/stm/detach.c b/c8/stm/detach.c
--- a/c8/stm/detach.c
+++ b/c8/stm/detach.c
@@ -32,6 +32,7 @@
 
 void _stm_leave_noninevitable_transactional_zone(void)
 {
+    dprintf(("leave_noninevitable_transactional_zone\n"));
     _stm_become_inevitable(MSG_INEV_DONT_SLEEP);
 
     /* did it work? */
@@ -61,11 +62,13 @@
        disappear.  XXX could be done even earlier, as soon as we have
        read the shadowstack inside the minor collection. */
     STM_SEGMENT->running_thread = NULL;
+
+    _core_commit_transaction();
+
+
     write_fence();
     assert(_stm_detached_inevitable_from_thread == -1);
     _stm_detached_inevitable_from_thread = 0;
-
-    _core_commit_transaction();
 }
 
 void _stm_reattach_transaction(intptr_t old, stm_thread_local_t *tl)
@@ -75,6 +78,7 @@
         if (old == -1) {
             /* busy-loop: wait until _stm_detached_inevitable_from_thread
                is reset to a value different from -1 */
+            dprintf(("reattach_transaction: busy wait...\n"));
             while (_stm_detached_inevitable_from_thread == -1)
                 spin_loop();
 
diff --git a/c8/stm/nursery.c b/c8/stm/nursery.c
--- a/c8/stm/nursery.c
+++ b/c8/stm/nursery.c
@@ -309,6 +309,7 @@
     else
         assert(finalbase <= ssbase && ssbase <= current);
 
+    dprintf(("collect_roots_in_nursery:\n"));
     while (current > ssbase) {
         --current;
         uintptr_t x = (uintptr_t)current->ss;
@@ -320,6 +321,7 @@
         else {
             /* it is an odd-valued marker, ignore */
         }
+        dprintf(("    %p: %p -> %p\n", current, (void *)x, current->ss));
     }
 
     minor_trace_if_young(&tl->thread_local_obj);
@@ -519,6 +521,7 @@
 static void _do_minor_collection(bool commit)
 {
     dprintf(("minor_collection commit=%d\n", (int)commit));
+    assert(!STM_SEGMENT->no_safe_point_here);
 
     STM_PSEGMENT->minor_collect_will_commit_now = commit;
 
diff --git a/c8/stm/setup.c b/c8/stm/setup.c
--- a/c8/stm/setup.c
+++ b/c8/stm/setup.c
@@ -137,6 +137,9 @@
     setup_detach();
 
     set_gs_register(get_segment_base(0));
+
+    dprintf(("nursery: %p -> %p\n", (void *)NURSERY_START,
+                                    (void *)NURSERY_END));
 }
 
 void stm_teardown(void)
diff --git a/c8/stm/sync.c b/c8/stm/sync.c
--- a/c8/stm/sync.c
+++ b/c8/stm/sync.c
@@ -383,6 +383,7 @@
             break;    /* no safe point requested */
 
         dprintf(("enter safe point\n"));
+        assert(!STM_SEGMENT->no_safe_point_here);
         assert(STM_SEGMENT->nursery_end == NSE_SIGPAUSE);
         assert(pause_signalled);
 
@@ -397,6 +398,7 @@
         cond_wait(C_REQUEST_REMOVED);
         STM_PSEGMENT->safe_point = SP_RUNNING;
         timing_event(STM_SEGMENT->running_thread, STM_WAIT_DONE);
+        assert(!STM_SEGMENT->no_safe_point_here);
         dprintf(("left safe point\n"));
     }
 }
diff --git a/c8/stmgc.h b/c8/stmgc.h
--- a/c8/stmgc.h
+++ b/c8/stmgc.h
@@ -40,6 +40,7 @@
 
 struct stm_segment_info_s {
     uint8_t transaction_read_version;
+    uint8_t no_safe_point_here;    /* set from outside, triggers an assert */
     int segment_num;
     char *segment_base;
     stm_char *nursery_current;
@@ -420,10 +421,12 @@
    far more efficient than constantly starting and committing
    transactions.
 */
+#include <stdio.h>
 static inline void stm_enter_transactional_zone(stm_thread_local_t *tl) {
     intptr_t old;
     atomic_exchange(&_stm_detached_inevitable_from_thread, old, -1);
     if (old == (intptr_t)tl) {
+        fprintf(stderr, "stm_enter_transactional_zone fast path\n");
         _stm_detached_inevitable_from_thread = 0;
     }
     else {
@@ -434,10 +437,13 @@
 }
 static inline void stm_leave_transactional_zone(stm_thread_local_t *tl) {
     assert(STM_SEGMENT->running_thread == tl);
-    if (stm_is_inevitable())
+    if (stm_is_inevitable()) {
+        fprintf(stderr, "stm_leave_transactional_zone fast path\n");
         _stm_detach_inevitable_transaction(tl);
-    else
+    }
+    else {
         _stm_leave_noninevitable_transactional_zone();
+    }
 }
 
 /* stm_force_transaction_break() is in theory equivalent to
@@ -461,8 +467,9 @@
     assert(STM_SEGMENT->running_thread == tl);
     if (!stm_is_inevitable())
         _stm_become_inevitable(msg);
-    /* now, we're running the inevitable transaction, so: */
-    assert(_stm_detached_inevitable_from_thread == 0);
+    /* now, we're running the inevitable transaction, so this var should
+       be 0 (but can occasionally be -1 for a tiny amount of time) */
+    assert(((_stm_detached_inevitable_from_thread + 1) & ~1) == 0);
 }
 
 /* Forces a safe-point if needed.  Normally not needed: this is
diff --git a/duhton-c8/duhton.c b/duhton-c8/duhton.c
--- a/duhton-c8/duhton.c
+++ b/duhton-c8/duhton.c
@@ -41,7 +41,8 @@
             printf("))) ");
             fflush(stdout);
         }
-        stm_start_inevitable_transaction(&stm_thread_local);
+        stm_enter_transactional_zone(&stm_thread_local);
+        stm_become_inevitable(&stm_thread_local, "starting point");
         DuObject *code = Du_Compile(filename, interactive);
 
         if (code == NULL) {
@@ -58,7 +59,7 @@
         //stm_collect(0);   /* hack... */
         //_du_restore1(stm_thread_local_obj);
 
-        stm_commit_transaction();
+        stm_leave_transactional_zone(&stm_thread_local);
 
         Du_TransactionRun();
         if (!interactive)
diff --git a/duhton-c8/glob.c b/duhton-c8/glob.c
--- a/duhton-c8/glob.c
+++ b/duhton-c8/glob.c
@@ -713,11 +713,12 @@
     //stm_collect(0);       /* hack... */
     //_du_restore1(stm_thread_local_obj);
 
-    stm_commit_transaction();
+    stm_leave_transactional_zone(&stm_thread_local);
 
     Du_TransactionRun();
 
-    stm_start_inevitable_transaction(&stm_thread_local);
+    stm_enter_transactional_zone(&stm_thread_local);
+    stm_become_inevitable(&stm_thread_local, "run-transactions finished");
     return Du_None;
 }
 
@@ -809,7 +810,8 @@
     /* prebuilt objs stay on the shadowstack forever */
 
     stm_register_thread_local(&stm_thread_local);
-    stm_start_inevitable_transaction(&stm_thread_local);
+    stm_enter_transactional_zone(&stm_thread_local);
+    stm_become_inevitable(&stm_thread_local, "initialization");
 
     all_threads_count = num_threads;
     all_threads = (pthread_t*)malloc(sizeof(pthread_t) * num_threads);
@@ -857,7 +859,7 @@
     DuFrame_SetBuiltinMacro(Du_Globals, "pair?", du_pair);
     DuFrame_SetBuiltinMacro(Du_Globals, "assert", du_assert);
     DuFrame_SetSymbolStr(Du_Globals, "None", Du_None);
-    stm_commit_transaction();
+    stm_leave_transactional_zone(&stm_thread_local);
 }
 
 void Du_Finalize(void)
diff --git a/duhton-c8/transaction.c b/duhton-c8/transaction.c
--- a/duhton-c8/transaction.c
+++ b/duhton-c8/transaction.c
@@ -58,14 +58,14 @@
     if (TLOBJ == NULL)
         return;
 
-    stm_start_inevitable_transaction(&stm_thread_local);
+    stm_enter_transactional_zone(&stm_thread_local);
 
     DuConsObject *root = du_pending_transactions;
     _du_write1(root);
     root->cdr = TLOBJ;
 
     TLOBJ = NULL;
-    stm_commit_transaction();
+    stm_leave_transactional_zone(&stm_thread_local);
 
     run_all_threads();
 }

From noreply at buildbot.pypy.org  Thu Jun 11 22:32:41 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 11 Jun 2015 22:32:41 +0200 (CEST)
Subject: [pypy-commit] stmgc c8-gil-like: demo_random.c starts to work
Message-ID: <20150611203241.D70E51C0EB1@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: c8-gil-like
Changeset: r1806:e2e9d6b116b9
Date: 2015-06-11 17:41 +0200
http://bitbucket.org/pypy/stmgc/changeset/e2e9d6b116b9/

Log:	demo_random.c starts to work

diff --git a/c8/stm/core.c b/c8/stm/core.c
--- a/c8/stm/core.c
+++ b/c8/stm/core.c
@@ -1287,28 +1287,41 @@
 
 void _stm_commit_transaction(void)
 {
+    assert(STM_PSEGMENT->running_pthread == pthread_self());
+    _core_commit_transaction(/*external=*/ false);
+}
+
+static void _core_commit_transaction(bool external)
+{
     exec_local_finalizers();
 
     assert(!_has_mutex());
     assert(STM_PSEGMENT->safe_point == SP_RUNNING);
-    assert(STM_PSEGMENT->running_pthread == pthread_self());
 
-    dprintf(("> stm_commit_transaction()\n"));
-    minor_collection(1);
+    dprintf(("> stm_commit_transaction(external=%d)\n", (int)external));
+    minor_collection(/*commit=*/ true, external);
 
-    _core_commit_transaction();
-}
-
-static void _core_commit_transaction(void)
-{
     push_large_overflow_objects_to_other_segments();
     /* push before validate. otherwise they are reachable too early */
 
+    if (external) {
+        /* from this point on, unlink the original 'stm_thread_local_t *'
+           from its segment.  Better do it as soon as possible, because
+           other threads might be spin-looping, waiting for the -1 to
+           disappear. */
+        STM_SEGMENT->running_thread = NULL;
+        write_fence();
+        assert(_stm_detached_inevitable_from_thread == -1);
+        _stm_detached_inevitable_from_thread = 0;
+    }
+
     bool was_inev = STM_PSEGMENT->transaction_state == TS_INEVITABLE;
     _validate_and_add_to_commit_log();
 
-    if (!was_inev)
+    if (!was_inev) {
+        assert(!external);
         stm_rewind_jmp_forget(STM_SEGMENT->running_thread);
+    }
 
     /* XXX do we still need a s_mutex_lock() section here? */
     s_mutex_lock();
@@ -1326,12 +1339,14 @@
     invoke_and_clear_user_callbacks(0);   /* for commit */
 
     /* >>>>> there may be a FORK() happening in the safepoint below <<<<<*/
-    enter_safe_point_if_requested();
-    assert(STM_SEGMENT->nursery_end == NURSERY_END);
+    if (!external) {
+        enter_safe_point_if_requested();
+        assert(STM_SEGMENT->nursery_end == NURSERY_END);
 
-    /* if a major collection is required, do it here */
-    if (is_major_collection_requested()) {
-        major_collection_with_mutex();
+        /* if a major collection is required, do it here */
+        if (is_major_collection_requested()) {
+            major_collection_with_mutex();
+        }
     }
 
     _verify_cards_cleared_in_all_lists(get_priv_segment(STM_SEGMENT->segment_num));
@@ -1342,6 +1357,7 @@
 
     /* done */
     stm_thread_local_t *tl = STM_SEGMENT->running_thread;
+    assert(external == (tl == NULL));
     _finish_transaction(STM_TRANSACTION_COMMIT);
     /* cannot access STM_SEGMENT or STM_PSEGMENT from here ! */
 
diff --git a/c8/stm/core.h b/c8/stm/core.h
--- a/c8/stm/core.h
+++ b/c8/stm/core.h
@@ -300,7 +300,7 @@
 
 static void _signal_handler(int sig, siginfo_t *siginfo, void *context);
 static bool _stm_validate(void);
-static void _core_commit_transaction(void);
+static void _core_commit_transaction(bool external);
 
 static inline bool was_read_remote(char *base, object_t *obj)
 {
diff --git a/c8/stm/detach.c b/c8/stm/detach.c
--- a/c8/stm/detach.c
+++ b/c8/stm/detach.c
@@ -49,26 +49,8 @@
 
 static void commit_external_inevitable_transaction(void)
 {
-    assert(!_has_mutex());
-    assert(STM_PSEGMENT->safe_point == SP_RUNNING);
     assert(STM_PSEGMENT->transaction_state == TS_INEVITABLE); /* can't abort */
-
-    exec_local_finalizers();
-    minor_collection(1);
-
-    /* from this point on, unlink the original 'stm_thread_local_t *'
-       from its segment.  Better do it as soon as possible, because
-       other threads might be spin-looping, waiting for the -1 to
-       disappear.  XXX could be done even earlier, as soon as we have
-       read the shadowstack inside the minor collection. */
-    STM_SEGMENT->running_thread = NULL;
-
-    _core_commit_transaction();
-
-
-    write_fence();
-    assert(_stm_detached_inevitable_from_thread == -1);
-    _stm_detached_inevitable_from_thread = 0;
+    _core_commit_transaction(/*external=*/ true);
 }
 
 void _stm_reattach_transaction(intptr_t old, stm_thread_local_t *tl)
@@ -92,6 +74,7 @@
         dprintf(("reattach_transaction: commit detached from seg %d\n",
                  remote_seg_num));
 
+        tl->last_associated_segment_num = remote_seg_num;
         ensure_gs_register(remote_seg_num);
         commit_external_inevitable_transaction();
     }
diff --git a/c8/stm/nursery.c b/c8/stm/nursery.c
--- a/c8/stm/nursery.c
+++ b/c8/stm/nursery.c
@@ -564,11 +564,12 @@
     assert(MINOR_NOTHING_TO_DO(STM_PSEGMENT));
 }
 
-static void minor_collection(bool commit)
+static void minor_collection(bool commit, bool external)
 {
     assert(!_has_mutex());
 
-    stm_safe_point();
+    if (!external)
+        stm_safe_point();
 
     timing_event(STM_SEGMENT->running_thread, STM_GC_MINOR_START);
 
@@ -582,7 +583,7 @@
     if (level > 0)
         force_major_collection_request();
 
-    minor_collection(/*commit=*/ false);
+    minor_collection(/*commit=*/ false, /*external=*/ false);
 
 #ifdef STM_TESTS
     /* tests don't want aborts in stm_allocate, thus
diff --git a/c8/stm/nursery.h b/c8/stm/nursery.h
--- a/c8/stm/nursery.h
+++ b/c8/stm/nursery.h
@@ -10,7 +10,7 @@
                                 object_t *obj, uint8_t mark_value,
                                 bool mark_all, bool really_clear);
 
-static void minor_collection(bool commit);
+static void minor_collection(bool commit, bool external);
 static void check_nursery_at_transaction_start(void);
 static size_t throw_away_nursery(struct stm_priv_segment_info_s *pseg);
 static void major_do_validation_and_minor_collections(void);

From noreply at buildbot.pypy.org  Thu Jun 11 22:32:42 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 11 Jun 2015 22:32:42 +0200 (CEST)
Subject: [pypy-commit] stmgc c8-gil-like: Tweaks: remove atomic_exchange()
 again, and use the regular
Message-ID: <20150611203242.D7B2B1C0EB1@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: c8-gil-like
Changeset: r1807:a30818fb55ed
Date: 2015-06-11 17:59 +0200
http://bitbucket.org/pypy/stmgc/changeset/a30818fb55ed/

Log:	Tweaks: remove atomic_exchange() again, and use the regular
	__sync_bool_compare_and_swap() instead. Avoids an issue with the
	value 0 changing to -1 and back to 0, which could overwrite by
	mistake a _stm_detach_inevitable_transaction().

diff --git a/c8/demo/demo_random.c b/c8/demo/demo_random.c
--- a/c8/demo/demo_random.c
+++ b/c8/demo/demo_random.c
@@ -8,6 +8,7 @@
 #include <sys/wait.h>
 
 #include "stmgc.h"
+#include "stm/fprintcolor.h"
 
 #define NUMTHREADS 2
 #define STEPS_PER_THREAD 500
@@ -145,10 +146,10 @@
         }
     }
 
-    fprintf(stderr, "stm_is_inevitable() = %d\n", (int)stm_is_inevitable());
+    dprintf(("stm_is_inevitable() = %d\n", (int)stm_is_inevitable()));
     for (i = 0; i < td.num_roots_at_transaction_start; i++) {
         if (td.roots[i]) {
-            fprintf(stderr, "root %d: %p\n", i, td.roots[i]);
+            dprintf(("root %d: %p\n", i, td.roots[i]));
             STM_PUSH_ROOT(stm_thread_local, td.roots[i]);
         }
     }
@@ -390,9 +391,9 @@
                     /* Nothing here; it's unlikely that a different thread
                        manages to steal the detached inev transaction.
                        Give them a little chance with a usleep(). */
-                    fprintf(stderr, "sleep...\n");
+                    dprintf(("sleep...\n"));
                     usleep(1);
-                    fprintf(stderr, "sleep done\n");
+                    dprintf(("sleep done\n"));
                     td.num_roots_at_transaction_start = td.num_roots;
                     stm_enter_transactional_zone(&stm_thread_local);
                 }
diff --git a/c8/stm/atomic.h b/c8/stm/atomic.h
--- a/c8/stm/atomic.h
+++ b/c8/stm/atomic.h
@@ -26,18 +26,18 @@
 
   static inline void spin_loop(void) { asm("pause" : : : "memory"); }
   static inline void write_fence(void) { asm("" : : : "memory"); }
-# define atomic_exchange(ptr, old, new)  do {           \
-        (old) = __sync_lock_test_and_set(ptr, new);     \
-    } while (0)
+/*# define atomic_exchange(ptr, old, new)  do {         \
+          (old) = __sync_lock_test_and_set(ptr, new);   \
+      } while (0)*/
 
 #else
 
   static inline void spin_loop(void) { asm("" : : : "memory"); }
   static inline void write_fence(void) { __sync_synchronize(); }
 
-# define atomic_exchange(ptr, old, new)  do {           \
-        (old) = *(ptr);                                 \
-    } while (UNLIKELY(!__sync_bool_compare_and_swap(ptr, old, new)));
+/*# define atomic_exchange(ptr, old, new)  do {           \
+          (old) = *(ptr);                                 \
+      } while (UNLIKELY(!__sync_bool_compare_and_swap(ptr, old, new))); */
 
 #endif
 
diff --git a/c8/stm/detach.c b/c8/stm/detach.c
--- a/c8/stm/detach.c
+++ b/c8/stm/detach.c
@@ -53,9 +53,11 @@
     _core_commit_transaction(/*external=*/ true);
 }
 
-void _stm_reattach_transaction(intptr_t old, stm_thread_local_t *tl)
+void _stm_reattach_transaction(stm_thread_local_t *tl)
 {
+    intptr_t old;
  restart:
+    old = _stm_detached_inevitable_from_thread;
     if (old != 0) {
         if (old == -1) {
             /* busy-loop: wait until _stm_detached_inevitable_from_thread
@@ -65,10 +67,13 @@
                 spin_loop();
 
             /* then retry */
-            atomic_exchange(&_stm_detached_inevitable_from_thread, old, -1);
             goto restart;
         }
 
+        if (!__sync_bool_compare_and_swap(&_stm_detached_inevitable_from_thread,
+                                          old, -1))
+            goto restart;
+
         stm_thread_local_t *old_tl = (stm_thread_local_t *)old;
         int remote_seg_num = old_tl->last_associated_segment_num;
         dprintf(("reattach_transaction: commit detached from seg %d\n",
@@ -78,10 +83,6 @@
         ensure_gs_register(remote_seg_num);
         commit_external_inevitable_transaction();
     }
-    else {
-        assert(_stm_detached_inevitable_from_thread == -1);
-        _stm_detached_inevitable_from_thread = 0;
-    }
     dprintf(("reattach_transaction: start a new transaction\n"));
     _stm_start_transaction(tl);
 }
@@ -102,14 +103,6 @@
     if (cur == 0) {    /* fast-path */
         return 0;   /* _stm_detached_inevitable_from_thread not changed */
     }
-    if (cur != -1) {
-        atomic_exchange(&_stm_detached_inevitable_from_thread, cur, -1);
-        if (cur == 0) {
-            /* found 0, so change from -1 to 0 again and return */
-            _stm_detached_inevitable_from_thread = 0;
-            return 0;
-        }
-    }
     if (cur == -1) {
         /* busy-loop: wait until _stm_detached_inevitable_from_thread
            is reset to a value different from -1 */
@@ -117,6 +110,10 @@
             spin_loop();
         goto restart;
     }
+    if (!__sync_bool_compare_and_swap(&_stm_detached_inevitable_from_thread,
+                                      cur, -1))
+        goto restart;
+
     /* this is the only case where we grabbed a detached transaction.
        _stm_detached_inevitable_from_thread is still -1, until
        commit_fetched_detached_transaction() is called. */
diff --git a/c8/stm/fprintcolor.h b/c8/stm/fprintcolor.h
--- a/c8/stm/fprintcolor.h
+++ b/c8/stm/fprintcolor.h
@@ -37,5 +37,6 @@
 /* ------------------------------------------------------------ */
 
 
+__attribute__((unused))
 static void stm_fatalerror(const char *format, ...)
      __attribute__((format (printf, 1, 2), noreturn));
diff --git a/c8/stmgc.h b/c8/stmgc.h
--- a/c8/stmgc.h
+++ b/c8/stmgc.h
@@ -94,7 +94,7 @@
     assert(_stm_detached_inevitable_from_thread == 0);          \
     _stm_detached_inevitable_from_thread = (intptr_t)(tl);      \
 } while (0)
-void _stm_reattach_transaction(intptr_t old, stm_thread_local_t *tl);
+void _stm_reattach_transaction(stm_thread_local_t *tl);
 void _stm_become_inevitable(const char*);
 void _stm_collectable_safe_point(void);
 
@@ -421,24 +421,29 @@
    far more efficient than constantly starting and committing
    transactions.
 */
+#ifdef STM_DEBUGPRINT
 #include <stdio.h>
+#endif
 static inline void stm_enter_transactional_zone(stm_thread_local_t *tl) {
-    intptr_t old;
-    atomic_exchange(&_stm_detached_inevitable_from_thread, old, -1);
-    if (old == (intptr_t)tl) {
+    if (__sync_bool_compare_and_swap(&_stm_detached_inevitable_from_thread,
+                                     (intptr_t)tl, 0)) {
+#ifdef STM_DEBUGPRINT
         fprintf(stderr, "stm_enter_transactional_zone fast path\n");
-        _stm_detached_inevitable_from_thread = 0;
+#endif
     }
     else {
-        _stm_reattach_transaction(old, tl);
+        _stm_reattach_transaction(tl);
         /* _stm_detached_inevitable_from_thread should be 0 here, but
-           it can already have been changed from a parallel thread */
+           it can already have been changed from a parallel thread
+           (assuming we're not inevitable ourselves) */
     }
 }
 static inline void stm_leave_transactional_zone(stm_thread_local_t *tl) {
     assert(STM_SEGMENT->running_thread == tl);
     if (stm_is_inevitable()) {
+#ifdef STM_DEBUGPRINT
         fprintf(stderr, "stm_leave_transactional_zone fast path\n");
+#endif
         _stm_detach_inevitable_transaction(tl);
     }
     else {
@@ -467,9 +472,8 @@
     assert(STM_SEGMENT->running_thread == tl);
     if (!stm_is_inevitable())
         _stm_become_inevitable(msg);
-    /* now, we're running the inevitable transaction, so this var should
-       be 0 (but can occasionally be -1 for a tiny amount of time) */
-    assert(((_stm_detached_inevitable_from_thread + 1) & ~1) == 0);
+    /* now, we're running the inevitable transaction, so this var should be 0 */
+    assert(_stm_detached_inevitable_from_thread == 0);
 }
 
 /* Forces a safe-point if needed.  Normally not needed: this is

From noreply at buildbot.pypy.org  Thu Jun 11 22:32:43 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 11 Jun 2015 22:32:43 +0200 (CEST)
Subject: [pypy-commit] stmgc c8-gil-like: fix
Message-ID: <20150611203243.D164F1C0EB1@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: c8-gil-like
Changeset: r1808:4b5e5bbe333f
Date: 2015-06-11 18:02 +0200
http://bitbucket.org/pypy/stmgc/changeset/4b5e5bbe333f/

Log:	fix

diff --git a/c8/demo/demo_random.c b/c8/demo/demo_random.c
--- a/c8/demo/demo_random.c
+++ b/c8/demo/demo_random.c
@@ -376,6 +376,7 @@
 
         assert(p == NULL || p_sig == SIGNATURE);
         (void)local_seg;
+        (void)p_sig;
 
         p = do_step(p);
 

From noreply at buildbot.pypy.org  Thu Jun 11 22:37:11 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 11 Jun 2015 22:37:11 +0200 (CEST)
Subject: [pypy-commit] stmgc c8-gil-like: These two disabled cases work too
Message-ID: <20150611203711.26E631C0F4B@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: c8-gil-like
Changeset: r1809:b5442bfc9534
Date: 2015-06-11 22:37 +0200
http://bitbucket.org/pypy/stmgc/changeset/b5442bfc9534/

Log:	These two disabled cases work too

diff --git a/c8/demo/demo_random.c b/c8/demo/demo_random.c
--- a/c8/demo/demo_random.c
+++ b/c8/demo/demo_random.c
@@ -318,9 +318,7 @@
         stm_become_inevitable(&stm_thread_local, "please");
         pop_roots();
         return NULL;
-    } else if (0 &&            // XXXXXXXXXXXXXXXXXXXXX
-                                                       
-               get_rand(240) == 1) {
+    } else if (get_rand(240) == 1) {
         push_roots();
         stm_become_globally_unique_transaction(&stm_thread_local, "really");
         fprintf(stderr, "[GUT/%d]", (int)STM_SEGMENT->segment_num);
@@ -384,9 +382,7 @@
             push_roots();
 
             long call_fork = (arg != NULL && *(long *)arg);
-            if (1 ||   // XXXXXXXXXXXXXXXX
-                                          
-                call_fork == 0) {   /* common case */
+            if (call_fork == 0) {   /* common case */
                 if (get_rand(100) < 50) {
                     stm_leave_transactional_zone(&stm_thread_local);
                     /* Nothing here; it's unlikely that a different thread

From noreply at buildbot.pypy.org  Thu Jun 11 23:16:53 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Thu, 11 Jun 2015 23:16:53 +0200 (CEST)
Subject: [pypy-commit] pypy dtypes-compatability: sort out bogus tests
 differentiation between pypy and cpython
Message-ID: <20150611211653.3A6C71C0EB1@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: dtypes-compatability
Changeset: r78033:23fdb8434415
Date: 2015-06-10 23:54 +0300
http://bitbucket.org/pypy/pypy/changeset/23fdb8434415/

Log:	sort out bogus tests differentiation between pypy and cpython

diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py
--- a/pypy/module/micronumpy/test/test_dtypes.py
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -688,16 +688,8 @@
                                       numpy.integer, numpy.number,
                                       numpy.generic, object]
         import sys
-        if '__pypy__' not in sys.builtin_module_names:
-            # These tests pass "by chance" on numpy, things that are larger than
-            # platform long (i.e. a python int), don't get put in a normal box,
-            # instead they become an object array containing a long, we don't have
-            # yet, so these can't pass.
-            assert numpy.uint64(9223372036854775808) == 9223372036854775808
-            assert numpy.uint64(18446744073709551615) == 18446744073709551615
-        else:
-            raises(OverflowError, numpy.int64, 9223372036854775808)
-            raises(OverflowError, numpy.int64, 18446744073709551615)
+        raises(OverflowError, numpy.int64, 9223372036854775808)
+        raises(OverflowError, numpy.int64, 18446744073709551615)
         raises(OverflowError, numpy.uint64, 18446744073709551616)
         assert numpy.uint64((2<<63) - 1) == (2<<63) - 1
 

From noreply at buildbot.pypy.org  Thu Jun 11 23:16:54 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Thu, 11 Jun 2015 23:16:54 +0200 (CEST)
Subject: [pypy-commit] pypy dtypes-compatability: fix translation
Message-ID: <20150611211654.643081C0EB1@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: dtypes-compatability
Changeset: r78034:3eda966e35cc
Date: 2015-06-11 08:18 +0300
http://bitbucket.org/pypy/pypy/changeset/3eda966e35cc/

Log:	fix translation

diff --git a/pypy/module/micronumpy/descriptor.py b/pypy/module/micronumpy/descriptor.py
--- a/pypy/module/micronumpy/descriptor.py
+++ b/pypy/module/micronumpy/descriptor.py
@@ -176,7 +176,10 @@
         return dtype
 
     def get_name(self):
-        name = self.w_box_type.name
+        from pypy.objspace.std.typeobject import W_TypeObject
+        w_box_type = self.w_box_type
+        assert isinstance(w_box_type, W_TypeObject)
+        name = w_box_type.getname(self.itemtype.space)
         if name.startswith('numpy.'):
             name = name[6:]
         if name.endswith('_'):

From noreply at buildbot.pypy.org  Thu Jun 11 23:16:55 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Thu, 11 Jun 2015 23:16:55 +0200 (CEST)
Subject: [pypy-commit] pypy default: tagging only after buildbot runs
	produces a cleaner flow
Message-ID: <20150611211655.7CE681C0EB1@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: 
Changeset: r78035:95bb26f8adf4
Date: 2015-06-12 00:12 +0300
http://bitbucket.org/pypy/pypy/changeset/95bb26f8adf4/

Log:	tagging only after buildbot runs produces a cleaner flow

diff --git a/pypy/doc/how-to-release.rst b/pypy/doc/how-to-release.rst
--- a/pypy/doc/how-to-release.rst
+++ b/pypy/doc/how-to-release.rst
@@ -15,8 +15,7 @@
 Release Steps
 -------------
 
-* At code freeze make a release branch using release-x.x.x in mercurial
-  and add a release-specific tag
+* If needed, make a release branch
 * Bump the
   pypy version number in module/sys/version.py and in
   module/cpyext/include/patchlevel.h and . The branch

From noreply at buildbot.pypy.org  Thu Jun 11 23:16:56 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Thu, 11 Jun 2015 23:16:56 +0200 (CEST)
Subject: [pypy-commit] pypy release-2.6.x: tagging only after buildbot runs
 produces a cleaner flow
Message-ID: <20150611211656.A9C131C0EB1@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: release-2.6.x
Changeset: r78036:93fb3c29652e
Date: 2015-06-12 00:12 +0300
http://bitbucket.org/pypy/pypy/changeset/93fb3c29652e/

Log:	tagging only after buildbot runs produces a cleaner flow

diff --git a/pypy/doc/how-to-release.rst b/pypy/doc/how-to-release.rst
--- a/pypy/doc/how-to-release.rst
+++ b/pypy/doc/how-to-release.rst
@@ -15,8 +15,7 @@
 Release Steps
 -------------
 
-* At code freeze make a release branch using release-x.x.x in mercurial
-  and add a release-specific tag
+* If needed, make a release branch
 * Bump the
   pypy version number in module/sys/version.py and in
   module/cpyext/include/patchlevel.h and . The branch

From noreply at buildbot.pypy.org  Fri Jun 12 00:02:46 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 12 Jun 2015 00:02:46 +0200 (CEST)
Subject: [pypy-commit] pypy default: Issue #2060: one more case
Message-ID: <20150611220246.1732B1C06D1@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78037:14b17ac4fe7f
Date: 2015-06-12 00:02 +0200
http://bitbucket.org/pypy/pypy/changeset/14b17ac4fe7f/

Log:	Issue #2060: one more case

diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -2434,7 +2434,7 @@
         while i < nbytes:
             addr = addr_add(base_loc, startindex_loc, baseofs + i, scale)
             current = nbytes - i
-            if current >= 16:
+            if current >= 16 and self.cpu.supports_floats:
                 current = 16
                 if not null_reg_cleared:
                     self.mc.XORPS_xx(null_loc.value, null_loc.value)

From noreply at buildbot.pypy.org  Fri Jun 12 05:39:33 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Fri, 12 Jun 2015 05:39:33 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Fix a bug that caused e.g. None.__eq__ to
 evaluate to an unbound function.
Message-ID: <20150612033933.438331C0186@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob <me at manueljacob.de>
Branch: py3k
Changeset: r78038:36f6f45e781a
Date: 2015-06-12 05:39 +0200
http://bitbucket.org/pypy/pypy/changeset/36f6f45e781a/

Log:	Fix a bug that caused e.g. None.__eq__ to evaluate to an unbound
	function.

diff --git a/pypy/module/__builtin__/test/test_builtin.py b/pypy/module/__builtin__/test/test_builtin.py
--- a/pypy/module/__builtin__/test/test_builtin.py
+++ b/pypy/module/__builtin__/test/test_builtin.py
@@ -839,6 +839,22 @@
         raises(TypeError, setattr, A(), 42, 'x')
         raises(TypeError, delattr, A(), 42)
 
+    def test_getattr_None(self):
+        from types import FunctionType, MethodType
+        assert isinstance(getattr(type(None), '__eq__'), FunctionType)
+        assert isinstance(getattr(None, '__eq__'), MethodType)
+
+    def test_getattr_userobject(self):
+        from types import FunctionType, MethodType
+        class A(object):
+            def __eq__(self, other):
+                pass
+        a = A()
+        assert isinstance(getattr(A, '__eq__'), FunctionType)
+        assert isinstance(getattr(a, '__eq__'), MethodType)
+        a.__eq__ = 42
+        assert a.__eq__ == 42
+
 
 class AppTestGetattrWithGetAttributeShortcut(AppTestGetattr):
     spaceconfig = {"objspace.std.getattributeshortcut": True}
diff --git a/pypy/objspace/descroperation.py b/pypy/objspace/descroperation.py
--- a/pypy/objspace/descroperation.py
+++ b/pypy/objspace/descroperation.py
@@ -102,6 +102,13 @@
         if w_value is not None:
             return w_value
         if w_descr is not None:
+            typ = type(w_descr)
+            if typ is Function or typ is FunctionWithFixedCode:
+                # This shortcut is necessary if w_obj is None.  Otherwise e.g.
+                # None.__eq__ would return an unbound function because calling
+                # __get__ with None as the first argument returns the attribute
+                # as if it was accessed through the owner (type(None).__eq__).
+                return Method(space, w_descr, w_obj)
             return space.get(w_descr, w_obj)
         raiseattrerror(space, w_obj, w_name)
 
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -2,6 +2,7 @@
 from pypy.interpreter import special
 from pypy.interpreter.baseobjspace import ObjSpace, W_Root
 from pypy.interpreter.error import OperationError, oefmt
+from pypy.interpreter.function import Function, Method, FunctionWithFixedCode
 from pypy.interpreter.typedef import get_unique_interplevel_subclass
 from pypy.objspace.std import frame, transparent, callmethod
 from pypy.objspace.descroperation import (
@@ -572,6 +573,13 @@
                     return w_value
                 if not is_data:
                     w_get = self.lookup(w_descr, "__get__")
+            typ = type(w_descr)
+            if typ is Function or typ is FunctionWithFixedCode:
+                # This shortcut is necessary if w_obj is None.  Otherwise e.g.
+                # None.__eq__ would return an unbound function because calling
+                # __get__ with None as the first argument returns the attribute
+                # as if it was accessed through the owner (type(None).__eq__).
+                return Method(self, w_descr, w_obj)
             if w_get is not None:
                 # __get__ is allowed to raise an AttributeError to trigger
                 # use of __getattr__.

From noreply at buildbot.pypy.org  Fri Jun 12 09:54:38 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 12 Jun 2015 09:54:38 +0200 (CEST)
Subject: [pypy-commit] stmgc c8-gil-like: Really remove
 stm_become_globally_unique_transaction and adapt
Message-ID: <20150612075438.58FFE1C024E@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: c8-gil-like
Changeset: r1810:246e3b318782
Date: 2015-06-12 09:55 +0200
http://bitbucket.org/pypy/stmgc/changeset/246e3b318782/

Log:	Really remove stm_become_globally_unique_transaction and adapt
	demo_random to test for it. Rare crashes at the moment...

diff --git a/c8/demo/demo_random.c b/c8/demo/demo_random.c
--- a/c8/demo/demo_random.c
+++ b/c8/demo/demo_random.c
@@ -49,8 +49,10 @@
     int num_roots;
     int num_roots_at_transaction_start;
     int steps_left;
+    long globally_unique;
 };
 __thread struct thread_data td;
+static long progress = 1;
 
 struct thread_data *_get_td(void)
 {
@@ -300,6 +302,15 @@
     return p;
 }
 
+static void end_gut(void)
+{
+    if (td.globally_unique != 0) {
+        fprintf(stderr, "[GUT END]");
+        assert(progress == td.globally_unique);
+        td.globally_unique = 0;
+        stm_resume_all_other_threads();
+    }
+}
 
 objptr_t do_step(objptr_t p)
 {
@@ -320,8 +331,14 @@
         return NULL;
     } else if (get_rand(240) == 1) {
         push_roots();
-        stm_become_globally_unique_transaction(&stm_thread_local, "really");
-        fprintf(stderr, "[GUT/%d]", (int)STM_SEGMENT->segment_num);
+        if (td.globally_unique == 0) {
+            stm_stop_all_other_threads();
+            td.globally_unique = progress;
+            fprintf(stderr, "[GUT/%d]", (int)STM_SEGMENT->segment_num);
+        }
+        else {
+            end_gut();
+        }
         pop_roots();
         return NULL;
     }
@@ -376,10 +393,14 @@
         (void)local_seg;
         (void)p_sig;
 
+        if (!td.globally_unique)
+            ++progress;   /* racy, but good enough */
+
         p = do_step(p);
 
         if (p == (objptr_t)-1) {
             push_roots();
+            end_gut();
 
             long call_fork = (arg != NULL && *(long *)arg);
             if (call_fork == 0) {   /* common case */
@@ -425,6 +446,7 @@
         }
     }
     push_roots();
+    end_gut();
     stm_force_transaction_break(&stm_thread_local);
 
     /* even out the shadow stack before leaveframe: */
diff --git a/c8/stm/core.c b/c8/stm/core.c
--- a/c8/stm/core.c
+++ b/c8/stm/core.c
@@ -1297,6 +1297,10 @@
 
     assert(!_has_mutex());
     assert(STM_PSEGMENT->safe_point == SP_RUNNING);
+    if (globally_unique_transaction) {
+        stm_fatalerror("cannot commit between stm_stop_all_other_threads "
+                       "and stm_resume_all_other_threads");
+    }
 
     dprintf(("> stm_commit_transaction(external=%d)\n", (int)external));
     minor_collection(/*commit=*/ true, external);
@@ -1351,10 +1355,6 @@
 
     _verify_cards_cleared_in_all_lists(get_priv_segment(STM_SEGMENT->segment_num));
 
-    if (globally_unique_transaction && was_inev) {
-        committed_globally_unique_transaction();
-    }
-
     /* done */
     stm_thread_local_t *tl = STM_SEGMENT->running_thread;
     assert(external == (tl == NULL));
@@ -1549,16 +1549,17 @@
     invoke_and_clear_user_callbacks(0);   /* for commit */
 }
 
+#if 0
 void stm_become_globally_unique_transaction(stm_thread_local_t *tl,
                                             const char *msg)
 {
-    stm_become_inevitable(tl, msg);   /* may still abort */
+    stm_become_inevitable(tl, msg);
 
     s_mutex_lock();
     synchronize_all_threads(STOP_OTHERS_AND_BECOME_GLOBALLY_UNIQUE);
     s_mutex_unlock();
 }
-
+#endif
 
 void stm_stop_all_other_threads(void)
 {
diff --git a/c8/stmgc.h b/c8/stmgc.h
--- a/c8/stmgc.h
+++ b/c8/stmgc.h
@@ -528,8 +528,8 @@
    other threads.  A very heavy-handed way to make sure that no other
    transaction is running concurrently.  Avoid as much as possible.
    Other transactions will continue running only after this transaction
-   commits.  (xxx deprecated and may be removed) */
-void stm_become_globally_unique_transaction(stm_thread_local_t *tl, const char *msg);
+   commits.  (deprecated, not working any more according to demo_random2) */
+//void stm_become_globally_unique_transaction(stm_thread_local_t *tl, const char *msg);
 
 /* Moves the transaction forward in time by validating the read and
    write set with all commits that happened since the last validation

From noreply at buildbot.pypy.org  Fri Jun 12 10:48:03 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 12 Jun 2015 10:48:03 +0200 (CEST)
Subject: [pypy-commit] stmgc c8-gil-like: fix
Message-ID: <20150612084803.9FDA51C024E@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: c8-gil-like
Changeset: r1811:4a03c8a56068
Date: 2015-06-12 10:48 +0200
http://bitbucket.org/pypy/stmgc/changeset/4a03c8a56068/

Log:	fix

diff --git a/c8/demo/demo_random.c b/c8/demo/demo_random.c
--- a/c8/demo/demo_random.c
+++ b/c8/demo/demo_random.c
@@ -9,6 +9,7 @@
 
 #include "stmgc.h"
 #include "stm/fprintcolor.h"
+#include "stm/fprintcolor.c"
 
 #define NUMTHREADS 2
 #define STEPS_PER_THREAD 500
diff --git a/c8/stm/detach.c b/c8/stm/detach.c
--- a/c8/stm/detach.c
+++ b/c8/stm/detach.c
@@ -132,16 +132,17 @@
        conditions.
     */
     int mysegnum = STM_SEGMENT->segment_num;
+    bool sp_running = (STM_PSEGMENT->safe_point == SP_RUNNING);
     int segnum = ((stm_thread_local_t *)old)->last_associated_segment_num;
     dprintf(("commit_fetched_detached_transaction from seg %d\n", segnum));
     assert(segnum > 0);
 
     if (segnum != mysegnum) {
-        s_mutex_lock();
-        assert(STM_PSEGMENT->safe_point == SP_RUNNING);
-        STM_PSEGMENT->safe_point = SP_COMMIT_OTHER_DETACHED;
-        s_mutex_unlock();
-
+        if (sp_running) {
+            s_mutex_lock();
+            STM_PSEGMENT->safe_point = SP_COMMIT_OTHER_DETACHED;
+            s_mutex_unlock();
+        }
         set_gs_register(get_segment_base(segnum));
     }
     commit_external_inevitable_transaction();
@@ -149,10 +150,12 @@
     if (segnum != mysegnum) {
         set_gs_register(get_segment_base(mysegnum));
 
-        s_mutex_lock();
-        assert(STM_PSEGMENT->safe_point == SP_COMMIT_OTHER_DETACHED);
-        STM_PSEGMENT->safe_point = SP_RUNNING;
-        s_mutex_unlock();
+        if (sp_running) {
+            s_mutex_lock();
+            assert(STM_PSEGMENT->safe_point == SP_COMMIT_OTHER_DETACHED);
+            STM_PSEGMENT->safe_point = SP_RUNNING;
+            s_mutex_unlock();
+        }
     }
 }
 

From noreply at buildbot.pypy.org  Fri Jun 12 10:48:09 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Fri, 12 Jun 2015 10:48:09 +0200 (CEST)
Subject: [pypy-commit] pypy dtypes-compatability: fix z_translation,
	test_compile (arigato)
Message-ID: <20150612084809.B3A0C1C024E@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: dtypes-compatability
Changeset: r78039:781c31ed6f8d
Date: 2015-06-12 11:48 +0300
http://bitbucket.org/pypy/pypy/changeset/781c31ed6f8d/

Log:	fix z_translation, test_compile (arigato)

diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py
--- a/pypy/module/micronumpy/compile.py
+++ b/pypy/module/micronumpy/compile.py
@@ -47,6 +47,9 @@
     def lookup(self, name):
         return self.getdictvalue(self, name)
 
+    def getname(self, space):
+        return self.name
+
 class FakeSpace(ObjSpace):
     w_ValueError = W_TypeObject("ValueError")
     w_TypeError = W_TypeObject("TypeError")
diff --git a/pypy/module/micronumpy/descriptor.py b/pypy/module/micronumpy/descriptor.py
--- a/pypy/module/micronumpy/descriptor.py
+++ b/pypy/module/micronumpy/descriptor.py
@@ -176,12 +176,7 @@
         return dtype
 
     def get_name(self):
-        from pypy.objspace.std.typeobject import W_TypeObject
-        w_box_type = self.w_box_type
-        assert isinstance(w_box_type, W_TypeObject)
-        name = w_box_type.getname(self.itemtype.space)
-        if name.startswith('numpy.'):
-            name = name[6:]
+        name = self.w_box_type.getname(self.itemtype.space)
         if name.endswith('_'):
             name = name[:-1]
         return name
diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py
--- a/pypy/objspace/fake/objspace.py
+++ b/pypy/objspace/fake/objspace.py
@@ -72,6 +72,10 @@
     def get_module(self):
         return w_some_obj()
 
+
+    def getname(self, space):
+        return self.name
+
 def w_some_obj():
     if NonConstant(False):
         return W_Root()

From noreply at buildbot.pypy.org  Fri Jun 12 11:47:40 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 12 Jun 2015 11:47:40 +0200 (CEST)
Subject: [pypy-commit] stmgc c8-gil-like: Port demo_random2. Apparently fix
 things by removing this hack. I'm
Message-ID: <20150612094740.30A1A1C024E@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: c8-gil-like
Changeset: r1812:759833fd3bcc
Date: 2015-06-12 11:39 +0200
http://bitbucket.org/pypy/stmgc/changeset/759833fd3bcc/

Log:	Port demo_random2. Apparently fix things by removing this hack.
	I'm waiting until the next time I see it going into an infinite loop
	to find a different solution.

diff --git a/c8/demo/demo_random2.c b/c8/demo/demo_random2.c
--- a/c8/demo/demo_random2.c
+++ b/c8/demo/demo_random2.c
@@ -8,6 +8,8 @@
 #include <sys/wait.h>
 
 #include "stmgc.h"
+#include "stm/fprintcolor.h"
+#include "stm/fprintcolor.c"
 
 #define NUMTHREADS 3
 #define STEPS_PER_THREAD 50000
@@ -52,8 +54,10 @@
     int active_roots_num;
     long roots_on_ss;
     long roots_on_ss_at_tr_start;
+    long globally_unique;
 };
 __thread struct thread_data td;
+static long progress = 1;
 
 struct thread_data *_get_td(void)
 {
@@ -61,9 +65,16 @@
 }
 
 
+long check_size(long size)
+{
+    assert(size >= sizeof(struct node_s));
+    assert(size <= sizeof(struct node_s) + 4096*70);
+    return size;
+}
+
 ssize_t stmcb_size_rounded_up(struct object_s *ob)
 {
-    return ((struct node_s*)ob)->my_size;
+    return check_size(((struct node_s*)ob)->my_size);
 }
 
 void stmcb_trace(struct object_s *obj, void visit(object_t **))
@@ -73,7 +84,8 @@
 
     /* and the same value at the end: */
     /* note, ->next may be the same as last_next */
-    nodeptr_t *last_next = (nodeptr_t*)((char*)n + n->my_size - sizeof(void*));
+    nodeptr_t *last_next = (nodeptr_t*)((char*)n + check_size(n->my_size)
+                                        - sizeof(void*));
 
     assert(n->next == *last_next);
 
@@ -193,7 +205,8 @@
         nodeptr_t n = (nodeptr_t)p;
 
         /* and the same value at the end: */
-        nodeptr_t TLPREFIX *last_next = (nodeptr_t TLPREFIX *)((stm_char*)n + n->my_size - sizeof(void*));
+        nodeptr_t TLPREFIX *last_next = (nodeptr_t TLPREFIX *)((stm_char*)n +
+                                       check_size(n->my_size) - sizeof(void*));
         assert(n->next == *last_next);
         n->next = (nodeptr_t)v;
         *last_next = (nodeptr_t)v;
@@ -205,7 +218,8 @@
     nodeptr_t n = (nodeptr_t)p;
 
     /* and the same value at the end: */
-    nodeptr_t TLPREFIX *last_next = (nodeptr_t TLPREFIX *)((stm_char*)n + n->my_size - sizeof(void*));
+    nodeptr_t TLPREFIX *last_next = (nodeptr_t TLPREFIX *)((stm_char*)n +
+                                       check_size(n->my_size) - sizeof(void*));
     OPT_ASSERT(n->next == *last_next);
 
     return n->next;
@@ -239,6 +253,7 @@
             sizeof(struct node_s)+32, sizeof(struct node_s)+48,
             sizeof(struct node_s) + (get_rand(100000) & ~15)};
         size_t size = sizes[get_rand(sizeof(sizes) / sizeof(size_t))];
+        size = check_size(size);
         p = stm_allocate(size);
         nodeptr_t n = (nodeptr_t)p;
         n->sig = SIGNATURE;
@@ -296,6 +311,16 @@
     return p;
 }
 
+static void end_gut(void)
+{
+    if (td.globally_unique != 0) {
+        fprintf(stderr, "[GUT END]");
+        assert(progress == td.globally_unique);
+        td.globally_unique = 0;
+        stm_resume_all_other_threads();
+    }
+}
+
 void frame_loop();
 objptr_t do_step(objptr_t p)
 {
@@ -309,13 +334,22 @@
         p = simple_events(p, _r);
     } else if (get_rand(20) == 1) {
         long pushed = push_roots();
-        stm_commit_transaction();
-        td.roots_on_ss_at_tr_start = td.roots_on_ss;
-
-        if (get_rand(100) < 98) {
-            stm_start_transaction(&stm_thread_local);
-        } else {
-            stm_start_inevitable_transaction(&stm_thread_local);
+        end_gut();
+        if (get_rand(100) < 95) {
+            stm_leave_transactional_zone(&stm_thread_local);
+            /* Nothing here; it's unlikely that a different thread
+               manages to steal the detached inev transaction.
+               Give them a little chance with a usleep(). */
+            dprintf(("sleep...\n"));
+            usleep(1);
+            dprintf(("sleep done\n"));
+            td.roots_on_ss_at_tr_start = td.roots_on_ss;
+            stm_enter_transactional_zone(&stm_thread_local);
+        }
+        else {
+            _stm_commit_transaction();
+            td.roots_on_ss_at_tr_start = td.roots_on_ss;
+            _stm_start_transaction(&stm_thread_local);
         }
         td.roots_on_ss = td.roots_on_ss_at_tr_start;
         td.active_roots_num = 0;
@@ -336,10 +370,16 @@
         p= NULL;
     } else if (get_rand(20) == 1) {
         p = (objptr_t)-1; // possibly fork
-    } else if (get_rand(20) == 1) {
+    } else if (get_rand(100) == 1) {
         long pushed = push_roots();
-        stm_become_globally_unique_transaction(&stm_thread_local, "really");
-        fprintf(stderr, "[GUT/%d]", (int)STM_SEGMENT->segment_num);
+        if (td.globally_unique == 0) {
+            stm_stop_all_other_threads();
+            td.globally_unique = progress;
+            fprintf(stderr, "[GUT/%d]", (int)STM_SEGMENT->segment_num);
+        }
+        else {
+            end_gut();
+        }
         pop_roots(pushed);
         p = NULL;
     }
@@ -364,6 +404,8 @@
 
         p = do_step(p);
 
+        if (!td.globally_unique)
+            ++progress;   /* racy, but good enough */
 
         if (p == (objptr_t)-1) {
             p = NULL;
@@ -371,6 +413,7 @@
             long call_fork = (thread_may_fork != NULL && *(long *)thread_may_fork);
             if (call_fork) {   /* common case */
                 long pushed = push_roots();
+                end_gut();
                 /* run a fork() inside the transaction */
                 printf("==========   FORK  =========\n");
                 *(long*)thread_may_fork = 0;
@@ -426,7 +469,7 @@
     setup_thread();
 
     td.roots_on_ss_at_tr_start = 0;
-    stm_start_transaction(&stm_thread_local);
+    stm_enter_transactional_zone(&stm_thread_local);
     td.roots_on_ss = td.roots_on_ss_at_tr_start;
     td.active_roots_num = 0;
 
@@ -435,7 +478,8 @@
         frame_loop();
     }
 
-    stm_commit_transaction();
+    end_gut();
+    stm_leave_transactional_zone(&stm_thread_local);
 
     stm_rewind_jmp_leaveframe(&stm_thread_local, &rjbuf);
     stm_unregister_thread_local(&stm_thread_local);
diff --git a/c8/stm/core.h b/c8/stm/core.h
--- a/c8/stm/core.h
+++ b/c8/stm/core.h
@@ -162,7 +162,6 @@
 #ifdef STM_TESTS
     SP_WAIT_FOR_OTHER_THREAD,
 #endif
-    SP_COMMIT_OTHER_DETACHED,
 };
 
 enum /* transaction_state */ {
diff --git a/c8/stm/detach.c b/c8/stm/detach.c
--- a/c8/stm/detach.c
+++ b/c8/stm/detach.c
@@ -132,30 +132,17 @@
        conditions.
     */
     int mysegnum = STM_SEGMENT->segment_num;
-    bool sp_running = (STM_PSEGMENT->safe_point == SP_RUNNING);
     int segnum = ((stm_thread_local_t *)old)->last_associated_segment_num;
     dprintf(("commit_fetched_detached_transaction from seg %d\n", segnum));
     assert(segnum > 0);
 
     if (segnum != mysegnum) {
-        if (sp_running) {
-            s_mutex_lock();
-            STM_PSEGMENT->safe_point = SP_COMMIT_OTHER_DETACHED;
-            s_mutex_unlock();
-        }
         set_gs_register(get_segment_base(segnum));
     }
     commit_external_inevitable_transaction();
 
     if (segnum != mysegnum) {
         set_gs_register(get_segment_base(mysegnum));
-
-        if (sp_running) {
-            s_mutex_lock();
-            assert(STM_PSEGMENT->safe_point == SP_COMMIT_OTHER_DETACHED);
-            STM_PSEGMENT->safe_point = SP_RUNNING;
-            s_mutex_unlock();
-        }
     }
 }
 

From noreply at buildbot.pypy.org  Fri Jun 12 12:15:53 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 12 Jun 2015 12:15:53 +0200 (CEST)
Subject: [pypy-commit] stmgc c8-gil-like: Next crash: if I manually add
 usleep(100) here, boom
Message-ID: <20150612101553.B05871C0EB1@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: c8-gil-like
Changeset: r1813:dff74dc51438
Date: 2015-06-12 12:15 +0200
http://bitbucket.org/pypy/stmgc/changeset/dff74dc51438/

Log:	Next crash: if I manually add usleep(100) here, boom

diff --git a/c8/stm/core.c b/c8/stm/core.c
--- a/c8/stm/core.c
+++ b/c8/stm/core.c
@@ -628,6 +628,8 @@
 
     new = _create_commit_log_entry();
     if (STM_PSEGMENT->transaction_state == TS_INEVITABLE) {
+        assert(_stm_detached_inevitable_from_thread == 0);  /* running it */
+
         old = STM_PSEGMENT->last_commit_log_entry;
         new->rev_num = old->rev_num + 1;
         OPT_ASSERT(old->next == INEV_RUNNING);
@@ -643,6 +645,8 @@
         list_clear(STM_PSEGMENT->modified_old_objects);
         STM_PSEGMENT->last_commit_log_entry = new;
 
+        usleep(100);  //XXX
+        
         /* do it: */
         bool yes = __sync_bool_compare_and_swap(&old->next, INEV_RUNNING, new);
         OPT_ASSERT(yes);

From noreply at buildbot.pypy.org  Fri Jun 12 13:25:02 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 12 Jun 2015 13:25:02 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: check only for gc stuff
Message-ID: <20150612112502.B69A01C0987@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r78040:afd28a605eaf
Date: 2015-06-12 13:25 +0200
http://bitbucket.org/pypy/pypy/changeset/afd28a605eaf/

Log:	check only for gc stuff

diff --git a/rpython/jit/backend/llsupport/llmodel.py b/rpython/jit/backend/llsupport/llmodel.py
--- a/rpython/jit/backend/llsupport/llmodel.py
+++ b/rpython/jit/backend/llsupport/llmodel.py
@@ -577,18 +577,21 @@
     @specialize.argtype(1)
     def bh_setfield_gc_i(self, struct, newvalue, fielddescr):
         ofs, size, _ = self.unpack_fielddescr_size(fielddescr)
-        fielddescr.check_correct_type(struct)
+        if isinstance(lltype.typeOf(struct), lltype.Ptr):
+            fielddescr.check_correct_type(struct)
         self.write_int_at_mem(struct, ofs, size, newvalue)
 
     def bh_setfield_gc_r(self, struct, newvalue, fielddescr):
         ofs = self.unpack_fielddescr(fielddescr)
-        fielddescr.check_correct_type(struct)
+        if isinstance(lltype.typeOf(struct), lltype.Ptr):
+            fielddescr.check_correct_type(struct)
         self.write_ref_at_mem(struct, ofs, newvalue)
 
     @specialize.argtype(1)
     def bh_setfield_gc_f(self, struct, newvalue, fielddescr):
         ofs = self.unpack_fielddescr(fielddescr)
-        fielddescr.check_correct_type(struct)
+        if isinstance(lltype.typeOf(struct), lltype.Ptr):
+            fielddescr.check_correct_type(struct)
         self.write_float_at_mem(struct, ofs, newvalue)
 
     bh_setfield_raw_i = bh_setfield_gc_i

From noreply at buildbot.pypy.org  Fri Jun 12 13:38:23 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 12 Jun 2015 13:38:23 +0200 (CEST)
Subject: [pypy-commit] stmgc c8-gil-like: improve
Message-ID: <20150612113823.7F7D41C0987@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: c8-gil-like
Changeset: r1814:adcb141e20ad
Date: 2015-06-12 12:54 +0200
http://bitbucket.org/pypy/stmgc/changeset/adcb141e20ad/

Log:	improve

diff --git a/c7/gdb/gdb_stm.py b/c7/gdb/gdb_stm.py
--- a/c7/gdb/gdb_stm.py
+++ b/c7/gdb/gdb_stm.py
@@ -74,11 +74,13 @@
 def thread_to_segment_id(thread_id):
     base = int(gdb.parse_and_eval('stm_object_pages'))
     for j in range(1, get_nb_segments() + 1):
-        ts = get_psegment(j, '->transaction_state')
-        if int(ts) != 0:
-            ti = get_psegment(j, '->pub.running_thread->creating_pthread[0]')
-            if int(ti) == thread_id:
-                return j
+        #ti = get_psegment(j, '->pub.running_thread->creating_pthread[0]')
+        ti = get_psegment(j, '->running_pthread')
+        if int(ti) == thread_id:
+            ts = get_psegment(j, '->transaction_state')
+            if int(ts) == 0:
+                print >> sys.stderr, "note: transaction_state == 0"
+            return j
     raise Exception("thread not found: %r" % (thread_id,))
 
 def interactive_segment_base(thread=None):
@@ -106,11 +108,13 @@
     sb = interactive_segment_base(thread)
     if p is not None and p.type.code == gdb.TYPE_CODE_PTR:
         return gdb.Value(sb + int(p)).cast(p.type).dereference()
-    elif p is None or int(p) == 0:
+    else:
+        if p is None:
+            p = 0
+        else:
+            p = int(p)
         T = gdb.lookup_type('char').pointer()
-        return gdb.Value(sb).cast(T)
-    else:
-        raise TypeError("gc() first argument must be a GC pointer or 0")
+        return gdb.Value(sb + p).cast(T)
 
 @gdb_function
 def psegment(thread=None):

From noreply at buildbot.pypy.org  Fri Jun 12 13:38:24 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 12 Jun 2015 13:38:24 +0200 (CEST)
Subject: [pypy-commit] stmgc c8-gil-like: Tentative fix: remove the two
 places setting 'TS_NONE' without the
Message-ID: <20150612113824.A5AB21C0987@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: c8-gil-like
Changeset: r1815:48f9b949581f
Date: 2015-06-12 13:39 +0200
http://bitbucket.org/pypy/stmgc/changeset/48f9b949581f/

Log:	Tentative fix: remove the two places setting 'TS_NONE' without the
	mutex_lock, which creates race conditions.

	Move the major collection at commit a bit earlier, at a point where
	the transaction being committed is still running like usual.

	Commit no longer does a safe_point at the end: if a safe point is
	requested, we now signal C_AT_SAFE_POINT but just leave.

diff --git a/c8/stm/core.c b/c8/stm/core.c
--- a/c8/stm/core.c
+++ b/c8/stm/core.c
@@ -324,10 +324,7 @@
     /* Don't check this 'cl'. This entry is already checked */
 
     if (STM_PSEGMENT->transaction_state == TS_INEVITABLE) {
-        //assert(first_cl->next == INEV_RUNNING);
-        /* the above assert may fail when running a major collection
-           while the commit of the inevitable transaction is in progress
-           and the element is already attached */
+        assert(first_cl->next == INEV_RUNNING);
         return true;
     }
 
@@ -606,9 +603,6 @@
 
     if (is_commit) {
         /* compare with _validate_and_add_to_commit_log */
-        STM_PSEGMENT->transaction_state = TS_NONE;
-        STM_PSEGMENT->safe_point = SP_NO_TRANSACTION;
-
         list_clear(STM_PSEGMENT->modified_old_objects);
         STM_PSEGMENT->last_commit_log_entry = new;
         release_modification_lock_wr(STM_SEGMENT->segment_num);
@@ -640,16 +634,15 @@
                                       STM_PSEGMENT->modified_old_objects);
 
         /* compare with _validate_and_attach: */
-        STM_PSEGMENT->transaction_state = TS_NONE;
-        STM_PSEGMENT->safe_point = SP_NO_TRANSACTION;
+        acquire_modification_lock_wr(STM_SEGMENT->segment_num);
         list_clear(STM_PSEGMENT->modified_old_objects);
         STM_PSEGMENT->last_commit_log_entry = new;
 
-        usleep(100);  //XXX
-        
         /* do it: */
         bool yes = __sync_bool_compare_and_swap(&old->next, INEV_RUNNING, new);
         OPT_ASSERT(yes);
+
+        release_modification_lock_wr(STM_SEGMENT->segment_num);
     }
     else {
         _validate_and_attach(new, /*can_sleep=*/true);
@@ -1232,6 +1225,7 @@
 {
     stm_thread_local_t *tl = STM_SEGMENT->running_thread;
 
+    assert(_has_mutex());
     STM_PSEGMENT->safe_point = SP_NO_TRANSACTION;
     STM_PSEGMENT->transaction_state = TS_NONE;
 
@@ -1242,6 +1236,13 @@
     if (tl != NULL)
         timing_event(tl, event);
 
+    /* If somebody is waiting for us to reach a safe point, we simply
+       signal it now and leave this transaction.  This should be enough
+       for synchronize_all_threads() to retry and notice that we are
+       no longer SP_RUNNING. */
+    if (STM_SEGMENT->nursery_end != NURSERY_END)
+        cond_signal(C_AT_SAFE_POINT);
+
     release_thread_segment(tl);
     /* cannot access STM_SEGMENT or STM_PSEGMENT from here ! */
 }
@@ -1301,6 +1302,7 @@
 
     assert(!_has_mutex());
     assert(STM_PSEGMENT->safe_point == SP_RUNNING);
+    assert(STM_PSEGMENT->transaction_state != TS_NONE);
     if (globally_unique_transaction) {
         stm_fatalerror("cannot commit between stm_stop_all_other_threads "
                        "and stm_resume_all_other_threads");
@@ -1308,6 +1310,13 @@
 
     dprintf(("> stm_commit_transaction(external=%d)\n", (int)external));
     minor_collection(/*commit=*/ true, external);
+    if (!external && is_major_collection_requested()) {
+        s_mutex_lock();
+        if (is_major_collection_requested()) {   /* if still true */
+            major_collection_with_mutex();
+        }
+        s_mutex_unlock();
+    }
 
     push_large_overflow_objects_to_other_segments();
     /* push before validate. otherwise they are reachable too early */
@@ -1346,19 +1355,6 @@
 
     invoke_and_clear_user_callbacks(0);   /* for commit */
 
-    /* >>>>> there may be a FORK() happening in the safepoint below <<<<<*/
-    if (!external) {
-        enter_safe_point_if_requested();
-        assert(STM_SEGMENT->nursery_end == NURSERY_END);
-
-        /* if a major collection is required, do it here */
-        if (is_major_collection_requested()) {
-            major_collection_with_mutex();
-        }
-    }
-
-    _verify_cards_cleared_in_all_lists(get_priv_segment(STM_SEGMENT->segment_num));
-
     /* done */
     stm_thread_local_t *tl = STM_SEGMENT->running_thread;
     assert(external == (tl == NULL));

From noreply at buildbot.pypy.org  Fri Jun 12 14:06:47 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 12 Jun 2015 14:06:47 +0200 (CEST)
Subject: [pypy-commit] stmgc c8-gil-like: Add the logic here,
	but commented out for now
Message-ID: <20150612120647.BE3EF1C124A@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: c8-gil-like
Changeset: r1816:f97d3ff65683
Date: 2015-06-12 14:07 +0200
http://bitbucket.org/pypy/stmgc/changeset/f97d3ff65683/

Log:	Add the logic here, but commented out for now

diff --git a/c8/stm/gcpage.c b/c8/stm/gcpage.c
--- a/c8/stm/gcpage.c
+++ b/c8/stm/gcpage.c
@@ -188,6 +188,13 @@
 
 static void major_collection_with_mutex(void)
 {
+    /* XXX unclear: force-commit a detached transaction before major GC, for
+       the purpose of letting other threads notice that they are doomed anyway
+    */
+    //intptr_t detached = fetch_detached_transaction();
+    //if (detached != 0) {
+    //    commit_fetched_detached_transaction(detached);
+
     timing_event(STM_SEGMENT->running_thread, STM_GC_MAJOR_START);
 
     synchronize_all_threads(STOP_OTHERS_UNTIL_MUTEX_UNLOCK);

From noreply at buildbot.pypy.org  Fri Jun 12 14:27:29 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 12 Jun 2015 14:27:29 +0200 (CEST)
Subject: [pypy-commit] stmgc c8-gil-like: Back out f97d3ff65683: we're in a
 synchronize_all_threads() section, 
Message-ID: <20150612122729.76B6C1C0987@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: c8-gil-like
Changeset: r1817:c867506dce0a
Date: 2015-06-12 14:28 +0200
http://bitbucket.org/pypy/stmgc/changeset/c867506dce0a/

Log:	Back out f97d3ff65683: we're in a synchronize_all_threads() section,
	which means the detached inevitable transaction was forcefully
	committed already.

diff --git a/c8/stm/gcpage.c b/c8/stm/gcpage.c
--- a/c8/stm/gcpage.c
+++ b/c8/stm/gcpage.c
@@ -188,13 +188,6 @@
 
 static void major_collection_with_mutex(void)
 {
-    /* XXX unclear: force-commit a detached transaction before major GC, for
-       the purpose of letting other threads notice that they are doomed anyway
-    */
-    //intptr_t detached = fetch_detached_transaction();
-    //if (detached != 0) {
-    //    commit_fetched_detached_transaction(detached);
-
     timing_event(STM_SEGMENT->running_thread, STM_GC_MAJOR_START);
 
     synchronize_all_threads(STOP_OTHERS_UNTIL_MUTEX_UNLOCK);

From noreply at buildbot.pypy.org  Fri Jun 12 14:46:58 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 12 Jun 2015 14:46:58 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: strgetitem handles self.make_equal_to
	already
Message-ID: <20150612124658.947A81C0FB8@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r78041:aad6cb4f869e
Date: 2015-06-12 14:47 +0200
http://bitbucket.org/pypy/pypy/changeset/aad6cb4f869e/

Log:	strgetitem handles self.make_equal_to already

diff --git a/rpython/jit/metainterp/optimizeopt/vstring.py b/rpython/jit/metainterp/optimizeopt/vstring.py
--- a/rpython/jit/metainterp/optimizeopt/vstring.py
+++ b/rpython/jit/metainterp/optimizeopt/vstring.py
@@ -552,9 +552,7 @@
         self._optimize_STRGETITEM(op, mode_unicode)
 
     def _optimize_STRGETITEM(self, op, mode):
-        res = self.strgetitem(op, op.getarg(0), op.getarg(1), mode)
-        if res is not None and not isinstance(res, AbstractResOp):
-            self.make_equal_to(op, res)
+        self.strgetitem(op, op.getarg(0), op.getarg(1), mode)
 
     def strgetitem(self, op, s, index, mode):
         self.make_nonnull_str(s, mode)

From noreply at buildbot.pypy.org  Fri Jun 12 15:51:32 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 12 Jun 2015 15:51:32 +0200 (CEST)
Subject: [pypy-commit] stmgc c8-gil-like: Fix test_extra
Message-ID: <20150612135132.43B6A1C0987@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: c8-gil-like
Changeset: r1818:db81f0821c99
Date: 2015-06-12 15:52 +0200
http://bitbucket.org/pypy/stmgc/changeset/db81f0821c99/

Log:	Fix test_extra

diff --git a/c8/test/support.py b/c8/test/support.py
--- a/c8/test/support.py
+++ b/c8/test/support.py
@@ -85,7 +85,7 @@
 bool _check_commit_transaction(void);
 bool _check_abort_transaction(void);
 bool _check_become_inevitable(stm_thread_local_t *tl);
-bool _check_become_globally_unique_transaction(stm_thread_local_t *tl);
+//bool _check_become_globally_unique_transaction(stm_thread_local_t *tl);
 bool _check_stop_all_other_threads(void);
 void stm_resume_all_other_threads(void);
 int stm_is_inevitable(void);
@@ -306,10 +306,6 @@
     CHECKED(stm_become_inevitable(tl, "TEST"));
 }
 
-bool _check_become_globally_unique_transaction(stm_thread_local_t *tl) {
-    CHECKED(stm_become_globally_unique_transaction(tl, "TESTGUT"));
-}
-
 bool _check_stop_all_other_threads(void) {
     CHECKED(stm_stop_all_other_threads());
 }
@@ -884,6 +880,7 @@
             raise Conflict()
 
     def become_globally_unique_transaction(self):
+        import py; py.test.skip("this function was removed")
         tl = self.tls[self.current_thread]
         if lib._check_become_globally_unique_transaction(tl):
             raise Conflict()
diff --git a/c8/test/test_extra.py b/c8/test/test_extra.py
--- a/c8/test/test_extra.py
+++ b/c8/test/test_extra.py
@@ -202,6 +202,9 @@
         assert lib.stm_is_inevitable()
         #
         py.test.raises(Conflict, self.switch, 0)
+        #
+        self.switch(1)
+        self.resume_all_other_threads()
 
     def test_stm_stop_all_other_threads_2(self):
         self.start_transaction()

From noreply at buildbot.pypy.org  Fri Jun 12 16:01:05 2015
From: noreply at buildbot.pypy.org (amauryfa)
Date: Fri, 12 Jun 2015 16:01:05 +0200 (CEST)
Subject: [pypy-commit] pypy vendor/stdlib: Upgrade 2.7 stdlib to 2.7.10
Message-ID: <20150612140105.3957D1C0EB1@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: vendor/stdlib
Changeset: r78042:a3b827ca23fa
Date: 2015-06-12 13:58 +0200
http://bitbucket.org/pypy/pypy/changeset/a3b827ca23fa/

Log:	Upgrade 2.7 stdlib to 2.7.10

diff too long, truncating to 2000 out of 10083 lines

diff --git a/lib-python/2.7/Cookie.py b/lib-python/2.7/Cookie.py
--- a/lib-python/2.7/Cookie.py
+++ b/lib-python/2.7/Cookie.py
@@ -528,12 +528,13 @@
 # result, the parsing rules here are less strict.
 #
 
-_LegalCharsPatt  = r"[\w\d!#%&'~_`><@,:/\$\*\+\-\.\^\|\)\(\?\}\{\=]"
+_LegalKeyChars  = r"\w\d!#%&'~_`><@,:/\$\*\+\-\.\^\|\)\(\?\}\{\="
+_LegalValueChars = _LegalKeyChars + r"\[\]"
 _CookiePattern = re.compile(
     r"(?x)"                       # This is a Verbose pattern
     r"\s*"                        # Optional whitespace at start of cookie
     r"(?P<key>"                   # Start of group 'key'
-    ""+ _LegalCharsPatt +"+?"     # Any word of at least one letter, nongreedy
+    "["+ _LegalKeyChars +"]+?"     # Any word of at least one letter, nongreedy
     r")"                          # End of group 'key'
     r"("                          # Optional group: there may not be a value.
     r"\s*=\s*"                    # Equal Sign
@@ -542,7 +543,7 @@
     r"|"                            # or
     r"\w{3},\s[\s\w\d-]{9,11}\s[\d:]{8}\sGMT" # Special case for "expires" attr
     r"|"                            # or
-    ""+ _LegalCharsPatt +"*"        # Any word or empty string
+    "["+ _LegalValueChars +"]*"        # Any word or empty string
     r")"                          # End of group 'val'
     r")?"                         # End of optional value group
     r"\s*"                        # Any number of spaces.
diff --git a/lib-python/2.7/SimpleHTTPServer.py b/lib-python/2.7/SimpleHTTPServer.py
--- a/lib-python/2.7/SimpleHTTPServer.py
+++ b/lib-python/2.7/SimpleHTTPServer.py
@@ -14,6 +14,7 @@
 import posixpath
 import BaseHTTPServer
 import urllib
+import urlparse
 import cgi
 import sys
 import shutil
@@ -68,10 +69,14 @@
         path = self.translate_path(self.path)
         f = None
         if os.path.isdir(path):
-            if not self.path.endswith('/'):
+            parts = urlparse.urlsplit(self.path)
+            if not parts.path.endswith('/'):
                 # redirect browser - doing basically what apache does
                 self.send_response(301)
-                self.send_header("Location", self.path + "/")
+                new_parts = (parts[0], parts[1], parts[2] + '/',
+                             parts[3], parts[4])
+                new_url = urlparse.urlunsplit(new_parts)
+                self.send_header("Location", new_url)
                 self.end_headers()
                 return None
             for index in "index.html", "index.htm":
diff --git a/lib-python/2.7/_LWPCookieJar.py b/lib-python/2.7/_LWPCookieJar.py
--- a/lib-python/2.7/_LWPCookieJar.py
+++ b/lib-python/2.7/_LWPCookieJar.py
@@ -18,7 +18,7 @@
                        iso2time, time2isoz)
 
 def lwp_cookie_str(cookie):
-    """Return string representation of Cookie in an the LWP cookie file format.
+    """Return string representation of Cookie in the LWP cookie file format.
 
     Actually, the format is extended a bit -- see module docstring.
 
diff --git a/lib-python/2.7/_abcoll.py b/lib-python/2.7/_abcoll.py
--- a/lib-python/2.7/_abcoll.py
+++ b/lib-python/2.7/_abcoll.py
@@ -548,23 +548,25 @@
             If E present and lacks .keys() method, does:     for (k, v) in E: D[k] = v
             In either case, this is followed by: for k, v in F.items(): D[k] = v
         '''
-        if len(args) > 2:
-            raise TypeError("update() takes at most 2 positional "
-                            "arguments ({} given)".format(len(args)))
-        elif not args:
-            raise TypeError("update() takes at least 1 argument (0 given)")
+        if not args:
+            raise TypeError("descriptor 'update' of 'MutableMapping' object "
+                            "needs an argument")
         self = args[0]
-        other = args[1] if len(args) >= 2 else ()
-
-        if isinstance(other, Mapping):
-            for key in other:
-                self[key] = other[key]
-        elif hasattr(other, "keys"):
-            for key in other.keys():
-                self[key] = other[key]
-        else:
-            for key, value in other:
-                self[key] = value
+        args = args[1:]
+        if len(args) > 1:
+            raise TypeError('update expected at most 1 arguments, got %d' %
+                            len(args))
+        if args:
+            other = args[0]
+            if isinstance(other, Mapping):
+                for key in other:
+                    self[key] = other[key]
+            elif hasattr(other, "keys"):
+                for key in other.keys():
+                    self[key] = other[key]
+            else:
+                for key, value in other:
+                    self[key] = value
         for key, value in kwds.items():
             self[key] = value
 
diff --git a/lib-python/2.7/_pyio.py b/lib-python/2.7/_pyio.py
--- a/lib-python/2.7/_pyio.py
+++ b/lib-python/2.7/_pyio.py
@@ -25,8 +25,8 @@
 DEFAULT_BUFFER_SIZE = 8 * 1024  # bytes
 
 # NOTE: Base classes defined here are registered with the "official" ABCs
-# defined in io.py. We don't use real inheritance though, because we don't
-# want to inherit the C implementations.
+# defined in io.py. We don't use real inheritance though, because we don't want
+# to inherit the C implementations.
 
 
 class BlockingIOError(IOError):
@@ -775,7 +775,7 @@
         clsname = self.__class__.__name__
         try:
             name = self.name
-        except AttributeError:
+        except Exception:
             return "<_pyio.{0}>".format(clsname)
         else:
             return "<_pyio.{0} name={1!r}>".format(clsname, name)
@@ -1216,8 +1216,10 @@
         return self.writer.flush()
 
     def close(self):
-        self.writer.close()
-        self.reader.close()
+        try:
+            self.writer.close()
+        finally:
+            self.reader.close()
 
     def isatty(self):
         return self.reader.isatty() or self.writer.isatty()
@@ -1538,7 +1540,7 @@
     def __repr__(self):
         try:
             name = self.name
-        except AttributeError:
+        except Exception:
             return "<_pyio.TextIOWrapper encoding='{0}'>".format(self.encoding)
         else:
             return "<_pyio.TextIOWrapper name={0!r} encoding='{1}'>".format(
diff --git a/lib-python/2.7/_strptime.py b/lib-python/2.7/_strptime.py
--- a/lib-python/2.7/_strptime.py
+++ b/lib-python/2.7/_strptime.py
@@ -335,9 +335,9 @@
     # though
     week_of_year = -1
     week_of_year_start = -1
-    # weekday and julian defaulted to -1 so as to signal need to calculate
+    # weekday and julian defaulted to None so as to signal need to calculate
     # values
-    weekday = julian = -1
+    weekday = julian = None
     found_dict = found.groupdict()
     for group_key in found_dict.iterkeys():
         # Directives not explicitly handled below:
@@ -434,14 +434,14 @@
         year = 1900
     # If we know the week of the year and what day of that week, we can figure
     # out the Julian day of the year.
-    if julian == -1 and week_of_year != -1 and weekday != -1:
+    if julian is None and week_of_year != -1 and weekday is not None:
         week_starts_Mon = True if week_of_year_start == 0 else False
         julian = _calc_julian_from_U_or_W(year, week_of_year, weekday,
                                             week_starts_Mon)
     # Cannot pre-calculate datetime_date() since can change in Julian
     # calculation and thus could have different value for the day of the week
     # calculation.
-    if julian == -1:
+    if julian is None:
         # Need to add 1 to result since first day of the year is 1, not 0.
         julian = datetime_date(year, month, day).toordinal() - \
                   datetime_date(year, 1, 1).toordinal() + 1
@@ -451,7 +451,7 @@
         year = datetime_result.year
         month = datetime_result.month
         day = datetime_result.day
-    if weekday == -1:
+    if weekday is None:
         weekday = datetime_date(year, month, day).weekday()
     if leap_year_fix:
         # the caller didn't supply a year but asked for Feb 29th. We couldn't
diff --git a/lib-python/2.7/aifc.py b/lib-python/2.7/aifc.py
--- a/lib-python/2.7/aifc.py
+++ b/lib-python/2.7/aifc.py
@@ -357,10 +357,13 @@
         self._soundpos = 0
 
     def close(self):
-        if self._decomp:
-            self._decomp.CloseDecompressor()
-            self._decomp = None
-        self._file.close()
+        decomp = self._decomp
+        try:
+            if decomp:
+                self._decomp = None
+                decomp.CloseDecompressor()
+        finally:
+            self._file.close()
 
     def tell(self):
         return self._soundpos
diff --git a/lib-python/2.7/binhex.py b/lib-python/2.7/binhex.py
--- a/lib-python/2.7/binhex.py
+++ b/lib-python/2.7/binhex.py
@@ -32,7 +32,8 @@
     pass
 
 # States (what have we written)
-[_DID_HEADER, _DID_DATA, _DID_RSRC] = range(3)
+_DID_HEADER = 0
+_DID_DATA = 1
 
 # Various constants
 REASONABLY_LARGE=32768  # Minimal amount we pass the rle-coder
@@ -235,17 +236,22 @@
         self._write(data)
 
     def close(self):
-        if self.state < _DID_DATA:
-            self.close_data()
-        if self.state != _DID_DATA:
-            raise Error, 'Close at the wrong time'
-        if self.rlen != 0:
-            raise Error, \
-                  "Incorrect resource-datasize, diff=%r" % (self.rlen,)
-        self._writecrc()
-        self.ofp.close()
-        self.state = None
-        del self.ofp
+        if self.state is None:
+            return
+        try:
+            if self.state < _DID_DATA:
+                self.close_data()
+            if self.state != _DID_DATA:
+                raise Error, 'Close at the wrong time'
+            if self.rlen != 0:
+                raise Error, \
+                    "Incorrect resource-datasize, diff=%r" % (self.rlen,)
+            self._writecrc()
+        finally:
+            self.state = None
+            ofp = self.ofp
+            del self.ofp
+            ofp.close()
 
 def binhex(inp, out):
     """(infilename, outfilename) - Create binhex-encoded copy of a file"""
@@ -463,11 +469,15 @@
         return self._read(n)
 
     def close(self):
-        if self.rlen:
-            dummy = self.read_rsrc(self.rlen)
-        self._checkcrc()
-        self.state = _DID_RSRC
-        self.ifp.close()
+        if self.state is None:
+            return
+        try:
+            if self.rlen:
+                dummy = self.read_rsrc(self.rlen)
+            self._checkcrc()
+        finally:
+            self.state = None
+            self.ifp.close()
 
 def hexbin(inp, out):
     """(infilename, outfilename) - Decode binhexed file"""
diff --git a/lib-python/2.7/bsddb/test/test_all.py b/lib-python/2.7/bsddb/test/test_all.py
--- a/lib-python/2.7/bsddb/test/test_all.py
+++ b/lib-python/2.7/bsddb/test/test_all.py
@@ -412,9 +412,6 @@
         def get_dbp(self) :
             return self._db
 
-    import string
-    string.letters=[chr(i) for i in xrange(65,91)]
-
     bsddb._db.DBEnv_orig = bsddb._db.DBEnv
     bsddb._db.DB_orig = bsddb._db.DB
     if bsddb.db.version() <= (4, 3) :
diff --git a/lib-python/2.7/bsddb/test/test_basics.py b/lib-python/2.7/bsddb/test/test_basics.py
--- a/lib-python/2.7/bsddb/test/test_basics.py
+++ b/lib-python/2.7/bsddb/test/test_basics.py
@@ -999,7 +999,7 @@
         for x in "The quick brown fox jumped over the lazy dog".split():
             d2.put(x, self.makeData(x))
 
-        for x in string.letters:
+        for x in string.ascii_letters:
             d3.put(x, x*70)
 
         d1.sync()
@@ -1047,7 +1047,7 @@
             if verbose:
                 print rec
             rec = c3.next()
-        self.assertEqual(count, len(string.letters))
+        self.assertEqual(count, len(string.ascii_letters))
 
 
         c1.close()
diff --git a/lib-python/2.7/bsddb/test/test_dbshelve.py b/lib-python/2.7/bsddb/test/test_dbshelve.py
--- a/lib-python/2.7/bsddb/test/test_dbshelve.py
+++ b/lib-python/2.7/bsddb/test/test_dbshelve.py
@@ -59,7 +59,7 @@
             return bytes(key, "iso8859-1")  # 8 bits
 
     def populateDB(self, d):
-        for x in string.letters:
+        for x in string.ascii_letters:
             d[self.mk('S' + x)] = 10 * x           # add a string
             d[self.mk('I' + x)] = ord(x)           # add an integer
             d[self.mk('L' + x)] = [x] * 10         # add a list
diff --git a/lib-python/2.7/bsddb/test/test_get_none.py b/lib-python/2.7/bsddb/test/test_get_none.py
--- a/lib-python/2.7/bsddb/test/test_get_none.py
+++ b/lib-python/2.7/bsddb/test/test_get_none.py
@@ -26,14 +26,14 @@
         d.open(self.filename, db.DB_BTREE, db.DB_CREATE)
         d.set_get_returns_none(1)
 
-        for x in string.letters:
+        for x in string.ascii_letters:
             d.put(x, x * 40)
 
         data = d.get('bad key')
         self.assertEqual(data, None)
 
-        data = d.get(string.letters[0])
-        self.assertEqual(data, string.letters[0]*40)
+        data = d.get(string.ascii_letters[0])
+        self.assertEqual(data, string.ascii_letters[0]*40)
 
         count = 0
         c = d.cursor()
@@ -43,7 +43,7 @@
             rec = c.next()
 
         self.assertEqual(rec, None)
-        self.assertEqual(count, len(string.letters))
+        self.assertEqual(count, len(string.ascii_letters))
 
         c.close()
         d.close()
@@ -54,14 +54,14 @@
         d.open(self.filename, db.DB_BTREE, db.DB_CREATE)
         d.set_get_returns_none(0)
 
-        for x in string.letters:
+        for x in string.ascii_letters:
             d.put(x, x * 40)
 
         self.assertRaises(db.DBNotFoundError, d.get, 'bad key')
         self.assertRaises(KeyError, d.get, 'bad key')
 
-        data = d.get(string.letters[0])
-        self.assertEqual(data, string.letters[0]*40)
+        data = d.get(string.ascii_letters[0])
+        self.assertEqual(data, string.ascii_letters[0]*40)
 
         count = 0
         exceptionHappened = 0
@@ -77,7 +77,7 @@
 
         self.assertNotEqual(rec, None)
         self.assertTrue(exceptionHappened)
-        self.assertEqual(count, len(string.letters))
+        self.assertEqual(count, len(string.ascii_letters))
 
         c.close()
         d.close()
diff --git a/lib-python/2.7/bsddb/test/test_queue.py b/lib-python/2.7/bsddb/test/test_queue.py
--- a/lib-python/2.7/bsddb/test/test_queue.py
+++ b/lib-python/2.7/bsddb/test/test_queue.py
@@ -10,7 +10,6 @@
 
 #----------------------------------------------------------------------
 
- at unittest.skip("fails on Windows; see issue 22943")
 class SimpleQueueTestCase(unittest.TestCase):
     def setUp(self):
         self.filename = get_new_database_path()
@@ -37,17 +36,17 @@
             print "before appends" + '-' * 30
             pprint(d.stat())
 
-        for x in string.letters:
+        for x in string.ascii_letters:
             d.append(x * 40)
 
-        self.assertEqual(len(d), len(string.letters))
+        self.assertEqual(len(d), len(string.ascii_letters))
 
         d.put(100, "some more data")
         d.put(101, "and some more ")
         d.put(75,  "out of order")
         d.put(1,   "replacement data")
 
-        self.assertEqual(len(d), len(string.letters)+3)
+        self.assertEqual(len(d), len(string.ascii_letters)+3)
 
         if verbose:
             print "before close" + '-' * 30
@@ -108,17 +107,17 @@
             print "before appends" + '-' * 30
             pprint(d.stat())
 
-        for x in string.letters:
+        for x in string.ascii_letters:
             d.append(x * 40)
 
-        self.assertEqual(len(d), len(string.letters))
+        self.assertEqual(len(d), len(string.ascii_letters))
 
         d.put(100, "some more data")
         d.put(101, "and some more ")
         d.put(75,  "out of order")
         d.put(1,   "replacement data")
 
-        self.assertEqual(len(d), len(string.letters)+3)
+        self.assertEqual(len(d), len(string.ascii_letters)+3)
 
         if verbose:
             print "before close" + '-' * 30
diff --git a/lib-python/2.7/bsddb/test/test_recno.py b/lib-python/2.7/bsddb/test/test_recno.py
--- a/lib-python/2.7/bsddb/test/test_recno.py
+++ b/lib-python/2.7/bsddb/test/test_recno.py
@@ -4,12 +4,11 @@
 import os, sys
 import errno
 from pprint import pprint
+import string
 import unittest
 
 from test_all import db, test_support, verbose, get_new_environment_path, get_new_database_path
 
-letters = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
-
 
 #----------------------------------------------------------------------
 
@@ -39,7 +38,7 @@
 
         d.open(self.filename, db.DB_RECNO, db.DB_CREATE)
 
-        for x in letters:
+        for x in string.ascii_letters:
             recno = d.append(x * 60)
             self.assertIsInstance(recno, int)
             self.assertGreaterEqual(recno, 1)
@@ -270,7 +269,7 @@
         d.set_re_pad(45)  # ...test both int and char
         d.open(self.filename, db.DB_RECNO, db.DB_CREATE)
 
-        for x in letters:
+        for x in string.ascii_letters:
             d.append(x * 35)    # These will be padded
 
         d.append('.' * 40)      # this one will be exact
diff --git a/lib-python/2.7/chunk.py b/lib-python/2.7/chunk.py
--- a/lib-python/2.7/chunk.py
+++ b/lib-python/2.7/chunk.py
@@ -85,8 +85,10 @@
 
     def close(self):
         if not self.closed:
-            self.skip()
-            self.closed = True
+            try:
+                self.skip()
+            finally:
+                self.closed = True
 
     def isatty(self):
         if self.closed:
diff --git a/lib-python/2.7/codecs.py b/lib-python/2.7/codecs.py
--- a/lib-python/2.7/codecs.py
+++ b/lib-python/2.7/codecs.py
@@ -20,8 +20,14 @@
            "BOM_LE", "BOM32_BE", "BOM32_LE", "BOM64_BE", "BOM64_LE",
            "BOM_UTF8", "BOM_UTF16", "BOM_UTF16_LE", "BOM_UTF16_BE",
            "BOM_UTF32", "BOM_UTF32_LE", "BOM_UTF32_BE",
+           "CodecInfo", "Codec", "IncrementalEncoder", "IncrementalDecoder",
+           "StreamReader", "StreamWriter",
+           "StreamReaderWriter", "StreamRecoder",
+           "getencoder", "getdecoder", "getincrementalencoder",
+           "getincrementaldecoder", "getreader", "getwriter",
+           "encode", "decode", "iterencode", "iterdecode",
            "strict_errors", "ignore_errors", "replace_errors",
-           "xmlcharrefreplace_errors",
+           "xmlcharrefreplace_errors", "backslashreplace_errors",
            "register_error", "lookup_error"]
 
 ### Constants
@@ -1051,7 +1057,7 @@
         during translation.
 
         One example where this happens is cp875.py which decodes
-        multiple character to \u001a.
+        multiple character to \\u001a.
 
     """
     m = {}
diff --git a/lib-python/2.7/collections.py b/lib-python/2.7/collections.py
--- a/lib-python/2.7/collections.py
+++ b/lib-python/2.7/collections.py
@@ -35,12 +35,17 @@
     # The sentinel element never gets deleted (this simplifies the algorithm).
     # Each link is stored as a list of length three:  [PREV, NEXT, KEY].
 
-    def __init__(self, *args, **kwds):
+    def __init__(*args, **kwds):
         '''Initialize an ordered dictionary.  The signature is the same as
         regular dictionaries, but keyword arguments are not recommended because
         their insertion order is arbitrary.
 
         '''
+        if not args:
+            raise TypeError("descriptor '__init__' of 'OrderedDict' object "
+                            "needs an argument")
+        self = args[0]
+        args = args[1:]
         if len(args) > 1:
             raise TypeError('expected at most 1 arguments, got %d' % len(args))
         try:
@@ -438,7 +443,7 @@
     #   http://code.activestate.com/recipes/259174/
     #   Knuth, TAOCP Vol. II section 4.6.3
 
-    def __init__(self, iterable=None, **kwds):
+    def __init__(*args, **kwds):
         '''Create a new, empty Counter object.  And if given, count elements
         from an input iterable.  Or, initialize the count from another mapping
         of elements to their counts.
@@ -449,8 +454,15 @@
         >>> c = Counter(a=4, b=2)                   # a new counter from keyword args
 
         '''
+        if not args:
+            raise TypeError("descriptor '__init__' of 'Counter' object "
+                            "needs an argument")
+        self = args[0]
+        args = args[1:]
+        if len(args) > 1:
+            raise TypeError('expected at most 1 arguments, got %d' % len(args))
         super(Counter, self).__init__()
-        self.update(iterable, **kwds)
+        self.update(*args, **kwds)
 
     def __missing__(self, key):
         'The count of elements not in the Counter is zero.'
@@ -501,7 +513,7 @@
         raise NotImplementedError(
             'Counter.fromkeys() is undefined.  Use Counter(iterable) instead.')
 
-    def update(self, iterable=None, **kwds):
+    def update(*args, **kwds):
         '''Like dict.update() but add counts instead of replacing them.
 
         Source can be an iterable, a dictionary, or another Counter instance.
@@ -521,6 +533,14 @@
         # contexts.  Instead, we implement straight-addition.  Both the inputs
         # and outputs are allowed to contain zero and negative counts.
 
+        if not args:
+            raise TypeError("descriptor 'update' of 'Counter' object "
+                            "needs an argument")
+        self = args[0]
+        args = args[1:]
+        if len(args) > 1:
+            raise TypeError('expected at most 1 arguments, got %d' % len(args))
+        iterable = args[0] if args else None
         if iterable is not None:
             if isinstance(iterable, Mapping):
                 if self:
@@ -536,7 +556,7 @@
         if kwds:
             self.update(kwds)
 
-    def subtract(self, iterable=None, **kwds):
+    def subtract(*args, **kwds):
         '''Like dict.update() but subtracts counts instead of replacing them.
         Counts can be reduced below zero.  Both the inputs and outputs are
         allowed to contain zero and negative counts.
@@ -552,6 +572,14 @@
         -1
 
         '''
+        if not args:
+            raise TypeError("descriptor 'subtract' of 'Counter' object "
+                            "needs an argument")
+        self = args[0]
+        args = args[1:]
+        if len(args) > 1:
+            raise TypeError('expected at most 1 arguments, got %d' % len(args))
+        iterable = args[0] if args else None
         if iterable is not None:
             self_get = self.get
             if isinstance(iterable, Mapping):
diff --git a/lib-python/2.7/cookielib.py b/lib-python/2.7/cookielib.py
--- a/lib-python/2.7/cookielib.py
+++ b/lib-python/2.7/cookielib.py
@@ -464,26 +464,42 @@
     for ns_header in ns_headers:
         pairs = []
         version_set = False
-        for ii, param in enumerate(re.split(r";\s*", ns_header)):
-            param = param.rstrip()
-            if param == "": continue
-            if "=" not in param:
-                k, v = param, None
-            else:
-                k, v = re.split(r"\s*=\s*", param, 1)
-                k = k.lstrip()
+
+        # XXX: The following does not strictly adhere to RFCs in that empty
+        # names and values are legal (the former will only appear once and will
+        # be overwritten if multiple occurrences are present). This is
+        # mostly to deal with backwards compatibility.
+        for ii, param in enumerate(ns_header.split(';')):
+            param = param.strip()
+
+            key, sep, val = param.partition('=')
+            key = key.strip()
+
+            if not key:
+                if ii == 0:
+                    break
+                else:
+                    continue
+
+            # allow for a distinction between present and empty and missing
+            # altogether
+            val = val.strip() if sep else None
+
             if ii != 0:
-                lc = k.lower()
+                lc = key.lower()
                 if lc in known_attrs:
-                    k = lc
-                if k == "version":
+                    key = lc
+
+                if key == "version":
                     # This is an RFC 2109 cookie.
-                    v = _strip_quotes(v)
+                    if val is not None:
+                        val = _strip_quotes(val)
                     version_set = True
-                if k == "expires":
+                elif key == "expires":
                     # convert expires date to seconds since epoch
-                    v = http2time(_strip_quotes(v))  # None if invalid
-            pairs.append((k, v))
+                    if val is not None:
+                        val = http2time(_strip_quotes(val))  # None if invalid
+            pairs.append((key, val))
 
         if pairs:
             if not version_set:
diff --git a/lib-python/2.7/ctypes/macholib/fetch_macholib.bat b/lib-python/2.7/ctypes/macholib/fetch_macholib.bat
--- a/lib-python/2.7/ctypes/macholib/fetch_macholib.bat
+++ b/lib-python/2.7/ctypes/macholib/fetch_macholib.bat
@@ -1,1 +1,1 @@
-svn export --force http://svn.red-bean.com/bob/macholib/trunk/macholib/ .
+svn export --force http://svn.red-bean.com/bob/macholib/trunk/macholib/ .
diff --git a/lib-python/2.7/ctypes/test/test_find.py b/lib-python/2.7/ctypes/test/test_find.py
--- a/lib-python/2.7/ctypes/test/test_find.py
+++ b/lib-python/2.7/ctypes/test/test_find.py
@@ -32,15 +32,24 @@
     def setUp(self):
         self.gl = self.glu = self.gle = None
         if lib_gl:
-            self.gl = CDLL(lib_gl, mode=RTLD_GLOBAL)
+            try:
+                self.gl = CDLL(lib_gl, mode=RTLD_GLOBAL)
+            except OSError:
+                pass
         if lib_glu:
-            self.glu = CDLL(lib_glu, RTLD_GLOBAL)
+            try:
+                self.glu = CDLL(lib_glu, RTLD_GLOBAL)
+            except OSError:
+                pass
         if lib_gle:
             try:
                 self.gle = CDLL(lib_gle)
             except OSError:
                 pass
 
+    def tearDown(self):
+        self.gl = self.glu = self.gle = None
+
     @unittest.skipUnless(lib_gl, 'lib_gl not available')
     def test_gl(self):
         if self.gl:
diff --git a/lib-python/2.7/ctypes/test/test_pickling.py b/lib-python/2.7/ctypes/test/test_pickling.py
--- a/lib-python/2.7/ctypes/test/test_pickling.py
+++ b/lib-python/2.7/ctypes/test/test_pickling.py
@@ -14,9 +14,9 @@
 class Y(X):
     _fields_ = [("str", c_char_p)]
 
-class PickleTest(unittest.TestCase):
+class PickleTest:
     def dumps(self, item):
-        return pickle.dumps(item)
+        return pickle.dumps(item, self.proto)
 
     def loads(self, item):
         return pickle.loads(item)
@@ -67,17 +67,15 @@
             self.assertRaises(ValueError, lambda: self.dumps(item))
 
     def test_wchar(self):
-        pickle.dumps(c_char("x"))
+        self.dumps(c_char(b"x"))
         # Issue 5049
-        pickle.dumps(c_wchar(u"x"))
+        self.dumps(c_wchar(u"x"))
 
-class PickleTest_1(PickleTest):
-    def dumps(self, item):
-        return pickle.dumps(item, 1)
-
-class PickleTest_2(PickleTest):
-    def dumps(self, item):
-        return pickle.dumps(item, 2)
+for proto in range(pickle.HIGHEST_PROTOCOL + 1):
+    name = 'PickleTest_%s' % proto
+    globals()[name] = type(name,
+                           (PickleTest, unittest.TestCase),
+                           {'proto': proto})
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/lib-python/2.7/ctypes/test/test_pointers.py b/lib-python/2.7/ctypes/test/test_pointers.py
--- a/lib-python/2.7/ctypes/test/test_pointers.py
+++ b/lib-python/2.7/ctypes/test/test_pointers.py
@@ -7,8 +7,6 @@
                  c_long, c_ulong, c_longlong, c_ulonglong, c_double, c_float]
 python_types = [int, int, int, int, int, long,
                 int, long, long, long, float, float]
-LargeNamedType = type('T' * 2 ** 25, (Structure,), {})
-large_string = 'T' * 2 ** 25
 
 class PointersTestCase(unittest.TestCase):
 
@@ -191,9 +189,11 @@
             self.assertEqual(bool(mth), True)
 
     def test_pointer_type_name(self):
+        LargeNamedType = type('T' * 2 ** 25, (Structure,), {})
         self.assertTrue(POINTER(LargeNamedType))
 
     def test_pointer_type_str_name(self):
+        large_string = 'T' * 2 ** 25
         self.assertTrue(POINTER(large_string))
 
 if __name__ == '__main__':
diff --git a/lib-python/2.7/ctypes/util.py b/lib-python/2.7/ctypes/util.py
--- a/lib-python/2.7/ctypes/util.py
+++ b/lib-python/2.7/ctypes/util.py
@@ -177,7 +177,7 @@
             res = re.findall(expr, data)
             if not res:
                 return _get_soname(_findLib_gcc(name))
-            res.sort(cmp= lambda x,y: cmp(_num_version(x), _num_version(y)))
+            res.sort(key=_num_version)
             return res[-1]
 
     elif sys.platform == "sunos5":
diff --git a/lib-python/2.7/distutils/__init__.py b/lib-python/2.7/distutils/__init__.py
--- a/lib-python/2.7/distutils/__init__.py
+++ b/lib-python/2.7/distutils/__init__.py
@@ -15,5 +15,5 @@
 # Updated automatically by the Python release process.
 #
 #--start constants--
-__version__ = "2.7.9"
+__version__ = "2.7.10"
 #--end constants--
diff --git a/lib-python/2.7/distutils/command/check.py b/lib-python/2.7/distutils/command/check.py
--- a/lib-python/2.7/distutils/command/check.py
+++ b/lib-python/2.7/distutils/command/check.py
@@ -126,7 +126,7 @@
         """Returns warnings when the provided data doesn't compile."""
         source_path = StringIO()
         parser = Parser()
-        settings = frontend.OptionParser().get_default_values()
+        settings = frontend.OptionParser(components=(Parser,)).get_default_values()
         settings.tab_width = 4
         settings.pep_references = None
         settings.rfc_references = None
@@ -142,8 +142,8 @@
         document.note_source(source_path, -1)
         try:
             parser.parse(data, document)
-        except AttributeError:
-            reporter.messages.append((-1, 'Could not finish the parsing.',
-                                      '', {}))
+        except AttributeError as e:
+            reporter.messages.append(
+                (-1, 'Could not finish the parsing: %s.' % e, '', {}))
 
         return reporter.messages
diff --git a/lib-python/2.7/distutils/dir_util.py b/lib-python/2.7/distutils/dir_util.py
--- a/lib-python/2.7/distutils/dir_util.py
+++ b/lib-python/2.7/distutils/dir_util.py
@@ -83,7 +83,7 @@
     """Create all the empty directories under 'base_dir' needed to put 'files'
     there.
 
-    'base_dir' is just the a name of a directory which doesn't necessarily
+    'base_dir' is just the name of a directory which doesn't necessarily
     exist yet; 'files' is a list of filenames to be interpreted relative to
     'base_dir'.  'base_dir' + the directory portion of every file in 'files'
     will be created if it doesn't already exist.  'mode', 'verbose' and
diff --git a/lib-python/2.7/distutils/tests/test_check.py b/lib-python/2.7/distutils/tests/test_check.py
--- a/lib-python/2.7/distutils/tests/test_check.py
+++ b/lib-python/2.7/distutils/tests/test_check.py
@@ -1,5 +1,6 @@
 # -*- encoding: utf8 -*-
 """Tests for distutils.command.check."""
+import textwrap
 import unittest
 from test.test_support import run_unittest
 
@@ -93,6 +94,36 @@
         cmd = self._run(metadata, strict=1, restructuredtext=1)
         self.assertEqual(cmd._warnings, 0)
 
+    @unittest.skipUnless(HAS_DOCUTILS, "won't test without docutils")
+    def test_check_restructuredtext_with_syntax_highlight(self):
+        # Don't fail if there is a `code` or `code-block` directive
+
+        example_rst_docs = []
+        example_rst_docs.append(textwrap.dedent("""\
+            Here's some code:
+
+            .. code:: python
+
+                def foo():
+                    pass
+            """))
+        example_rst_docs.append(textwrap.dedent("""\
+            Here's some code:
+
+            .. code-block:: python
+
+                def foo():
+                    pass
+            """))
+
+        for rest_with_code in example_rst_docs:
+            pkg_info, dist = self.create_dist(long_description=rest_with_code)
+            cmd = check(dist)
+            cmd.check_restructuredtext()
+            self.assertEqual(cmd._warnings, 0)
+            msgs = cmd._check_rst_data(rest_with_code)
+            self.assertEqual(len(msgs), 0)
+
     def test_check_all(self):
 
         metadata = {'url': 'xxx', 'author': 'xxx'}
diff --git a/lib-python/2.7/distutils/text_file.py b/lib-python/2.7/distutils/text_file.py
--- a/lib-python/2.7/distutils/text_file.py
+++ b/lib-python/2.7/distutils/text_file.py
@@ -124,11 +124,11 @@
     def close (self):
         """Close the current file and forget everything we know about it
            (filename, current line number)."""
-
-        self.file.close ()
+        file = self.file
         self.file = None
         self.filename = None
         self.current_line = None
+        file.close()
 
 
     def gen_error (self, msg, line=None):
diff --git a/lib-python/2.7/dumbdbm.py b/lib-python/2.7/dumbdbm.py
--- a/lib-python/2.7/dumbdbm.py
+++ b/lib-python/2.7/dumbdbm.py
@@ -21,6 +21,7 @@
 
 """
 
+import ast as _ast
 import os as _os
 import __builtin__
 import UserDict
@@ -85,7 +86,7 @@
             with f:
                 for line in f:
                     line = line.rstrip()
-                    key, pos_and_siz_pair = eval(line)
+                    key, pos_and_siz_pair = _ast.literal_eval(line)
                     self._index[key] = pos_and_siz_pair
 
     # Write the index dict to the directory file.  The original directory
@@ -208,8 +209,10 @@
         return len(self._index)
 
     def close(self):
-        self._commit()
-        self._index = self._datfile = self._dirfile = self._bakfile = None
+        try:
+            self._commit()
+        finally:
+            self._index = self._datfile = self._dirfile = self._bakfile = None
 
     __del__ = close
 
diff --git a/lib-python/2.7/encodings/uu_codec.py b/lib-python/2.7/encodings/uu_codec.py
--- a/lib-python/2.7/encodings/uu_codec.py
+++ b/lib-python/2.7/encodings/uu_codec.py
@@ -84,7 +84,7 @@
             data = a2b_uu(s)
         except binascii.Error, v:
             # Workaround for broken uuencoders by /Fredrik Lundh
-            nbytes = (((ord(s[0])-32) & 63) * 4 + 5) / 3
+            nbytes = (((ord(s[0])-32) & 63) * 4 + 5) // 3
             data = a2b_uu(s[:nbytes])
             #sys.stderr.write("Warning: %s\n" % str(v))
         write(data)
diff --git a/lib-python/2.7/ensurepip/__init__.py b/lib-python/2.7/ensurepip/__init__.py
--- a/lib-python/2.7/ensurepip/__init__.py
+++ b/lib-python/2.7/ensurepip/__init__.py
@@ -12,9 +12,9 @@
 __all__ = ["version", "bootstrap"]
 
 
-_SETUPTOOLS_VERSION = "7.0"
+_SETUPTOOLS_VERSION = "15.2"
 
-_PIP_VERSION = "1.5.6"
+_PIP_VERSION = "6.1.1"
 
 # pip currently requires ssl support, so we try to provide a nicer
 # error message when that is missing (http://bugs.python.org/issue19744)
diff --git a/lib-python/2.7/ensurepip/_bundled/pip-1.5.6-py2.py3-none-any.whl b/lib-python/2.7/ensurepip/_bundled/pip-1.5.6-py2.py3-none-any.whl
deleted file mode 100644
Binary file lib-python/2.7/ensurepip/_bundled/pip-1.5.6-py2.py3-none-any.whl has changed
diff --git a/lib-python/2.7/ensurepip/_bundled/pip-6.1.1-py2.py3-none-any.whl b/lib-python/2.7/ensurepip/_bundled/pip-6.1.1-py2.py3-none-any.whl
new file mode 100644
index 0000000000000000000000000000000000000000..e59694a019051d58b9a378a1adfc9461b8cec9c3
GIT binary patch

[cut]

diff --git a/lib-python/2.7/ensurepip/_bundled/setuptools-15.2-py2.py3-none-any.whl b/lib-python/2.7/ensurepip/_bundled/setuptools-15.2-py2.py3-none-any.whl
new file mode 100644
index 0000000000000000000000000000000000000000..f153ed376684275e08fcfebdb2de8352fb074171
GIT binary patch

[cut]

diff --git a/lib-python/2.7/ensurepip/_bundled/setuptools-7.0-py2.py3-none-any.whl b/lib-python/2.7/ensurepip/_bundled/setuptools-7.0-py2.py3-none-any.whl
deleted file mode 100644
Binary file lib-python/2.7/ensurepip/_bundled/setuptools-7.0-py2.py3-none-any.whl has changed
diff --git a/lib-python/2.7/fileinput.py b/lib-python/2.7/fileinput.py
--- a/lib-python/2.7/fileinput.py
+++ b/lib-python/2.7/fileinput.py
@@ -233,8 +233,10 @@
         self.close()
 
     def close(self):
-        self.nextfile()
-        self._files = ()
+        try:
+            self.nextfile()
+        finally:
+            self._files = ()
 
     def __iter__(self):
         return self
@@ -270,23 +272,25 @@
 
         output = self._output
         self._output = 0
-        if output:
-            output.close()
+        try:
+            if output:
+                output.close()
+        finally:
+            file = self._file
+            self._file = 0
+            try:
+                if file and not self._isstdin:
+                    file.close()
+            finally:
+                backupfilename = self._backupfilename
+                self._backupfilename = 0
+                if backupfilename and not self._backup:
+                    try: os.unlink(backupfilename)
+                    except OSError: pass
 
-        file = self._file
-        self._file = 0
-        if file and not self._isstdin:
-            file.close()
-
-        backupfilename = self._backupfilename
-        self._backupfilename = 0
-        if backupfilename and not self._backup:
-            try: os.unlink(backupfilename)
-            except OSError: pass
-
-        self._isstdin = False
-        self._buffer = []
-        self._bufindex = 0
+                self._isstdin = False
+                self._buffer = []
+                self._bufindex = 0
 
     def readline(self):
         try:
diff --git a/lib-python/2.7/fnmatch.py b/lib-python/2.7/fnmatch.py
--- a/lib-python/2.7/fnmatch.py
+++ b/lib-python/2.7/fnmatch.py
@@ -47,12 +47,14 @@
     import os,posixpath
     result=[]
     pat=os.path.normcase(pat)
-    if not pat in _cache:
+    try:
+        re_pat = _cache[pat]
+    except KeyError:
         res = translate(pat)
         if len(_cache) >= _MAXCACHE:
             _cache.clear()
-        _cache[pat] = re.compile(res)
-    match=_cache[pat].match
+        _cache[pat] = re_pat = re.compile(res)
+    match = re_pat.match
     if os.path is posixpath:
         # normcase on posix is NOP. Optimize it away from the loop.
         for name in names:
@@ -71,12 +73,14 @@
     its arguments.
     """
 
-    if not pat in _cache:
+    try:
+        re_pat = _cache[pat]
+    except KeyError:
         res = translate(pat)
         if len(_cache) >= _MAXCACHE:
             _cache.clear()
-        _cache[pat] = re.compile(res)
-    return _cache[pat].match(name) is not None
+        _cache[pat] = re_pat = re.compile(res)
+    return re_pat.match(name) is not None
 
 def translate(pat):
     """Translate a shell PATTERN to a regular expression.
diff --git a/lib-python/2.7/ftplib.py b/lib-python/2.7/ftplib.py
--- a/lib-python/2.7/ftplib.py
+++ b/lib-python/2.7/ftplib.py
@@ -594,11 +594,16 @@
 
     def close(self):
         '''Close the connection without assuming anything about it.'''
-        if self.file is not None:
-            self.file.close()
-        if self.sock is not None:
-            self.sock.close()
-        self.file = self.sock = None
+        try:
+            file = self.file
+            self.file = None
+            if file is not None:
+                file.close()
+        finally:
+            sock = self.sock
+            self.sock = None
+            if sock is not None:
+                sock.close()
 
 try:
     import ssl
@@ -638,12 +643,24 @@
         '221 Goodbye.'
         >>>
         '''
-        ssl_version = ssl.PROTOCOL_TLSv1
+        ssl_version = ssl.PROTOCOL_SSLv23
 
         def __init__(self, host='', user='', passwd='', acct='', keyfile=None,
-                     certfile=None, timeout=_GLOBAL_DEFAULT_TIMEOUT):
+                     certfile=None, context=None,
+                     timeout=_GLOBAL_DEFAULT_TIMEOUT, source_address=None):
+            if context is not None and keyfile is not None:
+                raise ValueError("context and keyfile arguments are mutually "
+                                 "exclusive")
+            if context is not None and certfile is not None:
+                raise ValueError("context and certfile arguments are mutually "
+                                 "exclusive")
             self.keyfile = keyfile
             self.certfile = certfile
+            if context is None:
+                context = ssl._create_stdlib_context(self.ssl_version,
+                                                     certfile=certfile,
+                                                     keyfile=keyfile)
+            self.context = context
             self._prot_p = False
             FTP.__init__(self, host, user, passwd, acct, timeout)
 
@@ -656,12 +673,12 @@
             '''Set up secure control connection by using TLS/SSL.'''
             if isinstance(self.sock, ssl.SSLSocket):
                 raise ValueError("Already using TLS")
-            if self.ssl_version == ssl.PROTOCOL_TLSv1:
+            if self.ssl_version >= ssl.PROTOCOL_SSLv23:
                 resp = self.voidcmd('AUTH TLS')
             else:
                 resp = self.voidcmd('AUTH SSL')
-            self.sock = ssl.wrap_socket(self.sock, self.keyfile, self.certfile,
-                                        ssl_version=self.ssl_version)
+            self.sock = self.context.wrap_socket(self.sock,
+                                                 server_hostname=self.host)
             self.file = self.sock.makefile(mode='rb')
             return resp
 
@@ -692,8 +709,8 @@
         def ntransfercmd(self, cmd, rest=None):
             conn, size = FTP.ntransfercmd(self, cmd, rest)
             if self._prot_p:
-                conn = ssl.wrap_socket(conn, self.keyfile, self.certfile,
-                                       ssl_version=self.ssl_version)
+                conn = self.context.wrap_socket(conn,
+                                                server_hostname=self.host)
             return conn, size
 
         def retrbinary(self, cmd, callback, blocksize=8192, rest=None):
diff --git a/lib-python/2.7/genericpath.py b/lib-python/2.7/genericpath.py
--- a/lib-python/2.7/genericpath.py
+++ b/lib-python/2.7/genericpath.py
@@ -10,6 +10,14 @@
            'getsize', 'isdir', 'isfile']
 
 
+try:
+    _unicode = unicode
+except NameError:
+    # If Python is built without Unicode support, the unicode type
+    # will not exist. Fake one.
+    class _unicode(object):
+        pass
+
 # Does a path exist?
 # This is false for dangling symbolic links on systems that support them.
 def exists(path):
diff --git a/lib-python/2.7/gettext.py b/lib-python/2.7/gettext.py
--- a/lib-python/2.7/gettext.py
+++ b/lib-python/2.7/gettext.py
@@ -52,7 +52,9 @@
 
 __all__ = ['NullTranslations', 'GNUTranslations', 'Catalog',
            'find', 'translation', 'install', 'textdomain', 'bindtextdomain',
-           'dgettext', 'dngettext', 'gettext', 'ngettext',
+           'bind_textdomain_codeset',
+           'dgettext', 'dngettext', 'gettext', 'lgettext', 'ldgettext',
+           'ldngettext', 'lngettext', 'ngettext',
            ]
 
 _default_localedir = os.path.join(sys.prefix, 'share', 'locale')
@@ -294,11 +296,12 @@
             # See if we're looking at GNU .mo conventions for metadata
             if mlen == 0:
                 # Catalog description
-                lastk = k = None
+                lastk = None
                 for item in tmsg.splitlines():
                     item = item.strip()
                     if not item:
                         continue
+                    k = v = None
                     if ':' in item:
                         k, v = item.split(':', 1)
                         k = k.strip().lower()
diff --git a/lib-python/2.7/gzip.py b/lib-python/2.7/gzip.py
--- a/lib-python/2.7/gzip.py
+++ b/lib-python/2.7/gzip.py
@@ -238,9 +238,9 @@
             data = data.tobytes()
 
         if len(data) > 0:
-            self.size = self.size + len(data)
+            self.fileobj.write(self.compress.compress(data))
+            self.size += len(data)
             self.crc = zlib.crc32(data, self.crc) & 0xffffffffL
-            self.fileobj.write( self.compress.compress(data) )
             self.offset += len(data)
 
         return len(data)
@@ -369,19 +369,21 @@
         return self.fileobj is None
 
     def close(self):
-        if self.fileobj is None:
+        fileobj = self.fileobj
+        if fileobj is None:
             return
-        if self.mode == WRITE:
-            self.fileobj.write(self.compress.flush())
-            write32u(self.fileobj, self.crc)
-            # self.size may exceed 2GB, or even 4GB
-            write32u(self.fileobj, self.size & 0xffffffffL)
-            self.fileobj = None
-        elif self.mode == READ:
-            self.fileobj = None
-        if self.myfileobj:
-            self.myfileobj.close()
-            self.myfileobj = None
+        self.fileobj = None
+        try:
+            if self.mode == WRITE:
+                fileobj.write(self.compress.flush())
+                write32u(fileobj, self.crc)
+                # self.size may exceed 2GB, or even 4GB
+                write32u(fileobj, self.size & 0xffffffffL)
+        finally:
+            myfileobj = self.myfileobj
+            if myfileobj:
+                self.myfileobj = None
+                myfileobj.close()
 
     def flush(self,zlib_mode=zlib.Z_SYNC_FLUSH):
         self._check_closed()
diff --git a/lib-python/2.7/hashlib.py b/lib-python/2.7/hashlib.py
--- a/lib-python/2.7/hashlib.py
+++ b/lib-python/2.7/hashlib.py
@@ -187,7 +187,7 @@
 
         def prf(msg, inner=inner, outer=outer):
             # PBKDF2_HMAC uses the password as key. We can re-use the same
-            # digest objects and and just update copies to skip initialization.
+            # digest objects and just update copies to skip initialization.
             icpy = inner.copy()
             ocpy = outer.copy()
             icpy.update(msg)
diff --git a/lib-python/2.7/htmlentitydefs.py b/lib-python/2.7/htmlentitydefs.py
--- a/lib-python/2.7/htmlentitydefs.py
+++ b/lib-python/2.7/htmlentitydefs.py
@@ -1,6 +1,6 @@
 """HTML character entity references."""
 
-# maps the HTML entity name to the Unicode codepoint
+# maps the HTML entity name to the Unicode code point
 name2codepoint = {
     'AElig':    0x00c6, # latin capital letter AE = latin capital ligature AE, U+00C6 ISOlat1
     'Aacute':   0x00c1, # latin capital letter A with acute, U+00C1 ISOlat1
@@ -256,7 +256,7 @@
     'zwnj':     0x200c, # zero width non-joiner, U+200C NEW RFC 2070
 }
 
-# maps the Unicode codepoint to the HTML entity name
+# maps the Unicode code point to the HTML entity name
 codepoint2name = {}
 
 # maps the HTML entity name to the character
diff --git a/lib-python/2.7/httplib.py b/lib-python/2.7/httplib.py
--- a/lib-python/2.7/httplib.py
+++ b/lib-python/2.7/httplib.py
@@ -68,6 +68,7 @@
 
 from array import array
 import os
+import re
 import socket
 from sys import py3kwarning
 from urlparse import urlsplit
@@ -218,6 +219,38 @@
 # maximum amount of headers accepted
 _MAXHEADERS = 100
 
+# Header name/value ABNF (http://tools.ietf.org/html/rfc7230#section-3.2)
+#
+# VCHAR          = %x21-7E
+# obs-text       = %x80-FF
+# header-field   = field-name ":" OWS field-value OWS
+# field-name     = token
+# field-value    = *( field-content / obs-fold )
+# field-content  = field-vchar [ 1*( SP / HTAB ) field-vchar ]
+# field-vchar    = VCHAR / obs-text
+#
+# obs-fold       = CRLF 1*( SP / HTAB )
+#                ; obsolete line folding
+#                ; see Section 3.2.4
+
+# token          = 1*tchar
+#
+# tchar          = "!" / "#" / "$" / "%" / "&" / "'" / "*"
+#                / "+" / "-" / "." / "^" / "_" / "`" / "|" / "~"
+#                / DIGIT / ALPHA
+#                ; any VCHAR, except delimiters
+#
+# VCHAR defined in http://tools.ietf.org/html/rfc5234#appendix-B.1
+
+# the patterns for both name and value are more leniant than RFC
+# definitions to allow for backwards compatibility
+_is_legal_header_name = re.compile(r'\A[^:\s][^:\r\n]*\Z').match
+_is_illegal_header_value = re.compile(r'\n(?![ \t])|\r(?![ \t\n])').search
+
+# We always set the Content-Length header for these methods because some
+# servers will otherwise respond with a 411
+_METHODS_EXPECTING_BODY = {'PATCH', 'POST', 'PUT'}
+
 
 class HTTPMessage(mimetools.Message):
 
@@ -313,6 +346,11 @@
                 hlist.append(line)
                 self.addheader(headerseen, line[len(headerseen)+1:].strip())
                 continue
+            elif headerseen is not None:
+                # An empty header name. These aren't allowed in HTTP, but it's
+                # probably a benign mistake. Don't add the header, just keep
+                # going.
+                continue
             else:
                 # It's not a header line; throw it back and stop here.
                 if not self.dict:
@@ -522,9 +560,10 @@
         return True
 
     def close(self):
-        if self.fp:
-            self.fp.close()
+        fp = self.fp
+        if fp:
             self.fp = None
+            fp.close()
 
     def isclosed(self):
         # NOTE: it is possible that we will not ever call self.close(). This
@@ -723,7 +762,7 @@
         endpoint passed to set_tunnel. This is done by sending a HTTP CONNECT
         request to the proxy server when the connection is established.
 
-        This method must be called before the HTML connection has been
+        This method must be called before the HTTP connection has been
         established.
 
         The headers argument should be a mapping of extra HTTP headers
@@ -797,13 +836,17 @@
 
     def close(self):
         """Close the connection to the HTTP server."""
-        if self.sock:
-            self.sock.close()   # close it manually... there may be other refs
-            self.sock = None
-        if self.__response:
-            self.__response.close()
-            self.__response = None
         self.__state = _CS_IDLE
+        try:
+            sock = self.sock
+            if sock:
+                self.sock = None
+                sock.close()   # close it manually... there may be other refs
+        finally:
+            response = self.__response
+            if response:
+                self.__response = None
+                response.close()
 
     def send(self, data):
         """Send `data' to the server."""
@@ -978,7 +1021,16 @@
         if self.__state != _CS_REQ_STARTED:
             raise CannotSendHeader()
 
-        hdr = '%s: %s' % (header, '\r\n\t'.join([str(v) for v in values]))
+        header = '%s' % header
+        if not _is_legal_header_name(header):
+            raise ValueError('Invalid header name %r' % (header,))
+
+        values = [str(v) for v in values]
+        for one_value in values:
+            if _is_illegal_header_value(one_value):
+                raise ValueError('Invalid header value %r' % (one_value,))
+
+        hdr = '%s: %s' % (header, '\r\n\t'.join(values))
         self._output(hdr)
 
     def endheaders(self, message_body=None):
@@ -1000,19 +1052,25 @@
         """Send a complete request to the server."""
         self._send_request(method, url, body, headers)
 
-    def _set_content_length(self, body):
-        # Set the content-length based on the body.
+    def _set_content_length(self, body, method):
+        # Set the content-length based on the body. If the body is "empty", we
+        # set Content-Length: 0 for methods that expect a body (RFC 7230,
+        # Section 3.3.2). If the body is set for other methods, we set the
+        # header provided we can figure out what the length is.
         thelen = None
-        try:
-            thelen = str(len(body))
-        except TypeError, te:
-            # If this is a file-like object, try to
-            # fstat its file descriptor
+        if body is None and method.upper() in _METHODS_EXPECTING_BODY:
+            thelen = '0'
+        elif body is not None:
             try:
-                thelen = str(os.fstat(body.fileno()).st_size)
-            except (AttributeError, OSError):
-                # Don't send a length if this failed
-                if self.debuglevel > 0: print "Cannot stat!!"
+                thelen = str(len(body))
+            except TypeError:
+                # If this is a file-like object, try to
+                # fstat its file descriptor
+                try:
+                    thelen = str(os.fstat(body.fileno()).st_size)
+                except (AttributeError, OSError):
+                    # Don't send a length if this failed
+                    if self.debuglevel > 0: print "Cannot stat!!"
 
         if thelen is not None:
             self.putheader('Content-Length', thelen)
@@ -1028,8 +1086,8 @@
 
         self.putrequest(method, url, **skips)
 
-        if body is not None and 'content-length' not in header_names:
-            self._set_content_length(body)
+        if 'content-length' not in header_names:
+            self._set_content_length(body, method)
         for hdr, value in headers.iteritems():
             self.putheader(hdr, value)
         self.endheaders(body)
@@ -1070,18 +1128,22 @@
             kwds["buffering"] = True;
         response = self.response_class(*args, **kwds)
 
-        response.begin()
-        assert response.will_close != _UNKNOWN
-        self.__state = _CS_IDLE
+        try:
+            response.begin()
+            assert response.will_close != _UNKNOWN
+            self.__state = _CS_IDLE
 
-        if response.will_close:
-            # this effectively passes the connection to the response
-            self.close()
-        else:
-            # remember this, so we can tell when it is complete
-            self.__response = response
+            if response.will_close:
+                # this effectively passes the connection to the response
+                self.close()
+            else:
+                # remember this, so we can tell when it is complete
+                self.__response = response
 
-        return response
+            return response
+        except:
+            response.close()
+            raise
 
 
 class HTTP:
@@ -1125,7 +1187,7 @@
         "Accept arguments to set the host/port, since the superclass doesn't."
 
         if host is not None:
-            self._conn._set_hostport(host, port)
+            (self._conn.host, self._conn.port) = self._conn._get_hostport(host, port)
         self._conn.connect()
 
     def getfile(self):
diff --git a/lib-python/2.7/idlelib/CodeContext.py b/lib-python/2.7/idlelib/CodeContext.py
--- a/lib-python/2.7/idlelib/CodeContext.py
+++ b/lib-python/2.7/idlelib/CodeContext.py
@@ -15,8 +15,8 @@
 from sys import maxint as INFINITY
 from idlelib.configHandler import idleConf
 
-BLOCKOPENERS = set(["class", "def", "elif", "else", "except", "finally", "for",
-                    "if", "try", "while", "with"])
+BLOCKOPENERS = {"class", "def", "elif", "else", "except", "finally", "for",
+                    "if", "try", "while", "with"}
 UPDATEINTERVAL = 100 # millisec
 FONTUPDATEINTERVAL = 1000 # millisec
 
diff --git a/lib-python/2.7/idlelib/EditorWindow.py b/lib-python/2.7/idlelib/EditorWindow.py
--- a/lib-python/2.7/idlelib/EditorWindow.py
+++ b/lib-python/2.7/idlelib/EditorWindow.py
@@ -469,13 +469,10 @@
         ("format", "F_ormat"),
         ("run", "_Run"),
         ("options", "_Options"),
-        ("windows", "_Windows"),
+        ("windows", "_Window"),
         ("help", "_Help"),
     ]
 
-    if sys.platform == "darwin":
-        menu_specs[-2] = ("windows", "_Window")
-
 
     def createmenubar(self):
         mbar = self.menubar
diff --git a/lib-python/2.7/idlelib/FormatParagraph.py b/lib-python/2.7/idlelib/FormatParagraph.py
--- a/lib-python/2.7/idlelib/FormatParagraph.py
+++ b/lib-python/2.7/idlelib/FormatParagraph.py
@@ -44,9 +44,11 @@
 
         The length limit parameter is for testing with a known value.
         """
-        if limit == None:
+        if limit is None:
+            # The default length limit is that defined by pep8
             limit = idleConf.GetOption(
-                    'main', 'FormatParagraph', 'paragraph', type='int')
+                'extensions', 'FormatParagraph', 'max-width',
+                type='int', default=72)
         text = self.editwin.text
         first, last = self.editwin.get_selection_indices()
         if first and last:
diff --git a/lib-python/2.7/idlelib/PyShell.py b/lib-python/2.7/idlelib/PyShell.py
--- a/lib-python/2.7/idlelib/PyShell.py
+++ b/lib-python/2.7/idlelib/PyShell.py
@@ -871,13 +871,10 @@
         ("edit", "_Edit"),
         ("debug", "_Debug"),
         ("options", "_Options"),
-        ("windows", "_Windows"),
+        ("windows", "_Window"),
         ("help", "_Help"),
     ]
 
-    if sys.platform == "darwin":
-        menu_specs[-2] = ("windows", "_Window")
-
 
     # New classes
     from idlelib.IdleHistory import History
@@ -1350,7 +1347,7 @@
         if type(s) not in (unicode, str, bytearray):
             # See issue #19481
             if isinstance(s, unicode):
-                s = unicode.__getslice__(s, None, None)
+                s = unicode.__getitem__(s, slice(None))
             elif isinstance(s, str):
                 s = str.__str__(s)
             elif isinstance(s, bytearray):
diff --git a/lib-python/2.7/idlelib/SearchEngine.py b/lib-python/2.7/idlelib/SearchEngine.py
--- a/lib-python/2.7/idlelib/SearchEngine.py
+++ b/lib-python/2.7/idlelib/SearchEngine.py
@@ -191,7 +191,7 @@
 
     This is done by searching forwards until there is no match.
     Prog: compiled re object with a search method returning a match.
-    Chars: line of text, without \n.
+    Chars: line of text, without \\n.
     Col: stop index for the search; the limit for match.end().
     '''
     m = prog.search(chars)
diff --git a/lib-python/2.7/idlelib/config-extensions.def b/lib-python/2.7/idlelib/config-extensions.def
--- a/lib-python/2.7/idlelib/config-extensions.def
+++ b/lib-python/2.7/idlelib/config-extensions.def
@@ -66,6 +66,7 @@
 
 [FormatParagraph]
 enable=True
+max-width=72
 [FormatParagraph_cfgBindings]
 format-paragraph=<Alt-Key-q>
 
diff --git a/lib-python/2.7/idlelib/config-main.def b/lib-python/2.7/idlelib/config-main.def
--- a/lib-python/2.7/idlelib/config-main.def
+++ b/lib-python/2.7/idlelib/config-main.def
@@ -58,9 +58,6 @@
 font-bold= 0
 encoding= none
 
-[FormatParagraph]
-paragraph=72
-
 [Indent]
 use-spaces= 1
 num-spaces= 4
diff --git a/lib-python/2.7/idlelib/configDialog.py b/lib-python/2.7/idlelib/configDialog.py
--- a/lib-python/2.7/idlelib/configDialog.py
+++ b/lib-python/2.7/idlelib/configDialog.py
@@ -371,7 +371,6 @@
         parent = self.parent
         self.winWidth = StringVar(parent)
         self.winHeight = StringVar(parent)
-        self.paraWidth = StringVar(parent)
         self.startupEdit = IntVar(parent)
         self.autoSave = IntVar(parent)
         self.encoding = StringVar(parent)
@@ -387,7 +386,6 @@
         frameSave = LabelFrame(frame, borderwidth=2, relief=GROOVE,
                                text=' Autosave Preferences ')
         frameWinSize = Frame(frame, borderwidth=2, relief=GROOVE)
-        frameParaSize = Frame(frame, borderwidth=2, relief=GROOVE)
         frameEncoding = Frame(frame, borderwidth=2, relief=GROOVE)
         frameHelp = LabelFrame(frame, borderwidth=2, relief=GROOVE,
                                text=' Additional Help Sources ')
@@ -416,11 +414,6 @@
         labelWinHeightTitle = Label(frameWinSize, text='Height')
         entryWinHeight = Entry(
                 frameWinSize, textvariable=self.winHeight, width=3)
-        #paragraphFormatWidth
-        labelParaWidthTitle = Label(
-                frameParaSize, text='Paragraph reformat width (in characters)')
-        entryParaWidth = Entry(
-                frameParaSize, textvariable=self.paraWidth, width=3)
         #frameEncoding
         labelEncodingTitle = Label(
                 frameEncoding, text="Default Source Encoding")
@@ -458,7 +451,6 @@
         frameRun.pack(side=TOP, padx=5, pady=5, fill=X)
         frameSave.pack(side=TOP, padx=5, pady=5, fill=X)
         frameWinSize.pack(side=TOP, padx=5, pady=5, fill=X)
-        frameParaSize.pack(side=TOP, padx=5, pady=5, fill=X)
         frameEncoding.pack(side=TOP, padx=5, pady=5, fill=X)
         frameHelp.pack(side=TOP, padx=5, pady=5, expand=TRUE, fill=BOTH)
         #frameRun
@@ -475,9 +467,6 @@
         labelWinHeightTitle.pack(side=RIGHT, anchor=E, pady=5)
         entryWinWidth.pack(side=RIGHT, anchor=E, padx=10, pady=5)
         labelWinWidthTitle.pack(side=RIGHT, anchor=E, pady=5)
-        #paragraphFormatWidth
-        labelParaWidthTitle.pack(side=LEFT, anchor=W, padx=5, pady=5)
-        entryParaWidth.pack(side=RIGHT, anchor=E, padx=10, pady=5)
         #frameEncoding
         labelEncodingTitle.pack(side=LEFT, anchor=W, padx=5, pady=5)
         radioEncNone.pack(side=RIGHT, anchor=E, pady=5)
@@ -509,7 +498,6 @@
         self.keysAreBuiltin.trace_variable('w', self.VarChanged_keysAreBuiltin)
         self.winWidth.trace_variable('w', self.VarChanged_winWidth)
         self.winHeight.trace_variable('w', self.VarChanged_winHeight)
-        self.paraWidth.trace_variable('w', self.VarChanged_paraWidth)
         self.startupEdit.trace_variable('w', self.VarChanged_startupEdit)
         self.autoSave.trace_variable('w', self.VarChanged_autoSave)
         self.encoding.trace_variable('w', self.VarChanged_encoding)
@@ -594,10 +582,6 @@
         value = self.winHeight.get()
         self.AddChangedItem('main', 'EditorWindow', 'height', value)
 
-    def VarChanged_paraWidth(self, *params):
-        value = self.paraWidth.get()
-        self.AddChangedItem('main', 'FormatParagraph', 'paragraph', value)
-
     def VarChanged_startupEdit(self, *params):
         value = self.startupEdit.get()
         self.AddChangedItem('main', 'General', 'editor-on-startup', value)
@@ -1094,9 +1078,6 @@
                 'main', 'EditorWindow', 'width', type='int'))
         self.winHeight.set(idleConf.GetOption(
                 'main', 'EditorWindow', 'height', type='int'))
-        #initial paragraph reformat size
-        self.paraWidth.set(idleConf.GetOption(
-                'main', 'FormatParagraph', 'paragraph', type='int'))
         # default source encoding
         self.encoding.set(idleConf.GetOption(
                 'main', 'EditorWindow', 'encoding', default='none'))
diff --git a/lib-python/2.7/idlelib/help.txt b/lib-python/2.7/idlelib/help.txt
--- a/lib-python/2.7/idlelib/help.txt
+++ b/lib-python/2.7/idlelib/help.txt
@@ -100,7 +100,7 @@
 			  which is scrolling off the top or the window.
 			  (Not present in Shell window.)
 
-Windows Menu:
+Window Menu:
 
 	Zoom Height -- toggles the window between configured size
 	and maximum height.
diff --git a/lib-python/2.7/idlelib/idle.bat b/lib-python/2.7/idlelib/idle.bat
--- a/lib-python/2.7/idlelib/idle.bat
+++ b/lib-python/2.7/idlelib/idle.bat
@@ -1,4 +1,4 @@
- at echo off
-rem Start IDLE using the appropriate Python interpreter
-set CURRDIR=%~dp0
-start "IDLE" "%CURRDIR%..\..\pythonw.exe" "%CURRDIR%idle.pyw" %1 %2 %3 %4 %5 %6 %7 %8 %9
+ at echo off
+rem Start IDLE using the appropriate Python interpreter
+set CURRDIR=%~dp0
+start "IDLE" "%CURRDIR%..\..\pythonw.exe" "%CURRDIR%idle.pyw" %1 %2 %3 %4 %5 %6 %7 %8 %9
diff --git a/lib-python/2.7/idlelib/idle_test/test_calltips.py b/lib-python/2.7/idlelib/idle_test/test_calltips.py
--- a/lib-python/2.7/idlelib/idle_test/test_calltips.py
+++ b/lib-python/2.7/idlelib/idle_test/test_calltips.py
@@ -55,7 +55,8 @@
         def gtest(obj, out):
             self.assertEqual(signature(obj), out)
 
-        gtest(List, '()\n' + List.__doc__)
+        if List.__doc__ is not None:
+            gtest(List, '()\n' + List.__doc__)
         gtest(list.__new__,
                'T.__new__(S, ...) -> a new object with type S, a subtype of T')
         gtest(list.__init__,
@@ -70,7 +71,8 @@
 
     def test_signature_wrap(self):
         # This is also a test of an old-style class
-        self.assertEqual(signature(textwrap.TextWrapper), '''\
+        if textwrap.TextWrapper.__doc__ is not None:
+            self.assertEqual(signature(textwrap.TextWrapper), '''\
 (width=70, initial_indent='', subsequent_indent='', expand_tabs=True,
     replace_whitespace=True, fix_sentence_endings=False, break_long_words=True,
     drop_whitespace=True, break_on_hyphens=True)''')
@@ -106,20 +108,23 @@
         def t5(a, b=None, *args, **kwds): 'doc'
         t5.tip = "(a, b=None, *args, **kwargs)"
 
+        doc = '\ndoc' if t1.__doc__ is not None else ''
         for func in (t1, t2, t3, t4, t5, TC):
-            self.assertEqual(signature(func), func.tip + '\ndoc')
+            self.assertEqual(signature(func), func.tip + doc)
 
     def test_methods(self):
+        doc = '\ndoc' if TC.__doc__ is not None else ''
         for meth in (TC.t1, TC.t2, TC.t3, TC.t4, TC.t5, TC.t6, TC.__call__):
-            self.assertEqual(signature(meth), meth.tip + "\ndoc")
-        self.assertEqual(signature(TC.cm), "(a)\ndoc")
-        self.assertEqual(signature(TC.sm), "(b)\ndoc")
+            self.assertEqual(signature(meth), meth.tip + doc)
+        self.assertEqual(signature(TC.cm), "(a)" + doc)
+        self.assertEqual(signature(TC.sm), "(b)" + doc)
 
     def test_bound_methods(self):
         # test that first parameter is correctly removed from argspec
+        doc = '\ndoc' if TC.__doc__ is not None else ''
         for meth, mtip  in ((tc.t1, "()"), (tc.t4, "(*args)"), (tc.t6, "(self)"),
                             (tc.__call__, '(ci)'), (tc, '(ci)'), (TC.cm, "(a)"),):
-            self.assertEqual(signature(meth), mtip + "\ndoc")
+            self.assertEqual(signature(meth), mtip + doc)
 
     def test_starred_parameter(self):
         # test that starred first parameter is *not* removed from argspec
diff --git a/lib-python/2.7/idlelib/idle_test/test_io.py b/lib-python/2.7/idlelib/idle_test/test_io.py
new file mode 100644
--- /dev/null
+++ b/lib-python/2.7/idlelib/idle_test/test_io.py
@@ -0,0 +1,267 @@
+import unittest
+import io
+from idlelib.PyShell import PseudoInputFile, PseudoOutputFile
+from test import test_support as support
+
+
+class Base(object):
+    def __str__(self):
+        return '%s:str' % type(self).__name__
+    def __unicode__(self):
+        return '%s:unicode' % type(self).__name__
+    def __len__(self):
+        return 3
+    def __iter__(self):
+        return iter('abc')
+    def __getitem__(self, *args):
+        return '%s:item' % type(self).__name__
+    def __getslice__(self, *args):
+        return '%s:slice' % type(self).__name__
+
+class S(Base, str):
+    pass
+
+class U(Base, unicode):
+    pass
+
+class BA(Base, bytearray):
+    pass
+
+class MockShell:
+    def __init__(self):
+        self.reset()
+
+    def write(self, *args):
+        self.written.append(args)
+
+    def readline(self):
+        return self.lines.pop()
+
+    def close(self):
+        pass
+
+    def reset(self):
+        self.written = []
+
+    def push(self, lines):
+        self.lines = list(lines)[::-1]
+
+
+class PseudeOutputFilesTest(unittest.TestCase):
+    def test_misc(self):
+        shell = MockShell()
+        f = PseudoOutputFile(shell, 'stdout', 'utf-8')
+        self.assertIsInstance(f, io.TextIOBase)
+        self.assertEqual(f.encoding, 'utf-8')
+        self.assertIsNone(f.errors)
+        self.assertIsNone(f.newlines)
+        self.assertEqual(f.name, '<stdout>')
+        self.assertFalse(f.closed)
+        self.assertTrue(f.isatty())
+        self.assertFalse(f.readable())
+        self.assertTrue(f.writable())
+        self.assertFalse(f.seekable())
+
+    def test_unsupported(self):
+        shell = MockShell()
+        f = PseudoOutputFile(shell, 'stdout', 'utf-8')
+        self.assertRaises(IOError, f.fileno)
+        self.assertRaises(IOError, f.tell)
+        self.assertRaises(IOError, f.seek, 0)
+        self.assertRaises(IOError, f.read, 0)
+        self.assertRaises(IOError, f.readline, 0)
+
+    def test_write(self):
+        shell = MockShell()
+        f = PseudoOutputFile(shell, 'stdout', 'utf-8')
+        f.write('test')
+        self.assertEqual(shell.written, [('test', 'stdout')])
+        shell.reset()
+        f.write('t\xe8st')
+        self.assertEqual(shell.written, [('t\xe8st', 'stdout')])
+        shell.reset()
+        f.write(u't\xe8st')
+        self.assertEqual(shell.written, [(u't\xe8st', 'stdout')])
+        shell.reset()
+
+        f.write(S('t\xe8st'))
+        self.assertEqual(shell.written, [('t\xe8st', 'stdout')])
+        self.assertEqual(type(shell.written[0][0]), str)
+        shell.reset()
+        f.write(BA('t\xe8st'))
+        self.assertEqual(shell.written, [('t\xe8st', 'stdout')])
+        self.assertEqual(type(shell.written[0][0]), str)
+        shell.reset()
+        f.write(U(u't\xe8st'))
+        self.assertEqual(shell.written, [(u't\xe8st', 'stdout')])
+        self.assertEqual(type(shell.written[0][0]), unicode)
+        shell.reset()
+
+        self.assertRaises(TypeError, f.write)
+        self.assertEqual(shell.written, [])
+        self.assertRaises(TypeError, f.write, 123)
+        self.assertEqual(shell.written, [])
+        self.assertRaises(TypeError, f.write, 'test', 'spam')
+        self.assertEqual(shell.written, [])
+
+    def test_writelines(self):
+        shell = MockShell()
+        f = PseudoOutputFile(shell, 'stdout', 'utf-8')
+        f.writelines([])
+        self.assertEqual(shell.written, [])
+        shell.reset()
+        f.writelines(['one\n', 'two'])
+        self.assertEqual(shell.written,
+                         [('one\n', 'stdout'), ('two', 'stdout')])
+        shell.reset()
+        f.writelines(['on\xe8\n', 'tw\xf2'])
+        self.assertEqual(shell.written,
+                         [('on\xe8\n', 'stdout'), ('tw\xf2', 'stdout')])
+        shell.reset()
+        f.writelines([u'on\xe8\n', u'tw\xf2'])
+        self.assertEqual(shell.written,
+                         [(u'on\xe8\n', 'stdout'), (u'tw\xf2', 'stdout')])
+        shell.reset()
+
+        f.writelines([S('t\xe8st')])
+        self.assertEqual(shell.written, [('t\xe8st', 'stdout')])
+        self.assertEqual(type(shell.written[0][0]), str)
+        shell.reset()
+        f.writelines([BA('t\xe8st')])
+        self.assertEqual(shell.written, [('t\xe8st', 'stdout')])
+        self.assertEqual(type(shell.written[0][0]), str)
+        shell.reset()
+        f.writelines([U(u't\xe8st')])
+        self.assertEqual(shell.written, [(u't\xe8st', 'stdout')])
+        self.assertEqual(type(shell.written[0][0]), unicode)
+        shell.reset()
+
+        self.assertRaises(TypeError, f.writelines)
+        self.assertEqual(shell.written, [])
+        self.assertRaises(TypeError, f.writelines, 123)
+        self.assertEqual(shell.written, [])
+        self.assertRaises(TypeError, f.writelines, [123])
+        self.assertEqual(shell.written, [])
+        self.assertRaises(TypeError, f.writelines, [], [])
+        self.assertEqual(shell.written, [])
+
+    def test_close(self):
+        shell = MockShell()
+        f = PseudoOutputFile(shell, 'stdout', 'utf-8')
+        self.assertFalse(f.closed)
+        f.write('test')
+        f.close()
+        self.assertTrue(f.closed)
+        self.assertRaises(ValueError, f.write, 'x')
+        self.assertEqual(shell.written, [('test', 'stdout')])
+        f.close()
+        self.assertRaises(TypeError, f.close, 1)
+
+
+class PseudeInputFilesTest(unittest.TestCase):
+    def test_misc(self):
+        shell = MockShell()
+        f = PseudoInputFile(shell, 'stdin', 'utf-8')
+        self.assertIsInstance(f, io.TextIOBase)
+        self.assertEqual(f.encoding, 'utf-8')
+        self.assertIsNone(f.errors)
+        self.assertIsNone(f.newlines)
+        self.assertEqual(f.name, '<stdin>')
+        self.assertFalse(f.closed)
+        self.assertTrue(f.isatty())
+        self.assertTrue(f.readable())
+        self.assertFalse(f.writable())
+        self.assertFalse(f.seekable())
+
+    def test_unsupported(self):
+        shell = MockShell()
+        f = PseudoInputFile(shell, 'stdin', 'utf-8')
+        self.assertRaises(IOError, f.fileno)
+        self.assertRaises(IOError, f.tell)
+        self.assertRaises(IOError, f.seek, 0)
+        self.assertRaises(IOError, f.write, 'x')
+        self.assertRaises(IOError, f.writelines, ['x'])
+
+    def test_read(self):
+        shell = MockShell()
+        f = PseudoInputFile(shell, 'stdin', 'utf-8')
+        shell.push(['one\n', 'two\n', ''])
+        self.assertEqual(f.read(), 'one\ntwo\n')
+        shell.push(['one\n', 'two\n', ''])
+        self.assertEqual(f.read(-1), 'one\ntwo\n')
+        shell.push(['one\n', 'two\n', ''])
+        self.assertEqual(f.read(None), 'one\ntwo\n')
+        shell.push(['one\n', 'two\n', 'three\n', ''])
+        self.assertEqual(f.read(2), 'on')
+        self.assertEqual(f.read(3), 'e\nt')
+        self.assertEqual(f.read(10), 'wo\nthree\n')
+
+        shell.push(['one\n', 'two\n'])
+        self.assertEqual(f.read(0), '')
+        self.assertRaises(TypeError, f.read, 1.5)
+        self.assertRaises(TypeError, f.read, '1')
+        self.assertRaises(TypeError, f.read, 1, 1)
+
+    def test_readline(self):
+        shell = MockShell()
+        f = PseudoInputFile(shell, 'stdin', 'utf-8')
+        shell.push(['one\n', 'two\n', 'three\n', 'four\n'])
+        self.assertEqual(f.readline(), 'one\n')
+        self.assertEqual(f.readline(-1), 'two\n')
+        self.assertEqual(f.readline(None), 'three\n')
+        shell.push(['one\ntwo\n'])
+        self.assertEqual(f.readline(), 'one\n')
+        self.assertEqual(f.readline(), 'two\n')
+        shell.push(['one', 'two', 'three'])
+        self.assertEqual(f.readline(), 'one')
+        self.assertEqual(f.readline(), 'two')
+        shell.push(['one\n', 'two\n', 'three\n'])
+        self.assertEqual(f.readline(2), 'on')
+        self.assertEqual(f.readline(1), 'e')
+        self.assertEqual(f.readline(1), '\n')
+        self.assertEqual(f.readline(10), 'two\n')
+
+        shell.push(['one\n', 'two\n'])
+        self.assertEqual(f.readline(0), '')
+        self.assertRaises(TypeError, f.readlines, 1.5)
+        self.assertRaises(TypeError, f.readlines, '1')
+        self.assertRaises(TypeError, f.readlines, 1, 1)
+
+    def test_readlines(self):

From noreply at buildbot.pypy.org  Fri Jun 12 16:01:08 2015
From: noreply at buildbot.pypy.org (amauryfa)
Date: Fri, 12 Jun 2015 16:01:08 +0200 (CEST)
Subject: [pypy-commit] pypy stdlib-2.7.10: Merge vendor/stdlib: 2.7.10
Message-ID: <20150612140108.1666A1C0EB1@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: stdlib-2.7.10
Changeset: r78043:b1d5575eac56
Date: 2015-06-12 15:58 +0200
http://bitbucket.org/pypy/pypy/changeset/b1d5575eac56/

Log:	Merge vendor/stdlib: 2.7.10

diff too long, truncating to 2000 out of 10062 lines

diff --git a/lib-python/2.7/Cookie.py b/lib-python/2.7/Cookie.py
--- a/lib-python/2.7/Cookie.py
+++ b/lib-python/2.7/Cookie.py
@@ -528,12 +528,13 @@
 # result, the parsing rules here are less strict.
 #
 
-_LegalCharsPatt  = r"[\w\d!#%&'~_`><@,:/\$\*\+\-\.\^\|\)\(\?\}\{\=]"
+_LegalKeyChars  = r"\w\d!#%&'~_`><@,:/\$\*\+\-\.\^\|\)\(\?\}\{\="
+_LegalValueChars = _LegalKeyChars + r"\[\]"
 _CookiePattern = re.compile(
     r"(?x)"                       # This is a Verbose pattern
     r"\s*"                        # Optional whitespace at start of cookie
     r"(?P<key>"                   # Start of group 'key'
-    ""+ _LegalCharsPatt +"+?"     # Any word of at least one letter, nongreedy
+    "["+ _LegalKeyChars +"]+?"     # Any word of at least one letter, nongreedy
     r")"                          # End of group 'key'
     r"("                          # Optional group: there may not be a value.
     r"\s*=\s*"                    # Equal Sign
@@ -542,7 +543,7 @@
     r"|"                            # or
     r"\w{3},\s[\s\w\d-]{9,11}\s[\d:]{8}\sGMT" # Special case for "expires" attr
     r"|"                            # or
-    ""+ _LegalCharsPatt +"*"        # Any word or empty string
+    "["+ _LegalValueChars +"]*"        # Any word or empty string
     r")"                          # End of group 'val'
     r")?"                         # End of optional value group
     r"\s*"                        # Any number of spaces.
diff --git a/lib-python/2.7/SimpleHTTPServer.py b/lib-python/2.7/SimpleHTTPServer.py
--- a/lib-python/2.7/SimpleHTTPServer.py
+++ b/lib-python/2.7/SimpleHTTPServer.py
@@ -14,6 +14,7 @@
 import posixpath
 import BaseHTTPServer
 import urllib
+import urlparse
 import cgi
 import sys
 import shutil
@@ -68,10 +69,14 @@
         path = self.translate_path(self.path)
         f = None
         if os.path.isdir(path):
-            if not self.path.endswith('/'):
+            parts = urlparse.urlsplit(self.path)
+            if not parts.path.endswith('/'):
                 # redirect browser - doing basically what apache does
                 self.send_response(301)
-                self.send_header("Location", self.path + "/")
+                new_parts = (parts[0], parts[1], parts[2] + '/',
+                             parts[3], parts[4])
+                new_url = urlparse.urlunsplit(new_parts)
+                self.send_header("Location", new_url)
                 self.end_headers()
                 return None
             for index in "index.html", "index.htm":
diff --git a/lib-python/2.7/_LWPCookieJar.py b/lib-python/2.7/_LWPCookieJar.py
--- a/lib-python/2.7/_LWPCookieJar.py
+++ b/lib-python/2.7/_LWPCookieJar.py
@@ -18,7 +18,7 @@
                        iso2time, time2isoz)
 
 def lwp_cookie_str(cookie):
-    """Return string representation of Cookie in an the LWP cookie file format.
+    """Return string representation of Cookie in the LWP cookie file format.
 
     Actually, the format is extended a bit -- see module docstring.
 
diff --git a/lib-python/2.7/_abcoll.py b/lib-python/2.7/_abcoll.py
--- a/lib-python/2.7/_abcoll.py
+++ b/lib-python/2.7/_abcoll.py
@@ -548,23 +548,25 @@
             If E present and lacks .keys() method, does:     for (k, v) in E: D[k] = v
             In either case, this is followed by: for k, v in F.items(): D[k] = v
         '''
-        if len(args) > 2:
-            raise TypeError("update() takes at most 2 positional "
-                            "arguments ({} given)".format(len(args)))
-        elif not args:
-            raise TypeError("update() takes at least 1 argument (0 given)")
+        if not args:
+            raise TypeError("descriptor 'update' of 'MutableMapping' object "
+                            "needs an argument")
         self = args[0]
-        other = args[1] if len(args) >= 2 else ()
-
-        if isinstance(other, Mapping):
-            for key in other:
-                self[key] = other[key]
-        elif hasattr(other, "keys"):
-            for key in other.keys():
-                self[key] = other[key]
-        else:
-            for key, value in other:
-                self[key] = value
+        args = args[1:]
+        if len(args) > 1:
+            raise TypeError('update expected at most 1 arguments, got %d' %
+                            len(args))
+        if args:
+            other = args[0]
+            if isinstance(other, Mapping):
+                for key in other:
+                    self[key] = other[key]
+            elif hasattr(other, "keys"):
+                for key in other.keys():
+                    self[key] = other[key]
+            else:
+                for key, value in other:
+                    self[key] = value
         for key, value in kwds.items():
             self[key] = value
 
diff --git a/lib-python/2.7/_pyio.py b/lib-python/2.7/_pyio.py
--- a/lib-python/2.7/_pyio.py
+++ b/lib-python/2.7/_pyio.py
@@ -25,8 +25,8 @@
 DEFAULT_BUFFER_SIZE = 8 * 1024  # bytes
 
 # NOTE: Base classes defined here are registered with the "official" ABCs
-# defined in io.py. We don't use real inheritance though, because we don't
-# want to inherit the C implementations.
+# defined in io.py. We don't use real inheritance though, because we don't want
+# to inherit the C implementations.
 
 
 class BlockingIOError(IOError):
@@ -775,7 +775,7 @@
         clsname = self.__class__.__name__
         try:
             name = self.name
-        except AttributeError:
+        except Exception:
             return "<_pyio.{0}>".format(clsname)
         else:
             return "<_pyio.{0} name={1!r}>".format(clsname, name)
@@ -1216,8 +1216,10 @@
         return self.writer.flush()
 
     def close(self):
-        self.writer.close()
-        self.reader.close()
+        try:
+            self.writer.close()
+        finally:
+            self.reader.close()
 
     def isatty(self):
         return self.reader.isatty() or self.writer.isatty()
@@ -1538,7 +1540,7 @@
     def __repr__(self):
         try:
             name = self.name
-        except AttributeError:
+        except Exception:
             return "<_pyio.TextIOWrapper encoding='{0}'>".format(self.encoding)
         else:
             return "<_pyio.TextIOWrapper name={0!r} encoding='{1}'>".format(
diff --git a/lib-python/2.7/_strptime.py b/lib-python/2.7/_strptime.py
--- a/lib-python/2.7/_strptime.py
+++ b/lib-python/2.7/_strptime.py
@@ -335,9 +335,9 @@
     # though
     week_of_year = -1
     week_of_year_start = -1
-    # weekday and julian defaulted to -1 so as to signal need to calculate
+    # weekday and julian defaulted to None so as to signal need to calculate
     # values
-    weekday = julian = -1
+    weekday = julian = None
     found_dict = found.groupdict()
     for group_key in found_dict.iterkeys():
         # Directives not explicitly handled below:
@@ -434,14 +434,14 @@
         year = 1900
     # If we know the week of the year and what day of that week, we can figure
     # out the Julian day of the year.
-    if julian == -1 and week_of_year != -1 and weekday != -1:
+    if julian is None and week_of_year != -1 and weekday is not None:
         week_starts_Mon = True if week_of_year_start == 0 else False
         julian = _calc_julian_from_U_or_W(year, week_of_year, weekday,
                                             week_starts_Mon)
     # Cannot pre-calculate datetime_date() since can change in Julian
     # calculation and thus could have different value for the day of the week
     # calculation.
-    if julian == -1:
+    if julian is None:
         # Need to add 1 to result since first day of the year is 1, not 0.
         julian = datetime_date(year, month, day).toordinal() - \
                   datetime_date(year, 1, 1).toordinal() + 1
@@ -451,7 +451,7 @@
         year = datetime_result.year
         month = datetime_result.month
         day = datetime_result.day
-    if weekday == -1:
+    if weekday is None:
         weekday = datetime_date(year, month, day).weekday()
     if leap_year_fix:
         # the caller didn't supply a year but asked for Feb 29th. We couldn't
diff --git a/lib-python/2.7/aifc.py b/lib-python/2.7/aifc.py
--- a/lib-python/2.7/aifc.py
+++ b/lib-python/2.7/aifc.py
@@ -357,10 +357,13 @@
         self._soundpos = 0
 
     def close(self):
-        if self._decomp:
-            self._decomp.CloseDecompressor()
-            self._decomp = None
-        self._file.close()
+        decomp = self._decomp
+        try:
+            if decomp:
+                self._decomp = None
+                decomp.CloseDecompressor()
+        finally:
+            self._file.close()
 
     def tell(self):
         return self._soundpos
diff --git a/lib-python/2.7/binhex.py b/lib-python/2.7/binhex.py
--- a/lib-python/2.7/binhex.py
+++ b/lib-python/2.7/binhex.py
@@ -32,7 +32,8 @@
     pass
 
 # States (what have we written)
-[_DID_HEADER, _DID_DATA, _DID_RSRC] = range(3)
+_DID_HEADER = 0
+_DID_DATA = 1
 
 # Various constants
 REASONABLY_LARGE=32768  # Minimal amount we pass the rle-coder
@@ -235,17 +236,22 @@
         self._write(data)
 
     def close(self):
-        if self.state < _DID_DATA:
-            self.close_data()
-        if self.state != _DID_DATA:
-            raise Error, 'Close at the wrong time'
-        if self.rlen != 0:
-            raise Error, \
-                  "Incorrect resource-datasize, diff=%r" % (self.rlen,)
-        self._writecrc()
-        self.ofp.close()
-        self.state = None
-        del self.ofp
+        if self.state is None:
+            return
+        try:
+            if self.state < _DID_DATA:
+                self.close_data()
+            if self.state != _DID_DATA:
+                raise Error, 'Close at the wrong time'
+            if self.rlen != 0:
+                raise Error, \
+                    "Incorrect resource-datasize, diff=%r" % (self.rlen,)
+            self._writecrc()
+        finally:
+            self.state = None
+            ofp = self.ofp
+            del self.ofp
+            ofp.close()
 
 def binhex(inp, out):
     """(infilename, outfilename) - Create binhex-encoded copy of a file"""
@@ -463,11 +469,15 @@
         return self._read(n)
 
     def close(self):
-        if self.rlen:
-            dummy = self.read_rsrc(self.rlen)
-        self._checkcrc()
-        self.state = _DID_RSRC
-        self.ifp.close()
+        if self.state is None:
+            return
+        try:
+            if self.rlen:
+                dummy = self.read_rsrc(self.rlen)
+            self._checkcrc()
+        finally:
+            self.state = None
+            self.ifp.close()
 
 def hexbin(inp, out):
     """(infilename, outfilename) - Decode binhexed file"""
diff --git a/lib-python/2.7/bsddb/test/test_all.py b/lib-python/2.7/bsddb/test/test_all.py
--- a/lib-python/2.7/bsddb/test/test_all.py
+++ b/lib-python/2.7/bsddb/test/test_all.py
@@ -412,9 +412,6 @@
         def get_dbp(self) :
             return self._db
 
-    import string
-    string.letters=[chr(i) for i in xrange(65,91)]
-
     bsddb._db.DBEnv_orig = bsddb._db.DBEnv
     bsddb._db.DB_orig = bsddb._db.DB
     if bsddb.db.version() <= (4, 3) :
diff --git a/lib-python/2.7/bsddb/test/test_basics.py b/lib-python/2.7/bsddb/test/test_basics.py
--- a/lib-python/2.7/bsddb/test/test_basics.py
+++ b/lib-python/2.7/bsddb/test/test_basics.py
@@ -999,7 +999,7 @@
         for x in "The quick brown fox jumped over the lazy dog".split():
             d2.put(x, self.makeData(x))
 
-        for x in string.letters:
+        for x in string.ascii_letters:
             d3.put(x, x*70)
 
         d1.sync()
@@ -1047,7 +1047,7 @@
             if verbose:
                 print rec
             rec = c3.next()
-        self.assertEqual(count, len(string.letters))
+        self.assertEqual(count, len(string.ascii_letters))
 
 
         c1.close()
diff --git a/lib-python/2.7/bsddb/test/test_dbshelve.py b/lib-python/2.7/bsddb/test/test_dbshelve.py
--- a/lib-python/2.7/bsddb/test/test_dbshelve.py
+++ b/lib-python/2.7/bsddb/test/test_dbshelve.py
@@ -59,7 +59,7 @@
             return bytes(key, "iso8859-1")  # 8 bits
 
     def populateDB(self, d):
-        for x in string.letters:
+        for x in string.ascii_letters:
             d[self.mk('S' + x)] = 10 * x           # add a string
             d[self.mk('I' + x)] = ord(x)           # add an integer
             d[self.mk('L' + x)] = [x] * 10         # add a list
diff --git a/lib-python/2.7/bsddb/test/test_get_none.py b/lib-python/2.7/bsddb/test/test_get_none.py
--- a/lib-python/2.7/bsddb/test/test_get_none.py
+++ b/lib-python/2.7/bsddb/test/test_get_none.py
@@ -26,14 +26,14 @@
         d.open(self.filename, db.DB_BTREE, db.DB_CREATE)
         d.set_get_returns_none(1)
 
-        for x in string.letters:
+        for x in string.ascii_letters:
             d.put(x, x * 40)
 
         data = d.get('bad key')
         self.assertEqual(data, None)
 
-        data = d.get(string.letters[0])
-        self.assertEqual(data, string.letters[0]*40)
+        data = d.get(string.ascii_letters[0])
+        self.assertEqual(data, string.ascii_letters[0]*40)
 
         count = 0
         c = d.cursor()
@@ -43,7 +43,7 @@
             rec = c.next()
 
         self.assertEqual(rec, None)
-        self.assertEqual(count, len(string.letters))
+        self.assertEqual(count, len(string.ascii_letters))
 
         c.close()
         d.close()
@@ -54,14 +54,14 @@
         d.open(self.filename, db.DB_BTREE, db.DB_CREATE)
         d.set_get_returns_none(0)
 
-        for x in string.letters:
+        for x in string.ascii_letters:
             d.put(x, x * 40)
 
         self.assertRaises(db.DBNotFoundError, d.get, 'bad key')
         self.assertRaises(KeyError, d.get, 'bad key')
 
-        data = d.get(string.letters[0])
-        self.assertEqual(data, string.letters[0]*40)
+        data = d.get(string.ascii_letters[0])
+        self.assertEqual(data, string.ascii_letters[0]*40)
 
         count = 0
         exceptionHappened = 0
@@ -77,7 +77,7 @@
 
         self.assertNotEqual(rec, None)
         self.assertTrue(exceptionHappened)
-        self.assertEqual(count, len(string.letters))
+        self.assertEqual(count, len(string.ascii_letters))
 
         c.close()
         d.close()
diff --git a/lib-python/2.7/bsddb/test/test_queue.py b/lib-python/2.7/bsddb/test/test_queue.py
--- a/lib-python/2.7/bsddb/test/test_queue.py
+++ b/lib-python/2.7/bsddb/test/test_queue.py
@@ -10,7 +10,6 @@
 
 #----------------------------------------------------------------------
 
- at unittest.skip("fails on Windows; see issue 22943")
 class SimpleQueueTestCase(unittest.TestCase):
     def setUp(self):
         self.filename = get_new_database_path()
@@ -37,17 +36,17 @@
             print "before appends" + '-' * 30
             pprint(d.stat())
 
-        for x in string.letters:
+        for x in string.ascii_letters:
             d.append(x * 40)
 
-        self.assertEqual(len(d), len(string.letters))
+        self.assertEqual(len(d), len(string.ascii_letters))
 
         d.put(100, "some more data")
         d.put(101, "and some more ")
         d.put(75,  "out of order")
         d.put(1,   "replacement data")
 
-        self.assertEqual(len(d), len(string.letters)+3)
+        self.assertEqual(len(d), len(string.ascii_letters)+3)
 
         if verbose:
             print "before close" + '-' * 30
@@ -108,17 +107,17 @@
             print "before appends" + '-' * 30
             pprint(d.stat())
 
-        for x in string.letters:
+        for x in string.ascii_letters:
             d.append(x * 40)
 
-        self.assertEqual(len(d), len(string.letters))
+        self.assertEqual(len(d), len(string.ascii_letters))
 
         d.put(100, "some more data")
         d.put(101, "and some more ")
         d.put(75,  "out of order")
         d.put(1,   "replacement data")
 
-        self.assertEqual(len(d), len(string.letters)+3)
+        self.assertEqual(len(d), len(string.ascii_letters)+3)
 
         if verbose:
             print "before close" + '-' * 30
diff --git a/lib-python/2.7/bsddb/test/test_recno.py b/lib-python/2.7/bsddb/test/test_recno.py
--- a/lib-python/2.7/bsddb/test/test_recno.py
+++ b/lib-python/2.7/bsddb/test/test_recno.py
@@ -4,12 +4,11 @@
 import os, sys
 import errno
 from pprint import pprint
+import string
 import unittest
 
 from test_all import db, test_support, verbose, get_new_environment_path, get_new_database_path
 
-letters = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
-
 
 #----------------------------------------------------------------------
 
@@ -39,7 +38,7 @@
 
         d.open(self.filename, db.DB_RECNO, db.DB_CREATE)
 
-        for x in letters:
+        for x in string.ascii_letters:
             recno = d.append(x * 60)
             self.assertIsInstance(recno, int)
             self.assertGreaterEqual(recno, 1)
@@ -270,7 +269,7 @@
         d.set_re_pad(45)  # ...test both int and char
         d.open(self.filename, db.DB_RECNO, db.DB_CREATE)
 
-        for x in letters:
+        for x in string.ascii_letters:
             d.append(x * 35)    # These will be padded
 
         d.append('.' * 40)      # this one will be exact
diff --git a/lib-python/2.7/chunk.py b/lib-python/2.7/chunk.py
--- a/lib-python/2.7/chunk.py
+++ b/lib-python/2.7/chunk.py
@@ -85,8 +85,10 @@
 
     def close(self):
         if not self.closed:
-            self.skip()
-            self.closed = True
+            try:
+                self.skip()
+            finally:
+                self.closed = True
 
     def isatty(self):
         if self.closed:
diff --git a/lib-python/2.7/codecs.py b/lib-python/2.7/codecs.py
--- a/lib-python/2.7/codecs.py
+++ b/lib-python/2.7/codecs.py
@@ -20,8 +20,14 @@
            "BOM_LE", "BOM32_BE", "BOM32_LE", "BOM64_BE", "BOM64_LE",
            "BOM_UTF8", "BOM_UTF16", "BOM_UTF16_LE", "BOM_UTF16_BE",
            "BOM_UTF32", "BOM_UTF32_LE", "BOM_UTF32_BE",
+           "CodecInfo", "Codec", "IncrementalEncoder", "IncrementalDecoder",
+           "StreamReader", "StreamWriter",
+           "StreamReaderWriter", "StreamRecoder",
+           "getencoder", "getdecoder", "getincrementalencoder",
+           "getincrementaldecoder", "getreader", "getwriter",
+           "encode", "decode", "iterencode", "iterdecode",
            "strict_errors", "ignore_errors", "replace_errors",
-           "xmlcharrefreplace_errors",
+           "xmlcharrefreplace_errors", "backslashreplace_errors",
            "register_error", "lookup_error"]
 
 ### Constants
@@ -1051,7 +1057,7 @@
         during translation.
 
         One example where this happens is cp875.py which decodes
-        multiple character to \u001a.
+        multiple character to \\u001a.
 
     """
     m = {}
diff --git a/lib-python/2.7/collections.py b/lib-python/2.7/collections.py
--- a/lib-python/2.7/collections.py
+++ b/lib-python/2.7/collections.py
@@ -330,7 +330,7 @@
     #   http://code.activestate.com/recipes/259174/
     #   Knuth, TAOCP Vol. II section 4.6.3
 
-    def __init__(self, iterable=None, **kwds):
+    def __init__(*args, **kwds):
         '''Create a new, empty Counter object.  And if given, count elements
         from an input iterable.  Or, initialize the count from another mapping
         of elements to their counts.
@@ -341,8 +341,15 @@
         >>> c = Counter(a=4, b=2)                   # a new counter from keyword args
 
         '''
+        if not args:
+            raise TypeError("descriptor '__init__' of 'Counter' object "
+                            "needs an argument")
+        self = args[0]
+        args = args[1:]
+        if len(args) > 1:
+            raise TypeError('expected at most 1 arguments, got %d' % len(args))
         super(Counter, self).__init__()
-        self.update(iterable, **kwds)
+        self.update(*args, **kwds)
 
     def __missing__(self, key):
         'The count of elements not in the Counter is zero.'
@@ -393,7 +400,7 @@
         raise NotImplementedError(
             'Counter.fromkeys() is undefined.  Use Counter(iterable) instead.')
 
-    def update(self, iterable=None, **kwds):
+    def update(*args, **kwds):
         '''Like dict.update() but add counts instead of replacing them.
 
         Source can be an iterable, a dictionary, or another Counter instance.
@@ -413,6 +420,14 @@
         # contexts.  Instead, we implement straight-addition.  Both the inputs
         # and outputs are allowed to contain zero and negative counts.
 
+        if not args:
+            raise TypeError("descriptor 'update' of 'Counter' object "
+                            "needs an argument")
+        self = args[0]
+        args = args[1:]
+        if len(args) > 1:
+            raise TypeError('expected at most 1 arguments, got %d' % len(args))
+        iterable = args[0] if args else None
         if iterable is not None:
             if isinstance(iterable, Mapping):
                 if self:
@@ -428,7 +443,7 @@
         if kwds:
             self.update(kwds)
 
-    def subtract(self, iterable=None, **kwds):
+    def subtract(*args, **kwds):
         '''Like dict.update() but subtracts counts instead of replacing them.
         Counts can be reduced below zero.  Both the inputs and outputs are
         allowed to contain zero and negative counts.
@@ -444,6 +459,14 @@
         -1
 
         '''
+        if not args:
+            raise TypeError("descriptor 'subtract' of 'Counter' object "
+                            "needs an argument")
+        self = args[0]
+        args = args[1:]
+        if len(args) > 1:
+            raise TypeError('expected at most 1 arguments, got %d' % len(args))
+        iterable = args[0] if args else None
         if iterable is not None:
             self_get = self.get
             if isinstance(iterable, Mapping):
diff --git a/lib-python/2.7/cookielib.py b/lib-python/2.7/cookielib.py
--- a/lib-python/2.7/cookielib.py
+++ b/lib-python/2.7/cookielib.py
@@ -464,26 +464,42 @@
     for ns_header in ns_headers:
         pairs = []
         version_set = False
-        for ii, param in enumerate(re.split(r";\s*", ns_header)):
-            param = param.rstrip()
-            if param == "": continue
-            if "=" not in param:
-                k, v = param, None
-            else:
-                k, v = re.split(r"\s*=\s*", param, 1)
-                k = k.lstrip()
+
+        # XXX: The following does not strictly adhere to RFCs in that empty
+        # names and values are legal (the former will only appear once and will
+        # be overwritten if multiple occurrences are present). This is
+        # mostly to deal with backwards compatibility.
+        for ii, param in enumerate(ns_header.split(';')):
+            param = param.strip()
+
+            key, sep, val = param.partition('=')
+            key = key.strip()
+
+            if not key:
+                if ii == 0:
+                    break
+                else:
+                    continue
+
+            # allow for a distinction between present and empty and missing
+            # altogether
+            val = val.strip() if sep else None
+
             if ii != 0:
-                lc = k.lower()
+                lc = key.lower()
                 if lc in known_attrs:
-                    k = lc
-                if k == "version":
+                    key = lc
+
+                if key == "version":
                     # This is an RFC 2109 cookie.
-                    v = _strip_quotes(v)
+                    if val is not None:
+                        val = _strip_quotes(val)
                     version_set = True
-                if k == "expires":
+                elif key == "expires":
                     # convert expires date to seconds since epoch
-                    v = http2time(_strip_quotes(v))  # None if invalid
-            pairs.append((k, v))
+                    if val is not None:
+                        val = http2time(_strip_quotes(val))  # None if invalid
+            pairs.append((key, val))
 
         if pairs:
             if not version_set:
diff --git a/lib-python/2.7/ctypes/macholib/fetch_macholib.bat b/lib-python/2.7/ctypes/macholib/fetch_macholib.bat
--- a/lib-python/2.7/ctypes/macholib/fetch_macholib.bat
+++ b/lib-python/2.7/ctypes/macholib/fetch_macholib.bat
@@ -1,1 +1,1 @@
-svn export --force http://svn.red-bean.com/bob/macholib/trunk/macholib/ .
+svn export --force http://svn.red-bean.com/bob/macholib/trunk/macholib/ .
diff --git a/lib-python/2.7/ctypes/test/test_find.py b/lib-python/2.7/ctypes/test/test_find.py
--- a/lib-python/2.7/ctypes/test/test_find.py
+++ b/lib-python/2.7/ctypes/test/test_find.py
@@ -32,15 +32,24 @@
     def setUp(self):
         self.gl = self.glu = self.gle = None
         if lib_gl:
-            self.gl = CDLL(lib_gl, mode=RTLD_GLOBAL)
+            try:
+                self.gl = CDLL(lib_gl, mode=RTLD_GLOBAL)
+            except OSError:
+                pass
         if lib_glu:
-            self.glu = CDLL(lib_glu, RTLD_GLOBAL)
+            try:
+                self.glu = CDLL(lib_glu, RTLD_GLOBAL)
+            except OSError:
+                pass
         if lib_gle:
             try:
                 self.gle = CDLL(lib_gle)
             except OSError:
                 pass
 
+    def tearDown(self):
+        self.gl = self.glu = self.gle = None
+
     @unittest.skipUnless(lib_gl, 'lib_gl not available')
     def test_gl(self):
         if self.gl:
diff --git a/lib-python/2.7/ctypes/test/test_pickling.py b/lib-python/2.7/ctypes/test/test_pickling.py
--- a/lib-python/2.7/ctypes/test/test_pickling.py
+++ b/lib-python/2.7/ctypes/test/test_pickling.py
@@ -15,9 +15,9 @@
 class Y(X):
     _fields_ = [("str", c_char_p)]
 
-class PickleTest(unittest.TestCase):
+class PickleTest:
     def dumps(self, item):
-        return pickle.dumps(item)
+        return pickle.dumps(item, self.proto)
 
     def loads(self, item):
         return pickle.loads(item)
@@ -72,17 +72,15 @@
 
     @xfail
     def test_wchar(self):
-        pickle.dumps(c_char("x"))
+        self.dumps(c_char(b"x"))
         # Issue 5049
-        pickle.dumps(c_wchar(u"x"))
+        self.dumps(c_wchar(u"x"))
 
-class PickleTest_1(PickleTest):
-    def dumps(self, item):
-        return pickle.dumps(item, 1)
-
-class PickleTest_2(PickleTest):
-    def dumps(self, item):
-        return pickle.dumps(item, 2)
+for proto in range(pickle.HIGHEST_PROTOCOL + 1):
+    name = 'PickleTest_%s' % proto
+    globals()[name] = type(name,
+                           (PickleTest, unittest.TestCase),
+                           {'proto': proto})
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/lib-python/2.7/ctypes/test/test_pointers.py b/lib-python/2.7/ctypes/test/test_pointers.py
--- a/lib-python/2.7/ctypes/test/test_pointers.py
+++ b/lib-python/2.7/ctypes/test/test_pointers.py
@@ -7,8 +7,6 @@
                  c_long, c_ulong, c_longlong, c_ulonglong, c_double, c_float]
 python_types = [int, int, int, int, int, long,
                 int, long, long, long, float, float]
-LargeNamedType = type('T' * 2 ** 25, (Structure,), {})
-large_string = 'T' * 2 ** 25
 
 class PointersTestCase(unittest.TestCase):
 
@@ -191,9 +189,11 @@
             self.assertEqual(bool(mth), True)
 
     def test_pointer_type_name(self):
+        LargeNamedType = type('T' * 2 ** 25, (Structure,), {})
         self.assertTrue(POINTER(LargeNamedType))
 
     def test_pointer_type_str_name(self):
+        large_string = 'T' * 2 ** 25
         self.assertTrue(POINTER(large_string))
 
 if __name__ == '__main__':
diff --git a/lib-python/2.7/ctypes/util.py b/lib-python/2.7/ctypes/util.py
--- a/lib-python/2.7/ctypes/util.py
+++ b/lib-python/2.7/ctypes/util.py
@@ -178,7 +178,7 @@
             res = re.findall(expr, data)
             if not res:
                 return _get_soname(_findLib_gcc(name))
-            res.sort(cmp= lambda x,y: cmp(_num_version(x), _num_version(y)))
+            res.sort(key=_num_version)
             return res[-1]
 
     elif sys.platform == "sunos5":
diff --git a/lib-python/2.7/distutils/__init__.py b/lib-python/2.7/distutils/__init__.py
--- a/lib-python/2.7/distutils/__init__.py
+++ b/lib-python/2.7/distutils/__init__.py
@@ -15,5 +15,5 @@
 # Updated automatically by the Python release process.
 #
 #--start constants--
-__version__ = "2.7.9"
+__version__ = "2.7.10"
 #--end constants--
diff --git a/lib-python/2.7/distutils/command/check.py b/lib-python/2.7/distutils/command/check.py
--- a/lib-python/2.7/distutils/command/check.py
+++ b/lib-python/2.7/distutils/command/check.py
@@ -126,7 +126,7 @@
         """Returns warnings when the provided data doesn't compile."""
         source_path = StringIO()
         parser = Parser()
-        settings = frontend.OptionParser().get_default_values()
+        settings = frontend.OptionParser(components=(Parser,)).get_default_values()
         settings.tab_width = 4
         settings.pep_references = None
         settings.rfc_references = None
@@ -142,8 +142,8 @@
         document.note_source(source_path, -1)
         try:
             parser.parse(data, document)
-        except AttributeError:
-            reporter.messages.append((-1, 'Could not finish the parsing.',
-                                      '', {}))
+        except AttributeError as e:
+            reporter.messages.append(
+                (-1, 'Could not finish the parsing: %s.' % e, '', {}))
 
         return reporter.messages
diff --git a/lib-python/2.7/distutils/dir_util.py b/lib-python/2.7/distutils/dir_util.py
--- a/lib-python/2.7/distutils/dir_util.py
+++ b/lib-python/2.7/distutils/dir_util.py
@@ -83,7 +83,7 @@
     """Create all the empty directories under 'base_dir' needed to put 'files'
     there.
 
-    'base_dir' is just the a name of a directory which doesn't necessarily
+    'base_dir' is just the name of a directory which doesn't necessarily
     exist yet; 'files' is a list of filenames to be interpreted relative to
     'base_dir'.  'base_dir' + the directory portion of every file in 'files'
     will be created if it doesn't already exist.  'mode', 'verbose' and
diff --git a/lib-python/2.7/distutils/tests/test_check.py b/lib-python/2.7/distutils/tests/test_check.py
--- a/lib-python/2.7/distutils/tests/test_check.py
+++ b/lib-python/2.7/distutils/tests/test_check.py
@@ -1,5 +1,6 @@
 # -*- encoding: utf8 -*-
 """Tests for distutils.command.check."""
+import textwrap
 import unittest
 from test.test_support import run_unittest
 
@@ -93,6 +94,36 @@
         cmd = self._run(metadata, strict=1, restructuredtext=1)
         self.assertEqual(cmd._warnings, 0)
 
+    @unittest.skipUnless(HAS_DOCUTILS, "won't test without docutils")
+    def test_check_restructuredtext_with_syntax_highlight(self):
+        # Don't fail if there is a `code` or `code-block` directive
+
+        example_rst_docs = []
+        example_rst_docs.append(textwrap.dedent("""\
+            Here's some code:
+
+            .. code:: python
+
+                def foo():
+                    pass
+            """))
+        example_rst_docs.append(textwrap.dedent("""\
+            Here's some code:
+
+            .. code-block:: python
+
+                def foo():
+                    pass
+            """))
+
+        for rest_with_code in example_rst_docs:
+            pkg_info, dist = self.create_dist(long_description=rest_with_code)
+            cmd = check(dist)
+            cmd.check_restructuredtext()
+            self.assertEqual(cmd._warnings, 0)
+            msgs = cmd._check_rst_data(rest_with_code)
+            self.assertEqual(len(msgs), 0)
+
     def test_check_all(self):
 
         metadata = {'url': 'xxx', 'author': 'xxx'}
diff --git a/lib-python/2.7/distutils/text_file.py b/lib-python/2.7/distutils/text_file.py
--- a/lib-python/2.7/distutils/text_file.py
+++ b/lib-python/2.7/distutils/text_file.py
@@ -124,11 +124,11 @@
     def close (self):
         """Close the current file and forget everything we know about it
            (filename, current line number)."""
-
-        self.file.close ()
+        file = self.file
         self.file = None
         self.filename = None
         self.current_line = None
+        file.close()
 
 
     def gen_error (self, msg, line=None):
diff --git a/lib-python/2.7/dumbdbm.py b/lib-python/2.7/dumbdbm.py
--- a/lib-python/2.7/dumbdbm.py
+++ b/lib-python/2.7/dumbdbm.py
@@ -21,6 +21,7 @@
 
 """
 
+import ast as _ast
 import os as _os
 import __builtin__
 import UserDict
@@ -85,7 +86,7 @@
             with f:
                 for line in f:
                     line = line.rstrip()
-                    key, pos_and_siz_pair = eval(line)
+                    key, pos_and_siz_pair = _ast.literal_eval(line)
                     self._index[key] = pos_and_siz_pair
 
     # Write the index dict to the directory file.  The original directory
@@ -208,8 +209,10 @@
         return len(self._index)
 
     def close(self):
-        self._commit()
-        self._index = self._datfile = self._dirfile = self._bakfile = None
+        try:
+            self._commit()
+        finally:
+            self._index = self._datfile = self._dirfile = self._bakfile = None
 
     __del__ = close
 
diff --git a/lib-python/2.7/encodings/uu_codec.py b/lib-python/2.7/encodings/uu_codec.py
--- a/lib-python/2.7/encodings/uu_codec.py
+++ b/lib-python/2.7/encodings/uu_codec.py
@@ -84,7 +84,7 @@
             data = a2b_uu(s)
         except binascii.Error, v:
             # Workaround for broken uuencoders by /Fredrik Lundh
-            nbytes = (((ord(s[0])-32) & 63) * 4 + 5) / 3
+            nbytes = (((ord(s[0])-32) & 63) * 4 + 5) // 3
             data = a2b_uu(s[:nbytes])
             #sys.stderr.write("Warning: %s\n" % str(v))
         write(data)
diff --git a/lib-python/2.7/ensurepip/__init__.py b/lib-python/2.7/ensurepip/__init__.py
--- a/lib-python/2.7/ensurepip/__init__.py
+++ b/lib-python/2.7/ensurepip/__init__.py
@@ -12,9 +12,9 @@
 __all__ = ["version", "bootstrap"]
 
 
-_SETUPTOOLS_VERSION = "7.0"
+_SETUPTOOLS_VERSION = "15.2"
 
-_PIP_VERSION = "1.5.6"
+_PIP_VERSION = "6.1.1"
 
 # pip currently requires ssl support, so we try to provide a nicer
 # error message when that is missing (http://bugs.python.org/issue19744)
diff --git a/lib-python/2.7/ensurepip/_bundled/pip-1.5.6-py2.py3-none-any.whl b/lib-python/2.7/ensurepip/_bundled/pip-1.5.6-py2.py3-none-any.whl
deleted file mode 100644
Binary file lib-python/2.7/ensurepip/_bundled/pip-1.5.6-py2.py3-none-any.whl has changed
diff --git a/lib-python/2.7/ensurepip/_bundled/pip-6.1.1-py2.py3-none-any.whl b/lib-python/2.7/ensurepip/_bundled/pip-6.1.1-py2.py3-none-any.whl
new file mode 100644
index 0000000000000000000000000000000000000000..e59694a019051d58b9a378a1adfc9461b8cec9c3
GIT binary patch

[cut]

diff --git a/lib-python/2.7/ensurepip/_bundled/setuptools-15.2-py2.py3-none-any.whl b/lib-python/2.7/ensurepip/_bundled/setuptools-15.2-py2.py3-none-any.whl
new file mode 100644
index 0000000000000000000000000000000000000000..f153ed376684275e08fcfebdb2de8352fb074171
GIT binary patch

[cut]

diff --git a/lib-python/2.7/ensurepip/_bundled/setuptools-7.0-py2.py3-none-any.whl b/lib-python/2.7/ensurepip/_bundled/setuptools-7.0-py2.py3-none-any.whl
deleted file mode 100644
Binary file lib-python/2.7/ensurepip/_bundled/setuptools-7.0-py2.py3-none-any.whl has changed
diff --git a/lib-python/2.7/fileinput.py b/lib-python/2.7/fileinput.py
--- a/lib-python/2.7/fileinput.py
+++ b/lib-python/2.7/fileinput.py
@@ -233,8 +233,10 @@
         self.close()
 
     def close(self):
-        self.nextfile()
-        self._files = ()
+        try:
+            self.nextfile()
+        finally:
+            self._files = ()
 
     def __iter__(self):
         return self
@@ -270,23 +272,25 @@
 
         output = self._output
         self._output = 0
-        if output:
-            output.close()
+        try:
+            if output:
+                output.close()
+        finally:
+            file = self._file
+            self._file = 0
+            try:
+                if file and not self._isstdin:
+                    file.close()
+            finally:
+                backupfilename = self._backupfilename
+                self._backupfilename = 0
+                if backupfilename and not self._backup:
+                    try: os.unlink(backupfilename)
+                    except OSError: pass
 
-        file = self._file
-        self._file = 0
-        if file and not self._isstdin:
-            file.close()
-
-        backupfilename = self._backupfilename
-        self._backupfilename = 0
-        if backupfilename and not self._backup:
-            try: os.unlink(backupfilename)
-            except OSError: pass
-
-        self._isstdin = False
-        self._buffer = []
-        self._bufindex = 0
+                self._isstdin = False
+                self._buffer = []
+                self._bufindex = 0
 
     def readline(self):
         try:
diff --git a/lib-python/2.7/fnmatch.py b/lib-python/2.7/fnmatch.py
--- a/lib-python/2.7/fnmatch.py
+++ b/lib-python/2.7/fnmatch.py
@@ -47,12 +47,14 @@
     import os,posixpath
     result=[]
     pat=os.path.normcase(pat)
-    if not pat in _cache:
+    try:
+        re_pat = _cache[pat]
+    except KeyError:
         res = translate(pat)
         if len(_cache) >= _MAXCACHE:
             _cache.clear()
-        _cache[pat] = re.compile(res)
-    match=_cache[pat].match
+        _cache[pat] = re_pat = re.compile(res)
+    match = re_pat.match
     if os.path is posixpath:
         # normcase on posix is NOP. Optimize it away from the loop.
         for name in names:
@@ -71,12 +73,14 @@
     its arguments.
     """
 
-    if not pat in _cache:
+    try:
+        re_pat = _cache[pat]
+    except KeyError:
         res = translate(pat)
         if len(_cache) >= _MAXCACHE:
             _cache.clear()
-        _cache[pat] = re.compile(res)
-    return _cache[pat].match(name) is not None
+        _cache[pat] = re_pat = re.compile(res)
+    return re_pat.match(name) is not None
 
 def translate(pat):
     """Translate a shell PATTERN to a regular expression.
diff --git a/lib-python/2.7/ftplib.py b/lib-python/2.7/ftplib.py
--- a/lib-python/2.7/ftplib.py
+++ b/lib-python/2.7/ftplib.py
@@ -594,11 +594,16 @@
 
     def close(self):
         '''Close the connection without assuming anything about it.'''
-        if self.file is not None:
-            self.file.close()
-        if self.sock is not None:
-            self.sock.close()
-        self.file = self.sock = None
+        try:
+            file = self.file
+            self.file = None
+            if file is not None:
+                file.close()
+        finally:
+            sock = self.sock
+            self.sock = None
+            if sock is not None:
+                sock.close()
 
 try:
     import ssl
@@ -638,12 +643,24 @@
         '221 Goodbye.'
         >>>
         '''
-        ssl_version = ssl.PROTOCOL_TLSv1
+        ssl_version = ssl.PROTOCOL_SSLv23
 
         def __init__(self, host='', user='', passwd='', acct='', keyfile=None,
-                     certfile=None, timeout=_GLOBAL_DEFAULT_TIMEOUT):
+                     certfile=None, context=None,
+                     timeout=_GLOBAL_DEFAULT_TIMEOUT, source_address=None):
+            if context is not None and keyfile is not None:
+                raise ValueError("context and keyfile arguments are mutually "
+                                 "exclusive")
+            if context is not None and certfile is not None:
+                raise ValueError("context and certfile arguments are mutually "
+                                 "exclusive")
             self.keyfile = keyfile
             self.certfile = certfile
+            if context is None:
+                context = ssl._create_stdlib_context(self.ssl_version,
+                                                     certfile=certfile,
+                                                     keyfile=keyfile)
+            self.context = context
             self._prot_p = False
             FTP.__init__(self, host, user, passwd, acct, timeout)
 
@@ -656,12 +673,12 @@
             '''Set up secure control connection by using TLS/SSL.'''
             if isinstance(self.sock, ssl.SSLSocket):
                 raise ValueError("Already using TLS")
-            if self.ssl_version == ssl.PROTOCOL_TLSv1:
+            if self.ssl_version >= ssl.PROTOCOL_SSLv23:
                 resp = self.voidcmd('AUTH TLS')
             else:
                 resp = self.voidcmd('AUTH SSL')
-            self.sock = ssl.wrap_socket(self.sock, self.keyfile, self.certfile,
-                                        ssl_version=self.ssl_version)
+            self.sock = self.context.wrap_socket(self.sock,
+                                                 server_hostname=self.host)
             self.file = self.sock.makefile(mode='rb')
             return resp
 
@@ -692,8 +709,8 @@
         def ntransfercmd(self, cmd, rest=None):
             conn, size = FTP.ntransfercmd(self, cmd, rest)
             if self._prot_p:
-                conn = ssl.wrap_socket(conn, self.keyfile, self.certfile,
-                                       ssl_version=self.ssl_version)
+                conn = self.context.wrap_socket(conn,
+                                                server_hostname=self.host)
             return conn, size
 
         def retrbinary(self, cmd, callback, blocksize=8192, rest=None):
diff --git a/lib-python/2.7/genericpath.py b/lib-python/2.7/genericpath.py
--- a/lib-python/2.7/genericpath.py
+++ b/lib-python/2.7/genericpath.py
@@ -10,6 +10,14 @@
            'getsize', 'isdir', 'isfile']
 
 
+try:
+    _unicode = unicode
+except NameError:
+    # If Python is built without Unicode support, the unicode type
+    # will not exist. Fake one.
+    class _unicode(object):
+        pass
+
 # Does a path exist?
 # This is false for dangling symbolic links on systems that support them.
 def exists(path):
diff --git a/lib-python/2.7/gettext.py b/lib-python/2.7/gettext.py
--- a/lib-python/2.7/gettext.py
+++ b/lib-python/2.7/gettext.py
@@ -52,7 +52,9 @@
 
 __all__ = ['NullTranslations', 'GNUTranslations', 'Catalog',
            'find', 'translation', 'install', 'textdomain', 'bindtextdomain',
-           'dgettext', 'dngettext', 'gettext', 'ngettext',
+           'bind_textdomain_codeset',
+           'dgettext', 'dngettext', 'gettext', 'lgettext', 'ldgettext',
+           'ldngettext', 'lngettext', 'ngettext',
            ]
 
 _default_localedir = os.path.join(sys.prefix, 'share', 'locale')
@@ -294,11 +296,12 @@
             # See if we're looking at GNU .mo conventions for metadata
             if mlen == 0:
                 # Catalog description
-                lastk = k = None
+                lastk = None
                 for item in tmsg.splitlines():
                     item = item.strip()
                     if not item:
                         continue
+                    k = v = None
                     if ':' in item:
                         k, v = item.split(':', 1)
                         k = k.strip().lower()
diff --git a/lib-python/2.7/gzip.py b/lib-python/2.7/gzip.py
--- a/lib-python/2.7/gzip.py
+++ b/lib-python/2.7/gzip.py
@@ -238,9 +238,9 @@
             data = data.tobytes()
 
         if len(data) > 0:
-            self.size = self.size + len(data)
+            self.fileobj.write(self.compress.compress(data))
+            self.size += len(data)
             self.crc = zlib.crc32(data, self.crc) & 0xffffffffL
-            self.fileobj.write( self.compress.compress(data) )
             self.offset += len(data)
 
         return len(data)
@@ -369,19 +369,21 @@
         return self.fileobj is None
 
     def close(self):
-        if self.fileobj is None:
+        fileobj = self.fileobj
+        if fileobj is None:
             return
-        if self.mode == WRITE:
-            self.fileobj.write(self.compress.flush())
-            write32u(self.fileobj, self.crc)
-            # self.size may exceed 2GB, or even 4GB
-            write32u(self.fileobj, self.size & 0xffffffffL)
-            self.fileobj = None
-        elif self.mode == READ:
-            self.fileobj = None
-        if self.myfileobj:
-            self.myfileobj.close()
-            self.myfileobj = None
+        self.fileobj = None
+        try:
+            if self.mode == WRITE:
+                fileobj.write(self.compress.flush())
+                write32u(fileobj, self.crc)
+                # self.size may exceed 2GB, or even 4GB
+                write32u(fileobj, self.size & 0xffffffffL)
+        finally:
+            myfileobj = self.myfileobj
+            if myfileobj:
+                self.myfileobj = None
+                myfileobj.close()
 
     def flush(self,zlib_mode=zlib.Z_SYNC_FLUSH):
         self._check_closed()
diff --git a/lib-python/2.7/hashlib.py b/lib-python/2.7/hashlib.py
--- a/lib-python/2.7/hashlib.py
+++ b/lib-python/2.7/hashlib.py
@@ -187,7 +187,7 @@
 
         def prf(msg, inner=inner, outer=outer):
             # PBKDF2_HMAC uses the password as key. We can re-use the same
-            # digest objects and and just update copies to skip initialization.
+            # digest objects and just update copies to skip initialization.
             icpy = inner.copy()
             ocpy = outer.copy()
             icpy.update(msg)
diff --git a/lib-python/2.7/htmlentitydefs.py b/lib-python/2.7/htmlentitydefs.py
--- a/lib-python/2.7/htmlentitydefs.py
+++ b/lib-python/2.7/htmlentitydefs.py
@@ -1,6 +1,6 @@
 """HTML character entity references."""
 
-# maps the HTML entity name to the Unicode codepoint
+# maps the HTML entity name to the Unicode code point
 name2codepoint = {
     'AElig':    0x00c6, # latin capital letter AE = latin capital ligature AE, U+00C6 ISOlat1
     'Aacute':   0x00c1, # latin capital letter A with acute, U+00C1 ISOlat1
@@ -256,7 +256,7 @@
     'zwnj':     0x200c, # zero width non-joiner, U+200C NEW RFC 2070
 }
 
-# maps the Unicode codepoint to the HTML entity name
+# maps the Unicode code point to the HTML entity name
 codepoint2name = {}
 
 # maps the HTML entity name to the character
diff --git a/lib-python/2.7/httplib.py b/lib-python/2.7/httplib.py
--- a/lib-python/2.7/httplib.py
+++ b/lib-python/2.7/httplib.py
@@ -68,6 +68,7 @@
 
 from array import array
 import os
+import re
 import socket
 from sys import py3kwarning
 from urlparse import urlsplit
@@ -218,6 +219,38 @@
 # maximum amount of headers accepted
 _MAXHEADERS = 100
 
+# Header name/value ABNF (http://tools.ietf.org/html/rfc7230#section-3.2)
+#
+# VCHAR          = %x21-7E
+# obs-text       = %x80-FF
+# header-field   = field-name ":" OWS field-value OWS
+# field-name     = token
+# field-value    = *( field-content / obs-fold )
+# field-content  = field-vchar [ 1*( SP / HTAB ) field-vchar ]
+# field-vchar    = VCHAR / obs-text
+#
+# obs-fold       = CRLF 1*( SP / HTAB )
+#                ; obsolete line folding
+#                ; see Section 3.2.4
+
+# token          = 1*tchar
+#
+# tchar          = "!" / "#" / "$" / "%" / "&" / "'" / "*"
+#                / "+" / "-" / "." / "^" / "_" / "`" / "|" / "~"
+#                / DIGIT / ALPHA
+#                ; any VCHAR, except delimiters
+#
+# VCHAR defined in http://tools.ietf.org/html/rfc5234#appendix-B.1
+
+# the patterns for both name and value are more leniant than RFC
+# definitions to allow for backwards compatibility
+_is_legal_header_name = re.compile(r'\A[^:\s][^:\r\n]*\Z').match
+_is_illegal_header_value = re.compile(r'\n(?![ \t])|\r(?![ \t\n])').search
+
+# We always set the Content-Length header for these methods because some
+# servers will otherwise respond with a 411
+_METHODS_EXPECTING_BODY = {'PATCH', 'POST', 'PUT'}
+
 
 class HTTPMessage(mimetools.Message):
 
@@ -313,6 +346,11 @@
                 hlist.append(line)
                 self.addheader(headerseen, line[len(headerseen)+1:].strip())
                 continue
+            elif headerseen is not None:
+                # An empty header name. These aren't allowed in HTTP, but it's
+                # probably a benign mistake. Don't add the header, just keep
+                # going.
+                continue
             else:
                 # It's not a header line; throw it back and stop here.
                 if not self.dict:
@@ -522,9 +560,10 @@
         return True
 
     def close(self):
-        if self.fp:
-            self.fp.close()
+        fp = self.fp
+        if fp:
             self.fp = None
+            fp.close()
 
     def isclosed(self):
         # NOTE: it is possible that we will not ever call self.close(). This
@@ -723,7 +762,7 @@
         endpoint passed to set_tunnel. This is done by sending a HTTP CONNECT
         request to the proxy server when the connection is established.
 
-        This method must be called before the HTML connection has been
+        This method must be called before the HTTP connection has been
         established.
 
         The headers argument should be a mapping of extra HTTP headers
@@ -797,13 +836,17 @@
 
     def close(self):
         """Close the connection to the HTTP server."""
-        if self.sock:
-            self.sock.close()   # close it manually... there may be other refs
-            self.sock = None
-        if self.__response:
-            self.__response.close()
-            self.__response = None
         self.__state = _CS_IDLE
+        try:
+            sock = self.sock
+            if sock:
+                self.sock = None
+                sock.close()   # close it manually... there may be other refs
+        finally:
+            response = self.__response
+            if response:
+                self.__response = None
+                response.close()
 
     def send(self, data):
         """Send `data' to the server."""
@@ -978,7 +1021,16 @@
         if self.__state != _CS_REQ_STARTED:
             raise CannotSendHeader()
 
-        hdr = '%s: %s' % (header, '\r\n\t'.join([str(v) for v in values]))
+        header = '%s' % header
+        if not _is_legal_header_name(header):
+            raise ValueError('Invalid header name %r' % (header,))
+
+        values = [str(v) for v in values]
+        for one_value in values:
+            if _is_illegal_header_value(one_value):
+                raise ValueError('Invalid header value %r' % (one_value,))
+
+        hdr = '%s: %s' % (header, '\r\n\t'.join(values))
         self._output(hdr)
 
     def endheaders(self, message_body=None):
@@ -1000,19 +1052,25 @@
         """Send a complete request to the server."""
         self._send_request(method, url, body, headers)
 
-    def _set_content_length(self, body):
-        # Set the content-length based on the body.
+    def _set_content_length(self, body, method):
+        # Set the content-length based on the body. If the body is "empty", we
+        # set Content-Length: 0 for methods that expect a body (RFC 7230,
+        # Section 3.3.2). If the body is set for other methods, we set the
+        # header provided we can figure out what the length is.
         thelen = None
-        try:
-            thelen = str(len(body))
-        except TypeError, te:
-            # If this is a file-like object, try to
-            # fstat its file descriptor
+        if body is None and method.upper() in _METHODS_EXPECTING_BODY:
+            thelen = '0'
+        elif body is not None:
             try:
-                thelen = str(os.fstat(body.fileno()).st_size)
-            except (AttributeError, OSError):
-                # Don't send a length if this failed
-                if self.debuglevel > 0: print "Cannot stat!!"
+                thelen = str(len(body))
+            except TypeError:
+                # If this is a file-like object, try to
+                # fstat its file descriptor
+                try:
+                    thelen = str(os.fstat(body.fileno()).st_size)
+                except (AttributeError, OSError):
+                    # Don't send a length if this failed
+                    if self.debuglevel > 0: print "Cannot stat!!"
 
         if thelen is not None:
             self.putheader('Content-Length', thelen)
@@ -1028,8 +1086,8 @@
 
         self.putrequest(method, url, **skips)
 
-        if body is not None and 'content-length' not in header_names:
-            self._set_content_length(body)
+        if 'content-length' not in header_names:
+            self._set_content_length(body, method)
         for hdr, value in headers.iteritems():
             self.putheader(hdr, value)
         self.endheaders(body)
@@ -1072,20 +1130,20 @@
 
         try:
             response.begin()
+            assert response.will_close != _UNKNOWN
+            self.__state = _CS_IDLE
+
+            if response.will_close:
+                # this effectively passes the connection to the response
+                self.close()
+            else:
+                # remember this, so we can tell when it is complete
+                self.__response = response
+
+            return response
         except:
             response.close()
             raise
-        assert response.will_close != _UNKNOWN
-        self.__state = _CS_IDLE
-
-        if response.will_close:
-            # this effectively passes the connection to the response
-            self.close()
-        else:
-            # remember this, so we can tell when it is complete
-            self.__response = response
-
-        return response
 
 
 class HTTP:
@@ -1129,7 +1187,7 @@
         "Accept arguments to set the host/port, since the superclass doesn't."
 
         if host is not None:
-            self._conn._set_hostport(host, port)
+            (self._conn.host, self._conn.port) = self._conn._get_hostport(host, port)
         self._conn.connect()
 
     def getfile(self):
diff --git a/lib-python/2.7/idlelib/CodeContext.py b/lib-python/2.7/idlelib/CodeContext.py
--- a/lib-python/2.7/idlelib/CodeContext.py
+++ b/lib-python/2.7/idlelib/CodeContext.py
@@ -15,8 +15,8 @@
 from sys import maxint as INFINITY
 from idlelib.configHandler import idleConf
 
-BLOCKOPENERS = set(["class", "def", "elif", "else", "except", "finally", "for",
-                    "if", "try", "while", "with"])
+BLOCKOPENERS = {"class", "def", "elif", "else", "except", "finally", "for",
+                    "if", "try", "while", "with"}
 UPDATEINTERVAL = 100 # millisec
 FONTUPDATEINTERVAL = 1000 # millisec
 
diff --git a/lib-python/2.7/idlelib/EditorWindow.py b/lib-python/2.7/idlelib/EditorWindow.py
--- a/lib-python/2.7/idlelib/EditorWindow.py
+++ b/lib-python/2.7/idlelib/EditorWindow.py
@@ -469,13 +469,10 @@
         ("format", "F_ormat"),
         ("run", "_Run"),
         ("options", "_Options"),
-        ("windows", "_Windows"),
+        ("windows", "_Window"),
         ("help", "_Help"),
     ]
 
-    if sys.platform == "darwin":
-        menu_specs[-2] = ("windows", "_Window")
-
 
     def createmenubar(self):
         mbar = self.menubar
diff --git a/lib-python/2.7/idlelib/FormatParagraph.py b/lib-python/2.7/idlelib/FormatParagraph.py
--- a/lib-python/2.7/idlelib/FormatParagraph.py
+++ b/lib-python/2.7/idlelib/FormatParagraph.py
@@ -44,9 +44,11 @@
 
         The length limit parameter is for testing with a known value.
         """
-        if limit == None:
+        if limit is None:
+            # The default length limit is that defined by pep8
             limit = idleConf.GetOption(
-                    'main', 'FormatParagraph', 'paragraph', type='int')
+                'extensions', 'FormatParagraph', 'max-width',
+                type='int', default=72)
         text = self.editwin.text
         first, last = self.editwin.get_selection_indices()
         if first and last:
diff --git a/lib-python/2.7/idlelib/PyShell.py b/lib-python/2.7/idlelib/PyShell.py
--- a/lib-python/2.7/idlelib/PyShell.py
+++ b/lib-python/2.7/idlelib/PyShell.py
@@ -871,13 +871,10 @@
         ("edit", "_Edit"),
         ("debug", "_Debug"),
         ("options", "_Options"),
-        ("windows", "_Windows"),
+        ("windows", "_Window"),
         ("help", "_Help"),
     ]
 
-    if sys.platform == "darwin":
-        menu_specs[-2] = ("windows", "_Window")
-
 
     # New classes
     from idlelib.IdleHistory import History
@@ -1350,7 +1347,7 @@
         if type(s) not in (unicode, str, bytearray):
             # See issue #19481
             if isinstance(s, unicode):
-                s = unicode.__getslice__(s, None, None)
+                s = unicode.__getitem__(s, slice(None))
             elif isinstance(s, str):
                 s = str.__str__(s)
             elif isinstance(s, bytearray):
diff --git a/lib-python/2.7/idlelib/SearchEngine.py b/lib-python/2.7/idlelib/SearchEngine.py
--- a/lib-python/2.7/idlelib/SearchEngine.py
+++ b/lib-python/2.7/idlelib/SearchEngine.py
@@ -191,7 +191,7 @@
 
     This is done by searching forwards until there is no match.
     Prog: compiled re object with a search method returning a match.
-    Chars: line of text, without \n.
+    Chars: line of text, without \\n.
     Col: stop index for the search; the limit for match.end().
     '''
     m = prog.search(chars)
diff --git a/lib-python/2.7/idlelib/config-extensions.def b/lib-python/2.7/idlelib/config-extensions.def
--- a/lib-python/2.7/idlelib/config-extensions.def
+++ b/lib-python/2.7/idlelib/config-extensions.def
@@ -66,6 +66,7 @@
 
 [FormatParagraph]
 enable=True
+max-width=72
 [FormatParagraph_cfgBindings]
 format-paragraph=<Alt-Key-q>
 
diff --git a/lib-python/2.7/idlelib/config-main.def b/lib-python/2.7/idlelib/config-main.def
--- a/lib-python/2.7/idlelib/config-main.def
+++ b/lib-python/2.7/idlelib/config-main.def
@@ -58,9 +58,6 @@
 font-bold= 0
 encoding= none
 
-[FormatParagraph]
-paragraph=72
-
 [Indent]
 use-spaces= 1
 num-spaces= 4
diff --git a/lib-python/2.7/idlelib/configDialog.py b/lib-python/2.7/idlelib/configDialog.py
--- a/lib-python/2.7/idlelib/configDialog.py
+++ b/lib-python/2.7/idlelib/configDialog.py
@@ -371,7 +371,6 @@
         parent = self.parent
         self.winWidth = StringVar(parent)
         self.winHeight = StringVar(parent)
-        self.paraWidth = StringVar(parent)
         self.startupEdit = IntVar(parent)
         self.autoSave = IntVar(parent)
         self.encoding = StringVar(parent)
@@ -387,7 +386,6 @@
         frameSave = LabelFrame(frame, borderwidth=2, relief=GROOVE,
                                text=' Autosave Preferences ')
         frameWinSize = Frame(frame, borderwidth=2, relief=GROOVE)
-        frameParaSize = Frame(frame, borderwidth=2, relief=GROOVE)
         frameEncoding = Frame(frame, borderwidth=2, relief=GROOVE)
         frameHelp = LabelFrame(frame, borderwidth=2, relief=GROOVE,
                                text=' Additional Help Sources ')
@@ -416,11 +414,6 @@
         labelWinHeightTitle = Label(frameWinSize, text='Height')
         entryWinHeight = Entry(
                 frameWinSize, textvariable=self.winHeight, width=3)
-        #paragraphFormatWidth
-        labelParaWidthTitle = Label(
-                frameParaSize, text='Paragraph reformat width (in characters)')
-        entryParaWidth = Entry(
-                frameParaSize, textvariable=self.paraWidth, width=3)
         #frameEncoding
         labelEncodingTitle = Label(
                 frameEncoding, text="Default Source Encoding")
@@ -458,7 +451,6 @@
         frameRun.pack(side=TOP, padx=5, pady=5, fill=X)
         frameSave.pack(side=TOP, padx=5, pady=5, fill=X)
         frameWinSize.pack(side=TOP, padx=5, pady=5, fill=X)
-        frameParaSize.pack(side=TOP, padx=5, pady=5, fill=X)
         frameEncoding.pack(side=TOP, padx=5, pady=5, fill=X)
         frameHelp.pack(side=TOP, padx=5, pady=5, expand=TRUE, fill=BOTH)
         #frameRun
@@ -475,9 +467,6 @@
         labelWinHeightTitle.pack(side=RIGHT, anchor=E, pady=5)
         entryWinWidth.pack(side=RIGHT, anchor=E, padx=10, pady=5)
         labelWinWidthTitle.pack(side=RIGHT, anchor=E, pady=5)
-        #paragraphFormatWidth
-        labelParaWidthTitle.pack(side=LEFT, anchor=W, padx=5, pady=5)
-        entryParaWidth.pack(side=RIGHT, anchor=E, padx=10, pady=5)
         #frameEncoding
         labelEncodingTitle.pack(side=LEFT, anchor=W, padx=5, pady=5)
         radioEncNone.pack(side=RIGHT, anchor=E, pady=5)
@@ -509,7 +498,6 @@
         self.keysAreBuiltin.trace_variable('w', self.VarChanged_keysAreBuiltin)
         self.winWidth.trace_variable('w', self.VarChanged_winWidth)
         self.winHeight.trace_variable('w', self.VarChanged_winHeight)
-        self.paraWidth.trace_variable('w', self.VarChanged_paraWidth)
         self.startupEdit.trace_variable('w', self.VarChanged_startupEdit)
         self.autoSave.trace_variable('w', self.VarChanged_autoSave)
         self.encoding.trace_variable('w', self.VarChanged_encoding)
@@ -594,10 +582,6 @@
         value = self.winHeight.get()
         self.AddChangedItem('main', 'EditorWindow', 'height', value)
 
-    def VarChanged_paraWidth(self, *params):
-        value = self.paraWidth.get()
-        self.AddChangedItem('main', 'FormatParagraph', 'paragraph', value)
-
     def VarChanged_startupEdit(self, *params):
         value = self.startupEdit.get()
         self.AddChangedItem('main', 'General', 'editor-on-startup', value)
@@ -1094,9 +1078,6 @@
                 'main', 'EditorWindow', 'width', type='int'))
         self.winHeight.set(idleConf.GetOption(
                 'main', 'EditorWindow', 'height', type='int'))
-        #initial paragraph reformat size
-        self.paraWidth.set(idleConf.GetOption(
-                'main', 'FormatParagraph', 'paragraph', type='int'))
         # default source encoding
         self.encoding.set(idleConf.GetOption(
                 'main', 'EditorWindow', 'encoding', default='none'))
diff --git a/lib-python/2.7/idlelib/help.txt b/lib-python/2.7/idlelib/help.txt
--- a/lib-python/2.7/idlelib/help.txt
+++ b/lib-python/2.7/idlelib/help.txt
@@ -100,7 +100,7 @@
 			  which is scrolling off the top or the window.
 			  (Not present in Shell window.)
 
-Windows Menu:
+Window Menu:
 
 	Zoom Height -- toggles the window between configured size
 	and maximum height.
diff --git a/lib-python/2.7/idlelib/idle.bat b/lib-python/2.7/idlelib/idle.bat
--- a/lib-python/2.7/idlelib/idle.bat
+++ b/lib-python/2.7/idlelib/idle.bat
@@ -1,4 +1,4 @@
- at echo off
-rem Start IDLE using the appropriate Python interpreter
-set CURRDIR=%~dp0
-start "IDLE" "%CURRDIR%..\..\pythonw.exe" "%CURRDIR%idle.pyw" %1 %2 %3 %4 %5 %6 %7 %8 %9
+ at echo off
+rem Start IDLE using the appropriate Python interpreter
+set CURRDIR=%~dp0
+start "IDLE" "%CURRDIR%..\..\pythonw.exe" "%CURRDIR%idle.pyw" %1 %2 %3 %4 %5 %6 %7 %8 %9
diff --git a/lib-python/2.7/idlelib/idle_test/test_calltips.py b/lib-python/2.7/idlelib/idle_test/test_calltips.py
--- a/lib-python/2.7/idlelib/idle_test/test_calltips.py
+++ b/lib-python/2.7/idlelib/idle_test/test_calltips.py
@@ -55,7 +55,8 @@
         def gtest(obj, out):
             self.assertEqual(signature(obj), out)
 
-        gtest(List, '()\n' + List.__doc__)
+        if List.__doc__ is not None:
+            gtest(List, '()\n' + List.__doc__)
         gtest(list.__new__,
                'T.__new__(S, ...) -> a new object with type S, a subtype of T')
         gtest(list.__init__,
@@ -70,7 +71,8 @@
 
     def test_signature_wrap(self):
         # This is also a test of an old-style class
-        self.assertEqual(signature(textwrap.TextWrapper), '''\
+        if textwrap.TextWrapper.__doc__ is not None:
+            self.assertEqual(signature(textwrap.TextWrapper), '''\
 (width=70, initial_indent='', subsequent_indent='', expand_tabs=True,
     replace_whitespace=True, fix_sentence_endings=False, break_long_words=True,
     drop_whitespace=True, break_on_hyphens=True)''')
@@ -106,20 +108,23 @@
         def t5(a, b=None, *args, **kwds): 'doc'
         t5.tip = "(a, b=None, *args, **kwargs)"
 
+        doc = '\ndoc' if t1.__doc__ is not None else ''
         for func in (t1, t2, t3, t4, t5, TC):
-            self.assertEqual(signature(func), func.tip + '\ndoc')
+            self.assertEqual(signature(func), func.tip + doc)
 
     def test_methods(self):
+        doc = '\ndoc' if TC.__doc__ is not None else ''
         for meth in (TC.t1, TC.t2, TC.t3, TC.t4, TC.t5, TC.t6, TC.__call__):
-            self.assertEqual(signature(meth), meth.tip + "\ndoc")
-        self.assertEqual(signature(TC.cm), "(a)\ndoc")
-        self.assertEqual(signature(TC.sm), "(b)\ndoc")
+            self.assertEqual(signature(meth), meth.tip + doc)
+        self.assertEqual(signature(TC.cm), "(a)" + doc)
+        self.assertEqual(signature(TC.sm), "(b)" + doc)
 
     def test_bound_methods(self):
         # test that first parameter is correctly removed from argspec
+        doc = '\ndoc' if TC.__doc__ is not None else ''
         for meth, mtip  in ((tc.t1, "()"), (tc.t4, "(*args)"), (tc.t6, "(self)"),
                             (tc.__call__, '(ci)'), (tc, '(ci)'), (TC.cm, "(a)"),):
-            self.assertEqual(signature(meth), mtip + "\ndoc")
+            self.assertEqual(signature(meth), mtip + doc)
 
     def test_starred_parameter(self):
         # test that starred first parameter is *not* removed from argspec
diff --git a/lib-python/2.7/idlelib/idle_test/test_io.py b/lib-python/2.7/idlelib/idle_test/test_io.py
new file mode 100644
--- /dev/null
+++ b/lib-python/2.7/idlelib/idle_test/test_io.py
@@ -0,0 +1,267 @@
+import unittest
+import io
+from idlelib.PyShell import PseudoInputFile, PseudoOutputFile
+from test import test_support as support
+
+
+class Base(object):
+    def __str__(self):
+        return '%s:str' % type(self).__name__
+    def __unicode__(self):
+        return '%s:unicode' % type(self).__name__
+    def __len__(self):
+        return 3
+    def __iter__(self):
+        return iter('abc')
+    def __getitem__(self, *args):
+        return '%s:item' % type(self).__name__
+    def __getslice__(self, *args):
+        return '%s:slice' % type(self).__name__
+
+class S(Base, str):
+    pass
+
+class U(Base, unicode):
+    pass
+
+class BA(Base, bytearray):
+    pass
+
+class MockShell:
+    def __init__(self):
+        self.reset()
+
+    def write(self, *args):
+        self.written.append(args)
+
+    def readline(self):
+        return self.lines.pop()
+
+    def close(self):
+        pass
+
+    def reset(self):
+        self.written = []
+
+    def push(self, lines):
+        self.lines = list(lines)[::-1]
+
+
+class PseudeOutputFilesTest(unittest.TestCase):
+    def test_misc(self):
+        shell = MockShell()
+        f = PseudoOutputFile(shell, 'stdout', 'utf-8')
+        self.assertIsInstance(f, io.TextIOBase)
+        self.assertEqual(f.encoding, 'utf-8')
+        self.assertIsNone(f.errors)
+        self.assertIsNone(f.newlines)
+        self.assertEqual(f.name, '<stdout>')
+        self.assertFalse(f.closed)
+        self.assertTrue(f.isatty())
+        self.assertFalse(f.readable())
+        self.assertTrue(f.writable())
+        self.assertFalse(f.seekable())
+
+    def test_unsupported(self):
+        shell = MockShell()
+        f = PseudoOutputFile(shell, 'stdout', 'utf-8')
+        self.assertRaises(IOError, f.fileno)
+        self.assertRaises(IOError, f.tell)
+        self.assertRaises(IOError, f.seek, 0)
+        self.assertRaises(IOError, f.read, 0)
+        self.assertRaises(IOError, f.readline, 0)
+
+    def test_write(self):
+        shell = MockShell()
+        f = PseudoOutputFile(shell, 'stdout', 'utf-8')
+        f.write('test')
+        self.assertEqual(shell.written, [('test', 'stdout')])
+        shell.reset()
+        f.write('t\xe8st')
+        self.assertEqual(shell.written, [('t\xe8st', 'stdout')])
+        shell.reset()
+        f.write(u't\xe8st')
+        self.assertEqual(shell.written, [(u't\xe8st', 'stdout')])
+        shell.reset()
+
+        f.write(S('t\xe8st'))
+        self.assertEqual(shell.written, [('t\xe8st', 'stdout')])
+        self.assertEqual(type(shell.written[0][0]), str)
+        shell.reset()
+        f.write(BA('t\xe8st'))
+        self.assertEqual(shell.written, [('t\xe8st', 'stdout')])
+        self.assertEqual(type(shell.written[0][0]), str)
+        shell.reset()
+        f.write(U(u't\xe8st'))
+        self.assertEqual(shell.written, [(u't\xe8st', 'stdout')])
+        self.assertEqual(type(shell.written[0][0]), unicode)
+        shell.reset()
+
+        self.assertRaises(TypeError, f.write)
+        self.assertEqual(shell.written, [])
+        self.assertRaises(TypeError, f.write, 123)
+        self.assertEqual(shell.written, [])
+        self.assertRaises(TypeError, f.write, 'test', 'spam')
+        self.assertEqual(shell.written, [])
+
+    def test_writelines(self):
+        shell = MockShell()
+        f = PseudoOutputFile(shell, 'stdout', 'utf-8')
+        f.writelines([])
+        self.assertEqual(shell.written, [])
+        shell.reset()
+        f.writelines(['one\n', 'two'])
+        self.assertEqual(shell.written,
+                         [('one\n', 'stdout'), ('two', 'stdout')])
+        shell.reset()
+        f.writelines(['on\xe8\n', 'tw\xf2'])
+        self.assertEqual(shell.written,
+                         [('on\xe8\n', 'stdout'), ('tw\xf2', 'stdout')])
+        shell.reset()
+        f.writelines([u'on\xe8\n', u'tw\xf2'])
+        self.assertEqual(shell.written,
+                         [(u'on\xe8\n', 'stdout'), (u'tw\xf2', 'stdout')])
+        shell.reset()
+
+        f.writelines([S('t\xe8st')])
+        self.assertEqual(shell.written, [('t\xe8st', 'stdout')])
+        self.assertEqual(type(shell.written[0][0]), str)
+        shell.reset()
+        f.writelines([BA('t\xe8st')])
+        self.assertEqual(shell.written, [('t\xe8st', 'stdout')])
+        self.assertEqual(type(shell.written[0][0]), str)
+        shell.reset()
+        f.writelines([U(u't\xe8st')])
+        self.assertEqual(shell.written, [(u't\xe8st', 'stdout')])
+        self.assertEqual(type(shell.written[0][0]), unicode)
+        shell.reset()
+
+        self.assertRaises(TypeError, f.writelines)
+        self.assertEqual(shell.written, [])
+        self.assertRaises(TypeError, f.writelines, 123)
+        self.assertEqual(shell.written, [])
+        self.assertRaises(TypeError, f.writelines, [123])
+        self.assertEqual(shell.written, [])
+        self.assertRaises(TypeError, f.writelines, [], [])
+        self.assertEqual(shell.written, [])
+
+    def test_close(self):
+        shell = MockShell()
+        f = PseudoOutputFile(shell, 'stdout', 'utf-8')
+        self.assertFalse(f.closed)
+        f.write('test')
+        f.close()
+        self.assertTrue(f.closed)
+        self.assertRaises(ValueError, f.write, 'x')
+        self.assertEqual(shell.written, [('test', 'stdout')])
+        f.close()
+        self.assertRaises(TypeError, f.close, 1)
+
+
+class PseudeInputFilesTest(unittest.TestCase):
+    def test_misc(self):
+        shell = MockShell()
+        f = PseudoInputFile(shell, 'stdin', 'utf-8')
+        self.assertIsInstance(f, io.TextIOBase)
+        self.assertEqual(f.encoding, 'utf-8')
+        self.assertIsNone(f.errors)
+        self.assertIsNone(f.newlines)
+        self.assertEqual(f.name, '<stdin>')
+        self.assertFalse(f.closed)
+        self.assertTrue(f.isatty())
+        self.assertTrue(f.readable())
+        self.assertFalse(f.writable())
+        self.assertFalse(f.seekable())
+
+    def test_unsupported(self):
+        shell = MockShell()
+        f = PseudoInputFile(shell, 'stdin', 'utf-8')
+        self.assertRaises(IOError, f.fileno)
+        self.assertRaises(IOError, f.tell)
+        self.assertRaises(IOError, f.seek, 0)
+        self.assertRaises(IOError, f.write, 'x')
+        self.assertRaises(IOError, f.writelines, ['x'])
+
+    def test_read(self):
+        shell = MockShell()
+        f = PseudoInputFile(shell, 'stdin', 'utf-8')
+        shell.push(['one\n', 'two\n', ''])
+        self.assertEqual(f.read(), 'one\ntwo\n')
+        shell.push(['one\n', 'two\n', ''])
+        self.assertEqual(f.read(-1), 'one\ntwo\n')
+        shell.push(['one\n', 'two\n', ''])
+        self.assertEqual(f.read(None), 'one\ntwo\n')
+        shell.push(['one\n', 'two\n', 'three\n', ''])
+        self.assertEqual(f.read(2), 'on')
+        self.assertEqual(f.read(3), 'e\nt')
+        self.assertEqual(f.read(10), 'wo\nthree\n')
+
+        shell.push(['one\n', 'two\n'])
+        self.assertEqual(f.read(0), '')
+        self.assertRaises(TypeError, f.read, 1.5)
+        self.assertRaises(TypeError, f.read, '1')
+        self.assertRaises(TypeError, f.read, 1, 1)
+
+    def test_readline(self):
+        shell = MockShell()
+        f = PseudoInputFile(shell, 'stdin', 'utf-8')
+        shell.push(['one\n', 'two\n', 'three\n', 'four\n'])
+        self.assertEqual(f.readline(), 'one\n')
+        self.assertEqual(f.readline(-1), 'two\n')
+        self.assertEqual(f.readline(None), 'three\n')
+        shell.push(['one\ntwo\n'])
+        self.assertEqual(f.readline(), 'one\n')
+        self.assertEqual(f.readline(), 'two\n')
+        shell.push(['one', 'two', 'three'])
+        self.assertEqual(f.readline(), 'one')
+        self.assertEqual(f.readline(), 'two')
+        shell.push(['one\n', 'two\n', 'three\n'])
+        self.assertEqual(f.readline(2), 'on')
+        self.assertEqual(f.readline(1), 'e')
+        self.assertEqual(f.readline(1), '\n')
+        self.assertEqual(f.readline(10), 'two\n')
+
+        shell.push(['one\n', 'two\n'])
+        self.assertEqual(f.readline(0), '')
+        self.assertRaises(TypeError, f.readlines, 1.5)
+        self.assertRaises(TypeError, f.readlines, '1')
+        self.assertRaises(TypeError, f.readlines, 1, 1)
+
+    def test_readlines(self):
+        shell = MockShell()
+        f = PseudoInputFile(shell, 'stdin', 'utf-8')
+        shell.push(['one\n', 'two\n', ''])
+        self.assertEqual(f.readlines(), ['one\n', 'two\n'])
+        shell.push(['one\n', 'two\n', ''])
+        self.assertEqual(f.readlines(-1), ['one\n', 'two\n'])
+        shell.push(['one\n', 'two\n', ''])
+        self.assertEqual(f.readlines(None), ['one\n', 'two\n'])
+        shell.push(['one\n', 'two\n', ''])
+        self.assertEqual(f.readlines(0), ['one\n', 'two\n'])
+        shell.push(['one\n', 'two\n', ''])
+        self.assertEqual(f.readlines(3), ['one\n'])
+        shell.push(['one\n', 'two\n', ''])
+        self.assertEqual(f.readlines(4), ['one\n', 'two\n'])
+
+        shell.push(['one\n', 'two\n', ''])
+        self.assertRaises(TypeError, f.readlines, 1.5)
+        self.assertRaises(TypeError, f.readlines, '1')
+        self.assertRaises(TypeError, f.readlines, 1, 1)
+

From noreply at buildbot.pypy.org  Fri Jun 12 16:22:34 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 12 Jun 2015 16:22:34 +0200 (CEST)
Subject: [pypy-commit] stmgc c8-gil-like: Save and restore errno at the
	critical point
Message-ID: <20150612142234.CF6021C033F@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: c8-gil-like
Changeset: r1819:bf0dfe206de5
Date: 2015-06-12 16:23 +0200
http://bitbucket.org/pypy/stmgc/changeset/bf0dfe206de5/

Log:	Save and restore errno at the critical point

diff --git a/c8/stm/detach.c b/c8/stm/detach.c
--- a/c8/stm/detach.c
+++ b/c8/stm/detach.c
@@ -2,6 +2,9 @@
 # error "must be compiled via stmgc.c"
 #endif
 
+#include <errno.h>
+
+
 /* Idea: if stm_leave_transactional_zone() is quickly followed by
    stm_enter_transactional_zone() in the same thread, then we should
    simply try to have one inevitable transaction that does both sides.
@@ -32,6 +35,7 @@
 
 void _stm_leave_noninevitable_transactional_zone(void)
 {
+    int saved_errno = errno;
     dprintf(("leave_noninevitable_transactional_zone\n"));
     _stm_become_inevitable(MSG_INEV_DONT_SLEEP);
 
@@ -45,6 +49,7 @@
         dprintf(("leave_noninevitable_transactional_zone: commit\n"));
         _stm_commit_transaction();
     }
+    errno = saved_errno;
 }
 
 static void commit_external_inevitable_transaction(void)
@@ -56,6 +61,7 @@
 void _stm_reattach_transaction(stm_thread_local_t *tl)
 {
     intptr_t old;
+    int saved_errno = errno;
  restart:
     old = _stm_detached_inevitable_from_thread;
     if (old != 0) {
@@ -85,6 +91,7 @@
     }
     dprintf(("reattach_transaction: start a new transaction\n"));
     _stm_start_transaction(tl);
+    errno = saved_errno;
 }
 
 void stm_force_transaction_break(stm_thread_local_t *tl)
diff --git a/c8/stmgc.h b/c8/stmgc.h
--- a/c8/stmgc.h
+++ b/c8/stmgc.h
@@ -420,6 +420,9 @@
    stm_enter_transactional_zone() will try to reattach to it.  This is
    far more efficient than constantly starting and committing
    transactions.
+
+   stm_enter_transactional_zone() and stm_leave_transactional_zone()
+   preserve the value of errno.
 */
 #ifdef STM_DEBUGPRINT
 #include <stdio.h>

From noreply at buildbot.pypy.org  Fri Jun 12 16:27:59 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Fri, 12 Jun 2015 16:27:59 +0200 (CEST)
Subject: [pypy-commit] pypy dtypes-compatability: add tests for metadata,
	dtype-from-dict
Message-ID: <20150612142759.6EA551C1461@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: dtypes-compatability
Changeset: r78044:ff5c0d392b3c
Date: 2015-06-12 13:53 +0300
http://bitbucket.org/pypy/pypy/changeset/ff5c0d392b3c/

Log:	add tests for metadata, dtype-from-dict

diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py
--- a/pypy/module/micronumpy/test/test_dtypes.py
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -1144,10 +1144,7 @@
         import sys
         d = {'names': ['r','g','b','a'],
              'formats': [np.uint8, np.uint8, np.uint8, np.uint8]}
-        if '__pypy__' not in sys.builtin_module_names:
-            dt = np.dtype(d)
-        else:
-            raises(NotImplementedError, np.dtype, d)
+        dt = np.dtype(d)
 
     def test_create_subarrays(self):
         from numpy import dtype
@@ -1356,4 +1353,45 @@
         assert a[0] == 1
         assert (a + a)[1] == 4
 
+class AppTestMonsterType(BaseNumpyAppTest):
+    """Test deeply nested subtypes."""
+    def test1(self):
+        import numpy as np
+        simple1 = np.dtype({'names': ['r', 'b'], 'formats': ['u1', 'u1'],
+            'titles': ['Red pixel', 'Blue pixel']})
+        a = np.dtype([('yo', np.int), ('ye', simple1),
+            ('yi', np.dtype((np.int, (3, 2))))])
+        b = np.dtype([('yo', np.int), ('ye', simple1),
+            ('yi', np.dtype((np.int, (3, 2))))])
+        assert a == b
 
+        c = np.dtype([('yo', np.int), ('ye', simple1),
+            ('yi', np.dtype((a, (3, 2))))])
+        d = np.dtype([('yo', np.int), ('ye', simple1),
+            ('yi', np.dtype((a, (3, 2))))])
+        assert c == d
+
+
+class AppTestMetadata(BaseNumpyAppTest):
+    def test_no_metadata(self):
+        import numpy as np
+        d = np.dtype(int)
+        assert d.metadata is None
+
+    def test_metadata_takes_dict(self):
+        import numpy as np
+        d = np.dtype(int, metadata={'datum': 1})
+        assert d.metadata == {'datum': 1}
+
+    def test_metadata_rejects_nondict(self):
+        import numpy as np
+        raises(TypeError, np.dtype, int, metadata='datum')
+        raises(TypeError, np.dtype, int, metadata=1)
+        raises(TypeError, np.dtype, int, metadata=None)
+
+    def test_nested_metadata(self):
+        import numpy as np
+        d = np.dtype([('a', np.dtype(int, metadata={'datum': 1}))])
+        assert d['a'].metadata == {'datum': 1}
+
+

From noreply at buildbot.pypy.org  Fri Jun 12 16:28:00 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Fri, 12 Jun 2015 16:28:00 +0200 (CEST)
Subject: [pypy-commit] pypy run-create_cffi_imports: start to add a build
 task to create cffi_imports
Message-ID: <20150612142800.B21901C1461@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: run-create_cffi_imports
Changeset: r78045:ef61b9c1dbf1
Date: 2015-06-12 17:27 +0300
http://bitbucket.org/pypy/pypy/changeset/ef61b9c1dbf1/

Log:	start to add a build task to create cffi_imports

diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -1,6 +1,6 @@
 import py
 
-import os, sys
+import os, sys, subprocess
 
 import pypy
 from pypy.interpreter import gateway
@@ -26,6 +26,44 @@
     except OSError:
         pass     # bah, no working stderr :-(
 
+# HACKHACKHACK to build cffi import libraries after translation
+
+cffi_build_scripts = {
+    "sqlite3": "_sqlite3_build.py",
+    "audioop": "_audioop_build.py",
+    "tk": "_tkinter/tklib_build.py",
+    "curses": "_curses_build.py" if sys.platform != "win32" else None,
+    "syslog": "_syslog_build.py" if sys.platform != "win32" else None,
+    "gdbm": "_gdbm_build.py"  if sys.platform != "win32" else None,
+    "pwdgrp": "_pwdgrp_build.py" if sys.platform != "win32" else None,
+    "xx": None,    # for testing: 'None' should be completely ignored
+    }
+
+def create_cffi_import_libraries(pypy_c, options, basedir):
+    shutil.rmtree(str(basedir.join('lib_pypy', '__pycache__')),
+                  ignore_errors=True)
+    for key, module in sorted(cffi_build_scripts.items()):
+        if module is None or getattr(options, 'no_' + key, None):
+            continue
+        if module.endswith('.py'):
+            args = [str(pypy_c), module]
+            cwd = str(basedir.join('lib_pypy'))
+        else:
+            args = [str(pypy_c), '-c', 'import ' + module]
+            cwd = None
+        print >> sys.stderr, '*', ' '.join(args)
+        try:
+            subprocess.check_call(args, cwd=cwd)
+        except subprocess.CalledProcessError:
+            print >>sys.stderr, """!!!!!!!!!!\nBuilding {0} bindings failed.
+You can either install development headers package,
+add the --without-{0} option to skip packaging this
+binary CFFI extension, or say --without-cffi.""".format(key)
+            raise MissingDependenciesError(module)
+
+# HACKHACKHACK end
+
+
 # __________  Entry point  __________
 
 
@@ -295,6 +333,29 @@
         wrapstr = 'space.wrap(%r)' % (options)
         pypy.module.sys.Module.interpleveldefs['pypy_translation_info'] = wrapstr
 
+        # HACKHACKHACK
+        # ugly hack to modify target goal from compile_c to build_cffi_imports
+        # this should probably get cleaned up and merged with driver.create_exe
+        from rpython.translator.driver import taskdef
+        import types
+
+        def mkexename(name):
+            if sys.platform == 'win32':
+                name = name.new(ext='exe')
+            return name
+
+        @taskdef(['compile_c'], "Create cffi bindings for modules")
+        def task_build_cffi_imports(self):
+            ''' Use cffi to compile cffi interfaces to modules'''
+            exename = mkexename(self.compute_exe_name())
+            modules = self.config.objspace.usemodules.getpaths()
+            import pdb;pdb.set_trace()
+            create_cffi_import_libraries(exename, modules, exename.basename())      
+        driver.task_build_cffi_imports = types.MethodType(task_build_cffi_imports, driver)
+        driver.tasks['build_cffi_imports'] = task_build_cffi_imports, ['compile_c']
+        driver.default_goal = 'build_cffi_imports'
+        # HACKHACKHACK end
+
         return self.get_entry_point(config)
 
     def jitpolicy(self, driver):

From noreply at buildbot.pypy.org  Fri Jun 12 18:07:02 2015
From: noreply at buildbot.pypy.org (amauryfa)
Date: Fri, 12 Jun 2015 18:07:02 +0200 (CEST)
Subject: [pypy-commit] pypy stdlib-2.7.10: Fix "import ssl"
Message-ID: <20150612160702.D55D11C0EB1@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: stdlib-2.7.10
Changeset: r78046:87dfaaf41b2a
Date: 2015-06-12 17:44 +0200
http://bitbucket.org/pypy/pypy/changeset/87dfaaf41b2a/

Log:	Fix "import ssl"

diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -66,6 +66,7 @@
 constants["HAS_TLS_UNIQUE"] = HAVE_OPENSSL_FINISHED
 constants["HAS_ECDH"] = not OPENSSL_NO_ECDH
 constants["HAS_NPN"] = OPENSSL_NPN_NEGOTIATED
+constants["HAS_ALPN"] = HAVE_ALPN
 
 if not OPENSSL_NO_SSL2:
     constants["PROTOCOL_SSLv2"]  = PY_SSL_VERSION_SSL2
diff --git a/rpython/rlib/ropenssl.py b/rpython/rlib/ropenssl.py
--- a/rpython/rlib/ropenssl.py
+++ b/rpython/rlib/ropenssl.py
@@ -97,6 +97,7 @@
     if HAVE_TLSv1_2:
         SSL_OP_NO_TLSv1_1 = rffi_platform.ConstantInteger("SSL_OP_NO_TLSv1_1")
         SSL_OP_NO_TLSv1_2 = rffi_platform.ConstantInteger("SSL_OP_NO_TLSv1_2")
+    OPENSSL_NO_TLSEXT = rffi_platform.Defined("OPENSSL_NO_TLSEXT")
     SSL_OP_CIPHER_SERVER_PREFERENCE = rffi_platform.ConstantInteger(
         "SSL_OP_CIPHER_SERVER_PREFERENCE")
     SSL_OP_SINGLE_DH_USE = rffi_platform.ConstantInteger(
@@ -259,6 +260,7 @@
                              OPENSSL_VERSION_NUMBER != 0x00909000
 if OPENSSL_VERSION_NUMBER < 0x0090800f and not OPENSSL_NO_ECDH:
     OPENSSL_NO_ECDH = True
+HAVE_ALPN = OPENSSL_VERSION_NUMBER >= 0x1000200fL and not OPENSSL_NO_TLSEXT
 
 
 def external(name, argtypes, restype, **kw):

From noreply at buildbot.pypy.org  Fri Jun 12 18:17:29 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 12 Jun 2015 18:17:29 +0200 (CEST)
Subject: [pypy-commit] stmgc c8-gil-like: Do a "full" implementation of
 stm_should_break_transaction(), as opposed
Message-ID: <20150612161729.B31421C033F@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: c8-gil-like
Changeset: r1820:5af967809206
Date: 2015-06-12 18:18 +0200
http://bitbucket.org/pypy/stmgc/changeset/5af967809206/

Log:	Do a "full" implementation of stm_should_break_transaction(), as
	opposed to the one in pypy, which didn't account for minor
	collections

diff --git a/c8/stm/core.c b/c8/stm/core.c
--- a/c8/stm/core.c
+++ b/c8/stm/core.c
@@ -1154,7 +1154,7 @@
 #endif
     STM_PSEGMENT->shadowstack_at_start_of_transaction = tl->shadowstack;
     STM_PSEGMENT->threadlocal_at_start_of_transaction = tl->thread_local_obj;
-
+    STM_PSEGMENT->total_throw_away_nursery = 0;
 
     assert(list_is_empty(STM_PSEGMENT->modified_old_objects));
     assert(list_is_empty(STM_PSEGMENT->large_overflow_objects));
@@ -1195,15 +1195,26 @@
     stm_validate();
 }
 
+#ifdef STM_NO_AUTOMATIC_SETJMP
+static int did_abort = 0;
+#endif
+
 long _stm_start_transaction(stm_thread_local_t *tl)
 {
     s_mutex_lock();
 #ifdef STM_NO_AUTOMATIC_SETJMP
-    long repeat_count = 0;    /* test/support.py */
+    long repeat_count = did_abort;    /* test/support.py */
+    did_abort = 0;
 #else
     long repeat_count = stm_rewind_jmp_setjmp(tl);
 #endif
     _do_start_transaction(tl);
+
+    if (repeat_count == 0) {  /* else, 'nursery_mark' was already set
+                                 in abort_data_structures_from_segment_num() */
+        STM_SEGMENT->nursery_mark = ((stm_char *)_stm_nursery_start +
+                                     stm_fill_mark_nursery_bytes);
+    }
     return repeat_count;
 }
 
@@ -1427,7 +1438,7 @@
 
     abort_finalizers(pseg);
 
-    long bytes_in_nursery = throw_away_nursery(pseg);
+    throw_away_nursery(pseg);
 
     /* clear CARD_MARKED on objs (don't care about CARD_MARKED_OLD) */
     LIST_FOREACH_R(pseg->old_objects_with_cards_set, object_t * /*item*/,
@@ -1461,7 +1472,26 @@
     assert(tl->shadowstack == pseg->shadowstack_at_start_of_transaction);
 #endif
     tl->thread_local_obj = pseg->threadlocal_at_start_of_transaction;
-    tl->last_abort__bytes_in_nursery = bytes_in_nursery;
+
+
+    /* Set the next nursery_mark: first compute the value that
+       nursery_mark must have had at the start of the aborted transaction */
+    stm_char *old_mark =pseg->pub.nursery_mark + pseg->total_throw_away_nursery;
+
+    /* This means that the limit, in term of bytes, was: */
+    uintptr_t old_limit = old_mark - (stm_char *)_stm_nursery_start;
+
+    /* If 'total_throw_away_nursery' is smaller than old_limit, use that */
+    if (pseg->total_throw_away_nursery < old_limit)
+        old_limit = pseg->total_throw_away_nursery;
+
+    /* Now set the new limit to 90% of the old limit */
+    pseg->pub.nursery_mark = ((stm_char *)_stm_nursery_start +
+                              (uintptr_t)(old_limit * 0.9));
+
+#ifdef STM_NO_AUTOMATIC_SETJMP
+    did_abort = 1;
+#endif
 
     list_clear(pseg->objects_pointing_to_nursery);
     list_clear(pseg->old_objects_with_cards_set);
diff --git a/c8/stm/core.h b/c8/stm/core.h
--- a/c8/stm/core.h
+++ b/c8/stm/core.h
@@ -152,6 +152,9 @@
     stm_char *sq_fragments[SYNC_QUEUE_SIZE];
     int sq_fragsizes[SYNC_QUEUE_SIZE];
     int sq_len;
+
+    /* For nursery_mark */
+    uintptr_t total_throw_away_nursery;
 };
 
 enum /* safe_point */ {
diff --git a/c8/stm/nursery.c b/c8/stm/nursery.c
--- a/c8/stm/nursery.c
+++ b/c8/stm/nursery.c
@@ -11,8 +11,13 @@
 static uintptr_t _stm_nursery_start;
 
 
+#define DEFAULT_FILL_MARK_NURSERY_BYTES   (NURSERY_SIZE / 4)
+
+uintptr_t stm_fill_mark_nursery_bytes = DEFAULT_FILL_MARK_NURSERY_BYTES;
+
 /************************************************************/
 
+
 static void setup_nursery(void)
 {
     assert(_STM_FAST_ALLOC <= NURSERY_SIZE);
@@ -449,7 +454,7 @@
 }
 
 
-static size_t throw_away_nursery(struct stm_priv_segment_info_s *pseg)
+static void throw_away_nursery(struct stm_priv_segment_info_s *pseg)
 {
 #pragma push_macro("STM_PSEGMENT")
 #pragma push_macro("STM_SEGMENT")
@@ -482,7 +487,9 @@
 #endif
 #endif
 
+    pseg->total_throw_away_nursery += nursery_used;
     pseg->pub.nursery_current = (stm_char *)_stm_nursery_start;
+    pseg->pub.nursery_mark -= nursery_used;
 
     /* free any object left from 'young_outside_nursery' */
     if (!tree_is_cleared(pseg->young_outside_nursery)) {
@@ -507,8 +514,6 @@
     }
 
     tree_clear(pseg->nursery_objects_shadows);
-
-    return nursery_used;
 #pragma pop_macro("STM_SEGMENT")
 #pragma pop_macro("STM_PSEGMENT")
 }
diff --git a/c8/stm/nursery.h b/c8/stm/nursery.h
--- a/c8/stm/nursery.h
+++ b/c8/stm/nursery.h
@@ -12,7 +12,7 @@
 
 static void minor_collection(bool commit, bool external);
 static void check_nursery_at_transaction_start(void);
-static size_t throw_away_nursery(struct stm_priv_segment_info_s *pseg);
+static void throw_away_nursery(struct stm_priv_segment_info_s *pseg);
 static void major_do_validation_and_minor_collections(void);
 
 static void assert_memset_zero(void *s, size_t n);
diff --git a/c8/stmgc.h b/c8/stmgc.h
--- a/c8/stmgc.h
+++ b/c8/stmgc.h
@@ -44,6 +44,7 @@
     int segment_num;
     char *segment_base;
     stm_char *nursery_current;
+    stm_char *nursery_mark;
     uintptr_t nursery_end;
     struct stm_thread_local_s *running_thread;
 };
@@ -67,9 +68,6 @@
        the following raw region of memory is cleared. */
     char *mem_clear_on_abort;
     size_t mem_bytes_to_clear_on_abort;
-    /* after an abort, some details about the abort are stored there.
-       (this field is not modified on a successful commit) */
-    long last_abort__bytes_in_nursery;
     /* the next fields are handled internally by the library */
     int last_associated_segment_num;   /* always a valid seg num */
     int thread_local_counter;
@@ -459,7 +457,8 @@
    stm_enter_transactional_zone(); however, it is supposed to be
    called in CPU-heavy threads that had a transaction run for a while,
    and so it *always* forces a commit and starts the next transaction.
-   The new transaction is never inevitable. */
+   The new transaction is never inevitable.  See also
+   stm_should_break_transaction(). */
 void stm_force_transaction_break(stm_thread_local_t *tl);
 
 /* Abort the currently running transaction.  This function never
@@ -490,6 +489,23 @@
 void stm_collect(long level);
 
 
+/* A way to detect that we've run for a while and should call
+   stm_force_transaction_break() */
+static inline int stm_should_break_transaction(void)
+{
+    return ((intptr_t)STM_SEGMENT->nursery_current >=
+            (intptr_t)STM_SEGMENT->nursery_mark);
+}
+extern uintptr_t stm_fill_mark_nursery_bytes;
+/* ^^^ at the start of a transaction, 'nursery_mark' is initialized to
+   'stm_fill_mark_nursery_bytes' inside the nursery.  This value can
+   be larger than the nursery; every minor collection shifts the
+   current 'nursery_mark' down by one nursery-size.  After an abort
+   and restart, 'nursery_mark' is set to ~90% of the value it reached
+   in the last attempt.
+*/
+
+
 /* Prepare an immortal "prebuilt" object managed by the GC.  Takes a
    pointer to an 'object_t', which should not actually be a GC-managed
    structure but a real static structure.  Returns the equivalent
diff --git a/c8/test/support.py b/c8/test/support.py
--- a/c8/test/support.py
+++ b/c8/test/support.py
@@ -28,7 +28,6 @@
     object_t *thread_local_obj;
     char *mem_clear_on_abort;
     size_t mem_bytes_to_clear_on_abort;
-    long last_abort__bytes_in_nursery;
     int last_associated_segment_num;
     struct stm_thread_local_s *prev, *next;
     void *creating_pthread[2];
@@ -37,6 +36,7 @@
 
 char *stm_object_pages;
 char *stm_file_pages;
+uintptr_t stm_fill_mark_nursery_bytes;
 
 void stm_read(object_t *obj);
 /*void stm_write(object_t *obj); use _checked_stm_write() instead */
@@ -104,6 +104,8 @@
 long _check_stm_collect(long level);
 uint64_t _stm_total_allocated(void);
 
+long bytes_before_transaction_break(void);
+
 void _stm_set_nursery_free_count(uint64_t free_count);
 void _stm_largemalloc_init_arena(char *data_start, size_t data_size);
 int _stm_largemalloc_resize_arena(size_t new_size);
@@ -404,6 +406,11 @@
     return *field;
 }
 
+long bytes_before_transaction_break(void)
+{
+    return STM_SEGMENT->nursery_mark - STM_SEGMENT->nursery_current;
+}
+
 
 ssize_t stmcb_size_rounded_up(struct object_s *obj)
 {
diff --git a/c8/test/test_basic.py b/c8/test/test_basic.py
--- a/c8/test/test_basic.py
+++ b/c8/test/test_basic.py
@@ -720,19 +720,53 @@
         lp1 = self.pop_root()
         self.check_char_everywhere(lp1, 'X')
 
-    def test_last_abort__bytes_in_nursery(self):
+    def test_stm_should_break_transaction_1(self):
+        lib.stm_fill_mark_nursery_bytes = 100
+        #
         self.start_transaction()
-        stm_allocate(56)
+        self.commit_transaction()
+        self.start_transaction()
+        assert lib.bytes_before_transaction_break() == 100
+        stm_allocate(64)
+        assert lib.bytes_before_transaction_break() == 36
+        stm_allocate(64)
+        assert lib.bytes_before_transaction_break() == -28
         self.abort_transaction()
-        assert self.get_stm_thread_local().last_abort__bytes_in_nursery == 56
         self.start_transaction()
-        assert self.get_stm_thread_local().last_abort__bytes_in_nursery == 56
+        assert lib.bytes_before_transaction_break() == 90    # 100 * 0.9
+        stm_allocate(200)
+        self.abort_transaction()
+        self.start_transaction()
+        assert lib.bytes_before_transaction_break() == 81    # 90 * 0.9
         self.commit_transaction()
-        assert self.get_stm_thread_local().last_abort__bytes_in_nursery == 56
+        #
         self.start_transaction()
-        assert self.get_stm_thread_local().last_abort__bytes_in_nursery == 56
+        assert lib.bytes_before_transaction_break() == 100
+        stm_allocate(64)
+        assert lib.bytes_before_transaction_break() == 36
         self.abort_transaction()
-        assert self.get_stm_thread_local().last_abort__bytes_in_nursery == 0
+        self.start_transaction()
+        assert lib.bytes_before_transaction_break() == 57    # int(64 * 0.9)
+        stm_allocate(32)
+        assert lib.bytes_before_transaction_break() == 25
+        self.abort_transaction()
+        self.start_transaction()
+        assert lib.bytes_before_transaction_break() == 28    # int(32 * 0.9)
+        stm_allocate(64)
+        assert lib.bytes_before_transaction_break() == -36
+        self.commit_transaction()
+
+    def test_stm_should_break_transaction_2(self):
+        lib.stm_fill_mark_nursery_bytes = 10000000
+        #
+        n = 10000000
+        self.start_transaction()
+        self.commit_transaction()
+        self.start_transaction()
+        for i in range(1000):
+            assert lib.bytes_before_transaction_break() == n
+            stm_allocate(10000)
+            n -= 10000
 
     def test_bug(self):
         lp_char_5 = stm_allocate_old(384)

From noreply at buildbot.pypy.org  Fri Jun 12 18:33:39 2015
From: noreply at buildbot.pypy.org (amauryfa)
Date: Fri, 12 Jun 2015 18:33:39 +0200 (CEST)
Subject: [pypy-commit] pypy stdlib-2.7.10: Add support for "alpn" in SSL.
Message-ID: <20150612163339.971C51C033F@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: stdlib-2.7.10
Changeset: r78047:ae7a5e9438ee
Date: 2015-06-12 18:33 +0200
http://bitbucket.org/pypy/pypy/changeset/ae7a5e9438ee/

Log:	Add support for "alpn" in SSL.

	Not tested at all: my machine does not have the correct version.

diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -65,8 +65,8 @@
 constants["HAS_SNI"] = HAS_SNI
 constants["HAS_TLS_UNIQUE"] = HAVE_OPENSSL_FINISHED
 constants["HAS_ECDH"] = not OPENSSL_NO_ECDH
-constants["HAS_NPN"] = OPENSSL_NPN_NEGOTIATED
-constants["HAS_ALPN"] = HAVE_ALPN
+constants["HAS_NPN"] = HAS_NPN
+constants["HAS_ALPN"] = HAS_ALPN
 
 if not OPENSSL_NO_SSL2:
     constants["PROTOCOL_SSLv2"]  = PY_SSL_VERSION_SSL2
@@ -176,7 +176,45 @@
                                      client, client_len)
         return rffi.cast(rffi.INT, SSL_TLSEXT_ERR_OK)
 
+
+class SSLAlpnProtocols(object):
+
+    def __init__(self, ctx, protos):
+        self.protos = protos
+        self.buf, self.pinned, self.is_raw = rffi.get_nonmovingbuffer(protos)
+        ALPN_STORAGE.set(r_uint(rffi.cast(rffi.UINT, self.buf)), self)
+
+        with rffi.scoped_str2charp(protos) as protos_buf:
+            if libssl_SSL_CTX_set_alpn_protos(
+                    ctx, rffi.cast(rffi.UCHARP, protos_buf), len(protos)):
+                raise MemoryError
+        libssl_SSL_CTX_set_alpn_select_cb(
+            ctx, self.selectALPN_cb, self.buf)
+
+    def __del__(self):
+        rffi.free_nonmovingbuffer(
+            self.protos, self.buf, self.pinned, self.is_raw)
+
+    @staticmethod
+    def selectALPN_cb(s, out_ptr, outlen_ptr, client, client_len, args):
+        alpn = ALPN_STORAGE.get(r_uint(rffi.cast(rffi.UINT, args)))
+        if alpn and alpn.protos:
+            server = alpn.buf
+            server_len = len(alpn.protos)
+        else:
+            server = lltype.nullptr(rffi.CCHARP.TO)
+            server_len = 0
+
+        ret = libssl_SSL_select_next_proto(out_ptr, outlen_ptr,
+                                           server, server_len,
+                                           client, client_len)
+        if ret != OPENSSL_NPN_NEGOTIATED:
+            return rffi.cast(rffi.INT, SSL_TLSEXT_ERR_NOACK)
+        return rffi.cast(rffi.INT, SSL_TLSEXT_ERR_OK)
+
+
 NPN_STORAGE = RWeakValueDictionary(r_uint, SSLNpnProtocols)
+ALPN_STORAGE = RWeakValueDictionary(r_uint, SSLAlpnProtocols)
 
 SOCKET_STORAGE = RWeakValueDictionary(int, W_Root)
 
@@ -572,6 +610,18 @@
                     return space.wrap(
                         rffi.charpsize2str(out_ptr[0], intmask(len_ptr[0])))
 
+    def selected_alpn_protocol(self, space):
+        if not HAS_ALPN:
+            raise oefmt(space.w_NotImplementedError,
+                        "The ALPN extension requires OpenSSL 1.0.2 or later.")
+        with lltype.scoped_alloc(rffi.CCHARPP.TO, 1) as out_ptr:
+            with lltype.scoped_alloc(rffi.UINTP.TO, 1) as len_ptr:
+                libssl_SSL_get0_alpn_selected(self.ssl,
+                                              out_ptr, len_ptr)
+                if out_ptr[0]:
+                    return space.wrap(
+                        rffi.charpsize2str(out_ptr[0], intmask(len_ptr[0])))
+
     def compression_w(self, space):
         if not self.ssl:
             return space.w_None
@@ -635,6 +685,7 @@
     cipher=interp2app(_SSLSocket.cipher),
     shutdown=interp2app(_SSLSocket.shutdown),
     selected_npn_protocol = interp2app(_SSLSocket.selected_npn_protocol),
+    selected_alpn_protocol = interp2app(_SSLSocket.selected_alpn_protocol),
     compression = interp2app(_SSLSocket.compression_w),
     version = interp2app(_SSLSocket.version_w),
     tls_unique_cb = interp2app(_SSLSocket.tls_unique_cb_w),
@@ -1562,6 +1613,14 @@
 
         self.npn_protocols = SSLNpnProtocols(self.ctx, protos)
 
+    @unwrap_spec(protos='bufferstr')
+    def set_alpn_protocols_w(self, space, protos):
+        if not HAS_ALPN:
+            raise oefmt(space.w_NotImplementedError,
+                        "The ALPN extension requires OpenSSL 1.0.2 or later.")
+
+        self.alpn_protocols = SSLAlpnProtocols(self.ctx, protos)
+
     def get_ca_certs_w(self, space, w_binary_form=None):
         if w_binary_form and space.is_true(w_binary_form):
             binary_mode = True
@@ -1630,6 +1689,7 @@
     session_stats = interp2app(_SSLContext.session_stats_w),
     set_default_verify_paths=interp2app(_SSLContext.descr_set_default_verify_paths),
     _set_npn_protocols=interp2app(_SSLContext.set_npn_protocols_w),
+    _set_alpn_protocols=interp2app(_SSLContext.set_alpn_protocols_w),
     get_ca_certs=interp2app(_SSLContext.get_ca_certs_w),
     set_ecdh_curve=interp2app(_SSLContext.set_ecdh_curve_w),
     set_servername_callback=interp2app(_SSLContext.set_servername_callback_w),
diff --git a/rpython/rlib/ropenssl.py b/rpython/rlib/ropenssl.py
--- a/rpython/rlib/ropenssl.py
+++ b/rpython/rlib/ropenssl.py
@@ -134,6 +134,7 @@
     SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER = rffi_platform.ConstantInteger("SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER")
     SSL_TLSEXT_ERR_OK = rffi_platform.ConstantInteger("SSL_TLSEXT_ERR_OK")
     SSL_TLSEXT_ERR_ALERT_FATAL = rffi_platform.ConstantInteger("SSL_TLSEXT_ERR_ALERT_FATAL")
+    SSL_TLSEXT_ERR_NOACK = rffi_platform.ConstantInteger("SSL_TLSEXT_ERR_NOACK")
 
     SSL_AD_INTERNAL_ERROR = rffi_platform.ConstantInteger("SSL_AD_INTERNAL_ERROR")
     SSL_AD_HANDSHAKE_FAILURE = rffi_platform.ConstantInteger("SSL_AD_HANDSHAKE_FAILURE")
@@ -260,7 +261,7 @@
                              OPENSSL_VERSION_NUMBER != 0x00909000
 if OPENSSL_VERSION_NUMBER < 0x0090800f and not OPENSSL_NO_ECDH:
     OPENSSL_NO_ECDH = True
-HAVE_ALPN = OPENSSL_VERSION_NUMBER >= 0x1000200fL and not OPENSSL_NO_TLSEXT
+HAS_ALPN = OPENSSL_VERSION_NUMBER >= 0x1000200fL and not OPENSSL_NO_TLSEXT
 
 
 def external(name, argtypes, restype, **kw):
@@ -513,6 +514,17 @@
     ssl_external(
         'SSL_get0_next_proto_negotiated', [
             SSL, rffi.CCHARPP, rffi.UINTP], lltype.Void)
+if HAS_ALPN:
+    ssl_external('SSL_CTX_set_alpn_protos',
+                 [SSL_CTX, rffi.UCHARP, rffi.UINT], rffi.INT)
+    SSL_ALPN_SEL_CB = lltype.Ptr(lltype.FuncType(
+        [SSL, rffi.CCHARPP, rffi.UCHARP, rffi.CCHARP, rffi.UINT, rffi.VOIDP],
+        rffi.INT))
+    ssl_external('SSL_CTX_set_alpn_select_cb',
+                 [SSL_CTX, SSL_ALPN_SEL_CB, rffi.VOIDP], lltype.Void)
+    ssl_external(
+        'SSL_get0_alpn_selected', [
+            SSL, rffi.CCHARPP, rffi.UINTP], lltype.Void)
 
 EVP_MD_CTX = rffi.COpaquePtr('EVP_MD_CTX', compilation_info=eci)
 EVP_MD     = lltype.Ptr(EVP_MD_st)

From noreply at buildbot.pypy.org  Fri Jun 12 19:00:34 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 12 Jun 2015 19:00:34 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c8-gil-like: import stmgc,
	branch c8-gil-like
Message-ID: <20150612170034.0CE0F1C033F@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c8-gil-like
Changeset: r78048:ffc83930d682
Date: 2015-06-12 16:53 +0200
http://bitbucket.org/pypy/pypy/changeset/ffc83930d682/

Log:	import stmgc, branch c8-gil-like

diff --git a/rpython/translator/stm/src_stm/revision b/rpython/translator/stm/src_stm/revision
--- a/rpython/translator/stm/src_stm/revision
+++ b/rpython/translator/stm/src_stm/revision
@@ -1,1 +1,1 @@
-f0d995d5609d
+bf0dfe206de5
diff --git a/rpython/translator/stm/src_stm/stm/atomic.h b/rpython/translator/stm/src_stm/stm/atomic.h
--- a/rpython/translator/stm/src_stm/stm/atomic.h
+++ b/rpython/translator/stm/src_stm/stm/atomic.h
@@ -24,15 +24,21 @@
 
 #if defined(__i386__) || defined(__amd64__)
 
-# define HAVE_FULL_EXCHANGE_INSN
   static inline void spin_loop(void) { asm("pause" : : : "memory"); }
   static inline void write_fence(void) { asm("" : : : "memory"); }
+/*# define atomic_exchange(ptr, old, new)  do {         \
+          (old) = __sync_lock_test_and_set(ptr, new);   \
+      } while (0)*/
 
 #else
 
   static inline void spin_loop(void) { asm("" : : : "memory"); }
   static inline void write_fence(void) { __sync_synchronize(); }
 
+/*# define atomic_exchange(ptr, old, new)  do {           \
+          (old) = *(ptr);                                 \
+      } while (UNLIKELY(!__sync_bool_compare_and_swap(ptr, old, new))); */
+
 #endif
 
 
diff --git a/rpython/translator/stm/src_stm/stm/core.c b/rpython/translator/stm/src_stm/stm/core.c
--- a/rpython/translator/stm/src_stm/stm/core.c
+++ b/rpython/translator/stm/src_stm/stm/core.c
@@ -324,10 +324,7 @@
     /* Don't check this 'cl'. This entry is already checked */
 
     if (STM_PSEGMENT->transaction_state == TS_INEVITABLE) {
-        //assert(first_cl->next == INEV_RUNNING);
-        /* the above assert may fail when running a major collection
-           while the commit of the inevitable transaction is in progress
-           and the element is already attached */
+        assert(first_cl->next == INEV_RUNNING);
         return true;
     }
 
@@ -496,11 +493,23 @@
 
 static void wait_for_other_inevitable(struct stm_commit_log_entry_s *old)
 {
+    intptr_t detached = fetch_detached_transaction();
+    if (detached != 0) {
+        commit_fetched_detached_transaction(detached);
+        return;
+    }
+
     timing_event(STM_SEGMENT->running_thread, STM_WAIT_OTHER_INEVITABLE);
 
     while (old->next == INEV_RUNNING && !safe_point_requested()) {
         spin_loop();
         usleep(10);    /* XXXXXX */
+
+        detached = fetch_detached_transaction();
+        if (detached != 0) {
+            commit_fetched_detached_transaction(detached);
+            break;
+        }
     }
     timing_event(STM_SEGMENT->running_thread, STM_WAIT_DONE);
 }
@@ -509,7 +518,8 @@
 static void readd_wb_executed_flags(void);
 static void check_all_write_barrier_flags(char *segbase, struct list_s *list);
 
-static void _validate_and_attach(struct stm_commit_log_entry_s *new)
+static bool _validate_and_attach(struct stm_commit_log_entry_s *new,
+                                 bool can_sleep)
 {
     struct stm_commit_log_entry_s *old;
 
@@ -571,6 +581,8 @@
             /* XXXXXX for now just sleep.  We should really ask to inev
                transaction to do the commit for us, and then we can
                continue running. */
+            if (!can_sleep)
+                return false;
             dprintf(("_validate_and_attach(%p) failed, "
                      "waiting for inevitable\n", new));
             wait_for_other_inevitable(old);
@@ -591,18 +603,17 @@
 
     if (is_commit) {
         /* compare with _validate_and_add_to_commit_log */
-        STM_PSEGMENT->transaction_state = TS_NONE;
-        STM_PSEGMENT->safe_point = SP_NO_TRANSACTION;
-
         list_clear(STM_PSEGMENT->modified_old_objects);
         STM_PSEGMENT->last_commit_log_entry = new;
         release_modification_lock_wr(STM_SEGMENT->segment_num);
     }
+    return true;
 }
 
-static void _validate_and_turn_inevitable(void)
+static bool _validate_and_turn_inevitable(bool can_sleep)
 {
-    _validate_and_attach((struct stm_commit_log_entry_s *)INEV_RUNNING);
+    return _validate_and_attach((struct stm_commit_log_entry_s *)INEV_RUNNING,
+                                can_sleep);
 }
 
 static void _validate_and_add_to_commit_log(void)
@@ -611,6 +622,8 @@
 
     new = _create_commit_log_entry();
     if (STM_PSEGMENT->transaction_state == TS_INEVITABLE) {
+        assert(_stm_detached_inevitable_from_thread == 0);  /* running it */
+
         old = STM_PSEGMENT->last_commit_log_entry;
         new->rev_num = old->rev_num + 1;
         OPT_ASSERT(old->next == INEV_RUNNING);
@@ -621,17 +634,18 @@
                                       STM_PSEGMENT->modified_old_objects);
 
         /* compare with _validate_and_attach: */
-        STM_PSEGMENT->transaction_state = TS_NONE;
-        STM_PSEGMENT->safe_point = SP_NO_TRANSACTION;
+        acquire_modification_lock_wr(STM_SEGMENT->segment_num);
         list_clear(STM_PSEGMENT->modified_old_objects);
         STM_PSEGMENT->last_commit_log_entry = new;
 
         /* do it: */
         bool yes = __sync_bool_compare_and_swap(&old->next, INEV_RUNNING, new);
         OPT_ASSERT(yes);
+
+        release_modification_lock_wr(STM_SEGMENT->segment_num);
     }
     else {
-        _validate_and_attach(new);
+        _validate_and_attach(new, /*can_sleep=*/true);
     }
 }
 
@@ -1123,7 +1137,7 @@
 
 
 
-static void _stm_start_transaction(stm_thread_local_t *tl)
+static void _do_start_transaction(stm_thread_local_t *tl)
 {
     assert(!_stm_in_transaction(tl));
 
@@ -1181,7 +1195,7 @@
     stm_validate();
 }
 
-long stm_start_transaction(stm_thread_local_t *tl)
+long _stm_start_transaction(stm_thread_local_t *tl)
 {
     s_mutex_lock();
 #ifdef STM_NO_AUTOMATIC_SETJMP
@@ -1189,23 +1203,10 @@
 #else
     long repeat_count = stm_rewind_jmp_setjmp(tl);
 #endif
-    _stm_start_transaction(tl);
+    _do_start_transaction(tl);
     return repeat_count;
 }
 
-void stm_start_inevitable_transaction(stm_thread_local_t *tl)
-{
-    /* used to be more efficient, starting directly an inevitable transaction,
-       but there is no real point any more, I believe */
-    rewind_jmp_buf rjbuf;
-    stm_rewind_jmp_enterframe(tl, &rjbuf);
-
-    stm_start_transaction(tl);
-    stm_become_inevitable(tl, "start_inevitable_transaction");
-
-    stm_rewind_jmp_leaveframe(tl, &rjbuf);
-}
-
 #ifdef STM_NO_AUTOMATIC_SETJMP
 void _test_run_abort(stm_thread_local_t *tl) __attribute__((noreturn));
 int stm_is_inevitable(void)
@@ -1224,6 +1225,7 @@
 {
     stm_thread_local_t *tl = STM_SEGMENT->running_thread;
 
+    assert(_has_mutex());
     STM_PSEGMENT->safe_point = SP_NO_TRANSACTION;
     STM_PSEGMENT->transaction_state = TS_NONE;
 
@@ -1231,7 +1233,15 @@
     list_clear(STM_PSEGMENT->objects_pointing_to_nursery);
     list_clear(STM_PSEGMENT->old_objects_with_cards_set);
     list_clear(STM_PSEGMENT->large_overflow_objects);
-    timing_event(tl, event);
+    if (tl != NULL)
+        timing_event(tl, event);
+
+    /* If somebody is waiting for us to reach a safe point, we simply
+       signal it now and leave this transaction.  This should be enough
+       for synchronize_all_threads() to retry and notice that we are
+       no longer SP_RUNNING. */
+    if (STM_SEGMENT->nursery_end != NURSERY_END)
+        cond_signal(C_AT_SAFE_POINT);
 
     release_thread_segment(tl);
     /* cannot access STM_SEGMENT or STM_PSEGMENT from here ! */
@@ -1280,24 +1290,55 @@
 }
 
 
-void stm_commit_transaction(void)
+void _stm_commit_transaction(void)
+{
+    assert(STM_PSEGMENT->running_pthread == pthread_self());
+    _core_commit_transaction(/*external=*/ false);
+}
+
+static void _core_commit_transaction(bool external)
 {
     exec_local_finalizers();
 
     assert(!_has_mutex());
     assert(STM_PSEGMENT->safe_point == SP_RUNNING);
-    assert(STM_PSEGMENT->running_pthread == pthread_self());
+    assert(STM_PSEGMENT->transaction_state != TS_NONE);
+    if (globally_unique_transaction) {
+        stm_fatalerror("cannot commit between stm_stop_all_other_threads "
+                       "and stm_resume_all_other_threads");
+    }
 
-    dprintf(("> stm_commit_transaction()\n"));
-    minor_collection(1);
+    dprintf(("> stm_commit_transaction(external=%d)\n", (int)external));
+    minor_collection(/*commit=*/ true, external);
+    if (!external && is_major_collection_requested()) {
+        s_mutex_lock();
+        if (is_major_collection_requested()) {   /* if still true */
+            major_collection_with_mutex();
+        }
+        s_mutex_unlock();
+    }
 
     push_large_overflow_objects_to_other_segments();
     /* push before validate. otherwise they are reachable too early */
 
+    if (external) {
+        /* from this point on, unlink the original 'stm_thread_local_t *'
+           from its segment.  Better do it as soon as possible, because
+           other threads might be spin-looping, waiting for the -1 to
+           disappear. */
+        STM_SEGMENT->running_thread = NULL;
+        write_fence();
+        assert(_stm_detached_inevitable_from_thread == -1);
+        _stm_detached_inevitable_from_thread = 0;
+    }
+
     bool was_inev = STM_PSEGMENT->transaction_state == TS_INEVITABLE;
     _validate_and_add_to_commit_log();
 
-    stm_rewind_jmp_forget(STM_SEGMENT->running_thread);
+    if (!was_inev) {
+        assert(!external);
+        stm_rewind_jmp_forget(STM_SEGMENT->running_thread);
+    }
 
     /* XXX do we still need a s_mutex_lock() section here? */
     s_mutex_lock();
@@ -1314,23 +1355,9 @@
 
     invoke_and_clear_user_callbacks(0);   /* for commit */
 
-    /* >>>>> there may be a FORK() happening in the safepoint below <<<<<*/
-    enter_safe_point_if_requested();
-    assert(STM_SEGMENT->nursery_end == NURSERY_END);
-
-    /* if a major collection is required, do it here */
-    if (is_major_collection_requested()) {
-        major_collection_with_mutex();
-    }
-
-    _verify_cards_cleared_in_all_lists(get_priv_segment(STM_SEGMENT->segment_num));
-
-    if (globally_unique_transaction && was_inev) {
-        committed_globally_unique_transaction();
-    }
-
     /* done */
     stm_thread_local_t *tl = STM_SEGMENT->running_thread;
+    assert(external == (tl == NULL));
     _finish_transaction(STM_TRANSACTION_COMMIT);
     /* cannot access STM_SEGMENT or STM_PSEGMENT from here ! */
 
@@ -1338,7 +1365,8 @@
 
     /* between transactions, call finalizers. this will execute
        a transaction itself */
-    invoke_general_finalizers(tl);
+    if (tl != NULL)
+        invoke_general_finalizers(tl);
 }
 
 static void reset_modified_from_backup_copies(int segment_num)
@@ -1502,32 +1530,36 @@
 
 void _stm_become_inevitable(const char *msg)
 {
-    if (STM_PSEGMENT->transaction_state == TS_REGULAR) {
+    assert(STM_PSEGMENT->transaction_state == TS_REGULAR);
+    _stm_collectable_safe_point();
+
+    if (msg != MSG_INEV_DONT_SLEEP) {
         dprintf(("become_inevitable: %s\n", msg));
-        _stm_collectable_safe_point();
         timing_become_inevitable();
-
-        _validate_and_turn_inevitable();
-        STM_PSEGMENT->transaction_state = TS_INEVITABLE;
-
-        stm_rewind_jmp_forget(STM_SEGMENT->running_thread);
-        invoke_and_clear_user_callbacks(0);   /* for commit */
+        _validate_and_turn_inevitable(/*can_sleep=*/true);
     }
     else {
-        assert(STM_PSEGMENT->transaction_state == TS_INEVITABLE);
+        if (!_validate_and_turn_inevitable(/*can_sleep=*/false))
+            return;
+        timing_become_inevitable();
     }
+    STM_PSEGMENT->transaction_state = TS_INEVITABLE;
+
+    stm_rewind_jmp_forget(STM_SEGMENT->running_thread);
+    invoke_and_clear_user_callbacks(0);   /* for commit */
 }
 
+#if 0
 void stm_become_globally_unique_transaction(stm_thread_local_t *tl,
                                             const char *msg)
 {
-    stm_become_inevitable(tl, msg);   /* may still abort */
+    stm_become_inevitable(tl, msg);
 
     s_mutex_lock();
     synchronize_all_threads(STOP_OTHERS_AND_BECOME_GLOBALLY_UNIQUE);
     s_mutex_unlock();
 }
-
+#endif
 
 void stm_stop_all_other_threads(void)
 {
diff --git a/rpython/translator/stm/src_stm/stm/core.h b/rpython/translator/stm/src_stm/stm/core.h
--- a/rpython/translator/stm/src_stm/stm/core.h
+++ b/rpython/translator/stm/src_stm/stm/core.h
@@ -170,6 +170,12 @@
     TS_INEVITABLE,
 };
 
+#define MSG_INEV_DONT_SLEEP  ((const char *)1)
+
+#define in_transaction(tl)                                              \
+    (get_segment((tl)->last_associated_segment_num)->running_thread == (tl))
+
+
 /* Commit Log things */
 struct stm_undo_s {
   union {
@@ -293,6 +299,7 @@
 
 static void _signal_handler(int sig, siginfo_t *siginfo, void *context);
 static bool _stm_validate(void);
+static void _core_commit_transaction(bool external);
 
 static inline bool was_read_remote(char *base, object_t *obj)
 {
diff --git a/rpython/translator/stm/src_stm/stm/detach.c b/rpython/translator/stm/src_stm/stm/detach.c
new file mode 100644
--- /dev/null
+++ b/rpython/translator/stm/src_stm/stm/detach.c
@@ -0,0 +1,175 @@
+/* Imported by rpython/translator/stm/import_stmgc.py */
+#ifndef _STM_CORE_H_
+# error "must be compiled via stmgc.c"
+#endif
+#include <errno.h>
+
+
+/* Idea: if stm_leave_transactional_zone() is quickly followed by
+   stm_enter_transactional_zone() in the same thread, then we should
+   simply try to have one inevitable transaction that does both sides.
+   This is useful if there are many such small interruptions.
+
+   stm_leave_transactional_zone() tries to make sure the transaction
+   is inevitable, and then sticks the current 'stm_thread_local_t *'
+   into _stm_detached_inevitable_from_thread.
+   stm_enter_transactional_zone() has a fast-path if the same
+   'stm_thread_local_t *' is still there.
+
+   If a different thread grabs it, it atomically replaces the value in
+   _stm_detached_inevitable_from_thread with -1, commits it (this part
+   involves reading for example the shadowstack of the thread that
+   originally detached), and at the point where we know the original
+   stm_thread_local_t is no longer relevant, we reset
+   _stm_detached_inevitable_from_thread to 0.
+*/
+
+volatile intptr_t _stm_detached_inevitable_from_thread;
+
+
+static void setup_detach(void)
+{
+    _stm_detached_inevitable_from_thread = 0;
+}
+
+
+void _stm_leave_noninevitable_transactional_zone(void)
+{
+    int saved_errno = errno;
+    dprintf(("leave_noninevitable_transactional_zone\n"));
+    _stm_become_inevitable(MSG_INEV_DONT_SLEEP);
+
+    /* did it work? */
+    if (STM_PSEGMENT->transaction_state == TS_INEVITABLE) {   /* yes */
+        dprintf(("leave_noninevitable_transactional_zone: now inevitable\n"));
+        stm_thread_local_t *tl = STM_SEGMENT->running_thread;
+        _stm_detach_inevitable_transaction(tl);
+    }
+    else {   /* no */
+        dprintf(("leave_noninevitable_transactional_zone: commit\n"));
+        _stm_commit_transaction();
+    }
+    errno = saved_errno;
+}
+
+static void commit_external_inevitable_transaction(void)
+{
+    assert(STM_PSEGMENT->transaction_state == TS_INEVITABLE); /* can't abort */
+    _core_commit_transaction(/*external=*/ true);
+}
+
+void _stm_reattach_transaction(stm_thread_local_t *tl)
+{
+    intptr_t old;
+    int saved_errno = errno;
+ restart:
+    old = _stm_detached_inevitable_from_thread;
+    if (old != 0) {
+        if (old == -1) {
+            /* busy-loop: wait until _stm_detached_inevitable_from_thread
+               is reset to a value different from -1 */
+            dprintf(("reattach_transaction: busy wait...\n"));
+            while (_stm_detached_inevitable_from_thread == -1)
+                spin_loop();
+
+            /* then retry */
+            goto restart;
+        }
+
+        if (!__sync_bool_compare_and_swap(&_stm_detached_inevitable_from_thread,
+                                          old, -1))
+            goto restart;
+
+        stm_thread_local_t *old_tl = (stm_thread_local_t *)old;
+        int remote_seg_num = old_tl->last_associated_segment_num;
+        dprintf(("reattach_transaction: commit detached from seg %d\n",
+                 remote_seg_num));
+
+        tl->last_associated_segment_num = remote_seg_num;
+        ensure_gs_register(remote_seg_num);
+        commit_external_inevitable_transaction();
+    }
+    dprintf(("reattach_transaction: start a new transaction\n"));
+    _stm_start_transaction(tl);
+    errno = saved_errno;
+}
+
+void stm_force_transaction_break(stm_thread_local_t *tl)
+{
+    dprintf(("> stm_force_transaction_break()\n"));
+    assert(STM_SEGMENT->running_thread == tl);
+    _stm_commit_transaction();
+    _stm_start_transaction(tl);
+}
+
+static intptr_t fetch_detached_transaction(void)
+{
+    intptr_t cur;
+ restart:
+    cur = _stm_detached_inevitable_from_thread;
+    if (cur == 0) {    /* fast-path */
+        return 0;   /* _stm_detached_inevitable_from_thread not changed */
+    }
+    if (cur == -1) {
+        /* busy-loop: wait until _stm_detached_inevitable_from_thread
+           is reset to a value different from -1 */
+        while (_stm_detached_inevitable_from_thread == -1)
+            spin_loop();
+        goto restart;
+    }
+    if (!__sync_bool_compare_and_swap(&_stm_detached_inevitable_from_thread,
+                                      cur, -1))
+        goto restart;
+
+    /* this is the only case where we grabbed a detached transaction.
+       _stm_detached_inevitable_from_thread is still -1, until
+       commit_fetched_detached_transaction() is called. */
+    assert(_stm_detached_inevitable_from_thread == -1);
+    return cur;
+}
+
+static void commit_fetched_detached_transaction(intptr_t old)
+{
+    /* Here, 'seg_num' is the segment that contains the detached
+       inevitable transaction from fetch_detached_transaction(),
+       probably belonging to an unrelated thread.  We fetched it,
+       which means that nobody else can concurrently fetch it now, but
+       everybody will see that there is still a concurrent inevitable
+       transaction.  This should guarantee there are no race
+       conditions.
+    */
+    int mysegnum = STM_SEGMENT->segment_num;
+    int segnum = ((stm_thread_local_t *)old)->last_associated_segment_num;
+    dprintf(("commit_fetched_detached_transaction from seg %d\n", segnum));
+    assert(segnum > 0);
+
+    if (segnum != mysegnum) {
+        set_gs_register(get_segment_base(segnum));
+    }
+    commit_external_inevitable_transaction();
+
+    if (segnum != mysegnum) {
+        set_gs_register(get_segment_base(mysegnum));
+    }
+}
+
+static void commit_detached_transaction_if_from(stm_thread_local_t *tl)
+{
+    intptr_t old;
+ restart:
+    old = _stm_detached_inevitable_from_thread;
+    if (old == (intptr_t)tl) {
+        if (!__sync_bool_compare_and_swap(&_stm_detached_inevitable_from_thread,
+                                          old, -1))
+            goto restart;
+        commit_fetched_detached_transaction(old);
+        return;
+    }
+    if (old == -1) {
+        /* busy-loop: wait until _stm_detached_inevitable_from_thread
+           is reset to a value different from -1 */
+        while (_stm_detached_inevitable_from_thread == -1)
+            spin_loop();
+        goto restart;
+    }
+}
diff --git a/rpython/translator/stm/src_stm/stm/detach.h b/rpython/translator/stm/src_stm/stm/detach.h
new file mode 100644
--- /dev/null
+++ b/rpython/translator/stm/src_stm/stm/detach.h
@@ -0,0 +1,5 @@
+/* Imported by rpython/translator/stm/import_stmgc.py */
+static void setup_detach(void);
+static intptr_t fetch_detached_transaction(void);
+static void commit_fetched_detached_transaction(intptr_t old);
+static void commit_detached_transaction_if_from(stm_thread_local_t *tl);
diff --git a/rpython/translator/stm/src_stm/stm/extra.c b/rpython/translator/stm/src_stm/stm/extra.c
--- a/rpython/translator/stm/src_stm/stm/extra.c
+++ b/rpython/translator/stm/src_stm/stm/extra.c
@@ -8,7 +8,7 @@
 {
     dprintf(("register_callbacks: tl=%p key=%p callback=%p index=%ld\n",
              tl, key, callback, index));
-    if (tl->associated_segment_num == -1) {
+    if (!in_transaction(tl)) {
         /* check that the provided thread-local is really running a
            transaction, and do nothing otherwise. */
         dprintf(("  NOT IN TRANSACTION\n"));
diff --git a/rpython/translator/stm/src_stm/stm/finalizer.c b/rpython/translator/stm/src_stm/stm/finalizer.c
--- a/rpython/translator/stm/src_stm/stm/finalizer.c
+++ b/rpython/translator/stm/src_stm/stm/finalizer.c
@@ -494,11 +494,11 @@
 
     rewind_jmp_buf rjbuf;
     stm_rewind_jmp_enterframe(tl, &rjbuf);
-    stm_start_transaction(tl);
+    _stm_start_transaction(tl);
 
     _execute_finalizers(&g_finalizers);
 
-    stm_commit_transaction();
+    _stm_commit_transaction();
     stm_rewind_jmp_leaveframe(tl, &rjbuf);
 
     __sync_lock_release(&lock);
diff --git a/rpython/translator/stm/src_stm/stm/forksupport.c b/rpython/translator/stm/src_stm/stm/forksupport.c
--- a/rpython/translator/stm/src_stm/stm/forksupport.c
+++ b/rpython/translator/stm/src_stm/stm/forksupport.c
@@ -40,7 +40,8 @@
 
     bool was_in_transaction = _stm_in_transaction(this_tl);
     if (!was_in_transaction)
-        stm_start_transaction(this_tl);
+        _stm_start_transaction(this_tl);
+    assert(in_transaction(this_tl));
 
     stm_become_inevitable(this_tl, "fork");
     /* Note that the line above can still fail and abort, which should
@@ -72,7 +73,7 @@
     s_mutex_unlock();
 
     if (!was_in_transaction) {
-        stm_commit_transaction();
+        _stm_commit_transaction();
     }
 
     dprintf(("forksupport_parent: continuing to run\n"));
@@ -83,7 +84,8 @@
     struct stm_priv_segment_info_s *pr = get_priv_segment(i);
     stm_thread_local_t *tl = pr->pub.running_thread;
     dprintf(("forksupport_child: abort in seg%ld\n", i));
-    assert(tl->associated_segment_num == i);
+    assert(tl->last_associated_segment_num == i);
+    assert(in_transaction(tl));
     assert(pr->transaction_state != TS_INEVITABLE);
     set_gs_register(get_segment_base(i));
     assert(STM_SEGMENT->segment_num == i);
@@ -150,14 +152,14 @@
 
     /* Restore a few things: the new pthread_self(), and the %gs
        register */
-    int segnum = fork_this_tl->associated_segment_num;
+    int segnum = fork_this_tl->last_associated_segment_num;
     assert(1 <= segnum && segnum < NB_SEGMENTS);
     *_get_cpth(fork_this_tl) = pthread_self();
     set_gs_register(get_segment_base(segnum));
     assert(STM_SEGMENT->segment_num == segnum);
 
     if (!fork_was_in_transaction) {
-        stm_commit_transaction();
+        _stm_commit_transaction();
     }
 
     /* Done */
diff --git a/rpython/translator/stm/src_stm/stm/fprintcolor.h b/rpython/translator/stm/src_stm/stm/fprintcolor.h
--- a/rpython/translator/stm/src_stm/stm/fprintcolor.h
+++ b/rpython/translator/stm/src_stm/stm/fprintcolor.h
@@ -37,5 +37,6 @@
 /* ------------------------------------------------------------ */
 
 
+__attribute__((unused))
 static void stm_fatalerror(const char *format, ...)
      __attribute__((format (printf, 1, 2), noreturn));
diff --git a/rpython/translator/stm/src_stm/stm/nursery.c b/rpython/translator/stm/src_stm/stm/nursery.c
--- a/rpython/translator/stm/src_stm/stm/nursery.c
+++ b/rpython/translator/stm/src_stm/stm/nursery.c
@@ -309,6 +309,7 @@
     else
         assert(finalbase <= ssbase && ssbase <= current);
 
+    dprintf(("collect_roots_in_nursery:\n"));
     while (current > ssbase) {
         --current;
         uintptr_t x = (uintptr_t)current->ss;
@@ -320,6 +321,7 @@
         else {
             /* it is an odd-valued marker, ignore */
         }
+        dprintf(("    %p: %p -> %p\n", current, (void *)x, current->ss));
     }
 
     minor_trace_if_young(&tl->thread_local_obj);
@@ -519,6 +521,7 @@
 static void _do_minor_collection(bool commit)
 {
     dprintf(("minor_collection commit=%d\n", (int)commit));
+    assert(!STM_SEGMENT->no_safe_point_here);
 
     STM_PSEGMENT->minor_collect_will_commit_now = commit;
 
@@ -561,11 +564,12 @@
     assert(MINOR_NOTHING_TO_DO(STM_PSEGMENT));
 }
 
-static void minor_collection(bool commit)
+static void minor_collection(bool commit, bool external)
 {
     assert(!_has_mutex());
 
-    stm_safe_point();
+    if (!external)
+        stm_safe_point();
 
     timing_event(STM_SEGMENT->running_thread, STM_GC_MINOR_START);
 
@@ -579,7 +583,7 @@
     if (level > 0)
         force_major_collection_request();
 
-    minor_collection(/*commit=*/ false);
+    minor_collection(/*commit=*/ false, /*external=*/ false);
 
 #ifdef STM_TESTS
     /* tests don't want aborts in stm_allocate, thus
diff --git a/rpython/translator/stm/src_stm/stm/nursery.h b/rpython/translator/stm/src_stm/stm/nursery.h
--- a/rpython/translator/stm/src_stm/stm/nursery.h
+++ b/rpython/translator/stm/src_stm/stm/nursery.h
@@ -10,7 +10,7 @@
                                 object_t *obj, uint8_t mark_value,
                                 bool mark_all, bool really_clear);
 
-static void minor_collection(bool commit);
+static void minor_collection(bool commit, bool external);
 static void check_nursery_at_transaction_start(void);
 static size_t throw_away_nursery(struct stm_priv_segment_info_s *pseg);
 static void major_do_validation_and_minor_collections(void);
diff --git a/rpython/translator/stm/src_stm/stm/setup.c b/rpython/translator/stm/src_stm/stm/setup.c
--- a/rpython/translator/stm/src_stm/stm/setup.c
+++ b/rpython/translator/stm/src_stm/stm/setup.c
@@ -134,8 +134,12 @@
     setup_pages();
     setup_forksupport();
     setup_finalizer();
+    setup_detach();
 
     set_gs_register(get_segment_base(0));
+
+    dprintf(("nursery: %p -> %p\n", (void *)NURSERY_START,
+                                    (void *)NURSERY_END));
 }
 
 void stm_teardown(void)
@@ -244,7 +248,6 @@
     /* assign numbers consecutively, but that's for tests; we could also
        assign the same number to all of them and they would get their own
        numbers automatically. */
-    tl->associated_segment_num = -1;
     tl->last_associated_segment_num = num + 1;
     tl->thread_local_counter = ++thread_local_counters;
     *_get_cpth(tl) = pthread_self();
@@ -264,6 +267,8 @@
 
 void stm_unregister_thread_local(stm_thread_local_t *tl)
 {
+    commit_detached_transaction_if_from(tl);
+
     s_mutex_lock();
     assert(tl->prev != NULL);
     assert(tl->next != NULL);
diff --git a/rpython/translator/stm/src_stm/stm/sync.c b/rpython/translator/stm/src_stm/stm/sync.c
--- a/rpython/translator/stm/src_stm/stm/sync.c
+++ b/rpython/translator/stm/src_stm/stm/sync.c
@@ -2,6 +2,7 @@
 #include <sys/syscall.h>
 #include <sys/prctl.h>
 #include <asm/prctl.h>
+#include <time.h>
 #ifndef _STM_CORE_H_
 # error "must be compiled via stmgc.c"
 #endif
@@ -21,25 +22,29 @@
 
 static void setup_sync(void)
 {
-    if (pthread_mutex_init(&sync_ctl.global_mutex, NULL) != 0)
-        stm_fatalerror("mutex initialization: %m");
+    int err = pthread_mutex_init(&sync_ctl.global_mutex, NULL);
+    if (err != 0)
+        stm_fatalerror("mutex initialization: %d", err);
 
     long i;
     for (i = 0; i < _C_TOTAL; i++) {
-        if (pthread_cond_init(&sync_ctl.cond[i], NULL) != 0)
-            stm_fatalerror("cond initialization: %m");
+        err = pthread_cond_init(&sync_ctl.cond[i], NULL);
+        if (err != 0)
+            stm_fatalerror("cond initialization: %d", err);
     }
 }
 
 static void teardown_sync(void)
 {
-    if (pthread_mutex_destroy(&sync_ctl.global_mutex) != 0)
-        stm_fatalerror("mutex destroy: %m");
+    int err = pthread_mutex_destroy(&sync_ctl.global_mutex);
+    if (err != 0)
+        stm_fatalerror("mutex destroy: %d", err);
 
     long i;
     for (i = 0; i < _C_TOTAL; i++) {
-        if (pthread_cond_destroy(&sync_ctl.cond[i]) != 0)
-            stm_fatalerror("cond destroy: %m");
+        err = pthread_cond_destroy(&sync_ctl.cond[i]);
+        if (err != 0)
+            stm_fatalerror("cond destroy: %d", err);
     }
 
     memset(&sync_ctl, 0, sizeof(sync_ctl));
@@ -59,19 +64,30 @@
         stm_fatalerror("syscall(arch_prctl, ARCH_SET_GS): %m");
 }
 
+static void ensure_gs_register(long segnum)
+{
+    /* XXX use this instead of set_gs_register() in many places */
+    if (STM_SEGMENT->segment_num != segnum) {
+        set_gs_register(get_segment_base(segnum));
+        assert(STM_SEGMENT->segment_num == segnum);
+    }
+}
+
 static inline void s_mutex_lock(void)
 {
     assert(!_has_mutex_here);
-    if (UNLIKELY(pthread_mutex_lock(&sync_ctl.global_mutex) != 0))
-        stm_fatalerror("pthread_mutex_lock: %m");
+    int err = pthread_mutex_lock(&sync_ctl.global_mutex);
+    if (UNLIKELY(err != 0))
+        stm_fatalerror("pthread_mutex_lock: %d", err);
     assert((_has_mutex_here = true, 1));
 }
 
 static inline void s_mutex_unlock(void)
 {
     assert(_has_mutex_here);
-    if (UNLIKELY(pthread_mutex_unlock(&sync_ctl.global_mutex) != 0))
-        stm_fatalerror("pthread_mutex_unlock: %m");
+    int err = pthread_mutex_unlock(&sync_ctl.global_mutex);
+    if (UNLIKELY(err != 0))
+        stm_fatalerror("pthread_mutex_unlock: %d", err);
     assert((_has_mutex_here = false, 1));
 }
 
@@ -83,26 +99,70 @@
 #endif
 
     assert(_has_mutex_here);
-    if (UNLIKELY(pthread_cond_wait(&sync_ctl.cond[ctype],
-                                   &sync_ctl.global_mutex) != 0))
-        stm_fatalerror("pthread_cond_wait/%d: %m", (int)ctype);
+    int err = pthread_cond_wait(&sync_ctl.cond[ctype],
+                                &sync_ctl.global_mutex);
+    if (UNLIKELY(err != 0))
+        stm_fatalerror("pthread_cond_wait/%d: %d", (int)ctype, err);
+}
+
+static inline void timespec_delay(struct timespec *t, double incr)
+{
+#ifdef CLOCK_REALTIME
+    clock_gettime(CLOCK_REALTIME, t);
+#else
+    struct timeval tv;
+    RPY_GETTIMEOFDAY(&tv);
+    t->tv_sec = tv.tv_sec;
+    t->tv_nsec = tv.tv_usec * 1000 + 999;
+#endif
+    /* assumes that "incr" is not too large, less than 1 second */
+    long nsec = t->tv_nsec + (long)(incr * 1000000000.0);
+    if (nsec >= 1000000000) {
+        t->tv_sec += 1;
+        nsec -= 1000000000;
+        assert(nsec < 1000000000);
+    }
+    t->tv_nsec = nsec;
+}
+
+static inline bool cond_wait_timeout(enum cond_type_e ctype, double delay)
+{
+#ifdef STM_NO_COND_WAIT
+    stm_fatalerror("*** cond_wait/%d called!", (int)ctype);
+#endif
+
+    assert(_has_mutex_here);
+
+    struct timespec t;
+    timespec_delay(&t, delay);
+
+    int err = pthread_cond_timedwait(&sync_ctl.cond[ctype],
+                                     &sync_ctl.global_mutex, &t);
+    if (err == 0)
+        return true;     /* success */
+    if (LIKELY(err == ETIMEDOUT))
+        return false;    /* timeout */
+    stm_fatalerror("pthread_cond_timedwait/%d: %d", (int)ctype, err);
 }
 
 static inline void cond_signal(enum cond_type_e ctype)
 {
-    if (UNLIKELY(pthread_cond_signal(&sync_ctl.cond[ctype]) != 0))
-        stm_fatalerror("pthread_cond_signal/%d: %m", (int)ctype);
+    int err = pthread_cond_signal(&sync_ctl.cond[ctype]);
+    if (UNLIKELY(err != 0))
+        stm_fatalerror("pthread_cond_signal/%d: %d", (int)ctype, err);
 }
 
 static inline void cond_broadcast(enum cond_type_e ctype)
 {
-    if (UNLIKELY(pthread_cond_broadcast(&sync_ctl.cond[ctype]) != 0))
-        stm_fatalerror("pthread_cond_broadcast/%d: %m", (int)ctype);
+    int err = pthread_cond_broadcast(&sync_ctl.cond[ctype]);
+    if (UNLIKELY(err != 0))
+        stm_fatalerror("pthread_cond_broadcast/%d: %d", (int)ctype, err);
 }
 
 /************************************************************/
 
 
+#if 0
 void stm_wait_for_current_inevitable_transaction(void)
 {
  restart:
@@ -125,7 +185,7 @@
     }
     s_mutex_unlock();
 }
-
+#endif
 
 
 static bool acquire_thread_segment(stm_thread_local_t *tl)
@@ -155,10 +215,12 @@
         num = (num+1) % (NB_SEGMENTS-1);
         if (sync_ctl.in_use1[num+1] == 0) {
             /* we're getting 'num', a different number. */
-            dprintf(("acquired different segment: %d->%d\n",
-                     tl->last_associated_segment_num, num+1));
+            int old_num = tl->last_associated_segment_num;
+            dprintf(("acquired different segment: %d->%d\n", old_num, num+1));
             tl->last_associated_segment_num = num+1;
             set_gs_register(get_segment_base(num+1));
+            dprintf(("                            %d->%d\n", old_num, num+1));
+            (void)old_num;
             goto got_num;
         }
     }
@@ -176,24 +238,31 @@
     sync_ctl.in_use1[num+1] = 1;
     assert(STM_SEGMENT->segment_num == num+1);
     assert(STM_SEGMENT->running_thread == NULL);
-    tl->associated_segment_num = tl->last_associated_segment_num;
+    assert(tl->last_associated_segment_num == STM_SEGMENT->segment_num);
+    assert(!in_transaction(tl));
     STM_SEGMENT->running_thread = tl;
+    assert(in_transaction(tl));
     return true;
 }
 
 static void release_thread_segment(stm_thread_local_t *tl)
 {
+    int segnum;
     assert(_has_mutex());
 
     cond_signal(C_SEGMENT_FREE);
 
     assert(STM_SEGMENT->running_thread == tl);
-    assert(tl->associated_segment_num == tl->last_associated_segment_num);
-    tl->associated_segment_num = -1;
-    STM_SEGMENT->running_thread = NULL;
+    segnum = STM_SEGMENT->segment_num;
+    if (tl != NULL) {
+        assert(tl->last_associated_segment_num == segnum);
+        assert(in_transaction(tl));
+        STM_SEGMENT->running_thread = NULL;
+        assert(!in_transaction(tl));
+    }
 
-    assert(sync_ctl.in_use1[tl->last_associated_segment_num] == 1);
-    sync_ctl.in_use1[tl->last_associated_segment_num] = 0;
+    assert(sync_ctl.in_use1[segnum] == 1);
+    sync_ctl.in_use1[segnum] = 0;
 }
 
 __attribute__((unused))
@@ -204,22 +273,15 @@
 
 bool _stm_in_transaction(stm_thread_local_t *tl)
 {
-    if (tl->associated_segment_num == -1) {
-        return false;
-    }
-    else {
-        int num = tl->associated_segment_num;
-        OPT_ASSERT(1 <= num && num < NB_SEGMENTS);
-        OPT_ASSERT(num == tl->last_associated_segment_num);
-        OPT_ASSERT(get_segment(num)->running_thread == tl);
-        return true;
-    }
+    int num = tl->last_associated_segment_num;
+    OPT_ASSERT(1 <= num && num < NB_SEGMENTS);
+    return in_transaction(tl);
 }
 
 void _stm_test_switch(stm_thread_local_t *tl)
 {
     assert(_stm_in_transaction(tl));
-    set_gs_register(get_segment_base(tl->associated_segment_num));
+    set_gs_register(get_segment_base(tl->last_associated_segment_num));
     assert(STM_SEGMENT->running_thread == tl);
     exec_local_finalizers();
 }
@@ -267,16 +329,19 @@
     }
     assert(!pause_signalled);
     pause_signalled = true;
+    dprintf(("request to pause\n"));
 }
 
 static inline long count_other_threads_sp_running(void)
 {
     /* Return the number of other threads in SP_RUNNING.
-       Asserts that SP_RUNNING threads still have the NSE_SIGxxx. */
+       Asserts that SP_RUNNING threads still have the NSE_SIGxxx.
+       (A detached inevitable transaction is still SP_RUNNING.) */
     long i;
     long result = 0;
-    int my_num = STM_SEGMENT->segment_num;
+    int my_num;
 
+    my_num = STM_SEGMENT->segment_num;
     for (i = 1; i < NB_SEGMENTS; i++) {
         if (i != my_num && get_priv_segment(i)->safe_point == SP_RUNNING) {
             assert(get_segment(i)->nursery_end <= _STM_NSE_SIGNAL_MAX);
@@ -299,6 +364,7 @@
         if (get_segment(i)->nursery_end == NSE_SIGPAUSE)
             get_segment(i)->nursery_end = NURSERY_END;
     }
+    dprintf(("request removed\n"));
     cond_broadcast(C_REQUEST_REMOVED);
 }
 
@@ -316,6 +382,8 @@
         if (STM_SEGMENT->nursery_end == NURSERY_END)
             break;    /* no safe point requested */
 
+        dprintf(("enter safe point\n"));
+        assert(!STM_SEGMENT->no_safe_point_here);
         assert(STM_SEGMENT->nursery_end == NSE_SIGPAUSE);
         assert(pause_signalled);
 
@@ -330,11 +398,15 @@
         cond_wait(C_REQUEST_REMOVED);
         STM_PSEGMENT->safe_point = SP_RUNNING;
         timing_event(STM_SEGMENT->running_thread, STM_WAIT_DONE);
+        assert(!STM_SEGMENT->no_safe_point_here);
+        dprintf(("left safe point\n"));
     }
 }
 
 static void synchronize_all_threads(enum sync_type_e sync_type)
 {
+ restart:
+    assert(_has_mutex());
     enter_safe_point_if_requested();
 
     /* Only one thread should reach this point concurrently.  This is
@@ -353,8 +425,19 @@
     /* If some other threads are SP_RUNNING, we cannot proceed now.
        Wait until all other threads are suspended. */
     while (count_other_threads_sp_running() > 0) {
+
+        intptr_t detached = fetch_detached_transaction();
+        if (detached != 0) {
+            remove_requests_for_safe_point();    /* => C_REQUEST_REMOVED */
+            s_mutex_unlock();
+            commit_fetched_detached_transaction(detached);
+            s_mutex_lock();
+            goto restart;
+        }
+
         STM_PSEGMENT->safe_point = SP_WAIT_FOR_C_AT_SAFE_POINT;
-        cond_wait(C_AT_SAFE_POINT);
+        cond_wait_timeout(C_AT_SAFE_POINT, 0.00001);
+        /* every 10 microsec, try again fetch_detached_transaction() */
         STM_PSEGMENT->safe_point = SP_RUNNING;
 
         if (must_abort()) {
diff --git a/rpython/translator/stm/src_stm/stm/sync.h b/rpython/translator/stm/src_stm/stm/sync.h
--- a/rpython/translator/stm/src_stm/stm/sync.h
+++ b/rpython/translator/stm/src_stm/stm/sync.h
@@ -17,6 +17,7 @@
 static bool _has_mutex(void);
 #endif
 static void set_gs_register(char *value);
+static void ensure_gs_register(long segnum);
 
 
 /* acquire and release one of the segments for running the given thread
diff --git a/rpython/translator/stm/src_stm/stmgc.c b/rpython/translator/stm/src_stm/stmgc.c
--- a/rpython/translator/stm/src_stm/stmgc.c
+++ b/rpython/translator/stm/src_stm/stmgc.c
@@ -19,6 +19,7 @@
 #include "stm/rewind_setjmp.h"
 #include "stm/finalizer.h"
 #include "stm/locks.h"
+#include "stm/detach.h"
 #include "stm/misc.c"
 #include "stm/list.c"
 #include "stm/smallmalloc.c"
@@ -41,3 +42,4 @@
 #include "stm/rewind_setjmp.c"
 #include "stm/finalizer.c"
 #include "stm/hashtable.c"
+#include "stm/detach.c"
diff --git a/rpython/translator/stm/src_stm/stmgc.h b/rpython/translator/stm/src_stm/stmgc.h
--- a/rpython/translator/stm/src_stm/stmgc.h
+++ b/rpython/translator/stm/src_stm/stmgc.h
@@ -13,6 +13,7 @@
 #include <limits.h>
 #include <unistd.h>
 
+#include "stm/atomic.h"
 #include "stm/rewind_setjmp.h"
 
 #if LONG_MAX == 2147483647
@@ -39,6 +40,7 @@
 
 struct stm_segment_info_s {
     uint8_t transaction_read_version;
+    uint8_t no_safe_point_here;    /* set from outside, triggers an assert */
     int segment_num;
     char *segment_base;
     stm_char *nursery_current;
@@ -69,8 +71,7 @@
        (this field is not modified on a successful commit) */
     long last_abort__bytes_in_nursery;
     /* the next fields are handled internally by the library */
-    int associated_segment_num;
-    int last_associated_segment_num;
+    int last_associated_segment_num;   /* always a valid seg num */
     int thread_local_counter;
     struct stm_thread_local_s *prev, *next;
     void *creating_pthread[2];
@@ -83,6 +84,17 @@
 void _stm_write_slowpath_card(object_t *, uintptr_t);
 object_t *_stm_allocate_slowpath(ssize_t);
 object_t *_stm_allocate_external(ssize_t);
+
+extern volatile intptr_t _stm_detached_inevitable_from_thread;
+long _stm_start_transaction(stm_thread_local_t *tl);
+void _stm_commit_transaction(void);
+void _stm_leave_noninevitable_transactional_zone(void);
+#define _stm_detach_inevitable_transaction(tl)  do {            \
+    write_fence();                                              \
+    assert(_stm_detached_inevitable_from_thread == 0);          \
+    _stm_detached_inevitable_from_thread = (intptr_t)(tl);      \
+} while (0)
+void _stm_reattach_transaction(stm_thread_local_t *tl);
 void _stm_become_inevitable(const char*);
 void _stm_collectable_safe_point(void);
 
@@ -380,23 +392,6 @@
     rewind_jmp_enum_shadowstack(&(tl)->rjthread, callback)
 
 
-/* Starting and ending transactions.  stm_read(), stm_write() and
-   stm_allocate() should only be called from within a transaction.
-   The stm_start_transaction() call returns the number of times it
-   returned, starting at 0.  If it is > 0, then the transaction was
-   aborted and restarted this number of times. */
-long stm_start_transaction(stm_thread_local_t *tl);
-void stm_start_inevitable_transaction(stm_thread_local_t *tl);
-void stm_commit_transaction(void);
-
-/* Temporary fix?  Call this outside a transaction.  If there is an
-   inevitable transaction running somewhere else, wait until it finishes. */
-void stm_wait_for_current_inevitable_transaction(void);
-
-/* Abort the currently running transaction.  This function never
-   returns: it jumps back to the stm_start_transaction(). */
-void stm_abort_transaction(void) __attribute__((noreturn));
-
 #ifdef STM_NO_AUTOMATIC_SETJMP
 int stm_is_inevitable(void);
 #else
@@ -405,6 +400,73 @@
 }
 #endif
 
+
+/* Entering and leaving a "transactional code zone": a (typically very
+   large) section in the code where we are running a transaction.
+   This is the STM equivalent to "acquire the GIL" and "release the
+   GIL", respectively.  stm_read(), stm_write(), stm_allocate(), and
+   other functions should only be called from within a transaction.
+
+   Note that transactions, in the STM sense, cover _at least_ one
+   transactional code zone.  They may be longer; for example, if one
+   thread does a lot of stm_enter_transactional_zone() +
+   stm_become_inevitable() + stm_leave_transactional_zone(), as is
+   typical in a thread that does a lot of C function calls, then we
+   get only a few bigger inevitable transactions that cover the many
+   short transactional zones.  This is done by having
+   stm_leave_transactional_zone() turn the current transaction
+   inevitable and detach it from the running thread (if there is no
+   other inevitable transaction running so far).  Then
+   stm_enter_transactional_zone() will try to reattach to it.  This is
+   far more efficient than constantly starting and committing
+   transactions.
+
+   stm_enter_transactional_zone() and stm_leave_transactional_zone()
+   preserve the value of errno.
+*/
+#ifdef STM_DEBUGPRINT
+#include <stdio.h>
+#endif
+static inline void stm_enter_transactional_zone(stm_thread_local_t *tl) {
+    if (__sync_bool_compare_and_swap(&_stm_detached_inevitable_from_thread,
+                                     (intptr_t)tl, 0)) {
+#ifdef STM_DEBUGPRINT
+        fprintf(stderr, "stm_enter_transactional_zone fast path\n");
+#endif
+    }
+    else {
+        _stm_reattach_transaction(tl);
+        /* _stm_detached_inevitable_from_thread should be 0 here, but
+           it can already have been changed from a parallel thread
+           (assuming we're not inevitable ourselves) */
+    }
+}
+static inline void stm_leave_transactional_zone(stm_thread_local_t *tl) {
+    assert(STM_SEGMENT->running_thread == tl);
+    if (stm_is_inevitable()) {
+#ifdef STM_DEBUGPRINT
+        fprintf(stderr, "stm_leave_transactional_zone fast path\n");
+#endif
+        _stm_detach_inevitable_transaction(tl);
+    }
+    else {
+        _stm_leave_noninevitable_transactional_zone();
+    }
+}
+
+/* stm_force_transaction_break() is in theory equivalent to
+   stm_leave_transactional_zone() immediately followed by
+   stm_enter_transactional_zone(); however, it is supposed to be
+   called in CPU-heavy threads that had a transaction run for a while,
+   and so it *always* forces a commit and starts the next transaction.
+   The new transaction is never inevitable. */
+void stm_force_transaction_break(stm_thread_local_t *tl);
+
+/* Abort the currently running transaction.  This function never
+   returns: it jumps back to the start of the transaction (which must
+   not be inevitable). */
+void stm_abort_transaction(void) __attribute__((noreturn));
+
 /* Turn the current transaction inevitable.
    stm_become_inevitable() itself may still abort the transaction instead
    of returning. */
@@ -413,6 +475,8 @@
     assert(STM_SEGMENT->running_thread == tl);
     if (!stm_is_inevitable())
         _stm_become_inevitable(msg);
+    /* now, we're running the inevitable transaction, so this var should be 0 */
+    assert(_stm_detached_inevitable_from_thread == 0);
 }
 
 /* Forces a safe-point if needed.  Normally not needed: this is
@@ -467,8 +531,8 @@
    other threads.  A very heavy-handed way to make sure that no other
    transaction is running concurrently.  Avoid as much as possible.
    Other transactions will continue running only after this transaction
-   commits.  (xxx deprecated and may be removed) */
-void stm_become_globally_unique_transaction(stm_thread_local_t *tl, const char *msg);
+   commits.  (deprecated, not working any more according to demo_random2) */
+//void stm_become_globally_unique_transaction(stm_thread_local_t *tl, const char *msg);
 
 /* Moves the transaction forward in time by validating the read and
    write set with all commits that happened since the last validation

From noreply at buildbot.pypy.org  Fri Jun 12 19:00:35 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 12 Jun 2015 19:00:35 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c8-gil-like: Yay,
	the first ztranslation test passes
Message-ID: <20150612170035.6CFCF1C033F@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c8-gil-like
Changeset: r78049:1fd7822491a3
Date: 2015-06-12 19:00 +0200
http://bitbucket.org/pypy/pypy/changeset/1fd7822491a3/

Log:	Yay, the first ztranslation test passes

diff --git a/rpython/rlib/rstm.py b/rpython/rlib/rstm.py
--- a/rpython/rlib/rstm.py
+++ b/rpython/rlib/rstm.py
@@ -43,10 +43,10 @@
 
 adr_pypy__rewind_jmp_copy_stack_slice = (
     CFlexSymbolic('((long)&pypy__rewind_jmp_copy_stack_slice)'))
-adr_pypy_stm_commit_if_not_atomic = (
-    CFlexSymbolic('((long)&pypy_stm_commit_if_not_atomic)'))
-adr_pypy_stm_start_if_not_atomic = (
-    CFlexSymbolic('((long)&pypy_stm_start_if_not_atomic)'))
+#adr_pypy_stm_commit_if_not_atomic = (
+#    CFlexSymbolic('((long)&pypy_stm_commit_if_not_atomic)'))
+#adr_pypy_stm_start_if_not_atomic = (
+#    CFlexSymbolic('((long)&pypy_stm_start_if_not_atomic)'))
 
 
 def rewind_jmp_frame():
@@ -123,16 +123,14 @@
 @dont_look_inside
 def before_external_call():
     if we_are_translated():
-        # this tries to commit, or becomes inevitable if atomic
-        llop.stm_commit_if_not_atomic(lltype.Void)
+        llop.stm_leave_transactional_zone(lltype.Void)
 before_external_call._dont_reach_me_in_del_ = True
 before_external_call._transaction_break_ = True
 
 @dont_look_inside
 def after_external_call():
     if we_are_translated():
-        # starts a new transaction if we are not atomic already
-        llop.stm_start_if_not_atomic(lltype.Void)
+        llop.stm_enter_transactional_zone(lltype.Void)
 after_external_call._dont_reach_me_in_del_ = True
 after_external_call._transaction_break_ = True
 
diff --git a/rpython/rtyper/llinterp.py b/rpython/rtyper/llinterp.py
--- a/rpython/rtyper/llinterp.py
+++ b/rpython/rtyper/llinterp.py
@@ -969,8 +969,8 @@
     op_stm_push_root = _stm_not_implemented
     op_stm_pop_root_into = _stm_not_implemented
     op_stm_get_root_stack_top = _stm_not_implemented
-    op_stm_start_if_not_atomic = _stm_not_implemented
-    op_stm_commit_if_not_atomic = _stm_not_implemented
+    op_stm_enter_transactional_zone = _stm_not_implemented
+    op_stm_leave_transactional_zone = _stm_not_implemented
     op_stm_enter_callback_call = _stm_not_implemented
     op_stm_leave_callback_call = _stm_not_implemented
     op_stm_get_atomic = _stm_not_implemented
diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -434,8 +434,8 @@
     'stm_become_inevitable':  LLOp(canmallocgc=True),
     'stm_push_root':          LLOp(),
     'stm_pop_root_into':      LLOp(),
-    'stm_commit_if_not_atomic':           LLOp(canmallocgc=True),
-    'stm_start_if_not_atomic':            LLOp(canmallocgc=True),
+    'stm_enter_transactional_zone':       LLOp(canmallocgc=True),
+    'stm_leave_transactional_zone':       LLOp(canmallocgc=True),
     'stm_abort_and_retry':                LLOp(canmallocgc=True),
     'stm_enter_callback_call':            LLOp(canmallocgc=True),
     'stm_leave_callback_call':            LLOp(),
diff --git a/rpython/translator/stm/breakfinder.py b/rpython/translator/stm/breakfinder.py
--- a/rpython/translator/stm/breakfinder.py
+++ b/rpython/translator/stm/breakfinder.py
@@ -3,8 +3,8 @@
 
 
 TRANSACTION_BREAK = set([
-    'stm_commit_if_not_atomic',
-    'stm_start_if_not_atomic',
+    'stm_enter_transactional_zone',
+    'stm_leave_transactional_zone',
     #'jit_assembler_call',
     'stm_enter_callback_call',
     'stm_leave_callback_call',
diff --git a/rpython/translator/stm/funcgen.py b/rpython/translator/stm/funcgen.py
--- a/rpython/translator/stm/funcgen.py
+++ b/rpython/translator/stm/funcgen.py
@@ -44,7 +44,7 @@
 
 
 def stm_hint_commit_soon(funcgen, op):
-    return 'stmcb_commit_soon();'
+    return '/* stmcb_commit_soon(); XXX */'
 
 def stm_register_thread_local(funcgen, op):
     return 'pypy_stm_register_thread_local();'
@@ -166,7 +166,7 @@
     except AttributeError:
         pass
     string_literal = c_string_constant(info)
-    return 'pypy_stm_become_inevitable(%s);' % (string_literal,)
+    return 'stm_become_inevitable(&stm_thread_local, %s);' % (string_literal,)
 
 def stm_stop_all_other_threads(funcgen, op):
     return 'stm_stop_all_other_threads();'
@@ -184,11 +184,11 @@
         return '/* %s = */ STM_POP_ROOT_RET(stm_thread_local);' % (arg0,)
     return 'STM_POP_ROOT(stm_thread_local, %s);' % (arg0,)
 
-def stm_commit_if_not_atomic(funcgen, op):
-   return 'pypy_stm_commit_if_not_atomic();'
+def stm_enter_transactional_zone(funcgen, op):
+    return 'stm_enter_transactional_zone(&stm_thread_local);'
 
-def stm_start_if_not_atomic(funcgen, op):
-    return 'pypy_stm_start_if_not_atomic();'
+def stm_leave_transactional_zone(funcgen, op):
+    return 'stm_leave_transactional_zone(&stm_thread_local);'
 
 def stm_enter_callback_call(funcgen, op):
     arg0 = funcgen.expr(op.args[0])
@@ -202,24 +202,23 @@
 
 def stm_should_break_transaction(funcgen, op):
     result = funcgen.expr(op.result)
-    return '%s = pypy_stm_should_break_transaction();' % (result,)
+    return '%s = stm_should_break_transaction();' % (result,)
 
 def stm_set_transaction_length(funcgen, op):
     arg0 = funcgen.expr(op.args[0])
-    return 'pypy_stm_set_transaction_length(%s);' % (arg0,)
+    return 'stm_fill_mark_nursery_bytes = %s;' % (arg0,)
 
 def stm_transaction_break(funcgen, op):
-    return 'pypy_stm_transaction_break();'
+    return 'stm_force_transaction_break(&stm_thread_local);'
 
 def stm_increment_atomic(funcgen, op):
-    return 'pypy_stm_increment_atomic();'
+    XXX
 
 def stm_decrement_atomic(funcgen, op):
-    return 'pypy_stm_decrement_atomic();'
+    XXX
 
 def stm_get_atomic(funcgen, op):
-    result = funcgen.expr(op.result)
-    return '%s = pypy_stm_get_atomic();' % (result,)
+    XXX
 
 def stm_is_inevitable(funcgen, op):
     result = funcgen.expr(op.result)
diff --git a/rpython/translator/stm/src_stm/extracode.h b/rpython/translator/stm/src_stm/extracode.h
--- a/rpython/translator/stm/src_stm/extracode.h
+++ b/rpython/translator/stm/src_stm/extracode.h
@@ -47,6 +47,7 @@
 void pypy_stm_unregister_thread_local(void)
 {
     stm_unregister_thread_local(&stm_thread_local);
+    stm_thread_local.shadowstack_base = NULL;
 }
 
 
diff --git a/rpython/translator/stm/src_stm/revision b/rpython/translator/stm/src_stm/revision
--- a/rpython/translator/stm/src_stm/revision
+++ b/rpython/translator/stm/src_stm/revision
@@ -1,1 +1,1 @@
-bf0dfe206de5
+5af967809206
diff --git a/rpython/translator/stm/src_stm/stm/core.c b/rpython/translator/stm/src_stm/stm/core.c
--- a/rpython/translator/stm/src_stm/stm/core.c
+++ b/rpython/translator/stm/src_stm/stm/core.c
@@ -1154,7 +1154,7 @@
 #endif
     STM_PSEGMENT->shadowstack_at_start_of_transaction = tl->shadowstack;
     STM_PSEGMENT->threadlocal_at_start_of_transaction = tl->thread_local_obj;
-
+    STM_PSEGMENT->total_throw_away_nursery = 0;
 
     assert(list_is_empty(STM_PSEGMENT->modified_old_objects));
     assert(list_is_empty(STM_PSEGMENT->large_overflow_objects));
@@ -1195,15 +1195,26 @@
     stm_validate();
 }
 
+#ifdef STM_NO_AUTOMATIC_SETJMP
+static int did_abort = 0;
+#endif
+
 long _stm_start_transaction(stm_thread_local_t *tl)
 {
     s_mutex_lock();
 #ifdef STM_NO_AUTOMATIC_SETJMP
-    long repeat_count = 0;    /* test/support.py */
+    long repeat_count = did_abort;    /* test/support.py */
+    did_abort = 0;
 #else
     long repeat_count = stm_rewind_jmp_setjmp(tl);
 #endif
     _do_start_transaction(tl);
+
+    if (repeat_count == 0) {  /* else, 'nursery_mark' was already set
+                                 in abort_data_structures_from_segment_num() */
+        STM_SEGMENT->nursery_mark = ((stm_char *)_stm_nursery_start +
+                                     stm_fill_mark_nursery_bytes);
+    }
     return repeat_count;
 }
 
@@ -1427,7 +1438,7 @@
 
     abort_finalizers(pseg);
 
-    long bytes_in_nursery = throw_away_nursery(pseg);
+    throw_away_nursery(pseg);
 
     /* clear CARD_MARKED on objs (don't care about CARD_MARKED_OLD) */
     LIST_FOREACH_R(pseg->old_objects_with_cards_set, object_t * /*item*/,
@@ -1461,7 +1472,26 @@
     assert(tl->shadowstack == pseg->shadowstack_at_start_of_transaction);
 #endif
     tl->thread_local_obj = pseg->threadlocal_at_start_of_transaction;
-    tl->last_abort__bytes_in_nursery = bytes_in_nursery;
+
+
+    /* Set the next nursery_mark: first compute the value that
+       nursery_mark must have had at the start of the aborted transaction */
+    stm_char *old_mark =pseg->pub.nursery_mark + pseg->total_throw_away_nursery;
+
+    /* This means that the limit, in term of bytes, was: */
+    uintptr_t old_limit = old_mark - (stm_char *)_stm_nursery_start;
+
+    /* If 'total_throw_away_nursery' is smaller than old_limit, use that */
+    if (pseg->total_throw_away_nursery < old_limit)
+        old_limit = pseg->total_throw_away_nursery;
+
+    /* Now set the new limit to 90% of the old limit */
+    pseg->pub.nursery_mark = ((stm_char *)_stm_nursery_start +
+                              (uintptr_t)(old_limit * 0.9));
+
+#ifdef STM_NO_AUTOMATIC_SETJMP
+    did_abort = 1;
+#endif
 
     list_clear(pseg->objects_pointing_to_nursery);
     list_clear(pseg->old_objects_with_cards_set);
diff --git a/rpython/translator/stm/src_stm/stm/core.h b/rpython/translator/stm/src_stm/stm/core.h
--- a/rpython/translator/stm/src_stm/stm/core.h
+++ b/rpython/translator/stm/src_stm/stm/core.h
@@ -152,6 +152,9 @@
     stm_char *sq_fragments[SYNC_QUEUE_SIZE];
     int sq_fragsizes[SYNC_QUEUE_SIZE];
     int sq_len;
+
+    /* For nursery_mark */
+    uintptr_t total_throw_away_nursery;
 };
 
 enum /* safe_point */ {
diff --git a/rpython/translator/stm/src_stm/stm/nursery.c b/rpython/translator/stm/src_stm/stm/nursery.c
--- a/rpython/translator/stm/src_stm/stm/nursery.c
+++ b/rpython/translator/stm/src_stm/stm/nursery.c
@@ -11,8 +11,13 @@
 static uintptr_t _stm_nursery_start;
 
 
+#define DEFAULT_FILL_MARK_NURSERY_BYTES   (NURSERY_SIZE / 4)
+
+uintptr_t stm_fill_mark_nursery_bytes = DEFAULT_FILL_MARK_NURSERY_BYTES;
+
 /************************************************************/
 
+
 static void setup_nursery(void)
 {
     assert(_STM_FAST_ALLOC <= NURSERY_SIZE);
@@ -449,7 +454,7 @@
 }
 
 
-static size_t throw_away_nursery(struct stm_priv_segment_info_s *pseg)
+static void throw_away_nursery(struct stm_priv_segment_info_s *pseg)
 {
 #pragma push_macro("STM_PSEGMENT")
 #pragma push_macro("STM_SEGMENT")
@@ -482,7 +487,9 @@
 #endif
 #endif
 
+    pseg->total_throw_away_nursery += nursery_used;
     pseg->pub.nursery_current = (stm_char *)_stm_nursery_start;
+    pseg->pub.nursery_mark -= nursery_used;
 
     /* free any object left from 'young_outside_nursery' */
     if (!tree_is_cleared(pseg->young_outside_nursery)) {
@@ -507,8 +514,6 @@
     }
 
     tree_clear(pseg->nursery_objects_shadows);
-
-    return nursery_used;
 #pragma pop_macro("STM_SEGMENT")
 #pragma pop_macro("STM_PSEGMENT")
 }
diff --git a/rpython/translator/stm/src_stm/stm/nursery.h b/rpython/translator/stm/src_stm/stm/nursery.h
--- a/rpython/translator/stm/src_stm/stm/nursery.h
+++ b/rpython/translator/stm/src_stm/stm/nursery.h
@@ -12,7 +12,7 @@
 
 static void minor_collection(bool commit, bool external);
 static void check_nursery_at_transaction_start(void);
-static size_t throw_away_nursery(struct stm_priv_segment_info_s *pseg);
+static void throw_away_nursery(struct stm_priv_segment_info_s *pseg);
 static void major_do_validation_and_minor_collections(void);
 
 static void assert_memset_zero(void *s, size_t n);
diff --git a/rpython/translator/stm/src_stm/stmgc.h b/rpython/translator/stm/src_stm/stmgc.h
--- a/rpython/translator/stm/src_stm/stmgc.h
+++ b/rpython/translator/stm/src_stm/stmgc.h
@@ -44,6 +44,7 @@
     int segment_num;
     char *segment_base;
     stm_char *nursery_current;
+    stm_char *nursery_mark;
     uintptr_t nursery_end;
     struct stm_thread_local_s *running_thread;
 };
@@ -67,9 +68,6 @@
        the following raw region of memory is cleared. */
     char *mem_clear_on_abort;
     size_t mem_bytes_to_clear_on_abort;
-    /* after an abort, some details about the abort are stored there.
-       (this field is not modified on a successful commit) */
-    long last_abort__bytes_in_nursery;
     /* the next fields are handled internally by the library */
     int last_associated_segment_num;   /* always a valid seg num */
     int thread_local_counter;
@@ -459,7 +457,8 @@
    stm_enter_transactional_zone(); however, it is supposed to be
    called in CPU-heavy threads that had a transaction run for a while,
    and so it *always* forces a commit and starts the next transaction.
-   The new transaction is never inevitable. */
+   The new transaction is never inevitable.  See also
+   stm_should_break_transaction(). */
 void stm_force_transaction_break(stm_thread_local_t *tl);
 
 /* Abort the currently running transaction.  This function never
@@ -490,6 +489,23 @@
 void stm_collect(long level);
 
 
+/* A way to detect that we've run for a while and should call
+   stm_force_transaction_break() */
+static inline int stm_should_break_transaction(void)
+{
+    return ((intptr_t)STM_SEGMENT->nursery_current >=
+            (intptr_t)STM_SEGMENT->nursery_mark);
+}
+extern uintptr_t stm_fill_mark_nursery_bytes;
+/* ^^^ at the start of a transaction, 'nursery_mark' is initialized to
+   'stm_fill_mark_nursery_bytes' inside the nursery.  This value can
+   be larger than the nursery; every minor collection shifts the
+   current 'nursery_mark' down by one nursery-size.  After an abort
+   and restart, 'nursery_mark' is set to ~90% of the value it reached
+   in the last attempt.
+*/
+
+
 /* Prepare an immortal "prebuilt" object managed by the GC.  Takes a
    pointer to an 'object_t', which should not actually be a GC-managed
    structure but a real static structure.  Returns the equivalent
diff --git a/rpython/translator/stm/src_stm/stmgcintf.c b/rpython/translator/stm/src_stm/stmgcintf.c
--- a/rpython/translator/stm/src_stm/stmgcintf.c
+++ b/rpython/translator/stm/src_stm/stmgcintf.c
@@ -2,13 +2,8 @@
 /* This is not meant to be compiled stand-alone, but with all
    of PyPy's #defines and #includes prepended. */
 
-__thread struct stm_thread_local_s stm_thread_local __attribute__((aligned(64)));
-
-/* 0 = not initialized; 1 = normal mode; 2 or more = atomic mode */
-__thread long pypy_stm_ready_atomic;
-__thread uintptr_t pypy_stm_nursery_low_fill_mark;
-__thread uintptr_t pypy_stm_nursery_low_fill_mark_saved;
-
+__thread
+struct stm_thread_local_s stm_thread_local __attribute__((aligned(64)));
 
 
 extern Signed pypy_stmcb_size_rounded_up(void*);
@@ -43,19 +38,6 @@
     pypy_stmcb_trace_cards(obj, (void(*)(void*))visit, start, stop);
 }
 
-inline void stmcb_commit_soon()
-{
-    if (pypy_stm_nursery_low_fill_mark == (uintptr_t)-1) {
-        /* atomic */
-        if (((long)pypy_stm_nursery_low_fill_mark_saved) > 0) {
-            pypy_stm_nursery_low_fill_mark_saved = 0;
-        }
-    } else if (((long)pypy_stm_nursery_low_fill_mark) > 0) {
-        /* if not set to unlimited by pypy_stm_setup() (s.b.) */
-        pypy_stm_nursery_low_fill_mark = 0;
-    }
-}
-
 
 /************************************************************/
 /* "include" the stmgc.c file here */
@@ -64,33 +46,22 @@
 /************************************************************/
 
 
-#define LOW_FILL_MARK  (NURSERY_SIZE / 2)
-
-static long pypy_transaction_length;
-
-
-void pypy_stm_set_transaction_length(double fraction)
-{
-    /* the value '1.0' means 'use the default'.  Other values are
-       interpreted proportionally, up to some maximum. */
-    long low_fill_mark = (long)(LOW_FILL_MARK * fraction);
-    if (low_fill_mark > (long)(NURSERY_SIZE * 3 / 4))
-        low_fill_mark = NURSERY_SIZE * 3 / 4;
-    pypy_transaction_length = low_fill_mark;
-}
-
 void pypy_stm_setup(void)
 {
     stm_setup();
     pypy_stm_setup_prebuilt();
 
     pypy_stm_register_thread_local();
-    pypy_stm_ready_atomic = 1;
-    /* set transaction length to unlimited until the first thread
-       starts. pypy_stm_set_transaction_length will then be called
+    /* set transaction length to a very large limit until the first
+       thread starts. stm_set_transaction_length() will then be called
        again by pypy. */
-    pypy_stm_set_transaction_length(-10000.0);
-    pypy_stm_start_inevitable_if_not_atomic();
+    stm_fill_mark_nursery_bytes = 1024 * NURSERY_SIZE;
+
+    rewind_jmp_buf rjbuf;
+    stm_rewind_jmp_enterframe(&stm_thread_local, &rjbuf);
+    stm_enter_transactional_zone(&stm_thread_local);
+    stm_become_inevitable(&stm_thread_local, "start-up");
+    stm_rewind_jmp_leaveframe(&stm_thread_local, &rjbuf);
 }
 
 void pypy_stm_teardown(void)
@@ -101,128 +72,48 @@
 
 long pypy_stm_enter_callback_call(void *rjbuf)
 {
-    if (pypy_stm_ready_atomic == 0) {
+    if (stm_thread_local.shadowstack_base == NULL) {
         /* first time we see this thread */
-        assert(pypy_transaction_length >= 0);
         int e = errno;
         pypy_stm_register_thread_local();
         stm_rewind_jmp_enterprepframe(&stm_thread_local,
                                       (rewind_jmp_buf *)rjbuf);
         errno = e;
-        pypy_stm_ready_atomic = 1;
-        pypy_stm_start_if_not_atomic();
+        stm_enter_transactional_zone(&stm_thread_local);
         return 1;
     }
     else {
         /* callback from C code, itself called from Python code */
         stm_rewind_jmp_enterprepframe(&stm_thread_local,
                                       (rewind_jmp_buf *)rjbuf);
-        pypy_stm_start_if_not_atomic();
+        stm_enter_transactional_zone(&stm_thread_local);
         return 0;
     }
 }
 
 void pypy_stm_leave_callback_call(void *rjbuf, long token)
 {
-    int e = errno;
+    stm_leave_transactional_zone(&stm_thread_local);
+    stm_rewind_jmp_leaveframe(&stm_thread_local, (rewind_jmp_buf *)rjbuf);
+
     if (token == 1) {
         /* if we're returning into foreign C code that was not itself
            called from Python code, then we're ignoring the atomic
            status and committing anyway. */
-        pypy_stm_ready_atomic = 1;
-        stm_commit_transaction();
-        pypy_stm_ready_atomic = 0;
-        stm_rewind_jmp_leaveframe(&stm_thread_local, (rewind_jmp_buf *)rjbuf);
+        int e = errno;
         pypy_stm_unregister_thread_local();
-    }
-    else {
-        pypy_stm_commit_if_not_atomic();
-        stm_rewind_jmp_leaveframe(&stm_thread_local, (rewind_jmp_buf *)rjbuf);
-    }
-    errno = e;
-}
-
-void _pypy_stm_initialize_nursery_low_fill_mark(long v_counter)
-{
-    /* If v_counter==0, initialize 'pypy_stm_nursery_low_fill_mark'
-       from the configured length limit.  If v_counter>0, we did an
-       abort, and we now configure 'pypy_stm_nursery_low_fill_mark'
-       to a value slightly smaller than the value at last abort.
-    */
-    long counter, limit;
-#ifdef HTM_INFO_AVAILABLE
-    if (_htm_info.use_gil)
-        counter = 0;            /* maybe we want the default size here... */
-    else
-        counter = _htm_info.retry_counter;
-    limit = pypy_transaction_length >> counter;
-#else
-    counter = v_counter;
-
-    if (counter == 0) {
-        limit = pypy_transaction_length;
-    }
-    else {
-        limit = stm_thread_local.last_abort__bytes_in_nursery;
-        limit -= (limit >> 4);
-    }
-#endif
-
-    pypy_stm_nursery_low_fill_mark = _stm_nursery_start + limit;
-}
-
-void _pypy_stm_start_transaction(void)
-{
-    pypy_stm_nursery_low_fill_mark = 1;  /* will be set to a correct value below */
-    long counter = stm_start_transaction(&stm_thread_local);
-
-    _pypy_stm_initialize_nursery_low_fill_mark(counter);
-
-    pypy_stm_ready_atomic = 1; /* reset after abort */
-}
-
-void _pypy_stm_start_transaction_save_errno_wait_inev(void)
-{
-    int e = errno;
-    stm_wait_for_current_inevitable_transaction();
-    _pypy_stm_start_transaction();
-    errno = e;
-}
-
-void pypy_stm_transaction_break(void)
-{
-    assert(pypy_stm_nursery_low_fill_mark != (uintptr_t) -1);
-    stm_commit_transaction();
-    _pypy_stm_start_transaction();
-}
-
-void _pypy_stm_inev_state(void)
-{
-    /* Reduce the limit so that inevitable transactions are generally
-       shorter. We depend a bit on stmcb_commit_soon() in order for
-       other transactions to signal us in case we block them. */
-    long t;
-    if (pypy_stm_ready_atomic == 1) {
-        t = (long)pypy_stm_nursery_low_fill_mark;
-        t = _stm_nursery_start + ((t - (long)_stm_nursery_start) >> 2);
-        pypy_stm_nursery_low_fill_mark = t;
-    }
-    else {
-        assert(pypy_stm_nursery_low_fill_mark == (uintptr_t) -1);
-        t = (long)pypy_stm_nursery_low_fill_mark_saved;
-        t = _stm_nursery_start + ((t - (long)_stm_nursery_start) >> 2);
-        pypy_stm_nursery_low_fill_mark_saved = t;
+        errno = e;
     }
 }
 
-void _pypy_stm_become_inevitable(const char *msg)
+/*void _pypy_stm_become_inevitable(const char *msg)
 {
     _pypy_stm_inev_state();
     if (msg == NULL) {
         msg = "return from JITted function";
     }
     _stm_become_inevitable(msg);
-}
+}*/
 
 long _pypy_stm_count(void)
 {
diff --git a/rpython/translator/stm/src_stm/stmgcintf.h b/rpython/translator/stm/src_stm/stmgcintf.h
--- a/rpython/translator/stm/src_stm/stmgcintf.h
+++ b/rpython/translator/stm/src_stm/stmgcintf.h
@@ -2,21 +2,10 @@
 #define _RPY_STMGCINTF_H
 
 
-/* meant to be #included after src_stm/stmgc.h */
-
 #include <errno.h>
 #include "stmgc.h"
-#include "stm/atomic.h"    /* for spin_loop(), write_fence(), spinlock_xxx() */
 
 extern __thread struct stm_thread_local_s stm_thread_local;
-extern __thread long pypy_stm_ready_atomic;
-extern __thread uintptr_t pypy_stm_nursery_low_fill_mark;
-extern __thread uintptr_t pypy_stm_nursery_low_fill_mark_saved;
-/* Invariant: if we're running a transaction:
-   - if it is atomic, pypy_stm_nursery_low_fill_mark == (uintptr_t) -1
-   - otherwise, if it is inevitable, pypy_stm_nursery_low_fill_mark == 0
-   - otherwise, it's a fraction of the nursery size strictly between 0 and 1
-*/
 
 void pypy_stm_setup(void);
 void pypy_stm_teardown(void);
@@ -26,13 +15,6 @@
 
 void pypy_stm_memclearinit(object_t *obj, size_t offset, size_t size);
 
-void _pypy_stm_initialize_nursery_low_fill_mark(long v_counter);
-void _pypy_stm_inev_state(void);
-void _pypy_stm_start_transaction(void);
-void _pypy_stm_start_transaction_save_errno_wait_inev(void);
-
-void _pypy_stm_become_inevitable(const char *);
-
 char *_pypy_stm_test_expand_marker(void);
 void pypy_stm_setup_expand_marker(long co_filename_ofs,
                                   long co_name_ofs,
@@ -41,88 +23,11 @@
 
 long _pypy_stm_count(void);
 
-
-
-/* C8: not implemented properly yet: */
-extern void stmcb_commit_soon(void);
-/* C8: not implemented properly yet ^^^^^^^^^^^^^^^^^^ */
-
-
-static inline void pypy_stm_become_inevitable(const char *msg)
-{
-    assert(STM_SEGMENT->running_thread == &stm_thread_local);
-    if (!stm_is_inevitable()) {
-        _pypy_stm_become_inevitable(msg);
-    }
-}
-static inline void pypy_stm_commit_if_not_atomic(void) {
-    int e = errno;
-    if (pypy_stm_ready_atomic == 1) {
-        stm_commit_transaction();
-    }
-    else {
-        pypy_stm_become_inevitable("commit_if_not_atomic in atomic");
-    }
-    errno = e;
-}
-static inline void pypy_stm_start_if_not_atomic(void) {
-    if (pypy_stm_ready_atomic == 1)
-        _pypy_stm_start_transaction_save_errno_wait_inev();
-}
-static inline void pypy_stm_start_inevitable_if_not_atomic(void) {
-    if (pypy_stm_ready_atomic == 1) {
-        int e = errno;
-        stm_start_inevitable_transaction(&stm_thread_local);
-        _pypy_stm_initialize_nursery_low_fill_mark(0);
-        _pypy_stm_inev_state();
-        errno = e;
-    }
-}
-static inline void pypy_stm_increment_atomic(void) {
-    switch (++pypy_stm_ready_atomic) {
-    case 2:
-        assert(pypy_stm_nursery_low_fill_mark != (uintptr_t) -1);
-        pypy_stm_nursery_low_fill_mark_saved = pypy_stm_nursery_low_fill_mark;
-        pypy_stm_nursery_low_fill_mark = (uintptr_t) -1;
-        break;
-    default:
-        break;
-    }
-}
-static inline void pypy_stm_decrement_atomic(void) {
-    switch (--pypy_stm_ready_atomic) {
-    case 1:
-        pypy_stm_nursery_low_fill_mark = pypy_stm_nursery_low_fill_mark_saved;
-        assert(pypy_stm_nursery_low_fill_mark != (uintptr_t) -1);
-        break;
-    case 0:
-        pypy_stm_ready_atomic = 1;
-        break;
-    default:
-        break;
-    }
-}
-static inline long pypy_stm_get_atomic(void) {
-    return pypy_stm_ready_atomic - 1;
-}
 long pypy_stm_enter_callback_call(void *);
 void pypy_stm_leave_callback_call(void *, long);
 void pypy_stm_set_transaction_length(double);
 void pypy_stm_transaction_break(void);
 
-static inline int pypy_stm_should_break_transaction(void)
-{
-    /* we should break the current transaction if we have used more than
-       some initial portion of the nursery, or if we are running inevitable
-       (in which case pypy_stm_nursery_low_fill_mark is set to 0).
-       If the transaction is atomic, pypy_stm_nursery_low_fill_mark is
-       instead set to (uintptr_t) -1, and the following check is never true.
-    */
-    uintptr_t current = (uintptr_t)STM_SEGMENT->nursery_current;
-    return current > pypy_stm_nursery_low_fill_mark;
-    /* NB. this logic is hard-coded in jit/backend/x86/assembler.py too */
-}
-
 static void pypy__rewind_jmp_copy_stack_slice(void)
 {
     _rewind_jmp_copy_stack_slice(&stm_thread_local.rjthread);
diff --git a/rpython/translator/stm/test/test_readbarrier.py b/rpython/translator/stm/test/test_readbarrier.py
--- a/rpython/translator/stm/test/test_readbarrier.py
+++ b/rpython/translator/stm/test/test_readbarrier.py
@@ -230,9 +230,9 @@
         def f1(f):
             x.a = f
             t = x.a # no read barrier
-            llop.stm_commit_if_not_atomic(lltype.Void)
+            llop.stm_leave_transactional_zone(lltype.Void)
             t += x.a
-            llop.stm_start_if_not_atomic(lltype.Void)
+            llop.stm_enter_transactional_zone(lltype.Void)
             t += x.a
             llop.stm_transaction_break(lltype.Void)
             t += x.a
diff --git a/rpython/translator/stm/test/transform_support.py b/rpython/translator/stm/test/transform_support.py
--- a/rpython/translator/stm/test/transform_support.py
+++ b/rpython/translator/stm/test/transform_support.py
@@ -123,10 +123,10 @@
     def op_stm_transaction_break(self):
         self.transaction_break()
 
-    def op_stm_commit_if_not_atomic(self):
+    def op_stm_leave_transactional_zone(self):
         self.transaction_break()
 
-    def op_stm_start_if_not_atomic(self):
+    def op_stm_enter_transactional_zone(self):
         self.transaction_break()
 
     def op_stm_enter_callback_call(self):

From noreply at buildbot.pypy.org  Fri Jun 12 19:01:11 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 12 Jun 2015 19:01:11 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: Add an assert
Message-ID: <20150612170111.D71671C033F@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: optresult
Changeset: r78050:85c11181735e
Date: 2015-06-12 10:28 +0200
http://bitbucket.org/pypy/pypy/changeset/85c11181735e/

Log:	Add an assert

diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py
--- a/rpython/jit/backend/llsupport/rewrite.py
+++ b/rpython/jit/backend/llsupport/rewrite.py
@@ -119,6 +119,7 @@
         #
         for i in range(len(operations)):
             op = operations[i]
+            assert op.get_forwarded() is None
             if op.getopnum() == rop.DEBUG_MERGE_POINT:
                 continue
             # ---------- turn NEWxxx into CALL_MALLOC_xxx ----------

From noreply at buildbot.pypy.org  Fri Jun 12 19:01:13 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 12 Jun 2015 19:01:13 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: merge heads
Message-ID: <20150612170113.1AC951C033F@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: optresult
Changeset: r78051:c751966834b2
Date: 2015-06-12 19:01 +0200
http://bitbucket.org/pypy/pypy/changeset/c751966834b2/

Log:	merge heads

diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py
--- a/rpython/jit/backend/llsupport/rewrite.py
+++ b/rpython/jit/backend/llsupport/rewrite.py
@@ -119,6 +119,7 @@
         #
         for i in range(len(operations)):
             op = operations[i]
+            assert op.get_forwarded() is None
             if op.getopnum() == rop.DEBUG_MERGE_POINT:
                 continue
             # ---------- turn NEWxxx into CALL_MALLOC_xxx ----------

From noreply at buildbot.pypy.org  Fri Jun 12 19:02:28 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 12 Jun 2015 19:02:28 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c8-gil-like: fix test
Message-ID: <20150612170228.AC61A1C033F@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c8-gil-like
Changeset: r78052:c16c27a09a61
Date: 2015-06-12 19:02 +0200
http://bitbucket.org/pypy/pypy/changeset/c16c27a09a61/

Log:	fix test

diff --git a/rpython/translator/stm/test/test_ztranslated.py b/rpython/translator/stm/test/test_ztranslated.py
--- a/rpython/translator/stm/test/test_ztranslated.py
+++ b/rpython/translator/stm/test/test_ztranslated.py
@@ -75,8 +75,7 @@
             glob.seen = None
             rthread.start_new_thread(threadfn, ())
             while glob.seen is None:
-                llop.stm_commit_if_not_atomic(lltype.Void)
-                llop.stm_start_if_not_atomic(lltype.Void)
+                llop.stm_transaction_break(lltype.Void)
             return glob.seen.value
         #
         t, cbuilder = self.compile(entry_point)

From noreply at buildbot.pypy.org  Fri Jun 12 19:12:49 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 12 Jun 2015 19:12:49 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c8-gil-like: Fix tests
Message-ID: <20150612171249.1B5781C024E@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c8-gil-like
Changeset: r78053:b64a78fbfd32
Date: 2015-06-12 19:12 +0200
http://bitbucket.org/pypy/pypy/changeset/b64a78fbfd32/

Log:	Fix tests

diff --git a/rpython/translator/stm/funcgen.py b/rpython/translator/stm/funcgen.py
--- a/rpython/translator/stm/funcgen.py
+++ b/rpython/translator/stm/funcgen.py
@@ -205,20 +205,21 @@
     return '%s = stm_should_break_transaction();' % (result,)
 
 def stm_set_transaction_length(funcgen, op):
-    arg0 = funcgen.expr(op.args[0])
-    return 'stm_fill_mark_nursery_bytes = %s;' % (arg0,)
+    arg0 = funcgen.expr(op.args[0])     # 'double': fraction
+    return 'pypy_stm_set_transaction_length(%s);' % (arg0,)
 
 def stm_transaction_break(funcgen, op):
     return 'stm_force_transaction_break(&stm_thread_local);'
 
 def stm_increment_atomic(funcgen, op):
-    XXX
+    return r'fprintf(stderr, "stm_increment_atomic: reimplement\n"); abort();'
 
 def stm_decrement_atomic(funcgen, op):
-    XXX
+    return r'fprintf(stderr, "stm_decrement_atomic: reimplement\n"); abort();'
 
 def stm_get_atomic(funcgen, op):
-    XXX
+    result = funcgen.expr(op.result)
+    return '%s = 0;  // XXX stm_get_atomic' % (result,)
 
 def stm_is_inevitable(funcgen, op):
     result = funcgen.expr(op.result)
diff --git a/rpython/translator/stm/src_stm/stmgcintf.c b/rpython/translator/stm/src_stm/stmgcintf.c
--- a/rpython/translator/stm/src_stm/stmgcintf.c
+++ b/rpython/translator/stm/src_stm/stmgcintf.c
@@ -64,6 +64,11 @@
     stm_rewind_jmp_leaveframe(&stm_thread_local, &rjbuf);
 }
 
+void pypy_stm_set_transaction_length(double fraction)
+{
+    stm_fill_mark_nursery_bytes = (uintptr_t)(NURSERY_SIZE * fraction / 4);
+}
+
 void pypy_stm_teardown(void)
 {
     pypy_stm_unregister_thread_local();
diff --git a/rpython/translator/stm/test/test_ztranslated.py b/rpython/translator/stm/test/test_ztranslated.py
--- a/rpython/translator/stm/test/test_ztranslated.py
+++ b/rpython/translator/stm/test/test_ztranslated.py
@@ -385,8 +385,7 @@
         def main(argv):
             lst[42] = 43
             lst2[999] = lst
-            llop.stm_commit_if_not_atomic(lltype.Void)
-            llop.stm_start_if_not_atomic(lltype.Void)
+            llop.stm_transaction_break(lltype.Void)
             print 'did not crash', lst2[999][42]
             return 0
 

From noreply at buildbot.pypy.org  Fri Jun 12 19:16:08 2015
From: noreply at buildbot.pypy.org (amauryfa)
Date: Fri, 12 Jun 2015 19:16:08 +0200 (CEST)
Subject: [pypy-commit] pypy stdlib-2.7.10: Close branch about to be merged
Message-ID: <20150612171608.499A91C024E@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: stdlib-2.7.10
Changeset: r78054:358cf80306cd
Date: 2015-06-12 19:13 +0200
http://bitbucket.org/pypy/pypy/changeset/358cf80306cd/

Log:	Close branch about to be merged


From noreply at buildbot.pypy.org  Fri Jun 12 19:16:11 2015
From: noreply at buildbot.pypy.org (amauryfa)
Date: Fri, 12 Jun 2015 19:16:11 +0200 (CEST)
Subject: [pypy-commit] pypy default: hg merge stdlib-2.7.10
Message-ID: <20150612171611.24B401C024E@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: 
Changeset: r78055:2198166ce926
Date: 2015-06-12 19:14 +0200
http://bitbucket.org/pypy/pypy/changeset/2198166ce926/

Log:	hg merge stdlib-2.7.10

diff too long, truncating to 2000 out of 10216 lines

diff --git a/lib-python/2.7/Cookie.py b/lib-python/2.7/Cookie.py
--- a/lib-python/2.7/Cookie.py
+++ b/lib-python/2.7/Cookie.py
@@ -528,12 +528,13 @@
 # result, the parsing rules here are less strict.
 #
 
-_LegalCharsPatt  = r"[\w\d!#%&'~_`><@,:/\$\*\+\-\.\^\|\)\(\?\}\{\=]"
+_LegalKeyChars  = r"\w\d!#%&'~_`><@,:/\$\*\+\-\.\^\|\)\(\?\}\{\="
+_LegalValueChars = _LegalKeyChars + r"\[\]"
 _CookiePattern = re.compile(
     r"(?x)"                       # This is a Verbose pattern
     r"\s*"                        # Optional whitespace at start of cookie
     r"(?P<key>"                   # Start of group 'key'
-    ""+ _LegalCharsPatt +"+?"     # Any word of at least one letter, nongreedy
+    "["+ _LegalKeyChars +"]+?"     # Any word of at least one letter, nongreedy
     r")"                          # End of group 'key'
     r"("                          # Optional group: there may not be a value.
     r"\s*=\s*"                    # Equal Sign
@@ -542,7 +543,7 @@
     r"|"                            # or
     r"\w{3},\s[\s\w\d-]{9,11}\s[\d:]{8}\sGMT" # Special case for "expires" attr
     r"|"                            # or
-    ""+ _LegalCharsPatt +"*"        # Any word or empty string
+    "["+ _LegalValueChars +"]*"        # Any word or empty string
     r")"                          # End of group 'val'
     r")?"                         # End of optional value group
     r"\s*"                        # Any number of spaces.
diff --git a/lib-python/2.7/SimpleHTTPServer.py b/lib-python/2.7/SimpleHTTPServer.py
--- a/lib-python/2.7/SimpleHTTPServer.py
+++ b/lib-python/2.7/SimpleHTTPServer.py
@@ -14,6 +14,7 @@
 import posixpath
 import BaseHTTPServer
 import urllib
+import urlparse
 import cgi
 import sys
 import shutil
@@ -68,10 +69,14 @@
         path = self.translate_path(self.path)
         f = None
         if os.path.isdir(path):
-            if not self.path.endswith('/'):
+            parts = urlparse.urlsplit(self.path)
+            if not parts.path.endswith('/'):
                 # redirect browser - doing basically what apache does
                 self.send_response(301)
-                self.send_header("Location", self.path + "/")
+                new_parts = (parts[0], parts[1], parts[2] + '/',
+                             parts[3], parts[4])
+                new_url = urlparse.urlunsplit(new_parts)
+                self.send_header("Location", new_url)
                 self.end_headers()
                 return None
             for index in "index.html", "index.htm":
diff --git a/lib-python/2.7/_LWPCookieJar.py b/lib-python/2.7/_LWPCookieJar.py
--- a/lib-python/2.7/_LWPCookieJar.py
+++ b/lib-python/2.7/_LWPCookieJar.py
@@ -18,7 +18,7 @@
                        iso2time, time2isoz)
 
 def lwp_cookie_str(cookie):
-    """Return string representation of Cookie in an the LWP cookie file format.
+    """Return string representation of Cookie in the LWP cookie file format.
 
     Actually, the format is extended a bit -- see module docstring.
 
diff --git a/lib-python/2.7/_abcoll.py b/lib-python/2.7/_abcoll.py
--- a/lib-python/2.7/_abcoll.py
+++ b/lib-python/2.7/_abcoll.py
@@ -548,23 +548,25 @@
             If E present and lacks .keys() method, does:     for (k, v) in E: D[k] = v
             In either case, this is followed by: for k, v in F.items(): D[k] = v
         '''
-        if len(args) > 2:
-            raise TypeError("update() takes at most 2 positional "
-                            "arguments ({} given)".format(len(args)))
-        elif not args:
-            raise TypeError("update() takes at least 1 argument (0 given)")
+        if not args:
+            raise TypeError("descriptor 'update' of 'MutableMapping' object "
+                            "needs an argument")
         self = args[0]
-        other = args[1] if len(args) >= 2 else ()
-
-        if isinstance(other, Mapping):
-            for key in other:
-                self[key] = other[key]
-        elif hasattr(other, "keys"):
-            for key in other.keys():
-                self[key] = other[key]
-        else:
-            for key, value in other:
-                self[key] = value
+        args = args[1:]
+        if len(args) > 1:
+            raise TypeError('update expected at most 1 arguments, got %d' %
+                            len(args))
+        if args:
+            other = args[0]
+            if isinstance(other, Mapping):
+                for key in other:
+                    self[key] = other[key]
+            elif hasattr(other, "keys"):
+                for key in other.keys():
+                    self[key] = other[key]
+            else:
+                for key, value in other:
+                    self[key] = value
         for key, value in kwds.items():
             self[key] = value
 
diff --git a/lib-python/2.7/_pyio.py b/lib-python/2.7/_pyio.py
--- a/lib-python/2.7/_pyio.py
+++ b/lib-python/2.7/_pyio.py
@@ -25,8 +25,8 @@
 DEFAULT_BUFFER_SIZE = 8 * 1024  # bytes
 
 # NOTE: Base classes defined here are registered with the "official" ABCs
-# defined in io.py. We don't use real inheritance though, because we don't
-# want to inherit the C implementations.
+# defined in io.py. We don't use real inheritance though, because we don't want
+# to inherit the C implementations.
 
 
 class BlockingIOError(IOError):
@@ -775,7 +775,7 @@
         clsname = self.__class__.__name__
         try:
             name = self.name
-        except AttributeError:
+        except Exception:
             return "<_pyio.{0}>".format(clsname)
         else:
             return "<_pyio.{0} name={1!r}>".format(clsname, name)
@@ -1216,8 +1216,10 @@
         return self.writer.flush()
 
     def close(self):
-        self.writer.close()
-        self.reader.close()
+        try:
+            self.writer.close()
+        finally:
+            self.reader.close()
 
     def isatty(self):
         return self.reader.isatty() or self.writer.isatty()
@@ -1538,7 +1540,7 @@
     def __repr__(self):
         try:
             name = self.name
-        except AttributeError:
+        except Exception:
             return "<_pyio.TextIOWrapper encoding='{0}'>".format(self.encoding)
         else:
             return "<_pyio.TextIOWrapper name={0!r} encoding='{1}'>".format(
diff --git a/lib-python/2.7/_strptime.py b/lib-python/2.7/_strptime.py
--- a/lib-python/2.7/_strptime.py
+++ b/lib-python/2.7/_strptime.py
@@ -335,9 +335,9 @@
     # though
     week_of_year = -1
     week_of_year_start = -1
-    # weekday and julian defaulted to -1 so as to signal need to calculate
+    # weekday and julian defaulted to None so as to signal need to calculate
     # values
-    weekday = julian = -1
+    weekday = julian = None
     found_dict = found.groupdict()
     for group_key in found_dict.iterkeys():
         # Directives not explicitly handled below:
@@ -434,14 +434,14 @@
         year = 1900
     # If we know the week of the year and what day of that week, we can figure
     # out the Julian day of the year.
-    if julian == -1 and week_of_year != -1 and weekday != -1:
+    if julian is None and week_of_year != -1 and weekday is not None:
         week_starts_Mon = True if week_of_year_start == 0 else False
         julian = _calc_julian_from_U_or_W(year, week_of_year, weekday,
                                             week_starts_Mon)
     # Cannot pre-calculate datetime_date() since can change in Julian
     # calculation and thus could have different value for the day of the week
     # calculation.
-    if julian == -1:
+    if julian is None:
         # Need to add 1 to result since first day of the year is 1, not 0.
         julian = datetime_date(year, month, day).toordinal() - \
                   datetime_date(year, 1, 1).toordinal() + 1
@@ -451,7 +451,7 @@
         year = datetime_result.year
         month = datetime_result.month
         day = datetime_result.day
-    if weekday == -1:
+    if weekday is None:
         weekday = datetime_date(year, month, day).weekday()
     if leap_year_fix:
         # the caller didn't supply a year but asked for Feb 29th. We couldn't
diff --git a/lib-python/2.7/aifc.py b/lib-python/2.7/aifc.py
--- a/lib-python/2.7/aifc.py
+++ b/lib-python/2.7/aifc.py
@@ -357,10 +357,13 @@
         self._soundpos = 0
 
     def close(self):
-        if self._decomp:
-            self._decomp.CloseDecompressor()
-            self._decomp = None
-        self._file.close()
+        decomp = self._decomp
+        try:
+            if decomp:
+                self._decomp = None
+                decomp.CloseDecompressor()
+        finally:
+            self._file.close()
 
     def tell(self):
         return self._soundpos
diff --git a/lib-python/2.7/binhex.py b/lib-python/2.7/binhex.py
--- a/lib-python/2.7/binhex.py
+++ b/lib-python/2.7/binhex.py
@@ -32,7 +32,8 @@
     pass
 
 # States (what have we written)
-[_DID_HEADER, _DID_DATA, _DID_RSRC] = range(3)
+_DID_HEADER = 0
+_DID_DATA = 1
 
 # Various constants
 REASONABLY_LARGE=32768  # Minimal amount we pass the rle-coder
@@ -235,17 +236,22 @@
         self._write(data)
 
     def close(self):
-        if self.state < _DID_DATA:
-            self.close_data()
-        if self.state != _DID_DATA:
-            raise Error, 'Close at the wrong time'
-        if self.rlen != 0:
-            raise Error, \
-                  "Incorrect resource-datasize, diff=%r" % (self.rlen,)
-        self._writecrc()
-        self.ofp.close()
-        self.state = None
-        del self.ofp
+        if self.state is None:
+            return
+        try:
+            if self.state < _DID_DATA:
+                self.close_data()
+            if self.state != _DID_DATA:
+                raise Error, 'Close at the wrong time'
+            if self.rlen != 0:
+                raise Error, \
+                    "Incorrect resource-datasize, diff=%r" % (self.rlen,)
+            self._writecrc()
+        finally:
+            self.state = None
+            ofp = self.ofp
+            del self.ofp
+            ofp.close()
 
 def binhex(inp, out):
     """(infilename, outfilename) - Create binhex-encoded copy of a file"""
@@ -463,11 +469,15 @@
         return self._read(n)
 
     def close(self):
-        if self.rlen:
-            dummy = self.read_rsrc(self.rlen)
-        self._checkcrc()
-        self.state = _DID_RSRC
-        self.ifp.close()
+        if self.state is None:
+            return
+        try:
+            if self.rlen:
+                dummy = self.read_rsrc(self.rlen)
+            self._checkcrc()
+        finally:
+            self.state = None
+            self.ifp.close()
 
 def hexbin(inp, out):
     """(infilename, outfilename) - Decode binhexed file"""
diff --git a/lib-python/2.7/bsddb/test/test_all.py b/lib-python/2.7/bsddb/test/test_all.py
--- a/lib-python/2.7/bsddb/test/test_all.py
+++ b/lib-python/2.7/bsddb/test/test_all.py
@@ -412,9 +412,6 @@
         def get_dbp(self) :
             return self._db
 
-    import string
-    string.letters=[chr(i) for i in xrange(65,91)]
-
     bsddb._db.DBEnv_orig = bsddb._db.DBEnv
     bsddb._db.DB_orig = bsddb._db.DB
     if bsddb.db.version() <= (4, 3) :
diff --git a/lib-python/2.7/bsddb/test/test_basics.py b/lib-python/2.7/bsddb/test/test_basics.py
--- a/lib-python/2.7/bsddb/test/test_basics.py
+++ b/lib-python/2.7/bsddb/test/test_basics.py
@@ -999,7 +999,7 @@
         for x in "The quick brown fox jumped over the lazy dog".split():
             d2.put(x, self.makeData(x))
 
-        for x in string.letters:
+        for x in string.ascii_letters:
             d3.put(x, x*70)
 
         d1.sync()
@@ -1047,7 +1047,7 @@
             if verbose:
                 print rec
             rec = c3.next()
-        self.assertEqual(count, len(string.letters))
+        self.assertEqual(count, len(string.ascii_letters))
 
 
         c1.close()
diff --git a/lib-python/2.7/bsddb/test/test_dbshelve.py b/lib-python/2.7/bsddb/test/test_dbshelve.py
--- a/lib-python/2.7/bsddb/test/test_dbshelve.py
+++ b/lib-python/2.7/bsddb/test/test_dbshelve.py
@@ -59,7 +59,7 @@
             return bytes(key, "iso8859-1")  # 8 bits
 
     def populateDB(self, d):
-        for x in string.letters:
+        for x in string.ascii_letters:
             d[self.mk('S' + x)] = 10 * x           # add a string
             d[self.mk('I' + x)] = ord(x)           # add an integer
             d[self.mk('L' + x)] = [x] * 10         # add a list
diff --git a/lib-python/2.7/bsddb/test/test_get_none.py b/lib-python/2.7/bsddb/test/test_get_none.py
--- a/lib-python/2.7/bsddb/test/test_get_none.py
+++ b/lib-python/2.7/bsddb/test/test_get_none.py
@@ -26,14 +26,14 @@
         d.open(self.filename, db.DB_BTREE, db.DB_CREATE)
         d.set_get_returns_none(1)
 
-        for x in string.letters:
+        for x in string.ascii_letters:
             d.put(x, x * 40)
 
         data = d.get('bad key')
         self.assertEqual(data, None)
 
-        data = d.get(string.letters[0])
-        self.assertEqual(data, string.letters[0]*40)
+        data = d.get(string.ascii_letters[0])
+        self.assertEqual(data, string.ascii_letters[0]*40)
 
         count = 0
         c = d.cursor()
@@ -43,7 +43,7 @@
             rec = c.next()
 
         self.assertEqual(rec, None)
-        self.assertEqual(count, len(string.letters))
+        self.assertEqual(count, len(string.ascii_letters))
 
         c.close()
         d.close()
@@ -54,14 +54,14 @@
         d.open(self.filename, db.DB_BTREE, db.DB_CREATE)
         d.set_get_returns_none(0)
 
-        for x in string.letters:
+        for x in string.ascii_letters:
             d.put(x, x * 40)
 
         self.assertRaises(db.DBNotFoundError, d.get, 'bad key')
         self.assertRaises(KeyError, d.get, 'bad key')
 
-        data = d.get(string.letters[0])
-        self.assertEqual(data, string.letters[0]*40)
+        data = d.get(string.ascii_letters[0])
+        self.assertEqual(data, string.ascii_letters[0]*40)
 
         count = 0
         exceptionHappened = 0
@@ -77,7 +77,7 @@
 
         self.assertNotEqual(rec, None)
         self.assertTrue(exceptionHappened)
-        self.assertEqual(count, len(string.letters))
+        self.assertEqual(count, len(string.ascii_letters))
 
         c.close()
         d.close()
diff --git a/lib-python/2.7/bsddb/test/test_queue.py b/lib-python/2.7/bsddb/test/test_queue.py
--- a/lib-python/2.7/bsddb/test/test_queue.py
+++ b/lib-python/2.7/bsddb/test/test_queue.py
@@ -10,7 +10,6 @@
 
 #----------------------------------------------------------------------
 
- at unittest.skip("fails on Windows; see issue 22943")
 class SimpleQueueTestCase(unittest.TestCase):
     def setUp(self):
         self.filename = get_new_database_path()
@@ -37,17 +36,17 @@
             print "before appends" + '-' * 30
             pprint(d.stat())
 
-        for x in string.letters:
+        for x in string.ascii_letters:
             d.append(x * 40)
 
-        self.assertEqual(len(d), len(string.letters))
+        self.assertEqual(len(d), len(string.ascii_letters))
 
         d.put(100, "some more data")
         d.put(101, "and some more ")
         d.put(75,  "out of order")
         d.put(1,   "replacement data")
 
-        self.assertEqual(len(d), len(string.letters)+3)
+        self.assertEqual(len(d), len(string.ascii_letters)+3)
 
         if verbose:
             print "before close" + '-' * 30
@@ -108,17 +107,17 @@
             print "before appends" + '-' * 30
             pprint(d.stat())
 
-        for x in string.letters:
+        for x in string.ascii_letters:
             d.append(x * 40)
 
-        self.assertEqual(len(d), len(string.letters))
+        self.assertEqual(len(d), len(string.ascii_letters))
 
         d.put(100, "some more data")
         d.put(101, "and some more ")
         d.put(75,  "out of order")
         d.put(1,   "replacement data")
 
-        self.assertEqual(len(d), len(string.letters)+3)
+        self.assertEqual(len(d), len(string.ascii_letters)+3)
 
         if verbose:
             print "before close" + '-' * 30
diff --git a/lib-python/2.7/bsddb/test/test_recno.py b/lib-python/2.7/bsddb/test/test_recno.py
--- a/lib-python/2.7/bsddb/test/test_recno.py
+++ b/lib-python/2.7/bsddb/test/test_recno.py
@@ -4,12 +4,11 @@
 import os, sys
 import errno
 from pprint import pprint
+import string
 import unittest
 
 from test_all import db, test_support, verbose, get_new_environment_path, get_new_database_path
 
-letters = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
-
 
 #----------------------------------------------------------------------
 
@@ -39,7 +38,7 @@
 
         d.open(self.filename, db.DB_RECNO, db.DB_CREATE)
 
-        for x in letters:
+        for x in string.ascii_letters:
             recno = d.append(x * 60)
             self.assertIsInstance(recno, int)
             self.assertGreaterEqual(recno, 1)
@@ -270,7 +269,7 @@
         d.set_re_pad(45)  # ...test both int and char
         d.open(self.filename, db.DB_RECNO, db.DB_CREATE)
 
-        for x in letters:
+        for x in string.ascii_letters:
             d.append(x * 35)    # These will be padded
 
         d.append('.' * 40)      # this one will be exact
diff --git a/lib-python/2.7/chunk.py b/lib-python/2.7/chunk.py
--- a/lib-python/2.7/chunk.py
+++ b/lib-python/2.7/chunk.py
@@ -85,8 +85,10 @@
 
     def close(self):
         if not self.closed:
-            self.skip()
-            self.closed = True
+            try:
+                self.skip()
+            finally:
+                self.closed = True
 
     def isatty(self):
         if self.closed:
diff --git a/lib-python/2.7/codecs.py b/lib-python/2.7/codecs.py
--- a/lib-python/2.7/codecs.py
+++ b/lib-python/2.7/codecs.py
@@ -20,8 +20,14 @@
            "BOM_LE", "BOM32_BE", "BOM32_LE", "BOM64_BE", "BOM64_LE",
            "BOM_UTF8", "BOM_UTF16", "BOM_UTF16_LE", "BOM_UTF16_BE",
            "BOM_UTF32", "BOM_UTF32_LE", "BOM_UTF32_BE",
+           "CodecInfo", "Codec", "IncrementalEncoder", "IncrementalDecoder",
+           "StreamReader", "StreamWriter",
+           "StreamReaderWriter", "StreamRecoder",
+           "getencoder", "getdecoder", "getincrementalencoder",
+           "getincrementaldecoder", "getreader", "getwriter",
+           "encode", "decode", "iterencode", "iterdecode",
            "strict_errors", "ignore_errors", "replace_errors",
-           "xmlcharrefreplace_errors",
+           "xmlcharrefreplace_errors", "backslashreplace_errors",
            "register_error", "lookup_error"]
 
 ### Constants
@@ -1051,7 +1057,7 @@
         during translation.
 
         One example where this happens is cp875.py which decodes
-        multiple character to \u001a.
+        multiple character to \\u001a.
 
     """
     m = {}
diff --git a/lib-python/2.7/collections.py b/lib-python/2.7/collections.py
--- a/lib-python/2.7/collections.py
+++ b/lib-python/2.7/collections.py
@@ -330,7 +330,7 @@
     #   http://code.activestate.com/recipes/259174/
     #   Knuth, TAOCP Vol. II section 4.6.3
 
-    def __init__(self, iterable=None, **kwds):
+    def __init__(*args, **kwds):
         '''Create a new, empty Counter object.  And if given, count elements
         from an input iterable.  Or, initialize the count from another mapping
         of elements to their counts.
@@ -341,8 +341,15 @@
         >>> c = Counter(a=4, b=2)                   # a new counter from keyword args
 
         '''
+        if not args:
+            raise TypeError("descriptor '__init__' of 'Counter' object "
+                            "needs an argument")
+        self = args[0]
+        args = args[1:]
+        if len(args) > 1:
+            raise TypeError('expected at most 1 arguments, got %d' % len(args))
         super(Counter, self).__init__()
-        self.update(iterable, **kwds)
+        self.update(*args, **kwds)
 
     def __missing__(self, key):
         'The count of elements not in the Counter is zero.'
@@ -393,7 +400,7 @@
         raise NotImplementedError(
             'Counter.fromkeys() is undefined.  Use Counter(iterable) instead.')
 
-    def update(self, iterable=None, **kwds):
+    def update(*args, **kwds):
         '''Like dict.update() but add counts instead of replacing them.
 
         Source can be an iterable, a dictionary, or another Counter instance.
@@ -413,6 +420,14 @@
         # contexts.  Instead, we implement straight-addition.  Both the inputs
         # and outputs are allowed to contain zero and negative counts.
 
+        if not args:
+            raise TypeError("descriptor 'update' of 'Counter' object "
+                            "needs an argument")
+        self = args[0]
+        args = args[1:]
+        if len(args) > 1:
+            raise TypeError('expected at most 1 arguments, got %d' % len(args))
+        iterable = args[0] if args else None
         if iterable is not None:
             if isinstance(iterable, Mapping):
                 if self:
@@ -428,7 +443,7 @@
         if kwds:
             self.update(kwds)
 
-    def subtract(self, iterable=None, **kwds):
+    def subtract(*args, **kwds):
         '''Like dict.update() but subtracts counts instead of replacing them.
         Counts can be reduced below zero.  Both the inputs and outputs are
         allowed to contain zero and negative counts.
@@ -444,6 +459,14 @@
         -1
 
         '''
+        if not args:
+            raise TypeError("descriptor 'subtract' of 'Counter' object "
+                            "needs an argument")
+        self = args[0]
+        args = args[1:]
+        if len(args) > 1:
+            raise TypeError('expected at most 1 arguments, got %d' % len(args))
+        iterable = args[0] if args else None
         if iterable is not None:
             self_get = self.get
             if isinstance(iterable, Mapping):
diff --git a/lib-python/2.7/cookielib.py b/lib-python/2.7/cookielib.py
--- a/lib-python/2.7/cookielib.py
+++ b/lib-python/2.7/cookielib.py
@@ -464,26 +464,42 @@
     for ns_header in ns_headers:
         pairs = []
         version_set = False
-        for ii, param in enumerate(re.split(r";\s*", ns_header)):
-            param = param.rstrip()
-            if param == "": continue
-            if "=" not in param:
-                k, v = param, None
-            else:
-                k, v = re.split(r"\s*=\s*", param, 1)
-                k = k.lstrip()
+
+        # XXX: The following does not strictly adhere to RFCs in that empty
+        # names and values are legal (the former will only appear once and will
+        # be overwritten if multiple occurrences are present). This is
+        # mostly to deal with backwards compatibility.
+        for ii, param in enumerate(ns_header.split(';')):
+            param = param.strip()
+
+            key, sep, val = param.partition('=')
+            key = key.strip()
+
+            if not key:
+                if ii == 0:
+                    break
+                else:
+                    continue
+
+            # allow for a distinction between present and empty and missing
+            # altogether
+            val = val.strip() if sep else None
+
             if ii != 0:
-                lc = k.lower()
+                lc = key.lower()
                 if lc in known_attrs:
-                    k = lc
-                if k == "version":
+                    key = lc
+
+                if key == "version":
                     # This is an RFC 2109 cookie.
-                    v = _strip_quotes(v)
+                    if val is not None:
+                        val = _strip_quotes(val)
                     version_set = True
-                if k == "expires":
+                elif key == "expires":
                     # convert expires date to seconds since epoch
-                    v = http2time(_strip_quotes(v))  # None if invalid
-            pairs.append((k, v))
+                    if val is not None:
+                        val = http2time(_strip_quotes(val))  # None if invalid
+            pairs.append((key, val))
 
         if pairs:
             if not version_set:
diff --git a/lib-python/2.7/ctypes/macholib/fetch_macholib.bat b/lib-python/2.7/ctypes/macholib/fetch_macholib.bat
--- a/lib-python/2.7/ctypes/macholib/fetch_macholib.bat
+++ b/lib-python/2.7/ctypes/macholib/fetch_macholib.bat
@@ -1,1 +1,1 @@
-svn export --force http://svn.red-bean.com/bob/macholib/trunk/macholib/ .
+svn export --force http://svn.red-bean.com/bob/macholib/trunk/macholib/ .
diff --git a/lib-python/2.7/ctypes/test/test_find.py b/lib-python/2.7/ctypes/test/test_find.py
--- a/lib-python/2.7/ctypes/test/test_find.py
+++ b/lib-python/2.7/ctypes/test/test_find.py
@@ -32,15 +32,24 @@
     def setUp(self):
         self.gl = self.glu = self.gle = None
         if lib_gl:
-            self.gl = CDLL(lib_gl, mode=RTLD_GLOBAL)
+            try:
+                self.gl = CDLL(lib_gl, mode=RTLD_GLOBAL)
+            except OSError:
+                pass
         if lib_glu:
-            self.glu = CDLL(lib_glu, RTLD_GLOBAL)
+            try:
+                self.glu = CDLL(lib_glu, RTLD_GLOBAL)
+            except OSError:
+                pass
         if lib_gle:
             try:
                 self.gle = CDLL(lib_gle)
             except OSError:
                 pass
 
+    def tearDown(self):
+        self.gl = self.glu = self.gle = None
+
     @unittest.skipUnless(lib_gl, 'lib_gl not available')
     def test_gl(self):
         if self.gl:
diff --git a/lib-python/2.7/ctypes/test/test_pickling.py b/lib-python/2.7/ctypes/test/test_pickling.py
--- a/lib-python/2.7/ctypes/test/test_pickling.py
+++ b/lib-python/2.7/ctypes/test/test_pickling.py
@@ -15,9 +15,9 @@
 class Y(X):
     _fields_ = [("str", c_char_p)]
 
-class PickleTest(unittest.TestCase):
+class PickleTest:
     def dumps(self, item):
-        return pickle.dumps(item)
+        return pickle.dumps(item, self.proto)
 
     def loads(self, item):
         return pickle.loads(item)
@@ -72,17 +72,15 @@
 
     @xfail
     def test_wchar(self):
-        pickle.dumps(c_char("x"))
+        self.dumps(c_char(b"x"))
         # Issue 5049
-        pickle.dumps(c_wchar(u"x"))
+        self.dumps(c_wchar(u"x"))
 
-class PickleTest_1(PickleTest):
-    def dumps(self, item):
-        return pickle.dumps(item, 1)
-
-class PickleTest_2(PickleTest):
-    def dumps(self, item):
-        return pickle.dumps(item, 2)
+for proto in range(pickle.HIGHEST_PROTOCOL + 1):
+    name = 'PickleTest_%s' % proto
+    globals()[name] = type(name,
+                           (PickleTest, unittest.TestCase),
+                           {'proto': proto})
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/lib-python/2.7/ctypes/test/test_pointers.py b/lib-python/2.7/ctypes/test/test_pointers.py
--- a/lib-python/2.7/ctypes/test/test_pointers.py
+++ b/lib-python/2.7/ctypes/test/test_pointers.py
@@ -7,8 +7,6 @@
                  c_long, c_ulong, c_longlong, c_ulonglong, c_double, c_float]
 python_types = [int, int, int, int, int, long,
                 int, long, long, long, float, float]
-LargeNamedType = type('T' * 2 ** 25, (Structure,), {})
-large_string = 'T' * 2 ** 25
 
 class PointersTestCase(unittest.TestCase):
 
@@ -191,9 +189,11 @@
             self.assertEqual(bool(mth), True)
 
     def test_pointer_type_name(self):
+        LargeNamedType = type('T' * 2 ** 25, (Structure,), {})
         self.assertTrue(POINTER(LargeNamedType))
 
     def test_pointer_type_str_name(self):
+        large_string = 'T' * 2 ** 25
         self.assertTrue(POINTER(large_string))
 
 if __name__ == '__main__':
diff --git a/lib-python/2.7/ctypes/util.py b/lib-python/2.7/ctypes/util.py
--- a/lib-python/2.7/ctypes/util.py
+++ b/lib-python/2.7/ctypes/util.py
@@ -178,7 +178,7 @@
             res = re.findall(expr, data)
             if not res:
                 return _get_soname(_findLib_gcc(name))
-            res.sort(cmp= lambda x,y: cmp(_num_version(x), _num_version(y)))
+            res.sort(key=_num_version)
             return res[-1]
 
     elif sys.platform == "sunos5":
diff --git a/lib-python/2.7/distutils/__init__.py b/lib-python/2.7/distutils/__init__.py
--- a/lib-python/2.7/distutils/__init__.py
+++ b/lib-python/2.7/distutils/__init__.py
@@ -15,5 +15,5 @@
 # Updated automatically by the Python release process.
 #
 #--start constants--
-__version__ = "2.7.9"
+__version__ = "2.7.10"
 #--end constants--
diff --git a/lib-python/2.7/distutils/command/check.py b/lib-python/2.7/distutils/command/check.py
--- a/lib-python/2.7/distutils/command/check.py
+++ b/lib-python/2.7/distutils/command/check.py
@@ -126,7 +126,7 @@
         """Returns warnings when the provided data doesn't compile."""
         source_path = StringIO()
         parser = Parser()
-        settings = frontend.OptionParser().get_default_values()
+        settings = frontend.OptionParser(components=(Parser,)).get_default_values()
         settings.tab_width = 4
         settings.pep_references = None
         settings.rfc_references = None
@@ -142,8 +142,8 @@
         document.note_source(source_path, -1)
         try:
             parser.parse(data, document)
-        except AttributeError:
-            reporter.messages.append((-1, 'Could not finish the parsing.',
-                                      '', {}))
+        except AttributeError as e:
+            reporter.messages.append(
+                (-1, 'Could not finish the parsing: %s.' % e, '', {}))
 
         return reporter.messages
diff --git a/lib-python/2.7/distutils/dir_util.py b/lib-python/2.7/distutils/dir_util.py
--- a/lib-python/2.7/distutils/dir_util.py
+++ b/lib-python/2.7/distutils/dir_util.py
@@ -83,7 +83,7 @@
     """Create all the empty directories under 'base_dir' needed to put 'files'
     there.
 
-    'base_dir' is just the a name of a directory which doesn't necessarily
+    'base_dir' is just the name of a directory which doesn't necessarily
     exist yet; 'files' is a list of filenames to be interpreted relative to
     'base_dir'.  'base_dir' + the directory portion of every file in 'files'
     will be created if it doesn't already exist.  'mode', 'verbose' and
diff --git a/lib-python/2.7/distutils/tests/test_check.py b/lib-python/2.7/distutils/tests/test_check.py
--- a/lib-python/2.7/distutils/tests/test_check.py
+++ b/lib-python/2.7/distutils/tests/test_check.py
@@ -1,5 +1,6 @@
 # -*- encoding: utf8 -*-
 """Tests for distutils.command.check."""
+import textwrap
 import unittest
 from test.test_support import run_unittest
 
@@ -93,6 +94,36 @@
         cmd = self._run(metadata, strict=1, restructuredtext=1)
         self.assertEqual(cmd._warnings, 0)
 
+    @unittest.skipUnless(HAS_DOCUTILS, "won't test without docutils")
+    def test_check_restructuredtext_with_syntax_highlight(self):
+        # Don't fail if there is a `code` or `code-block` directive
+
+        example_rst_docs = []
+        example_rst_docs.append(textwrap.dedent("""\
+            Here's some code:
+
+            .. code:: python
+
+                def foo():
+                    pass
+            """))
+        example_rst_docs.append(textwrap.dedent("""\
+            Here's some code:
+
+            .. code-block:: python
+
+                def foo():
+                    pass
+            """))
+
+        for rest_with_code in example_rst_docs:
+            pkg_info, dist = self.create_dist(long_description=rest_with_code)
+            cmd = check(dist)
+            cmd.check_restructuredtext()
+            self.assertEqual(cmd._warnings, 0)
+            msgs = cmd._check_rst_data(rest_with_code)
+            self.assertEqual(len(msgs), 0)
+
     def test_check_all(self):
 
         metadata = {'url': 'xxx', 'author': 'xxx'}
diff --git a/lib-python/2.7/distutils/text_file.py b/lib-python/2.7/distutils/text_file.py
--- a/lib-python/2.7/distutils/text_file.py
+++ b/lib-python/2.7/distutils/text_file.py
@@ -124,11 +124,11 @@
     def close (self):
         """Close the current file and forget everything we know about it
            (filename, current line number)."""
-
-        self.file.close ()
+        file = self.file
         self.file = None
         self.filename = None
         self.current_line = None
+        file.close()
 
 
     def gen_error (self, msg, line=None):
diff --git a/lib-python/2.7/dumbdbm.py b/lib-python/2.7/dumbdbm.py
--- a/lib-python/2.7/dumbdbm.py
+++ b/lib-python/2.7/dumbdbm.py
@@ -21,6 +21,7 @@
 
 """
 
+import ast as _ast
 import os as _os
 import __builtin__
 import UserDict
@@ -85,7 +86,7 @@
             with f:
                 for line in f:
                     line = line.rstrip()
-                    key, pos_and_siz_pair = eval(line)
+                    key, pos_and_siz_pair = _ast.literal_eval(line)
                     self._index[key] = pos_and_siz_pair
 
     # Write the index dict to the directory file.  The original directory
@@ -208,8 +209,10 @@
         return len(self._index)
 
     def close(self):
-        self._commit()
-        self._index = self._datfile = self._dirfile = self._bakfile = None
+        try:
+            self._commit()
+        finally:
+            self._index = self._datfile = self._dirfile = self._bakfile = None
 
     __del__ = close
 
diff --git a/lib-python/2.7/encodings/uu_codec.py b/lib-python/2.7/encodings/uu_codec.py
--- a/lib-python/2.7/encodings/uu_codec.py
+++ b/lib-python/2.7/encodings/uu_codec.py
@@ -84,7 +84,7 @@
             data = a2b_uu(s)
         except binascii.Error, v:
             # Workaround for broken uuencoders by /Fredrik Lundh
-            nbytes = (((ord(s[0])-32) & 63) * 4 + 5) / 3
+            nbytes = (((ord(s[0])-32) & 63) * 4 + 5) // 3
             data = a2b_uu(s[:nbytes])
             #sys.stderr.write("Warning: %s\n" % str(v))
         write(data)
diff --git a/lib-python/2.7/ensurepip/__init__.py b/lib-python/2.7/ensurepip/__init__.py
--- a/lib-python/2.7/ensurepip/__init__.py
+++ b/lib-python/2.7/ensurepip/__init__.py
@@ -12,9 +12,9 @@
 __all__ = ["version", "bootstrap"]
 
 
-_SETUPTOOLS_VERSION = "7.0"
+_SETUPTOOLS_VERSION = "15.2"
 
-_PIP_VERSION = "1.5.6"
+_PIP_VERSION = "6.1.1"
 
 # pip currently requires ssl support, so we try to provide a nicer
 # error message when that is missing (http://bugs.python.org/issue19744)
diff --git a/lib-python/2.7/ensurepip/_bundled/pip-1.5.6-py2.py3-none-any.whl b/lib-python/2.7/ensurepip/_bundled/pip-1.5.6-py2.py3-none-any.whl
deleted file mode 100644
Binary file lib-python/2.7/ensurepip/_bundled/pip-1.5.6-py2.py3-none-any.whl has changed
diff --git a/lib-python/2.7/ensurepip/_bundled/pip-6.1.1-py2.py3-none-any.whl b/lib-python/2.7/ensurepip/_bundled/pip-6.1.1-py2.py3-none-any.whl
new file mode 100644
index 0000000000000000000000000000000000000000..e59694a019051d58b9a378a1adfc9461b8cec9c3
GIT binary patch

[cut]

diff --git a/lib-python/2.7/ensurepip/_bundled/setuptools-15.2-py2.py3-none-any.whl b/lib-python/2.7/ensurepip/_bundled/setuptools-15.2-py2.py3-none-any.whl
new file mode 100644
index 0000000000000000000000000000000000000000..f153ed376684275e08fcfebdb2de8352fb074171
GIT binary patch

[cut]

diff --git a/lib-python/2.7/ensurepip/_bundled/setuptools-7.0-py2.py3-none-any.whl b/lib-python/2.7/ensurepip/_bundled/setuptools-7.0-py2.py3-none-any.whl
deleted file mode 100644
Binary file lib-python/2.7/ensurepip/_bundled/setuptools-7.0-py2.py3-none-any.whl has changed
diff --git a/lib-python/2.7/fileinput.py b/lib-python/2.7/fileinput.py
--- a/lib-python/2.7/fileinput.py
+++ b/lib-python/2.7/fileinput.py
@@ -233,8 +233,10 @@
         self.close()
 
     def close(self):
-        self.nextfile()
-        self._files = ()
+        try:
+            self.nextfile()
+        finally:
+            self._files = ()
 
     def __iter__(self):
         return self
@@ -270,23 +272,25 @@
 
         output = self._output
         self._output = 0
-        if output:
-            output.close()
+        try:
+            if output:
+                output.close()
+        finally:
+            file = self._file
+            self._file = 0
+            try:
+                if file and not self._isstdin:
+                    file.close()
+            finally:
+                backupfilename = self._backupfilename
+                self._backupfilename = 0
+                if backupfilename and not self._backup:
+                    try: os.unlink(backupfilename)
+                    except OSError: pass
 
-        file = self._file
-        self._file = 0
-        if file and not self._isstdin:
-            file.close()
-
-        backupfilename = self._backupfilename
-        self._backupfilename = 0
-        if backupfilename and not self._backup:
-            try: os.unlink(backupfilename)
-            except OSError: pass
-
-        self._isstdin = False
-        self._buffer = []
-        self._bufindex = 0
+                self._isstdin = False
+                self._buffer = []
+                self._bufindex = 0
 
     def readline(self):
         try:
diff --git a/lib-python/2.7/fnmatch.py b/lib-python/2.7/fnmatch.py
--- a/lib-python/2.7/fnmatch.py
+++ b/lib-python/2.7/fnmatch.py
@@ -47,12 +47,14 @@
     import os,posixpath
     result=[]
     pat=os.path.normcase(pat)
-    if not pat in _cache:
+    try:
+        re_pat = _cache[pat]
+    except KeyError:
         res = translate(pat)
         if len(_cache) >= _MAXCACHE:
             _cache.clear()
-        _cache[pat] = re.compile(res)
-    match=_cache[pat].match
+        _cache[pat] = re_pat = re.compile(res)
+    match = re_pat.match
     if os.path is posixpath:
         # normcase on posix is NOP. Optimize it away from the loop.
         for name in names:
@@ -71,12 +73,14 @@
     its arguments.
     """
 
-    if not pat in _cache:
+    try:
+        re_pat = _cache[pat]
+    except KeyError:
         res = translate(pat)
         if len(_cache) >= _MAXCACHE:
             _cache.clear()
-        _cache[pat] = re.compile(res)
-    return _cache[pat].match(name) is not None
+        _cache[pat] = re_pat = re.compile(res)
+    return re_pat.match(name) is not None
 
 def translate(pat):
     """Translate a shell PATTERN to a regular expression.
diff --git a/lib-python/2.7/ftplib.py b/lib-python/2.7/ftplib.py
--- a/lib-python/2.7/ftplib.py
+++ b/lib-python/2.7/ftplib.py
@@ -594,11 +594,16 @@
 
     def close(self):
         '''Close the connection without assuming anything about it.'''
-        if self.file is not None:
-            self.file.close()
-        if self.sock is not None:
-            self.sock.close()
-        self.file = self.sock = None
+        try:
+            file = self.file
+            self.file = None
+            if file is not None:
+                file.close()
+        finally:
+            sock = self.sock
+            self.sock = None
+            if sock is not None:
+                sock.close()
 
 try:
     import ssl
@@ -638,12 +643,24 @@
         '221 Goodbye.'
         >>>
         '''
-        ssl_version = ssl.PROTOCOL_TLSv1
+        ssl_version = ssl.PROTOCOL_SSLv23
 
         def __init__(self, host='', user='', passwd='', acct='', keyfile=None,
-                     certfile=None, timeout=_GLOBAL_DEFAULT_TIMEOUT):
+                     certfile=None, context=None,
+                     timeout=_GLOBAL_DEFAULT_TIMEOUT, source_address=None):
+            if context is not None and keyfile is not None:
+                raise ValueError("context and keyfile arguments are mutually "
+                                 "exclusive")
+            if context is not None and certfile is not None:
+                raise ValueError("context and certfile arguments are mutually "
+                                 "exclusive")
             self.keyfile = keyfile
             self.certfile = certfile
+            if context is None:
+                context = ssl._create_stdlib_context(self.ssl_version,
+                                                     certfile=certfile,
+                                                     keyfile=keyfile)
+            self.context = context
             self._prot_p = False
             FTP.__init__(self, host, user, passwd, acct, timeout)
 
@@ -656,12 +673,12 @@
             '''Set up secure control connection by using TLS/SSL.'''
             if isinstance(self.sock, ssl.SSLSocket):
                 raise ValueError("Already using TLS")
-            if self.ssl_version == ssl.PROTOCOL_TLSv1:
+            if self.ssl_version >= ssl.PROTOCOL_SSLv23:
                 resp = self.voidcmd('AUTH TLS')
             else:
                 resp = self.voidcmd('AUTH SSL')
-            self.sock = ssl.wrap_socket(self.sock, self.keyfile, self.certfile,
-                                        ssl_version=self.ssl_version)
+            self.sock = self.context.wrap_socket(self.sock,
+                                                 server_hostname=self.host)
             self.file = self.sock.makefile(mode='rb')
             return resp
 
@@ -692,8 +709,8 @@
         def ntransfercmd(self, cmd, rest=None):
             conn, size = FTP.ntransfercmd(self, cmd, rest)
             if self._prot_p:
-                conn = ssl.wrap_socket(conn, self.keyfile, self.certfile,
-                                       ssl_version=self.ssl_version)
+                conn = self.context.wrap_socket(conn,
+                                                server_hostname=self.host)
             return conn, size
 
         def retrbinary(self, cmd, callback, blocksize=8192, rest=None):
diff --git a/lib-python/2.7/genericpath.py b/lib-python/2.7/genericpath.py
--- a/lib-python/2.7/genericpath.py
+++ b/lib-python/2.7/genericpath.py
@@ -10,6 +10,14 @@
            'getsize', 'isdir', 'isfile']
 
 
+try:
+    _unicode = unicode
+except NameError:
+    # If Python is built without Unicode support, the unicode type
+    # will not exist. Fake one.
+    class _unicode(object):
+        pass
+
 # Does a path exist?
 # This is false for dangling symbolic links on systems that support them.
 def exists(path):
diff --git a/lib-python/2.7/gettext.py b/lib-python/2.7/gettext.py
--- a/lib-python/2.7/gettext.py
+++ b/lib-python/2.7/gettext.py
@@ -52,7 +52,9 @@
 
 __all__ = ['NullTranslations', 'GNUTranslations', 'Catalog',
            'find', 'translation', 'install', 'textdomain', 'bindtextdomain',
-           'dgettext', 'dngettext', 'gettext', 'ngettext',
+           'bind_textdomain_codeset',
+           'dgettext', 'dngettext', 'gettext', 'lgettext', 'ldgettext',
+           'ldngettext', 'lngettext', 'ngettext',
            ]
 
 _default_localedir = os.path.join(sys.prefix, 'share', 'locale')
@@ -294,11 +296,12 @@
             # See if we're looking at GNU .mo conventions for metadata
             if mlen == 0:
                 # Catalog description
-                lastk = k = None
+                lastk = None
                 for item in tmsg.splitlines():
                     item = item.strip()
                     if not item:
                         continue
+                    k = v = None
                     if ':' in item:
                         k, v = item.split(':', 1)
                         k = k.strip().lower()
diff --git a/lib-python/2.7/gzip.py b/lib-python/2.7/gzip.py
--- a/lib-python/2.7/gzip.py
+++ b/lib-python/2.7/gzip.py
@@ -238,9 +238,9 @@
             data = data.tobytes()
 
         if len(data) > 0:
-            self.size = self.size + len(data)
+            self.fileobj.write(self.compress.compress(data))
+            self.size += len(data)
             self.crc = zlib.crc32(data, self.crc) & 0xffffffffL
-            self.fileobj.write( self.compress.compress(data) )
             self.offset += len(data)
 
         return len(data)
@@ -369,19 +369,21 @@
         return self.fileobj is None
 
     def close(self):
-        if self.fileobj is None:
+        fileobj = self.fileobj
+        if fileobj is None:
             return
-        if self.mode == WRITE:
-            self.fileobj.write(self.compress.flush())
-            write32u(self.fileobj, self.crc)
-            # self.size may exceed 2GB, or even 4GB
-            write32u(self.fileobj, self.size & 0xffffffffL)
-            self.fileobj = None
-        elif self.mode == READ:
-            self.fileobj = None
-        if self.myfileobj:
-            self.myfileobj.close()
-            self.myfileobj = None
+        self.fileobj = None
+        try:
+            if self.mode == WRITE:
+                fileobj.write(self.compress.flush())
+                write32u(fileobj, self.crc)
+                # self.size may exceed 2GB, or even 4GB
+                write32u(fileobj, self.size & 0xffffffffL)
+        finally:
+            myfileobj = self.myfileobj
+            if myfileobj:
+                self.myfileobj = None
+                myfileobj.close()
 
     def flush(self,zlib_mode=zlib.Z_SYNC_FLUSH):
         self._check_closed()
diff --git a/lib-python/2.7/hashlib.py b/lib-python/2.7/hashlib.py
--- a/lib-python/2.7/hashlib.py
+++ b/lib-python/2.7/hashlib.py
@@ -187,7 +187,7 @@
 
         def prf(msg, inner=inner, outer=outer):
             # PBKDF2_HMAC uses the password as key. We can re-use the same
-            # digest objects and and just update copies to skip initialization.
+            # digest objects and just update copies to skip initialization.
             icpy = inner.copy()
             ocpy = outer.copy()
             icpy.update(msg)
diff --git a/lib-python/2.7/htmlentitydefs.py b/lib-python/2.7/htmlentitydefs.py
--- a/lib-python/2.7/htmlentitydefs.py
+++ b/lib-python/2.7/htmlentitydefs.py
@@ -1,6 +1,6 @@
 """HTML character entity references."""
 
-# maps the HTML entity name to the Unicode codepoint
+# maps the HTML entity name to the Unicode code point
 name2codepoint = {
     'AElig':    0x00c6, # latin capital letter AE = latin capital ligature AE, U+00C6 ISOlat1
     'Aacute':   0x00c1, # latin capital letter A with acute, U+00C1 ISOlat1
@@ -256,7 +256,7 @@
     'zwnj':     0x200c, # zero width non-joiner, U+200C NEW RFC 2070
 }
 
-# maps the Unicode codepoint to the HTML entity name
+# maps the Unicode code point to the HTML entity name
 codepoint2name = {}
 
 # maps the HTML entity name to the character
diff --git a/lib-python/2.7/httplib.py b/lib-python/2.7/httplib.py
--- a/lib-python/2.7/httplib.py
+++ b/lib-python/2.7/httplib.py
@@ -68,6 +68,7 @@
 
 from array import array
 import os
+import re
 import socket
 from sys import py3kwarning
 from urlparse import urlsplit
@@ -218,6 +219,38 @@
 # maximum amount of headers accepted
 _MAXHEADERS = 100
 
+# Header name/value ABNF (http://tools.ietf.org/html/rfc7230#section-3.2)
+#
+# VCHAR          = %x21-7E
+# obs-text       = %x80-FF
+# header-field   = field-name ":" OWS field-value OWS
+# field-name     = token
+# field-value    = *( field-content / obs-fold )
+# field-content  = field-vchar [ 1*( SP / HTAB ) field-vchar ]
+# field-vchar    = VCHAR / obs-text
+#
+# obs-fold       = CRLF 1*( SP / HTAB )
+#                ; obsolete line folding
+#                ; see Section 3.2.4
+
+# token          = 1*tchar
+#
+# tchar          = "!" / "#" / "$" / "%" / "&" / "'" / "*"
+#                / "+" / "-" / "." / "^" / "_" / "`" / "|" / "~"
+#                / DIGIT / ALPHA
+#                ; any VCHAR, except delimiters
+#
+# VCHAR defined in http://tools.ietf.org/html/rfc5234#appendix-B.1
+
+# the patterns for both name and value are more leniant than RFC
+# definitions to allow for backwards compatibility
+_is_legal_header_name = re.compile(r'\A[^:\s][^:\r\n]*\Z').match
+_is_illegal_header_value = re.compile(r'\n(?![ \t])|\r(?![ \t\n])').search
+
+# We always set the Content-Length header for these methods because some
+# servers will otherwise respond with a 411
+_METHODS_EXPECTING_BODY = {'PATCH', 'POST', 'PUT'}
+
 
 class HTTPMessage(mimetools.Message):
 
@@ -313,6 +346,11 @@
                 hlist.append(line)
                 self.addheader(headerseen, line[len(headerseen)+1:].strip())
                 continue
+            elif headerseen is not None:
+                # An empty header name. These aren't allowed in HTTP, but it's
+                # probably a benign mistake. Don't add the header, just keep
+                # going.
+                continue
             else:
                 # It's not a header line; throw it back and stop here.
                 if not self.dict:
@@ -522,9 +560,10 @@
         return True
 
     def close(self):
-        if self.fp:
-            self.fp.close()
+        fp = self.fp
+        if fp:
             self.fp = None
+            fp.close()
 
     def isclosed(self):
         # NOTE: it is possible that we will not ever call self.close(). This
@@ -723,7 +762,7 @@
         endpoint passed to set_tunnel. This is done by sending a HTTP CONNECT
         request to the proxy server when the connection is established.
 
-        This method must be called before the HTML connection has been
+        This method must be called before the HTTP connection has been
         established.
 
         The headers argument should be a mapping of extra HTTP headers
@@ -797,13 +836,17 @@
 
     def close(self):
         """Close the connection to the HTTP server."""
-        if self.sock:
-            self.sock.close()   # close it manually... there may be other refs
-            self.sock = None
-        if self.__response:
-            self.__response.close()
-            self.__response = None
         self.__state = _CS_IDLE
+        try:
+            sock = self.sock
+            if sock:
+                self.sock = None
+                sock.close()   # close it manually... there may be other refs
+        finally:
+            response = self.__response
+            if response:
+                self.__response = None
+                response.close()
 
     def send(self, data):
         """Send `data' to the server."""
@@ -978,7 +1021,16 @@
         if self.__state != _CS_REQ_STARTED:
             raise CannotSendHeader()
 
-        hdr = '%s: %s' % (header, '\r\n\t'.join([str(v) for v in values]))
+        header = '%s' % header
+        if not _is_legal_header_name(header):
+            raise ValueError('Invalid header name %r' % (header,))
+
+        values = [str(v) for v in values]
+        for one_value in values:
+            if _is_illegal_header_value(one_value):
+                raise ValueError('Invalid header value %r' % (one_value,))
+
+        hdr = '%s: %s' % (header, '\r\n\t'.join(values))
         self._output(hdr)
 
     def endheaders(self, message_body=None):
@@ -1000,19 +1052,25 @@
         """Send a complete request to the server."""
         self._send_request(method, url, body, headers)
 
-    def _set_content_length(self, body):
-        # Set the content-length based on the body.
+    def _set_content_length(self, body, method):
+        # Set the content-length based on the body. If the body is "empty", we
+        # set Content-Length: 0 for methods that expect a body (RFC 7230,
+        # Section 3.3.2). If the body is set for other methods, we set the
+        # header provided we can figure out what the length is.
         thelen = None
-        try:
-            thelen = str(len(body))
-        except TypeError, te:
-            # If this is a file-like object, try to
-            # fstat its file descriptor
+        if body is None and method.upper() in _METHODS_EXPECTING_BODY:
+            thelen = '0'
+        elif body is not None:
             try:
-                thelen = str(os.fstat(body.fileno()).st_size)
-            except (AttributeError, OSError):
-                # Don't send a length if this failed
-                if self.debuglevel > 0: print "Cannot stat!!"
+                thelen = str(len(body))
+            except TypeError:
+                # If this is a file-like object, try to
+                # fstat its file descriptor
+                try:
+                    thelen = str(os.fstat(body.fileno()).st_size)
+                except (AttributeError, OSError):
+                    # Don't send a length if this failed
+                    if self.debuglevel > 0: print "Cannot stat!!"
 
         if thelen is not None:
             self.putheader('Content-Length', thelen)
@@ -1028,8 +1086,8 @@
 
         self.putrequest(method, url, **skips)
 
-        if body is not None and 'content-length' not in header_names:
-            self._set_content_length(body)
+        if 'content-length' not in header_names:
+            self._set_content_length(body, method)
         for hdr, value in headers.iteritems():
             self.putheader(hdr, value)
         self.endheaders(body)
@@ -1072,20 +1130,20 @@
 
         try:
             response.begin()
+            assert response.will_close != _UNKNOWN
+            self.__state = _CS_IDLE
+
+            if response.will_close:
+                # this effectively passes the connection to the response
+                self.close()
+            else:
+                # remember this, so we can tell when it is complete
+                self.__response = response
+
+            return response
         except:
             response.close()
             raise
-        assert response.will_close != _UNKNOWN
-        self.__state = _CS_IDLE
-
-        if response.will_close:
-            # this effectively passes the connection to the response
-            self.close()
-        else:
-            # remember this, so we can tell when it is complete
-            self.__response = response
-
-        return response
 
 
 class HTTP:
@@ -1129,7 +1187,7 @@
         "Accept arguments to set the host/port, since the superclass doesn't."
 
         if host is not None:
-            self._conn._set_hostport(host, port)
+            (self._conn.host, self._conn.port) = self._conn._get_hostport(host, port)
         self._conn.connect()
 
     def getfile(self):
diff --git a/lib-python/2.7/idlelib/CodeContext.py b/lib-python/2.7/idlelib/CodeContext.py
--- a/lib-python/2.7/idlelib/CodeContext.py
+++ b/lib-python/2.7/idlelib/CodeContext.py
@@ -15,8 +15,8 @@
 from sys import maxint as INFINITY
 from idlelib.configHandler import idleConf
 
-BLOCKOPENERS = set(["class", "def", "elif", "else", "except", "finally", "for",
-                    "if", "try", "while", "with"])
+BLOCKOPENERS = {"class", "def", "elif", "else", "except", "finally", "for",
+                    "if", "try", "while", "with"}
 UPDATEINTERVAL = 100 # millisec
 FONTUPDATEINTERVAL = 1000 # millisec
 
diff --git a/lib-python/2.7/idlelib/EditorWindow.py b/lib-python/2.7/idlelib/EditorWindow.py
--- a/lib-python/2.7/idlelib/EditorWindow.py
+++ b/lib-python/2.7/idlelib/EditorWindow.py
@@ -469,13 +469,10 @@
         ("format", "F_ormat"),
         ("run", "_Run"),
         ("options", "_Options"),
-        ("windows", "_Windows"),
+        ("windows", "_Window"),
         ("help", "_Help"),
     ]
 
-    if sys.platform == "darwin":
-        menu_specs[-2] = ("windows", "_Window")
-
 
     def createmenubar(self):
         mbar = self.menubar
diff --git a/lib-python/2.7/idlelib/FormatParagraph.py b/lib-python/2.7/idlelib/FormatParagraph.py
--- a/lib-python/2.7/idlelib/FormatParagraph.py
+++ b/lib-python/2.7/idlelib/FormatParagraph.py
@@ -44,9 +44,11 @@
 
         The length limit parameter is for testing with a known value.
         """
-        if limit == None:
+        if limit is None:
+            # The default length limit is that defined by pep8
             limit = idleConf.GetOption(
-                    'main', 'FormatParagraph', 'paragraph', type='int')
+                'extensions', 'FormatParagraph', 'max-width',
+                type='int', default=72)
         text = self.editwin.text
         first, last = self.editwin.get_selection_indices()
         if first and last:
diff --git a/lib-python/2.7/idlelib/PyShell.py b/lib-python/2.7/idlelib/PyShell.py
--- a/lib-python/2.7/idlelib/PyShell.py
+++ b/lib-python/2.7/idlelib/PyShell.py
@@ -871,13 +871,10 @@
         ("edit", "_Edit"),
         ("debug", "_Debug"),
         ("options", "_Options"),
-        ("windows", "_Windows"),
+        ("windows", "_Window"),
         ("help", "_Help"),
     ]
 
-    if sys.platform == "darwin":
-        menu_specs[-2] = ("windows", "_Window")
-
 
     # New classes
     from idlelib.IdleHistory import History
@@ -1350,7 +1347,7 @@
         if type(s) not in (unicode, str, bytearray):
             # See issue #19481
             if isinstance(s, unicode):
-                s = unicode.__getslice__(s, None, None)
+                s = unicode.__getitem__(s, slice(None))
             elif isinstance(s, str):
                 s = str.__str__(s)
             elif isinstance(s, bytearray):
diff --git a/lib-python/2.7/idlelib/SearchEngine.py b/lib-python/2.7/idlelib/SearchEngine.py
--- a/lib-python/2.7/idlelib/SearchEngine.py
+++ b/lib-python/2.7/idlelib/SearchEngine.py
@@ -191,7 +191,7 @@
 
     This is done by searching forwards until there is no match.
     Prog: compiled re object with a search method returning a match.
-    Chars: line of text, without \n.
+    Chars: line of text, without \\n.
     Col: stop index for the search; the limit for match.end().
     '''
     m = prog.search(chars)
diff --git a/lib-python/2.7/idlelib/config-extensions.def b/lib-python/2.7/idlelib/config-extensions.def
--- a/lib-python/2.7/idlelib/config-extensions.def
+++ b/lib-python/2.7/idlelib/config-extensions.def
@@ -66,6 +66,7 @@
 
 [FormatParagraph]
 enable=True
+max-width=72
 [FormatParagraph_cfgBindings]
 format-paragraph=<Alt-Key-q>
 
diff --git a/lib-python/2.7/idlelib/config-main.def b/lib-python/2.7/idlelib/config-main.def
--- a/lib-python/2.7/idlelib/config-main.def
+++ b/lib-python/2.7/idlelib/config-main.def
@@ -58,9 +58,6 @@
 font-bold= 0
 encoding= none
 
-[FormatParagraph]
-paragraph=72
-
 [Indent]
 use-spaces= 1
 num-spaces= 4
diff --git a/lib-python/2.7/idlelib/configDialog.py b/lib-python/2.7/idlelib/configDialog.py
--- a/lib-python/2.7/idlelib/configDialog.py
+++ b/lib-python/2.7/idlelib/configDialog.py
@@ -371,7 +371,6 @@
         parent = self.parent
         self.winWidth = StringVar(parent)
         self.winHeight = StringVar(parent)
-        self.paraWidth = StringVar(parent)
         self.startupEdit = IntVar(parent)
         self.autoSave = IntVar(parent)
         self.encoding = StringVar(parent)
@@ -387,7 +386,6 @@
         frameSave = LabelFrame(frame, borderwidth=2, relief=GROOVE,
                                text=' Autosave Preferences ')
         frameWinSize = Frame(frame, borderwidth=2, relief=GROOVE)
-        frameParaSize = Frame(frame, borderwidth=2, relief=GROOVE)
         frameEncoding = Frame(frame, borderwidth=2, relief=GROOVE)
         frameHelp = LabelFrame(frame, borderwidth=2, relief=GROOVE,
                                text=' Additional Help Sources ')
@@ -416,11 +414,6 @@
         labelWinHeightTitle = Label(frameWinSize, text='Height')
         entryWinHeight = Entry(
                 frameWinSize, textvariable=self.winHeight, width=3)
-        #paragraphFormatWidth
-        labelParaWidthTitle = Label(
-                frameParaSize, text='Paragraph reformat width (in characters)')
-        entryParaWidth = Entry(
-                frameParaSize, textvariable=self.paraWidth, width=3)
         #frameEncoding
         labelEncodingTitle = Label(
                 frameEncoding, text="Default Source Encoding")
@@ -458,7 +451,6 @@
         frameRun.pack(side=TOP, padx=5, pady=5, fill=X)
         frameSave.pack(side=TOP, padx=5, pady=5, fill=X)
         frameWinSize.pack(side=TOP, padx=5, pady=5, fill=X)
-        frameParaSize.pack(side=TOP, padx=5, pady=5, fill=X)
         frameEncoding.pack(side=TOP, padx=5, pady=5, fill=X)
         frameHelp.pack(side=TOP, padx=5, pady=5, expand=TRUE, fill=BOTH)
         #frameRun
@@ -475,9 +467,6 @@
         labelWinHeightTitle.pack(side=RIGHT, anchor=E, pady=5)
         entryWinWidth.pack(side=RIGHT, anchor=E, padx=10, pady=5)
         labelWinWidthTitle.pack(side=RIGHT, anchor=E, pady=5)
-        #paragraphFormatWidth
-        labelParaWidthTitle.pack(side=LEFT, anchor=W, padx=5, pady=5)
-        entryParaWidth.pack(side=RIGHT, anchor=E, padx=10, pady=5)
         #frameEncoding
         labelEncodingTitle.pack(side=LEFT, anchor=W, padx=5, pady=5)
         radioEncNone.pack(side=RIGHT, anchor=E, pady=5)
@@ -509,7 +498,6 @@
         self.keysAreBuiltin.trace_variable('w', self.VarChanged_keysAreBuiltin)
         self.winWidth.trace_variable('w', self.VarChanged_winWidth)
         self.winHeight.trace_variable('w', self.VarChanged_winHeight)
-        self.paraWidth.trace_variable('w', self.VarChanged_paraWidth)
         self.startupEdit.trace_variable('w', self.VarChanged_startupEdit)
         self.autoSave.trace_variable('w', self.VarChanged_autoSave)
         self.encoding.trace_variable('w', self.VarChanged_encoding)
@@ -594,10 +582,6 @@
         value = self.winHeight.get()
         self.AddChangedItem('main', 'EditorWindow', 'height', value)
 
-    def VarChanged_paraWidth(self, *params):
-        value = self.paraWidth.get()
-        self.AddChangedItem('main', 'FormatParagraph', 'paragraph', value)
-
     def VarChanged_startupEdit(self, *params):
         value = self.startupEdit.get()
         self.AddChangedItem('main', 'General', 'editor-on-startup', value)
@@ -1094,9 +1078,6 @@
                 'main', 'EditorWindow', 'width', type='int'))
         self.winHeight.set(idleConf.GetOption(
                 'main', 'EditorWindow', 'height', type='int'))
-        #initial paragraph reformat size
-        self.paraWidth.set(idleConf.GetOption(
-                'main', 'FormatParagraph', 'paragraph', type='int'))
         # default source encoding
         self.encoding.set(idleConf.GetOption(
                 'main', 'EditorWindow', 'encoding', default='none'))
diff --git a/lib-python/2.7/idlelib/help.txt b/lib-python/2.7/idlelib/help.txt
--- a/lib-python/2.7/idlelib/help.txt
+++ b/lib-python/2.7/idlelib/help.txt
@@ -100,7 +100,7 @@
 			  which is scrolling off the top or the window.
 			  (Not present in Shell window.)
 
-Windows Menu:
+Window Menu:
 
 	Zoom Height -- toggles the window between configured size
 	and maximum height.
diff --git a/lib-python/2.7/idlelib/idle.bat b/lib-python/2.7/idlelib/idle.bat
--- a/lib-python/2.7/idlelib/idle.bat
+++ b/lib-python/2.7/idlelib/idle.bat
@@ -1,4 +1,4 @@
- at echo off
-rem Start IDLE using the appropriate Python interpreter
-set CURRDIR=%~dp0
-start "IDLE" "%CURRDIR%..\..\pythonw.exe" "%CURRDIR%idle.pyw" %1 %2 %3 %4 %5 %6 %7 %8 %9
+ at echo off
+rem Start IDLE using the appropriate Python interpreter
+set CURRDIR=%~dp0
+start "IDLE" "%CURRDIR%..\..\pythonw.exe" "%CURRDIR%idle.pyw" %1 %2 %3 %4 %5 %6 %7 %8 %9
diff --git a/lib-python/2.7/idlelib/idle_test/test_calltips.py b/lib-python/2.7/idlelib/idle_test/test_calltips.py
--- a/lib-python/2.7/idlelib/idle_test/test_calltips.py
+++ b/lib-python/2.7/idlelib/idle_test/test_calltips.py
@@ -55,7 +55,8 @@
         def gtest(obj, out):
             self.assertEqual(signature(obj), out)
 
-        gtest(List, '()\n' + List.__doc__)
+        if List.__doc__ is not None:
+            gtest(List, '()\n' + List.__doc__)
         gtest(list.__new__,
                'T.__new__(S, ...) -> a new object with type S, a subtype of T')
         gtest(list.__init__,
@@ -70,7 +71,8 @@
 
     def test_signature_wrap(self):
         # This is also a test of an old-style class
-        self.assertEqual(signature(textwrap.TextWrapper), '''\
+        if textwrap.TextWrapper.__doc__ is not None:
+            self.assertEqual(signature(textwrap.TextWrapper), '''\
 (width=70, initial_indent='', subsequent_indent='', expand_tabs=True,
     replace_whitespace=True, fix_sentence_endings=False, break_long_words=True,
     drop_whitespace=True, break_on_hyphens=True)''')
@@ -106,20 +108,23 @@
         def t5(a, b=None, *args, **kwds): 'doc'
         t5.tip = "(a, b=None, *args, **kwargs)"
 
+        doc = '\ndoc' if t1.__doc__ is not None else ''
         for func in (t1, t2, t3, t4, t5, TC):
-            self.assertEqual(signature(func), func.tip + '\ndoc')
+            self.assertEqual(signature(func), func.tip + doc)
 
     def test_methods(self):
+        doc = '\ndoc' if TC.__doc__ is not None else ''
         for meth in (TC.t1, TC.t2, TC.t3, TC.t4, TC.t5, TC.t6, TC.__call__):
-            self.assertEqual(signature(meth), meth.tip + "\ndoc")
-        self.assertEqual(signature(TC.cm), "(a)\ndoc")
-        self.assertEqual(signature(TC.sm), "(b)\ndoc")
+            self.assertEqual(signature(meth), meth.tip + doc)
+        self.assertEqual(signature(TC.cm), "(a)" + doc)
+        self.assertEqual(signature(TC.sm), "(b)" + doc)
 
     def test_bound_methods(self):
         # test that first parameter is correctly removed from argspec
+        doc = '\ndoc' if TC.__doc__ is not None else ''
         for meth, mtip  in ((tc.t1, "()"), (tc.t4, "(*args)"), (tc.t6, "(self)"),
                             (tc.__call__, '(ci)'), (tc, '(ci)'), (TC.cm, "(a)"),):
-            self.assertEqual(signature(meth), mtip + "\ndoc")
+            self.assertEqual(signature(meth), mtip + doc)
 
     def test_starred_parameter(self):
         # test that starred first parameter is *not* removed from argspec
diff --git a/lib-python/2.7/idlelib/idle_test/test_io.py b/lib-python/2.7/idlelib/idle_test/test_io.py
new file mode 100644
--- /dev/null
+++ b/lib-python/2.7/idlelib/idle_test/test_io.py
@@ -0,0 +1,267 @@
+import unittest
+import io
+from idlelib.PyShell import PseudoInputFile, PseudoOutputFile
+from test import test_support as support
+
+
+class Base(object):
+    def __str__(self):
+        return '%s:str' % type(self).__name__
+    def __unicode__(self):
+        return '%s:unicode' % type(self).__name__
+    def __len__(self):
+        return 3
+    def __iter__(self):
+        return iter('abc')
+    def __getitem__(self, *args):
+        return '%s:item' % type(self).__name__
+    def __getslice__(self, *args):
+        return '%s:slice' % type(self).__name__
+
+class S(Base, str):
+    pass
+
+class U(Base, unicode):
+    pass
+
+class BA(Base, bytearray):
+    pass
+
+class MockShell:
+    def __init__(self):
+        self.reset()
+
+    def write(self, *args):
+        self.written.append(args)
+
+    def readline(self):
+        return self.lines.pop()
+
+    def close(self):
+        pass
+
+    def reset(self):
+        self.written = []
+
+    def push(self, lines):
+        self.lines = list(lines)[::-1]
+
+
+class PseudeOutputFilesTest(unittest.TestCase):
+    def test_misc(self):
+        shell = MockShell()
+        f = PseudoOutputFile(shell, 'stdout', 'utf-8')
+        self.assertIsInstance(f, io.TextIOBase)
+        self.assertEqual(f.encoding, 'utf-8')
+        self.assertIsNone(f.errors)
+        self.assertIsNone(f.newlines)
+        self.assertEqual(f.name, '<stdout>')
+        self.assertFalse(f.closed)
+        self.assertTrue(f.isatty())
+        self.assertFalse(f.readable())
+        self.assertTrue(f.writable())
+        self.assertFalse(f.seekable())
+
+    def test_unsupported(self):
+        shell = MockShell()
+        f = PseudoOutputFile(shell, 'stdout', 'utf-8')
+        self.assertRaises(IOError, f.fileno)
+        self.assertRaises(IOError, f.tell)
+        self.assertRaises(IOError, f.seek, 0)
+        self.assertRaises(IOError, f.read, 0)
+        self.assertRaises(IOError, f.readline, 0)
+
+    def test_write(self):
+        shell = MockShell()
+        f = PseudoOutputFile(shell, 'stdout', 'utf-8')
+        f.write('test')
+        self.assertEqual(shell.written, [('test', 'stdout')])
+        shell.reset()
+        f.write('t\xe8st')
+        self.assertEqual(shell.written, [('t\xe8st', 'stdout')])
+        shell.reset()
+        f.write(u't\xe8st')
+        self.assertEqual(shell.written, [(u't\xe8st', 'stdout')])
+        shell.reset()
+
+        f.write(S('t\xe8st'))
+        self.assertEqual(shell.written, [('t\xe8st', 'stdout')])
+        self.assertEqual(type(shell.written[0][0]), str)
+        shell.reset()
+        f.write(BA('t\xe8st'))
+        self.assertEqual(shell.written, [('t\xe8st', 'stdout')])
+        self.assertEqual(type(shell.written[0][0]), str)
+        shell.reset()
+        f.write(U(u't\xe8st'))
+        self.assertEqual(shell.written, [(u't\xe8st', 'stdout')])
+        self.assertEqual(type(shell.written[0][0]), unicode)
+        shell.reset()
+
+        self.assertRaises(TypeError, f.write)
+        self.assertEqual(shell.written, [])
+        self.assertRaises(TypeError, f.write, 123)
+        self.assertEqual(shell.written, [])
+        self.assertRaises(TypeError, f.write, 'test', 'spam')
+        self.assertEqual(shell.written, [])
+
+    def test_writelines(self):
+        shell = MockShell()
+        f = PseudoOutputFile(shell, 'stdout', 'utf-8')
+        f.writelines([])
+        self.assertEqual(shell.written, [])
+        shell.reset()
+        f.writelines(['one\n', 'two'])
+        self.assertEqual(shell.written,
+                         [('one\n', 'stdout'), ('two', 'stdout')])
+        shell.reset()
+        f.writelines(['on\xe8\n', 'tw\xf2'])
+        self.assertEqual(shell.written,
+                         [('on\xe8\n', 'stdout'), ('tw\xf2', 'stdout')])
+        shell.reset()
+        f.writelines([u'on\xe8\n', u'tw\xf2'])
+        self.assertEqual(shell.written,
+                         [(u'on\xe8\n', 'stdout'), (u'tw\xf2', 'stdout')])
+        shell.reset()
+
+        f.writelines([S('t\xe8st')])
+        self.assertEqual(shell.written, [('t\xe8st', 'stdout')])
+        self.assertEqual(type(shell.written[0][0]), str)
+        shell.reset()
+        f.writelines([BA('t\xe8st')])
+        self.assertEqual(shell.written, [('t\xe8st', 'stdout')])
+        self.assertEqual(type(shell.written[0][0]), str)
+        shell.reset()
+        f.writelines([U(u't\xe8st')])
+        self.assertEqual(shell.written, [(u't\xe8st', 'stdout')])
+        self.assertEqual(type(shell.written[0][0]), unicode)
+        shell.reset()
+
+        self.assertRaises(TypeError, f.writelines)
+        self.assertEqual(shell.written, [])
+        self.assertRaises(TypeError, f.writelines, 123)
+        self.assertEqual(shell.written, [])
+        self.assertRaises(TypeError, f.writelines, [123])
+        self.assertEqual(shell.written, [])
+        self.assertRaises(TypeError, f.writelines, [], [])
+        self.assertEqual(shell.written, [])
+
+    def test_close(self):
+        shell = MockShell()
+        f = PseudoOutputFile(shell, 'stdout', 'utf-8')
+        self.assertFalse(f.closed)
+        f.write('test')
+        f.close()
+        self.assertTrue(f.closed)
+        self.assertRaises(ValueError, f.write, 'x')
+        self.assertEqual(shell.written, [('test', 'stdout')])
+        f.close()
+        self.assertRaises(TypeError, f.close, 1)
+
+
+class PseudeInputFilesTest(unittest.TestCase):
+    def test_misc(self):
+        shell = MockShell()
+        f = PseudoInputFile(shell, 'stdin', 'utf-8')
+        self.assertIsInstance(f, io.TextIOBase)
+        self.assertEqual(f.encoding, 'utf-8')
+        self.assertIsNone(f.errors)
+        self.assertIsNone(f.newlines)
+        self.assertEqual(f.name, '<stdin>')
+        self.assertFalse(f.closed)
+        self.assertTrue(f.isatty())
+        self.assertTrue(f.readable())
+        self.assertFalse(f.writable())
+        self.assertFalse(f.seekable())
+
+    def test_unsupported(self):
+        shell = MockShell()
+        f = PseudoInputFile(shell, 'stdin', 'utf-8')
+        self.assertRaises(IOError, f.fileno)
+        self.assertRaises(IOError, f.tell)
+        self.assertRaises(IOError, f.seek, 0)
+        self.assertRaises(IOError, f.write, 'x')
+        self.assertRaises(IOError, f.writelines, ['x'])
+
+    def test_read(self):
+        shell = MockShell()
+        f = PseudoInputFile(shell, 'stdin', 'utf-8')
+        shell.push(['one\n', 'two\n', ''])
+        self.assertEqual(f.read(), 'one\ntwo\n')
+        shell.push(['one\n', 'two\n', ''])
+        self.assertEqual(f.read(-1), 'one\ntwo\n')
+        shell.push(['one\n', 'two\n', ''])
+        self.assertEqual(f.read(None), 'one\ntwo\n')
+        shell.push(['one\n', 'two\n', 'three\n', ''])
+        self.assertEqual(f.read(2), 'on')
+        self.assertEqual(f.read(3), 'e\nt')
+        self.assertEqual(f.read(10), 'wo\nthree\n')
+
+        shell.push(['one\n', 'two\n'])
+        self.assertEqual(f.read(0), '')
+        self.assertRaises(TypeError, f.read, 1.5)
+        self.assertRaises(TypeError, f.read, '1')
+        self.assertRaises(TypeError, f.read, 1, 1)
+
+    def test_readline(self):
+        shell = MockShell()
+        f = PseudoInputFile(shell, 'stdin', 'utf-8')
+        shell.push(['one\n', 'two\n', 'three\n', 'four\n'])
+        self.assertEqual(f.readline(), 'one\n')
+        self.assertEqual(f.readline(-1), 'two\n')
+        self.assertEqual(f.readline(None), 'three\n')
+        shell.push(['one\ntwo\n'])
+        self.assertEqual(f.readline(), 'one\n')
+        self.assertEqual(f.readline(), 'two\n')
+        shell.push(['one', 'two', 'three'])
+        self.assertEqual(f.readline(), 'one')
+        self.assertEqual(f.readline(), 'two')
+        shell.push(['one\n', 'two\n', 'three\n'])
+        self.assertEqual(f.readline(2), 'on')
+        self.assertEqual(f.readline(1), 'e')
+        self.assertEqual(f.readline(1), '\n')
+        self.assertEqual(f.readline(10), 'two\n')
+
+        shell.push(['one\n', 'two\n'])
+        self.assertEqual(f.readline(0), '')
+        self.assertRaises(TypeError, f.readlines, 1.5)
+        self.assertRaises(TypeError, f.readlines, '1')
+        self.assertRaises(TypeError, f.readlines, 1, 1)
+
+    def test_readlines(self):
+        shell = MockShell()
+        f = PseudoInputFile(shell, 'stdin', 'utf-8')
+        shell.push(['one\n', 'two\n', ''])
+        self.assertEqual(f.readlines(), ['one\n', 'two\n'])
+        shell.push(['one\n', 'two\n', ''])
+        self.assertEqual(f.readlines(-1), ['one\n', 'two\n'])
+        shell.push(['one\n', 'two\n', ''])
+        self.assertEqual(f.readlines(None), ['one\n', 'two\n'])
+        shell.push(['one\n', 'two\n', ''])
+        self.assertEqual(f.readlines(0), ['one\n', 'two\n'])
+        shell.push(['one\n', 'two\n', ''])
+        self.assertEqual(f.readlines(3), ['one\n'])
+        shell.push(['one\n', 'two\n', ''])
+        self.assertEqual(f.readlines(4), ['one\n', 'two\n'])
+
+        shell.push(['one\n', 'two\n', ''])
+        self.assertRaises(TypeError, f.readlines, 1.5)
+        self.assertRaises(TypeError, f.readlines, '1')
+        self.assertRaises(TypeError, f.readlines, 1, 1)
+

From noreply at buildbot.pypy.org  Fri Jun 12 19:16:12 2015
From: noreply at buildbot.pypy.org (amauryfa)
Date: Fri, 12 Jun 2015 19:16:12 +0200 (CEST)
Subject: [pypy-commit] pypy default: Document merged branch
Message-ID: <20150612171612.4BE011C024E@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: 
Changeset: r78056:f6c8f214fa02
Date: 2015-06-12 19:15 +0200
http://bitbucket.org/pypy/pypy/changeset/f6c8f214fa02/

Log:	Document merged branch

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -7,3 +7,7 @@
 
 .. branch: use_min_scalar
 Correctly resolve the output dtype of ufunc(array, scalar) calls.
+
+.. branch: stdlib-2.7.10
+
+Update stdlib to version 2.7.10

From noreply at buildbot.pypy.org  Fri Jun 12 19:26:14 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 12 Jun 2015 19:26:14 +0200 (CEST)
Subject: [pypy-commit] cffi default: on pypy,
 the _cffi_backend module doesn't have a __file__ at all
Message-ID: <20150612172614.BEF3E1C024E@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2184:7cd6732be4e5
Date: 2015-06-12 19:26 +0200
http://bitbucket.org/cffi/cffi/changeset/7cd6732be4e5/

Log:	on pypy, the _cffi_backend module doesn't have a __file__ at all

diff --git a/testing/cffi0/test_zintegration.py b/testing/cffi0/test_zintegration.py
--- a/testing/cffi0/test_zintegration.py
+++ b/testing/cffi0/test_zintegration.py
@@ -35,6 +35,8 @@
         paths = []
         for module in modules:
             target = __import__(module, None, None, [])
+            if not hasattr(target, '__file__'):   # for _cffi_backend on pypy
+                continue
             src = os.path.abspath(target.__file__)
             for end in ['__init__.pyc', '__init__.pyo', '__init__.py']:
                 if src.lower().endswith(end):

From noreply at buildbot.pypy.org  Fri Jun 12 19:28:00 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 12 Jun 2015 19:28:00 +0200 (CEST)
Subject: [pypy-commit] pypy default: import cffi/7cd6732be4e5
Message-ID: <20150612172800.673071C024E@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78057:d2d8c9799524
Date: 2015-06-12 19:28 +0200
http://bitbucket.org/pypy/pypy/changeset/d2d8c9799524/

Log:	import cffi/7cd6732be4e5

diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py
--- a/lib_pypy/cffi/cparser.py
+++ b/lib_pypy/cffi/cparser.py
@@ -337,7 +337,7 @@
                 length = self._parse_constant(
                     typenode.dim, partial_length_ok=partial_length_ok)
             tp = self._get_type(typenode.type,
-                                partial_length_ok=(length == '...'))
+                                partial_length_ok=partial_length_ok)
             return model.ArrayType(tp, length)
         #
         if isinstance(typenode, pycparser.c_ast.PtrDecl):
diff --git a/lib_pypy/cffi/gc_weakref.py b/lib_pypy/cffi/gc_weakref.py
--- a/lib_pypy/cffi/gc_weakref.py
+++ b/lib_pypy/cffi/gc_weakref.py
@@ -5,7 +5,6 @@
     def __init__(self, ffi):
         self.ffi = ffi
         self.data = {}
-        self.nextindex = 0
 
     def build(self, cdata, destructor):
         # make a new cdata of the same type as the original one
@@ -18,7 +17,6 @@
             destructor(cdata)
         #
         key = ref(new_cdata, remove)
-        index = self.nextindex
-        self.nextindex = index + 1     # we're protected by the lock here
+        index = object()
         self.data[index] = key
         return new_cdata
diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py
--- a/lib_pypy/cffi/recompiler.py
+++ b/lib_pypy/cffi/recompiler.py
@@ -749,10 +749,12 @@
     # named structs or unions
 
     def _field_type(self, tp_struct, field_name, tp_field):
-        if isinstance(tp_field, model.ArrayType) and tp_field.length == '...':
-            ptr_struct_name = tp_struct.get_c_name('*')
-            actual_length = '_cffi_array_len(((%s)0)->%s)' % (
-                ptr_struct_name, field_name)
+        if isinstance(tp_field, model.ArrayType):
+            actual_length = tp_field.length
+            if actual_length == '...':
+                ptr_struct_name = tp_struct.get_c_name('*')
+                actual_length = '_cffi_array_len(((%s)0)->%s)' % (
+                    ptr_struct_name, field_name)
             tp_item = self._field_type(tp_struct, '%s[0]' % field_name,
                                        tp_field.item)
             tp_field = model.ArrayType(tp_item, actual_length)
@@ -1055,8 +1057,10 @@
     # global variables
 
     def _global_type(self, tp, global_name):
-        if isinstance(tp, model.ArrayType) and tp.length == '...':
-            actual_length = '_cffi_array_len(%s)' % (global_name,)
+        if isinstance(tp, model.ArrayType):
+            actual_length = tp.length
+            if actual_length == '...':
+                actual_length = '_cffi_array_len(%s)' % (global_name,)
             tp_item = self._global_type(tp.item, '%s[0]' % global_name)
             tp = model.ArrayType(tp_item, actual_length)
         return tp
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zintegration.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zintegration.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zintegration.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zintegration.py
@@ -36,6 +36,8 @@
         paths = []
         for module in modules:
             target = __import__(module, None, None, [])
+            if not hasattr(target, '__file__'):   # for _cffi_backend on pypy
+                continue
             src = os.path.abspath(target.__file__)
             for end in ['__init__.pyc', '__init__.pyo', '__init__.py']:
                 if src.lower().endswith(end):
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
@@ -410,6 +410,11 @@
     # 'x' is another <built-in method> object on lib, made very indirectly
     x = type(lib).__dir__.__get__(lib)
     py.test.raises(TypeError, ffi.typeof, x)
+    #
+    # present on built-in functions on CPython; must be emulated on PyPy:
+    assert lib.sin.__name__ == 'sin'
+    assert lib.sin.__module__ == '_CFFI_test_math_sin_type'
+    assert lib.sin.__doc__ == 'direct call to the C function of the same name'
 
 def test_verify_anonymous_struct_with_typedef():
     ffi = FFI()
@@ -925,6 +930,18 @@
     assert ffi.typeof(s.a) == ffi.typeof("int[5][8]")
     assert ffi.typeof(s.a[0]) == ffi.typeof("int[8]")
 
+def test_struct_array_guess_length_3():
+    ffi = FFI()
+    ffi.cdef("struct foo_s { int a[][...]; };")
+    lib = verify(ffi, 'test_struct_array_guess_length_3',
+                 "struct foo_s { int x; int a[5][7]; int y; };")
+    assert ffi.sizeof('struct foo_s') == 37 * ffi.sizeof('int')
+    s = ffi.new("struct foo_s *")
+    assert ffi.typeof(s.a) == ffi.typeof("int(*)[7]")
+    assert s.a[4][6] == 0
+    py.test.raises(IndexError, 's.a[4][7]')
+    assert ffi.typeof(s.a[0]) == ffi.typeof("int[7]")
+
 def test_global_var_array_2():
     ffi = FFI()
     ffi.cdef("int a[...][...];")
@@ -936,6 +953,27 @@
     assert ffi.typeof(lib.a) == ffi.typeof("int[10][8]")
     assert ffi.typeof(lib.a[0]) == ffi.typeof("int[8]")
 
+def test_global_var_array_3():
+    ffi = FFI()
+    ffi.cdef("int a[][...];")
+    lib = verify(ffi, 'test_global_var_array_3', 'int a[10][8];')
+    lib.a[9][7] = 123456
+    assert lib.a[9][7] == 123456
+    py.test.raises(IndexError, 'lib.a[0][8]')
+    assert ffi.typeof(lib.a) == ffi.typeof("int(*)[8]")
+    assert ffi.typeof(lib.a[0]) == ffi.typeof("int[8]")
+
+def test_global_var_array_4():
+    ffi = FFI()
+    ffi.cdef("int a[10][...];")
+    lib = verify(ffi, 'test_global_var_array_4', 'int a[10][8];')
+    lib.a[9][7] = 123456
+    assert lib.a[9][7] == 123456
+    py.test.raises(IndexError, 'lib.a[0][8]')
+    py.test.raises(IndexError, 'lib.a[10][8]')
+    assert ffi.typeof(lib.a) == ffi.typeof("int[10][8]")
+    assert ffi.typeof(lib.a[0]) == ffi.typeof("int[8]")
+
 def test_some_integer_type():
     ffi = FFI()
     ffi.cdef("""

From noreply at buildbot.pypy.org  Fri Jun 12 19:35:35 2015
From: noreply at buildbot.pypy.org (amauryfa)
Date: Fri, 12 Jun 2015 19:35:35 +0200 (CEST)
Subject: [pypy-commit] pypy default: Update version numbers as indicated in
 doc/how-to-release.rst
Message-ID: <20150612173535.E45211C024E@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: 
Changeset: r78058:a8ce50056e5c
Date: 2015-06-12 19:34 +0200
http://bitbucket.org/pypy/pypy/changeset/a8ce50056e5c/

Log:	Update version numbers as indicated in doc/how-to-release.rst

diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h
--- a/pypy/module/cpyext/include/patchlevel.h
+++ b/pypy/module/cpyext/include/patchlevel.h
@@ -21,12 +21,12 @@
 /* Version parsed out into numeric values */
 #define PY_MAJOR_VERSION	2
 #define PY_MINOR_VERSION	7
-#define PY_MICRO_VERSION	9
+#define PY_MICRO_VERSION	10
 #define PY_RELEASE_LEVEL	PY_RELEASE_LEVEL_FINAL
 #define PY_RELEASE_SERIAL	0
 
 /* Version as a string */
-#define PY_VERSION		"2.7.9"
+#define PY_VERSION		"2.7.10"
 
 /* PyPy version as a string */
 #define PYPY_VERSION "2.7.0-alpha0"
diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py
--- a/pypy/module/sys/version.py
+++ b/pypy/module/sys/version.py
@@ -6,7 +6,7 @@
 from pypy.interpreter import gateway
 
 #XXX # the release serial 42 is not in range(16)
-CPYTHON_VERSION            = (2, 7, 9, "final", 42)
+CPYTHON_VERSION            = (2, 7, 10, "final", 42)
 #XXX # sync CPYTHON_VERSION with patchlevel.h, package.py
 CPYTHON_API_VERSION        = 1013   #XXX # sync with include/modsupport.h
 

From noreply at buildbot.pypy.org  Fri Jun 12 22:58:04 2015
From: noreply at buildbot.pypy.org (amauryfa)
Date: Fri, 12 Jun 2015 22:58:04 +0200 (CEST)
Subject: [pypy-commit] pypy default: Add missing test,
	the test suite should start now.
Message-ID: <20150612205804.9B9291C024E@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: 
Changeset: r78059:b216b7b77272
Date: 2015-06-12 22:57 +0200
http://bitbucket.org/pypy/pypy/changeset/b216b7b77272/

Log:	Add missing test, the test suite should start now.

diff --git a/lib-python/conftest.py b/lib-python/conftest.py
--- a/lib-python/conftest.py
+++ b/lib-python/conftest.py
@@ -438,6 +438,7 @@
     RegrTest('test_threading_local.py', usemodules="thread", core=True),
     RegrTest('test_threadsignals.py', usemodules="thread"),
     RegrTest('test_time.py', core=True),
+    RegrTest('test_timeit.py'),
     RegrTest('test_timeout.py'),
     RegrTest('test_tk.py'),
     RegrTest('test_tokenize.py'),

From noreply at buildbot.pypy.org  Sat Jun 13 00:00:59 2015
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sat, 13 Jun 2015 00:00:59 +0200 (CEST)
Subject: [pypy-commit] pypy default: Fix some segfaults after a file is
	detached.
Message-ID: <20150612220059.26F421C124A@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: 
Changeset: r78060:cef7d7a0598d
Date: 2015-06-12 23:43 +0200
http://bitbucket.org/pypy/pypy/changeset/cef7d7a0598d/

Log:	Fix some segfaults after a file is detached.

diff --git a/pypy/module/_io/interp_bufferedio.py b/pypy/module/_io/interp_bufferedio.py
--- a/pypy/module/_io/interp_bufferedio.py
+++ b/pypy/module/_io/interp_bufferedio.py
@@ -196,9 +196,11 @@
         return space.getattr(self.w_raw, space.wrap("closed"))
 
     def name_get_w(self, space):
+        self._check_init(space)
         return space.getattr(self.w_raw, space.wrap("name"))
 
     def mode_get_w(self, space):
+        self._check_init(space)
         return space.getattr(self.w_raw, space.wrap("mode"))
 
     def readable_w(self, space):
@@ -214,6 +216,7 @@
         return space.call_method(self.w_raw, "seekable")
 
     def isatty_w(self, space):
+        self._check_init(space)
         return space.call_method(self.w_raw, "isatty")
 
     def repr_w(self, space):
@@ -221,7 +224,7 @@
         try:
             w_name = space.getattr(self, space.wrap("name"))
         except OperationError, e:
-            if not e.match(space, space.w_AttributeError):
+            if not e.match(space, space.w_Exception):
                 raise
             return space.wrap("<%s>" % (typename,))
         else:
diff --git a/pypy/module/_io/test/test_bufferedio.py b/pypy/module/_io/test/test_bufferedio.py
--- a/pypy/module/_io/test/test_bufferedio.py
+++ b/pypy/module/_io/test/test_bufferedio.py
@@ -180,6 +180,20 @@
         assert not raw.closed
         raw.close()
 
+    def test_detached(self):
+        import _io
+        class MockRawIO(_io._RawIOBase):
+            def readable(self):
+                return True
+        raw = MockRawIO()
+        buf = _io.BufferedReader(raw)
+        assert buf.detach() is raw
+        raises(ValueError, buf.detach)
+
+        raises(ValueError, getattr, buf, 'mode')
+        raises(ValueError, buf.isatty)
+        repr(buf)  # Should still work
+
     def test_tell(self):
         import _io
         raw = _io.FileIO(self.tmpfile)

From noreply at buildbot.pypy.org  Sat Jun 13 00:01:00 2015
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sat, 13 Jun 2015 00:01:00 +0200 (CEST)
Subject: [pypy-commit] pypy default: Avoid a crash when TextIOWrapper is not
 fully initialized
Message-ID: <20150612220100.6D2801C124A@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: 
Changeset: r78061:3f6c2b6ab6d5
Date: 2015-06-12 23:56 +0200
http://bitbucket.org/pypy/pypy/changeset/3f6c2b6ab6d5/

Log:	Avoid a crash when TextIOWrapper is not fully initialized

diff --git a/pypy/module/_io/interp_textio.py b/pypy/module/_io/interp_textio.py
--- a/pypy/module/_io/interp_textio.py
+++ b/pypy/module/_io/interp_textio.py
@@ -436,6 +436,7 @@
         W_TextIOBase._check_closed(self, space, message)
 
     def descr_repr(self, space):
+        self._check_init(space)
         w_name = space.findattr(self, space.wrap("name"))
         if w_name is None:
             w_name_str = space.wrap("")
diff --git a/pypy/module/_io/test/test_textio.py b/pypy/module/_io/test/test_textio.py
--- a/pypy/module/_io/test/test_textio.py
+++ b/pypy/module/_io/test/test_textio.py
@@ -252,6 +252,16 @@
         t = _io.TextIOWrapper(NonbytesStream(u'a'))
         t.read() == u'a'
 
+    def test_uninitialized(self):
+        import _io
+        t = _io.TextIOWrapper.__new__(_io.TextIOWrapper)
+        del t
+        t = _io.TextIOWrapper.__new__(_io.TextIOWrapper)
+        raises(Exception, repr, t)
+        raises(ValueError, t.read, 0)
+        t.__init__(_io.BytesIO())
+        assert t.read(0) == u''
+
 
 class AppTestIncrementalNewlineDecoder:
     def test_newline_decoder(self):

From noreply at buildbot.pypy.org  Sat Jun 13 00:49:54 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Sat, 13 Jun 2015 00:49:54 +0200 (CEST)
Subject: [pypy-commit] pypy numpy-docstrings: Start implementing
	np.add_docstring
Message-ID: <20150612224954.86F291C033F@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: numpy-docstrings
Changeset: r78062:caa48835941b
Date: 2015-06-12 17:48 +0100
http://bitbucket.org/pypy/pypy/changeset/caa48835941b/

Log:	Start implementing np.add_docstring

diff --git a/pypy/module/micronumpy/__init__.py b/pypy/module/micronumpy/__init__.py
--- a/pypy/module/micronumpy/__init__.py
+++ b/pypy/module/micronumpy/__init__.py
@@ -29,6 +29,8 @@
         'set_string_function': 'appbridge.set_string_function',
         'typeinfo': 'descriptor.get_dtype_cache(space).w_typeinfo',
         'nditer': 'nditer.W_NDIter',
+
+        'add_docstring': 'support.descr_add_docstring',
     }
     for c in ['MAXDIMS', 'CLIP', 'WRAP', 'RAISE']:
         interpleveldefs[c] = 'space.wrap(constants.%s)' % c
diff --git a/pypy/module/micronumpy/support.py b/pypy/module/micronumpy/support.py
--- a/pypy/module/micronumpy/support.py
+++ b/pypy/module/micronumpy/support.py
@@ -1,8 +1,9 @@
-from pypy.interpreter.error import OperationError, oefmt
 from rpython.rlib import jit
 from rpython.rlib.rarithmetic import ovfcheck
 from rpython.rtyper.lltypesystem import rffi, lltype
 
+from pypy.interpreter.error import OperationError, oefmt
+from pypy.interpreter.gateway import unwrap_spec, appdef
 
 def issequence_w(space, w_obj):
     from pypy.module.micronumpy.base import W_NDimArray
@@ -172,3 +173,17 @@
     elif req_order == 'A':
         return proto_order
 
+ at unwrap_spec(docstring=str)
+def descr_add_docstring(space, w_obj, docstring):
+    return _add_doc_w(space, w_obj, space.wrap(docstring))
+
+_add_doc_w = appdef("""add_docstring(obj, docstring):
+    old_doc = getattr(obj, '__doc__', None)
+    if old_doc is not None:
+        raise RuntimeError("object already has a docstring")
+    try:
+        obj.__doc__ = docstring
+    except:
+        if not isinstance(obj, type) and callable(obj):
+            raise TypeError("Cannot set a docstring for %s" % obj)
+""")

From noreply at buildbot.pypy.org  Sat Jun 13 00:49:55 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Sat, 13 Jun 2015 00:49:55 +0200 (CEST)
Subject: [pypy-commit] pypy numpy-docstrings: Allow adding a docstring to
	builtin types
Message-ID: <20150612224955.CA0E31C033F@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: numpy-docstrings
Changeset: r78063:b16ddaa55e90
Date: 2015-06-12 18:14 +0100
http://bitbucket.org/pypy/pypy/changeset/b16ddaa55e90/

Log:	Allow adding a docstring to builtin types (breaks test_ztranslation,
	but real translation works)

diff --git a/pypy/module/micronumpy/support.py b/pypy/module/micronumpy/support.py
--- a/pypy/module/micronumpy/support.py
+++ b/pypy/module/micronumpy/support.py
@@ -4,6 +4,7 @@
 
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.gateway import unwrap_spec, appdef
+from pypy.objspace.std.typeobject import W_TypeObject
 
 def issequence_w(space, w_obj):
     from pypy.module.micronumpy.base import W_NDimArray
@@ -175,15 +176,18 @@
 
 @unwrap_spec(docstring=str)
 def descr_add_docstring(space, w_obj, docstring):
-    return _add_doc_w(space, w_obj, space.wrap(docstring))
+    if isinstance(w_obj, W_TypeObject):
+        if w_obj.w_doc is None or space.is_none(w_obj.w_doc):
+            w_obj.w_doc = space.wrap(docstring)
+            return
+    _add_doc_w(space, w_obj, space.wrap(docstring))
 
 _add_doc_w = appdef("""add_docstring(obj, docstring):
     old_doc = getattr(obj, '__doc__', None)
     if old_doc is not None:
-        raise RuntimeError("object already has a docstring")
+        raise RuntimeError("%s already has a docstring" % obj)
     try:
         obj.__doc__ = docstring
     except:
-        if not isinstance(obj, type) and callable(obj):
-            raise TypeError("Cannot set a docstring for %s" % obj)
+        raise TypeError("Cannot set a docstring for %s" % obj)
 """)

From noreply at buildbot.pypy.org  Sat Jun 13 01:46:54 2015
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sat, 13 Jun 2015 01:46:54 +0200 (CEST)
Subject: [pypy-commit] pypy default: Fix imports in test_bz2
Message-ID: <20150612234654.76CAF1C033F@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: 
Changeset: r78064:77cff5abb0b0
Date: 2015-06-13 00:15 +0200
http://bitbucket.org/pypy/pypy/changeset/77cff5abb0b0/

Log:	Fix imports in test_bz2

diff --git a/lib-python/2.7/test/test_bz2.py b/lib-python/2.7/test/test_bz2.py
--- a/lib-python/2.7/test/test_bz2.py
+++ b/lib-python/2.7/test/test_bz2.py
@@ -51,7 +51,7 @@
         self.filename = TESTFN
 
     def tearDown(self):
-        test_support.gc_collect()
+        support.gc_collect()
         if os.path.isfile(self.filename):
             os.unlink(self.filename)
 
@@ -249,7 +249,7 @@
             o = BZ2File(self.filename)
             del o
             if i % 100 == 0:
-                test_support.gc_collect()
+                support.gc_collect()
 
     def testOpenNonexistent(self):
         # "Test opening a nonexistent file"
@@ -312,7 +312,7 @@
             with support.start_threads(threads):
                 pass
 
-    @test_support.impl_detail()
+    @support.impl_detail()
     def testMixedIterationReads(self):
         # Issue #8397: mixed iteration and reads should be forbidden.
         with bz2.BZ2File(self.filename, 'wb') as f:

From noreply at buildbot.pypy.org  Sat Jun 13 01:46:55 2015
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sat, 13 Jun 2015 01:46:55 +0200 (CEST)
Subject: [pypy-commit] pypy default: It is TypeError.
Message-ID: <20150612234655.95E171C033F@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: 
Changeset: r78065:48213fa62b6d
Date: 2015-06-13 00:16 +0200
http://bitbucket.org/pypy/pypy/changeset/48213fa62b6d/

Log:	It is TypeError.

diff --git a/lib-python/2.7/test/test_fileio.py b/lib-python/2.7/test/test_fileio.py
--- a/lib-python/2.7/test/test_fileio.py
+++ b/lib-python/2.7/test/test_fileio.py
@@ -145,11 +145,11 @@
         self.assertRaises(ValueError, self.f.write, 0)
         self.assertRaises(ValueError, self.f.seek, 0)
 
-        self.assertRaises(ValueError, self.f.readinto) # XXX should be TypeError?
+        self.assertRaises(TypeError, self.f.readinto)
         self.assertRaises(ValueError, self.f.readinto, bytearray(1))
-        self.assertRaises(ValueError, self.f.seek)
+        self.assertRaises(TypeError, self.f.seek)
         self.assertRaises(ValueError, self.f.seek, 0)
-        self.assertRaises(ValueError, self.f.write)
+        self.assertRaises(TypeError, self.f.write)
         self.assertRaises(ValueError, self.f.write, b'')
         self.assertRaises(TypeError, self.f.writelines)
         self.assertRaises(ValueError, self.f.writelines, b'')

From noreply at buildbot.pypy.org  Sat Jun 13 01:46:56 2015
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sat, 13 Jun 2015 01:46:56 +0200 (CEST)
Subject: [pypy-commit] pypy default: Add TextIOBase.newlines=None,
	requested in test_idle
Message-ID: <20150612234656.B245D1C033F@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: 
Changeset: r78066:1d09fc0a175a
Date: 2015-06-13 00:20 +0200
http://bitbucket.org/pypy/pypy/changeset/1d09fc0a175a/

Log:	Add TextIOBase.newlines=None, requested in test_idle

diff --git a/pypy/module/_io/interp_textio.py b/pypy/module/_io/interp_textio.py
--- a/pypy/module/_io/interp_textio.py
+++ b/pypy/module/_io/interp_textio.py
@@ -211,6 +211,8 @@
     def errors_get_w(self, space):
         return space.w_None
 
+    def newlines_get_w(self, space):
+        return space.w_None
 
     def _find_line_ending(self, line, start, end):
         size = end - start
@@ -262,6 +264,7 @@
     readline = interp2app(W_TextIOBase.readline_w),
     detach = interp2app(W_TextIOBase.detach_w),
     encoding = interp_attrproperty_w("w_encoding", W_TextIOBase),
+    newlines = GetSetProperty(W_TextIOBase.newlines_get_w),
     errors = GetSetProperty(W_TextIOBase.errors_get_w),
 )
 

From noreply at buildbot.pypy.org  Sat Jun 13 01:46:57 2015
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sat, 13 Jun 2015 01:46:57 +0200 (CEST)
Subject: [pypy-commit] pypy default: Another method which was not exposed.
Message-ID: <20150612234657.CE86B1C033F@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: 
Changeset: r78067:d95e53d9540e
Date: 2015-06-13 00:21 +0200
http://bitbucket.org/pypy/pypy/changeset/d95e53d9540e/

Log:	Another method which was not exposed.

diff --git a/pypy/module/_io/interp_textio.py b/pypy/module/_io/interp_textio.py
--- a/pypy/module/_io/interp_textio.py
+++ b/pypy/module/_io/interp_textio.py
@@ -262,6 +262,7 @@
 
     read = interp2app(W_TextIOBase.read_w),
     readline = interp2app(W_TextIOBase.readline_w),
+    write = interp2app(W_TextIOBase.write_w),
     detach = interp2app(W_TextIOBase.detach_w),
     encoding = interp_attrproperty_w("w_encoding", W_TextIOBase),
     newlines = GetSetProperty(W_TextIOBase.newlines_get_w),

From noreply at buildbot.pypy.org  Sat Jun 13 01:46:58 2015
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sat, 13 Jun 2015 01:46:58 +0200 (CEST)
Subject: [pypy-commit] pypy default: Raises ValueError on os.major(-1)
Message-ID: <20150612234658.EC5441C033F@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: 
Changeset: r78068:1767d7627fba
Date: 2015-06-13 00:28 +0200
http://bitbucket.org/pypy/pypy/changeset/1767d7627fba/

Log:	Raises ValueError on os.major(-1)

diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -1319,12 +1319,12 @@
     result = os.makedev(major, minor)
     return space.wrap(result)
 
- at unwrap_spec(device=c_int)
+ at unwrap_spec(device="c_uint")
 def major(space, device):
     result = os.major(device)
     return space.wrap(result)
 
- at unwrap_spec(device=c_int)
+ at unwrap_spec(device="c_uint")
 def minor(space, device):
     result = os.minor(device)
     return space.wrap(result)
diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py
--- a/pypy/module/posix/test/test_posix2.py
+++ b/pypy/module/posix/test/test_posix2.py
@@ -766,6 +766,7 @@
             assert os.minor(12345) == self.expected_minor_12345
             assert os.makedev(self.expected_major_12345,
                               self.expected_minor_12345) == 12345
+            raises((ValueError, OverflowError), os.major, -1)
 
     if hasattr(os, 'fsync'):
         def test_fsync(self):

From noreply at buildbot.pypy.org  Sat Jun 13 01:47:00 2015
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sat, 13 Jun 2015 01:47:00 +0200 (CEST)
Subject: [pypy-commit] pypy default: Add old buffer interface to memoryview,
 this is needed by the struct module.
Message-ID: <20150612234700.0C1E71C033F@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: 
Changeset: r78069:8e323987d093
Date: 2015-06-13 00:57 +0200
http://bitbucket.org/pypy/pypy/changeset/8e323987d093/

Log:	Add old buffer interface to memoryview, this is needed by the struct
	module.

diff --git a/pypy/module/struct/test/test_struct.py b/pypy/module/struct/test/test_struct.py
--- a/pypy/module/struct/test/test_struct.py
+++ b/pypy/module/struct/test/test_struct.py
@@ -449,6 +449,8 @@
         assert b[:] == ('\x00' * 2 +
                         self.struct.pack("ii", 17, 42) +
                         '\x00' * (19-sz-2))
+        m = memoryview(b)
+        self.struct.pack_into("ii", m, 2, 17, 42)
 
     def test_unpack_from(self):
         b = self.bytebuffer(19)
diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -23,6 +23,14 @@
         space.check_buf_flags(flags, self.buf.readonly)
         return self.buf
 
+    def readbuf_w(self, space):
+        return self.buf
+
+    def writebuf_w(self, space):
+        if self.buf.readonly:
+            raise oefmt(space.w_TypeError, "buffer is read-only")
+        return self.buf
+
     @staticmethod
     def descr_new_memoryview(space, w_subtype, w_object):
         return W_MemoryView(space.buffer_w(w_object, space.BUF_FULL_RO))

From noreply at buildbot.pypy.org  Sat Jun 13 01:47:01 2015
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sat, 13 Jun 2015 01:47:01 +0200 (CEST)
Subject: [pypy-commit] pypy default: Blindly copy a CPython change:
Message-ID: <20150612234701.2AF161C033F@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: 
Changeset: r78070:044b6da79c49
Date: 2015-06-13 01:45 +0200
http://bitbucket.org/pypy/pypy/changeset/044b6da79c49/

Log:	Blindly copy a CPython change: Issue #23048: Fix jumping out of an
	infinite while loop in the pdb.

	it crashed the test suite.

diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -527,7 +527,7 @@
             self.emit_jump(ops.JUMP_ABSOLUTE, loop, True)
             if test_constant == optimize.CONST_NOT_CONST:
                 self.use_next_block(anchor)
-                self.emit_op(ops.POP_BLOCK)
+            self.emit_op(ops.POP_BLOCK)
             self.pop_frame_block(F_BLOCK_LOOP, loop)
             self.visit_sequence(wh.orelse)
             self.use_next_block(end)

From noreply at buildbot.pypy.org  Sat Jun 13 01:47:02 2015
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sat, 13 Jun 2015 01:47:02 +0200 (CEST)
Subject: [pypy-commit] pypy default: Fix renamed import
Message-ID: <20150612234702.42DB71C033F@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: 
Changeset: r78071:468017773f37
Date: 2015-06-13 01:46 +0200
http://bitbucket.org/pypy/pypy/changeset/468017773f37/

Log:	Fix renamed import

diff --git a/lib-python/2.7/test/test_threading_local.py b/lib-python/2.7/test/test_threading_local.py
--- a/lib-python/2.7/test/test_threading_local.py
+++ b/lib-python/2.7/test/test_threading_local.py
@@ -168,7 +168,7 @@
         obj = cls()
         obj.x = 5
         self.assertEqual(obj.__dict__, {'x': 5})
-        if test_support.check_impl_detail():
+        if support.check_impl_detail():
             with self.assertRaises(AttributeError):
                 obj.__dict__ = {}
         with self.assertRaises(AttributeError):

From noreply at buildbot.pypy.org  Sat Jun 13 03:15:54 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Sat, 13 Jun 2015 03:15:54 +0200 (CEST)
Subject: [pypy-commit] pypy numpy-docstrings: Handle properties in
	add_docstring()
Message-ID: <20150613011554.1D7481C033F@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: numpy-docstrings
Changeset: r78073:8b8ecf5fd1ad
Date: 2015-06-13 01:43 +0100
http://bitbucket.org/pypy/pypy/changeset/8b8ecf5fd1ad/

Log:	Handle properties in add_docstring()

diff --git a/pypy/module/micronumpy/support.py b/pypy/module/micronumpy/support.py
--- a/pypy/module/micronumpy/support.py
+++ b/pypy/module/micronumpy/support.py
@@ -5,6 +5,7 @@
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.gateway import unwrap_spec, appdef
 from pypy.interpreter.function import Method
+from pypy.interpreter.typedef import GetSetProperty
 from pypy.objspace.std.typeobject import W_TypeObject
 
 def issequence_w(space, w_obj):
@@ -181,6 +182,10 @@
         if w_obj.w_doc is None or space.is_none(w_obj.w_doc):
             w_obj.w_doc = space.wrap(docstring)
             return
+    elif isinstance(w_obj, GetSetProperty):
+        if w_obj.doc is None or space.is_none(w_obj.doc):
+            w_obj.doc = space.wrap(docstring)
+            return
     _add_doc_w(space, w_obj, space.wrap(docstring))
 
 _add_doc_w = appdef("""add_docstring(obj, docstring):
diff --git a/pypy/module/micronumpy/test/test_support_app.py b/pypy/module/micronumpy/test/test_support_app.py
--- a/pypy/module/micronumpy/test/test_support_app.py
+++ b/pypy/module/micronumpy/test/test_support_app.py
@@ -20,3 +20,10 @@
         #raises(RuntimeError, np.add_docstring, int.bit_length, 'foo')
         np.add_docstring(int.bit_length,'foo')
         assert int.bit_length.__doc__ == 'foo'
+
+    def test_property_docstring(self):
+        # XXX: We cannot sensibly test np.add_docstring() being successful
+        import numpy as np
+        #raises(RuntimeError, np.add_docstring, int.bit_length, 'foo')
+        np.add_docstring(np.flatiter.base, 'foo')
+        assert np.flatiter.base.__doc__ == 'foo'

From noreply at buildbot.pypy.org  Sat Jun 13 03:15:52 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Sat, 13 Jun 2015 03:15:52 +0200 (CEST)
Subject: [pypy-commit] pypy numpy-docstrings: Handle methods in
	add_docstring()
Message-ID: <20150613011552.E4C711C033F@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: numpy-docstrings
Changeset: r78072:5b3d5ccedb57
Date: 2015-06-13 01:25 +0100
http://bitbucket.org/pypy/pypy/changeset/5b3d5ccedb57/

Log:	Handle methods in add_docstring()

diff --git a/pypy/module/micronumpy/support.py b/pypy/module/micronumpy/support.py
--- a/pypy/module/micronumpy/support.py
+++ b/pypy/module/micronumpy/support.py
@@ -4,6 +4,7 @@
 
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.gateway import unwrap_spec, appdef
+from pypy.interpreter.function import Method
 from pypy.objspace.std.typeobject import W_TypeObject
 
 def issequence_w(space, w_obj):
@@ -183,9 +184,14 @@
     _add_doc_w(space, w_obj, space.wrap(docstring))
 
 _add_doc_w = appdef("""add_docstring(obj, docstring):
+    import types
     old_doc = getattr(obj, '__doc__', None)
     if old_doc is not None:
-        raise RuntimeError("%s already has a docstring" % obj)
+        # raise RuntimeError("%s already has a docstring" % obj)
+        pass
+    if isinstance(obj, types.MethodType):
+        add_docstring(obj.im_func, docstring)
+        return
     try:
         obj.__doc__ = docstring
     except:
diff --git a/pypy/module/micronumpy/test/test_support_app.py b/pypy/module/micronumpy/test/test_support_app.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/micronumpy/test/test_support_app.py
@@ -0,0 +1,22 @@
+"""App-level tests for support.py"""
+from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest
+
+class AppTestSupport(BaseNumpyAppTest):
+    def test_add_docstring(self):
+        import numpy as np
+        foo = lambda: None
+        np.add_docstring(foo, "Does a thing")
+        assert foo.__doc__ == "Does a thing"
+
+    def test_type_docstring(self):
+        # XXX: We cannot sensibly test np.add_docstring() being successful
+        import numpy as np
+        import types
+        raises(RuntimeError, np.add_docstring, types.FunctionType, 'foo')
+
+    def test_method_docstring(self):
+        # XXX: We cannot sensibly test np.add_docstring() being successful
+        import numpy as np
+        #raises(RuntimeError, np.add_docstring, int.bit_length, 'foo')
+        np.add_docstring(int.bit_length,'foo')
+        assert int.bit_length.__doc__ == 'foo'

From noreply at buildbot.pypy.org  Sat Jun 13 03:15:55 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Sat, 13 Jun 2015 03:15:55 +0200 (CEST)
Subject: [pypy-commit] pypy numpy-docstrings: Create set_docstring() helper,
 move add_docstring() implementation to app-level and clean up
Message-ID: <20150613011555.47AED1C033F@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: numpy-docstrings
Changeset: r78074:116f25d26306
Date: 2015-06-13 02:15 +0100
http://bitbucket.org/pypy/pypy/changeset/116f25d26306/

Log:	Create set_docstring() helper, move add_docstring() implementation
	to app-level and clean up

diff --git a/pypy/module/micronumpy/__init__.py b/pypy/module/micronumpy/__init__.py
--- a/pypy/module/micronumpy/__init__.py
+++ b/pypy/module/micronumpy/__init__.py
@@ -2,7 +2,9 @@
 
 
 class MultiArrayModule(MixedModule):
-    appleveldefs = {'arange': 'app_numpy.arange'}
+    appleveldefs = {
+        'arange': 'app_numpy.arange',
+        'add_docstring': 'app_numpy.add_docstring'}
     interpleveldefs = {
         'ndarray': 'ndarray.W_NDimArray',
         'dtype': 'descriptor.W_Dtype',
@@ -30,7 +32,7 @@
         'typeinfo': 'descriptor.get_dtype_cache(space).w_typeinfo',
         'nditer': 'nditer.W_NDIter',
 
-        'add_docstring': 'support.descr_add_docstring',
+        'set_docstring': 'support.descr_set_docstring',
     }
     for c in ['MAXDIMS', 'CLIP', 'WRAP', 'RAISE']:
         interpleveldefs[c] = 'space.wrap(constants.%s)' % c
diff --git a/pypy/module/micronumpy/app_numpy.py b/pypy/module/micronumpy/app_numpy.py
--- a/pypy/module/micronumpy/app_numpy.py
+++ b/pypy/module/micronumpy/app_numpy.py
@@ -3,6 +3,7 @@
 import math
 
 import _numpypy
+from _numpypy.multiarray import set_docstring
 
 def arange(start, stop=None, step=1, dtype=None):
     '''arange([start], stop[, step], dtype=None)
@@ -22,3 +23,13 @@
         arr[j] = i
         i += step
     return arr
+
+
+def add_docstring(obj, docstring):
+    old_doc = getattr(obj, '__doc__', None)
+    if old_doc is not None:
+        raise RuntimeError("%s already has a docstring" % obj)
+    try:
+        set_docstring(obj, docstring)
+    except:
+        raise TypeError("Cannot set a docstring for %s" % obj)
diff --git a/pypy/module/micronumpy/support.py b/pypy/module/micronumpy/support.py
--- a/pypy/module/micronumpy/support.py
+++ b/pypy/module/micronumpy/support.py
@@ -4,7 +4,6 @@
 
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.gateway import unwrap_spec, appdef
-from pypy.interpreter.function import Method
 from pypy.interpreter.typedef import GetSetProperty
 from pypy.objspace.std.typeobject import W_TypeObject
 
@@ -188,17 +187,19 @@
             return
     _add_doc_w(space, w_obj, space.wrap(docstring))
 
-_add_doc_w = appdef("""add_docstring(obj, docstring):
+def descr_set_docstring(space, w_obj, w_docstring):
+    if isinstance(w_obj, W_TypeObject):
+        w_obj.w_doc = w_docstring
+        return
+    elif isinstance(w_obj, GetSetProperty):
+        w_obj.doc = w_docstring
+        return
+    app_set_docstring(space, w_obj, w_docstring)
+
+app_set_docstring = appdef("""app_set_docstring_(obj, docstring):
     import types
-    old_doc = getattr(obj, '__doc__', None)
-    if old_doc is not None:
-        # raise RuntimeError("%s already has a docstring" % obj)
-        pass
     if isinstance(obj, types.MethodType):
-        add_docstring(obj.im_func, docstring)
-        return
-    try:
+        obj.im_func.__doc__ = docstring
+    else:
         obj.__doc__ = docstring
-    except:
-        raise TypeError("Cannot set a docstring for %s" % obj)
 """)
diff --git a/pypy/module/micronumpy/test/test_support_app.py b/pypy/module/micronumpy/test/test_support_app.py
--- a/pypy/module/micronumpy/test/test_support_app.py
+++ b/pypy/module/micronumpy/test/test_support_app.py
@@ -1,7 +1,16 @@
 """App-level tests for support.py"""
+import sys
+import py
+
 from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest
+from pypy.conftest import option
 
 class AppTestSupport(BaseNumpyAppTest):
+    def setup_class(cls):
+        if option.runappdirect and '__pypy__' not in sys.builtin_module_names:
+            py.test.skip("pypy only test")
+        BaseNumpyAppTest.setup_class.im_func(cls)
+
     def test_add_docstring(self):
         import numpy as np
         foo = lambda: None
@@ -9,21 +18,31 @@
         assert foo.__doc__ == "Does a thing"
 
     def test_type_docstring(self):
-        # XXX: We cannot sensibly test np.add_docstring() being successful
         import numpy as np
         import types
+        doc = types.FunctionType.__doc__
+        try:
+            np.set_docstring(types.FunctionType, 'foo')
+            assert types.FunctionType.__doc__ == 'foo'
+        finally:
+            np.set_docstring(types.FunctionType, doc)
+
         raises(RuntimeError, np.add_docstring, types.FunctionType, 'foo')
 
     def test_method_docstring(self):
-        # XXX: We cannot sensibly test np.add_docstring() being successful
         import numpy as np
-        #raises(RuntimeError, np.add_docstring, int.bit_length, 'foo')
-        np.add_docstring(int.bit_length,'foo')
-        assert int.bit_length.__doc__ == 'foo'
+        doc = int.bit_length.__doc__
+        try:
+            np.set_docstring(int.bit_length, 'foo')
+            assert int.bit_length.__doc__ == 'foo'
+        finally:
+            np.set_docstring(int.bit_length, doc)
 
     def test_property_docstring(self):
-        # XXX: We cannot sensibly test np.add_docstring() being successful
         import numpy as np
-        #raises(RuntimeError, np.add_docstring, int.bit_length, 'foo')
-        np.add_docstring(np.flatiter.base, 'foo')
-        assert np.flatiter.base.__doc__ == 'foo'
+        doc = np.flatiter.base.__doc__
+        try:
+            np.set_docstring(np.flatiter.base, 'foo')
+            assert np.flatiter.base.__doc__ == 'foo'
+        finally:
+            np.set_docstring(np.flatiter.base, doc)

From noreply at buildbot.pypy.org  Sat Jun 13 09:28:17 2015
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sat, 13 Jun 2015 09:28:17 +0200 (CEST)
Subject: [pypy-commit] pypy default: Fix broken translation
Message-ID: <20150613072817.8C9331C033F@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: 
Changeset: r78075:d9c84544ac9f
Date: 2015-06-13 09:28 +0200
http://bitbucket.org/pypy/pypy/changeset/d9c84544ac9f/

Log:	Fix broken translation

diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -3,7 +3,7 @@
 
 from rpython.rlib import rposix, objectmodel, rurandom
 from rpython.rlib.objectmodel import specialize
-from rpython.rlib.rarithmetic import r_longlong
+from rpython.rlib.rarithmetic import r_longlong, intmask
 from rpython.rlib.unroll import unrolling_iterable
 from rpython.rtyper.module import ll_os_stat
 from rpython.rtyper.module.ll_os import RegisterOs
@@ -1321,12 +1321,12 @@
 
 @unwrap_spec(device="c_uint")
 def major(space, device):
-    result = os.major(device)
+    result = os.major(intmask(device))
     return space.wrap(result)
 
 @unwrap_spec(device="c_uint")
 def minor(space, device):
-    result = os.minor(device)
+    result = os.minor(intmask(device))
     return space.wrap(result)
 
 @unwrap_spec(inc=c_int)

From noreply at buildbot.pypy.org  Sat Jun 13 11:14:29 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 13 Jun 2015 11:14:29 +0200 (CEST)
Subject: [pypy-commit] pypy default: Arguably buggy corner cases of
 str.startswith() vs unicode.startswith()
Message-ID: <20150613091429.2307E1C033F@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78076:b385d5240694
Date: 2015-06-13 10:41 +0200
http://bitbucket.org/pypy/pypy/changeset/b385d5240694/

Log:	Arguably buggy corner cases of str.startswith() vs
	unicode.startswith() on CPython

diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -635,7 +635,8 @@
         first index of value'''
         # needs to be safe against eq_w() mutating the w_list behind our back
         size = self.length()
-        i, stop = unwrap_start_stop(space, size, w_start, w_stop, True)
+        i, stop = unwrap_start_stop(space, size, w_start, w_stop)
+        # note that 'i' and 'stop' can be bigger than the length of the list
         try:
             i = self.find(w_value, i, stop)
         except ValueError:
diff --git a/pypy/objspace/std/sliceobject.py b/pypy/objspace/std/sliceobject.py
--- a/pypy/objspace/std/sliceobject.py
+++ b/pypy/objspace/std/sliceobject.py
@@ -211,26 +211,14 @@
     assert index >= 0
     return index
 
-def adapt_bound(space, size, w_index):
-    index = adapt_lower_bound(space, size, w_index)
-    if index > size:
-        index = size
-    assert index >= 0
-    return index
-
- at specialize.arg(4)
-def unwrap_start_stop(space, size, w_start, w_end, upper_bound=False):
+def unwrap_start_stop(space, size, w_start, w_end):
     if space.is_none(w_start):
         start = 0
-    elif upper_bound:
-        start = adapt_bound(space, size, w_start)
     else:
         start = adapt_lower_bound(space, size, w_start)
 
     if space.is_none(w_end):
         end = size
-    elif upper_bound:
-        end = adapt_bound(space, size, w_end)
     else:
         end = adapt_lower_bound(space, size, w_end)
     return start, end
diff --git a/pypy/objspace/std/stringmethods.py b/pypy/objspace/std/stringmethods.py
--- a/pypy/objspace/std/stringmethods.py
+++ b/pypy/objspace/std/stringmethods.py
@@ -22,12 +22,10 @@
         #    return orig_obj
         return self._new(s[start:stop])
 
-    @specialize.arg(4)
-    def _convert_idx_params(self, space, w_start, w_end, upper_bound=False):
+    def _convert_idx_params(self, space, w_start, w_end):
         value = self._val(space)
         lenself = len(value)
-        start, end = unwrap_start_stop(space, lenself, w_start, w_end,
-                                       upper_bound=upper_bound)
+        start, end = unwrap_start_stop(space, lenself, w_start, w_end)
         return (value, start, end)
 
     def _multi_chr(self, c):
@@ -606,8 +604,7 @@
         return self._newlist_unwrapped(space, strs)
 
     def descr_startswith(self, space, w_prefix, w_start=None, w_end=None):
-        (value, start, end) = self._convert_idx_params(space, w_start, w_end,
-                                                       True)
+        (value, start, end) = self._convert_idx_params(space, w_start, w_end)
         if space.isinstance_w(w_prefix, space.w_tuple):
             for w_prefix in space.fixedview(w_prefix):
                 if self._startswith(space, value, w_prefix, start, end):
@@ -617,11 +614,17 @@
                                               end))
 
     def _startswith(self, space, value, w_prefix, start, end):
-        return startswith(value, self._op_val(space, w_prefix), start, end)
+        prefix = self._op_val(space, w_prefix)
+        if start > len(value):
+            return self._starts_ends_overflow(prefix)
+        return startswith(value, prefix, start, end)
+
+    def _starts_ends_overflow(self, prefix):
+        return False     # bug-to-bug compat: this is for strings and
+                         # bytearrays, but overridden for unicodes
 
     def descr_endswith(self, space, w_suffix, w_start=None, w_end=None):
-        (value, start, end) = self._convert_idx_params(space, w_start, w_end,
-                                                       True)
+        (value, start, end) = self._convert_idx_params(space, w_start, w_end)
         if space.isinstance_w(w_suffix, space.w_tuple):
             for w_suffix in space.fixedview(w_suffix):
                 if self._endswith(space, value, w_suffix, start, end):
@@ -631,7 +634,10 @@
                                             end))
 
     def _endswith(self, space, value, w_prefix, start, end):
-        return endswith(value, self._op_val(space, w_prefix), start, end)
+        prefix = self._op_val(space, w_prefix)
+        if start > len(value):
+            return self._starts_ends_overflow(prefix)
+        return endswith(value, prefix, start, end)
 
     def _strip(self, space, w_chars, left, right):
         "internal function called by str_xstrip methods"
diff --git a/pypy/objspace/std/test/test_bytearrayobject.py b/pypy/objspace/std/test/test_bytearrayobject.py
--- a/pypy/objspace/std/test/test_bytearrayobject.py
+++ b/pypy/objspace/std/test/test_bytearrayobject.py
@@ -195,6 +195,14 @@
         assert bytearray('hello').endswith(bytearray('lo'))
         assert bytearray('hello').endswith((bytearray('lo'), 'he'))
 
+    def test_startswith_too_large(self):
+        assert bytearray('ab').startswith(bytearray('b'), 1) is True
+        assert bytearray('ab').startswith(bytearray(''), 2) is True
+        assert bytearray('ab').startswith(bytearray(''), 3) is False
+        assert bytearray('ab').endswith(bytearray('b'), 1) is True
+        assert bytearray('ab').endswith(bytearray(''), 2) is True
+        assert bytearray('ab').endswith(bytearray(''), 3) is False
+
     def test_stringlike_conversions(self):
         # methods that should return bytearray (and not str)
         def check(result, expected):
diff --git a/pypy/objspace/std/test/test_bytesobject.py b/pypy/objspace/std/test/test_bytesobject.py
--- a/pypy/objspace/std/test/test_bytesobject.py
+++ b/pypy/objspace/std/test/test_bytesobject.py
@@ -308,6 +308,14 @@
         assert 'abc'.startswith('bc', 1, 2) is False
         assert 'abc'.startswith('c', -1, 4) is True
 
+    def test_startswith_too_large(self):
+        assert 'ab'.startswith('b', 1) is True
+        assert 'ab'.startswith('', 2) is True
+        assert 'ab'.startswith('', 3) is False
+        assert 'ab'.endswith('b', 1) is True
+        assert 'ab'.endswith('', 2) is True
+        assert 'ab'.endswith('', 3) is False
+
     def test_startswith_tuples(self):
         assert 'hello'.startswith(('he', 'ha'))
         assert not 'hello'.startswith(('lo', 'llo'))
diff --git a/pypy/objspace/std/test/test_unicodeobject.py b/pypy/objspace/std/test/test_unicodeobject.py
--- a/pypy/objspace/std/test/test_unicodeobject.py
+++ b/pypy/objspace/std/test/test_unicodeobject.py
@@ -409,6 +409,14 @@
         assert u'abc'.startswith(u'bc', 1, 2) is False
         assert u'abc'.startswith(u'c', -1, 4) is True
 
+    def test_startswith_too_large(self):
+        assert u'ab'.startswith(u'b', 1) is True
+        assert u'ab'.startswith(u'', 2) is True
+        assert u'ab'.startswith(u'', 3) is True   # not False
+        assert u'ab'.endswith(u'b', 1) is True
+        assert u'ab'.endswith(u'', 2) is True
+        assert u'ab'.endswith(u'', 3) is True   # not False
+
     def test_startswith_tuples(self):
         assert u'hello'.startswith((u'he', u'ha'))
         assert not u'hello'.startswith((u'lo', u'llo'))
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -392,6 +392,9 @@
                 cased = True
         return space.newbool(cased)
 
+    def _starts_ends_overflow(self, prefix):
+        return len(prefix) == 0
+
 
 def wrapunicode(space, uni):
     return W_UnicodeObject(uni)

From noreply at buildbot.pypy.org  Sat Jun 13 11:14:30 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 13 Jun 2015 11:14:30 +0200 (CEST)
Subject: [pypy-commit] pypy default: Run the tests here with "--jit
	disable_unrolling=9999"
Message-ID: <20150613091430.3CCB41C033F@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78077:c4e72b72dc51
Date: 2015-06-13 10:48 +0200
http://bitbucket.org/pypy/pypy/changeset/c4e72b72dc51/

Log:	Run the tests here with "--jit disable_unrolling=9999"

diff --git a/pypy/module/pypyjit/test_pypy_c/test_00_model.py b/pypy/module/pypyjit/test_pypy_c/test_00_model.py
--- a/pypy/module/pypyjit/test_pypy_c/test_00_model.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_00_model.py
@@ -28,6 +28,7 @@
     def run(self, func_or_src, args=[], import_site=False,
             discard_stdout_before_last_line=False, **jitopts):
         jitopts.setdefault('threshold', 200)
+        jitopts.setdefault('disable_unrolling', 9999)
         src = py.code.Source(func_or_src)
         if isinstance(func_or_src, types.FunctionType):
             funcname = func_or_src.func_name

From noreply at buildbot.pypy.org  Sat Jun 13 11:14:50 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 13 Jun 2015 11:14:50 +0200 (CEST)
Subject: [pypy-commit] pypy default: Forgot this
Message-ID: <20150613091450.761C41C033F@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78078:0938f6ed592f
Date: 2015-06-13 10:16 +0100
http://bitbucket.org/pypy/pypy/changeset/0938f6ed592f/

Log:	Forgot this

diff --git a/pypy/objspace/std/sliceobject.py b/pypy/objspace/std/sliceobject.py
--- a/pypy/objspace/std/sliceobject.py
+++ b/pypy/objspace/std/sliceobject.py
@@ -219,6 +219,7 @@
 
     if space.is_none(w_end):
         end = size
+        assert end >= 0
     else:
         end = adapt_lower_bound(space, size, w_end)
     return start, end

From noreply at buildbot.pypy.org  Sat Jun 13 11:22:36 2015
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sat, 13 Jun 2015 11:22:36 +0200 (CEST)
Subject: [pypy-commit] pypy default: Another import issue
Message-ID: <20150613092236.77B581C033F@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: 
Changeset: r78079:85d52522b437
Date: 2015-06-13 11:10 +0200
http://bitbucket.org/pypy/pypy/changeset/85d52522b437/

Log:	Another import issue

diff --git a/lib-python/2.7/test/test_capi.py b/lib-python/2.7/test/test_capi.py
--- a/lib-python/2.7/test/test_capi.py
+++ b/lib-python/2.7/test/test_capi.py
@@ -18,7 +18,7 @@
 
 
 skips = []
-if test_support.check_impl_detail(pypy=True):
+if support.check_impl_detail(pypy=True):
     skips += [
             'test_broken_memoryview',
             'test_capsule',

From noreply at buildbot.pypy.org  Sat Jun 13 11:22:37 2015
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sat, 13 Jun 2015 11:22:37 +0200 (CEST)
Subject: [pypy-commit] pypy default: Revert the changes in memoryview.
Message-ID: <20150613092237.938781C033F@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: 
Changeset: r78080:47e51aba1243
Date: 2015-06-13 11:21 +0200
http://bitbucket.org/pypy/pypy/changeset/47e51aba1243/

Log:	Revert the changes in memoryview. CPython 2.7.10 changed argument
	parsing of the struct module, and also some errors messages.

	(checked with running tests with -A, on a CPython-2.7.10
	interpreter)

diff --git a/pypy/module/struct/interp_struct.py b/pypy/module/struct/interp_struct.py
--- a/pypy/module/struct/interp_struct.py
+++ b/pypy/module/struct/interp_struct.py
@@ -61,7 +61,7 @@
 @unwrap_spec(format=str, offset=int)
 def pack_into(space, format, w_buffer, offset, args_w):
     res = _pack(space, format, args_w)
-    buf = space.writebuf_w(w_buffer)
+    buf = space.getarg_w('w*', w_buffer)
     if offset < 0:
         offset += buf.getlength()
     size = len(res)
diff --git a/pypy/module/struct/test/test_struct.py b/pypy/module/struct/test/test_struct.py
--- a/pypy/module/struct/test/test_struct.py
+++ b/pypy/module/struct/test/test_struct.py
@@ -390,9 +390,9 @@
                                       self.struct.pack("ii", 17, 42) +
                                       '\x00' * (19-sz-2))
         exc = raises(TypeError, self.struct.pack_into, "ii", buffer(b), 0, 17, 42)
-        assert str(exc.value) == "buffer is read-only"
+        assert str(exc.value) == "argument must be read-write buffer, not buffer"
         exc = raises(TypeError, self.struct.pack_into, "ii", 'test', 0, 17, 42)
-        assert str(exc.value) == "Cannot use string as modifiable buffer"
+        assert str(exc.value) == "argument must be read-write buffer, not str"
         exc = raises(self.struct.error, self.struct.pack_into, "ii", b[0:1], 0, 17, 42)
         assert str(exc.value) == "pack_into requires a buffer of at least 8 bytes"
 
diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -23,14 +23,6 @@
         space.check_buf_flags(flags, self.buf.readonly)
         return self.buf
 
-    def readbuf_w(self, space):
-        return self.buf
-
-    def writebuf_w(self, space):
-        if self.buf.readonly:
-            raise oefmt(space.w_TypeError, "buffer is read-only")
-        return self.buf
-
     @staticmethod
     def descr_new_memoryview(space, w_subtype, w_object):
         return W_MemoryView(space.buffer_w(w_object, space.BUF_FULL_RO))

From noreply at buildbot.pypy.org  Sat Jun 13 11:56:40 2015
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sat, 13 Jun 2015 11:56:40 +0200 (CEST)
Subject: [pypy-commit] pypy py3.3: Fix sys.thread_info
Message-ID: <20150613095640.BA0EC1C1C5C@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3.3
Changeset: r78081:38c4a6aa6559
Date: 2015-05-02 22:38 +0200
http://bitbucket.org/pypy/pypy/changeset/38c4a6aa6559/

Log:	Fix sys.thread_info

diff --git a/pypy/module/sys/__init__.py b/pypy/module/sys/__init__.py
--- a/pypy/module/sys/__init__.py
+++ b/pypy/module/sys/__init__.py
@@ -78,7 +78,7 @@
         'int_info'              : 'system.get_int_info(space)',
         'hash_info'             : 'system.get_hash_info(space)',
         'float_repr_style'      : 'system.get_float_repr_style(space)',
-        'thread_info'           : 'system.get_thread_info(space)'
+        'thread_info'           : 'system.get_thread_info(space)',
         }
 
     if sys.platform == 'win32':
diff --git a/pypy/module/sys/system.py b/pypy/module/sys/system.py
--- a/pypy/module/sys/system.py
+++ b/pypy/module/sys/system.py
@@ -1,5 +1,6 @@
 """Information about the current system."""
 import sys
+import os
 
 from pypy.objspace.std.complexobject import HASH_IMAG
 from pypy.objspace.std.floatobject import HASH_INF, HASH_NAN
@@ -91,10 +92,21 @@
     # every field
     if not space.config.objspace.usemodules.thread:
         return None
+    from rpython.rlib import rthread
+    if rthread.RPYTHREAD_NAME == "pthread":
+        w_lock = space.wrap("semaphore" if rthread.USE_SEMAPHORES
+                            else "mutex+cond")
+        if rthread.CS_GNU_LIBPTHREAD_VERSION is not None:
+            w_version = space.wrap(
+                os.confstr(rthread.CS_GNU_LIBPTHREAD_VERSION))
+        else:
+            w_version = space.w_None
+    else:
+        w_lock = space.w_None
+        w_version = space.w_None
     info_w = [
-        space.wrap(space.w_None),
-        space.wrap(space.w_None),
-        space.wrap(space.w_None),
+        space.wrap(rthread.RPYTHREAD_NAME),
+        w_lock, w_version,
     ]
     w_thread_info = app.wget(space, "thread_info")
     return space.call_function(w_thread_info, space.newtuple(info_w))
diff --git a/pypy/module/sys/test/test_sysmodule.py b/pypy/module/sys/test/test_sysmodule.py
--- a/pypy/module/sys/test/test_sysmodule.py
+++ b/pypy/module/sys/test/test_sysmodule.py
@@ -38,6 +38,9 @@
         space.setattr(w_sys, space.wrap('stderr'), w_sys.get('__stderr__'))
 
 class AppTestAppSysTests:
+    spaceconfig = {
+        "usemodules": ["thread"],
+    }
 
     def setup_class(cls):
         cls.w_appdirect = cls.space.wrap(cls.runappdirect)
@@ -201,6 +204,13 @@
         exc = raises(SystemExit, sys.exit, (1, 2, 3))
         assert exc.value.code == (1, 2, 3)
 
+    def test_sys_thread_info(self):
+        import sys
+        info = sys.thread_info
+        assert isinstance(info.name, str)
+        assert isinstance(info.lock, (str, type(None)))
+        assert isinstance(info.version, (str, type(None)))
+
 
 class AppTestSysModulePortedFromCPython:
     def setup_class(cls):
diff --git a/rpython/rlib/rthread.py b/rpython/rlib/rthread.py
--- a/rpython/rlib/rthread.py
+++ b/rpython/rlib/rthread.py
@@ -22,6 +22,16 @@
     include_dirs = [translator_c_dir],
 )
 
+class CConfig:
+    _compilation_info_ = eci
+    RPYTHREAD_NAME = rffi_platform.DefinedConstantString('RPYTHREAD_NAME')
+    USE_SEMAPHORES = rffi_platform.Defined('USE_SEMAPHORES')
+    CS_GNU_LIBPTHREAD_VERSION = rffi_platform.DefinedConstantInteger(
+        '_CS_GNU_LIBPTHREAD_VERSION')
+cconfig = rffi_platform.configure(CConfig)
+globals().update(cconfig)
+
+
 def llexternal(name, args, result, **kwds):
     kwds.setdefault('sandboxsafe', True)
     return rffi.llexternal(name, args, result, compilation_info=eci,
diff --git a/rpython/translator/c/src/thread.h b/rpython/translator/c/src/thread.h
--- a/rpython/translator/c/src/thread.h
+++ b/rpython/translator/c/src/thread.h
@@ -12,6 +12,7 @@
 } RPyLockStatus;
 
 #ifdef _WIN32
+#define RPYTHREAD_NAME "nt"
 #include "thread_nt.h"
 #define inline _inline
 #else
@@ -22,6 +23,7 @@
    always go ahead and use them, assuming they are supported on all
    platforms for which we care.  If not, do some detecting again.
 */
+#define RPYTHREAD_NAME "pthread"
 #include "thread_pthread.h"
 
 #endif /* !_WIN32 */

From noreply at buildbot.pypy.org  Sat Jun 13 12:46:14 2015
From: noreply at buildbot.pypy.org (yuyichao)
Date: Sat, 13 Jun 2015 12:46:14 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: fix strerror encoding
Message-ID: <20150613104614.BB8F61C11B5@cobra.cs.uni-duesseldorf.de>

Author: Yichao Yu <yyc1992 at gmail.com>
Branch: py3k
Changeset: r78082:7b8f47ac90ef
Date: 2015-06-11 23:23 -0400
http://bitbucket.org/pypy/pypy/changeset/7b8f47ac90ef/

Log:	fix strerror encoding

diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py
--- a/pypy/interpreter/error.py
+++ b/pypy/interpreter/error.py
@@ -14,6 +14,10 @@
 AUTO_DEBUG = os.getenv('PYPY_DEBUG')
 RECORD_INTERPLEVEL_TRACEBACK = True
 
+def strerror(errno):
+    """Translate an error code to a message string."""
+    from pypy.module._codecs.locale import str_decode_locale_surrogateescape
+    return str_decode_locale_surrogateescape(os.strerror(errno))
 
 class OperationError(Exception):
     """Interpreter-level exception that signals an exception that should be
@@ -530,9 +534,9 @@
         space.getexecutioncontext().checksignals()
 
     try:
-        msg = os.strerror(errno)
+        msg = strerror(errno)
     except ValueError:
-        msg = 'error %d' % errno
+        msg = u'error %d' % errno
     if w_exception_class is None:
         exc = getattr(space, exception_name)
     else:
@@ -562,7 +566,7 @@
     from rpython.rlib.rposix import get_saved_errno
 
     errno = get_saved_errno()
-    msg = os.strerror(errno)
+    msg = strerror(errno)
     w_error = space.call_function(w_type, space.wrap(errno), space.wrap(msg))
     return OperationError(w_type, w_error)
 
diff --git a/pypy/module/_codecs/locale.py b/pypy/module/_codecs/locale.py
--- a/pypy/module/_codecs/locale.py
+++ b/pypy/module/_codecs/locale.py
@@ -13,6 +13,8 @@
 from rpython.translator import cdir
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
 
+from pypy.interpreter.error import strerror as _strerror
+
 cwd = py.path.local(__file__).dirpath()
 eci = ExternalCompilationInfo(
     includes=[cwd.join('locale_codec.h')],
@@ -56,7 +58,7 @@
                 errorpos = rffi.cast(lltype.Signed, errorposp[0])
                 if errorpos == -1:
                     raise MemoryError
-                errmsg = _errmsg("pypy_wchar2char")
+                errmsg = _errmsg(u"pypy_wchar2char")
                 errorhandler('strict', 'filesystemencoding', errmsg, u,
                              errorpos, errorpos + 1)
             return rffi.charp2str(sbuf)
@@ -79,7 +81,7 @@
             ubuf = pypy_char2wchar(sbuf, sizep)
         try:
             if ubuf is None:
-                errmsg = _errmsg("pypy_char2wchar")
+                errmsg = _errmsg(u"pypy_char2wchar")
                 errorhandler('strict', 'filesystemencoding', errmsg, s, 0, 1)
             size = rffi.cast(lltype.Signed, sizep[0])
             return rawwcharp2unicoden(ubuf, size)
@@ -89,8 +91,8 @@
 
 def _errmsg(what):
     from rpython.rlib import rposix
-    errmsg = os.strerror(rposix.get_errno())
-    return "%s failed" % what if errmsg is None else errmsg
+    errmsg = _strerror(rposix.get_errno())
+    return u"%s failed" % what if errmsg is None else errmsg
 
 
 class scoped_unicode2rawwcharp:
diff --git a/pypy/module/cpyext/pyerrors.py b/pypy/module/cpyext/pyerrors.py
--- a/pypy/module/cpyext/pyerrors.py
+++ b/pypy/module/cpyext/pyerrors.py
@@ -1,7 +1,7 @@
 import os
 
 from rpython.rtyper.lltypesystem import rffi, lltype
-from pypy.interpreter.error import OperationError
+from pypy.interpreter.error import OperationError, strerror as _strerror
 from pypy.interpreter import pytraceback
 from pypy.module.cpyext.api import cpython_api, CANNOT_FAIL, CONST_STRING
 from pypy.module.exceptions.interp_exceptions import W_RuntimeWarning
@@ -164,7 +164,7 @@
     Return value: always NULL."""
     # XXX Doesn't actually do anything with PyErr_CheckSignals.
     errno = rffi.cast(lltype.Signed, rposix._get_errno())
-    msg = os.strerror(errno)
+    msg = _strerror(errno)
     if w_value:
         w_error = space.call_function(w_type,
                                       space.wrap(errno),
diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -9,7 +9,8 @@
 from rpython.rtyper.module.ll_os import RegisterOs
 
 from pypy.interpreter.gateway import unwrap_spec, WrappedDefault
-from pypy.interpreter.error import OperationError, wrap_oserror, wrap_oserror2
+from pypy.interpreter.error import (OperationError, wrap_oserror,
+                                    wrap_oserror2, strerror as _strerror)
 from pypy.interpreter.executioncontext import ExecutionContext
 
 
@@ -477,11 +478,10 @@
 def strerror(space, errno):
     """Translate an error code to a message string."""
     try:
-        text = os.strerror(errno)
+        return space.wrap(_strerror(errno))
     except ValueError:
         raise OperationError(space.w_ValueError,
                              space.wrap("strerror() argument out of range"))
-    return space.wrap(text)
 
 def getlogin(space):
     """Return the currently logged in user."""
diff --git a/pypy/module/time/interp_time.py b/pypy/module/time/interp_time.py
--- a/pypy/module/time/interp_time.py
+++ b/pypy/module/time/interp_time.py
@@ -1,6 +1,6 @@
 from rpython.rtyper.tool import rffi_platform as platform
 from rpython.rtyper.lltypesystem import rffi
-from pypy.interpreter.error import OperationError, oefmt
+from pypy.interpreter.error import OperationError, oefmt, strerror as _strerror
 from pypy.interpreter.gateway import unwrap_spec
 from rpython.rtyper.lltypesystem import lltype
 from rpython.rlib.rarithmetic import intmask
@@ -306,7 +306,7 @@
 
 def _get_error_msg():
     errno = rposix.get_saved_errno()
-    return os.strerror(errno)
+    return _strerror(errno)
 
 if sys.platform != 'win32':
     @unwrap_spec(secs=float)
@@ -404,7 +404,7 @@
         lltype.free(t_ref, flavor='raw')
         if not pbuf:
             raise OperationError(space.w_ValueError,
-                space.wrap(_get_error_msg()))
+                                 space.wrap(_get_error_msg()))
         return pbuf
 
     tup_w = space.fixedview(w_tup)

From noreply at buildbot.pypy.org  Sat Jun 13 12:46:16 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Sat, 13 Jun 2015 12:46:16 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Merged in yuyichao/pypy/py3k (pull request
	#326)
Message-ID: <20150613104616.0F8001C11B5@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob <me at manueljacob.de>
Branch: py3k
Changeset: r78083:c99fbaf556ce
Date: 2015-06-13 12:44 +0200
http://bitbucket.org/pypy/pypy/changeset/c99fbaf556ce/

Log:	Merged in yuyichao/pypy/py3k (pull request #326)

	fix strerror encoding

diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py
--- a/pypy/interpreter/error.py
+++ b/pypy/interpreter/error.py
@@ -14,6 +14,10 @@
 AUTO_DEBUG = os.getenv('PYPY_DEBUG')
 RECORD_INTERPLEVEL_TRACEBACK = True
 
+def strerror(errno):
+    """Translate an error code to a message string."""
+    from pypy.module._codecs.locale import str_decode_locale_surrogateescape
+    return str_decode_locale_surrogateescape(os.strerror(errno))
 
 class OperationError(Exception):
     """Interpreter-level exception that signals an exception that should be
@@ -530,9 +534,9 @@
         space.getexecutioncontext().checksignals()
 
     try:
-        msg = os.strerror(errno)
+        msg = strerror(errno)
     except ValueError:
-        msg = 'error %d' % errno
+        msg = u'error %d' % errno
     if w_exception_class is None:
         exc = getattr(space, exception_name)
     else:
@@ -562,7 +566,7 @@
     from rpython.rlib.rposix import get_saved_errno
 
     errno = get_saved_errno()
-    msg = os.strerror(errno)
+    msg = strerror(errno)
     w_error = space.call_function(w_type, space.wrap(errno), space.wrap(msg))
     return OperationError(w_type, w_error)
 
diff --git a/pypy/module/_codecs/locale.py b/pypy/module/_codecs/locale.py
--- a/pypy/module/_codecs/locale.py
+++ b/pypy/module/_codecs/locale.py
@@ -13,6 +13,8 @@
 from rpython.translator import cdir
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
 
+from pypy.interpreter.error import strerror as _strerror
+
 cwd = py.path.local(__file__).dirpath()
 eci = ExternalCompilationInfo(
     includes=[cwd.join('locale_codec.h')],
@@ -56,7 +58,7 @@
                 errorpos = rffi.cast(lltype.Signed, errorposp[0])
                 if errorpos == -1:
                     raise MemoryError
-                errmsg = _errmsg("pypy_wchar2char")
+                errmsg = _errmsg(u"pypy_wchar2char")
                 errorhandler('strict', 'filesystemencoding', errmsg, u,
                              errorpos, errorpos + 1)
             return rffi.charp2str(sbuf)
@@ -79,7 +81,7 @@
             ubuf = pypy_char2wchar(sbuf, sizep)
         try:
             if ubuf is None:
-                errmsg = _errmsg("pypy_char2wchar")
+                errmsg = _errmsg(u"pypy_char2wchar")
                 errorhandler('strict', 'filesystemencoding', errmsg, s, 0, 1)
             size = rffi.cast(lltype.Signed, sizep[0])
             return rawwcharp2unicoden(ubuf, size)
@@ -89,8 +91,8 @@
 
 def _errmsg(what):
     from rpython.rlib import rposix
-    errmsg = os.strerror(rposix.get_errno())
-    return "%s failed" % what if errmsg is None else errmsg
+    errmsg = _strerror(rposix.get_errno())
+    return u"%s failed" % what if errmsg is None else errmsg
 
 
 class scoped_unicode2rawwcharp:
diff --git a/pypy/module/cpyext/pyerrors.py b/pypy/module/cpyext/pyerrors.py
--- a/pypy/module/cpyext/pyerrors.py
+++ b/pypy/module/cpyext/pyerrors.py
@@ -1,7 +1,7 @@
 import os
 
 from rpython.rtyper.lltypesystem import rffi, lltype
-from pypy.interpreter.error import OperationError
+from pypy.interpreter.error import OperationError, strerror as _strerror
 from pypy.interpreter import pytraceback
 from pypy.module.cpyext.api import cpython_api, CANNOT_FAIL, CONST_STRING
 from pypy.module.exceptions.interp_exceptions import W_RuntimeWarning
@@ -164,7 +164,7 @@
     Return value: always NULL."""
     # XXX Doesn't actually do anything with PyErr_CheckSignals.
     errno = rffi.cast(lltype.Signed, rposix._get_errno())
-    msg = os.strerror(errno)
+    msg = _strerror(errno)
     if w_value:
         w_error = space.call_function(w_type,
                                       space.wrap(errno),
diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -9,7 +9,8 @@
 from rpython.rtyper.module.ll_os import RegisterOs
 
 from pypy.interpreter.gateway import unwrap_spec, WrappedDefault
-from pypy.interpreter.error import OperationError, wrap_oserror, wrap_oserror2
+from pypy.interpreter.error import (OperationError, wrap_oserror,
+                                    wrap_oserror2, strerror as _strerror)
 from pypy.interpreter.executioncontext import ExecutionContext
 
 
@@ -477,11 +478,10 @@
 def strerror(space, errno):
     """Translate an error code to a message string."""
     try:
-        text = os.strerror(errno)
+        return space.wrap(_strerror(errno))
     except ValueError:
         raise OperationError(space.w_ValueError,
                              space.wrap("strerror() argument out of range"))
-    return space.wrap(text)
 
 def getlogin(space):
     """Return the currently logged in user."""
diff --git a/pypy/module/time/interp_time.py b/pypy/module/time/interp_time.py
--- a/pypy/module/time/interp_time.py
+++ b/pypy/module/time/interp_time.py
@@ -1,6 +1,6 @@
 from rpython.rtyper.tool import rffi_platform as platform
 from rpython.rtyper.lltypesystem import rffi
-from pypy.interpreter.error import OperationError, oefmt
+from pypy.interpreter.error import OperationError, oefmt, strerror as _strerror
 from pypy.interpreter.gateway import unwrap_spec
 from rpython.rtyper.lltypesystem import lltype
 from rpython.rlib.rarithmetic import intmask
@@ -306,7 +306,7 @@
 
 def _get_error_msg():
     errno = rposix.get_saved_errno()
-    return os.strerror(errno)
+    return _strerror(errno)
 
 if sys.platform != 'win32':
     @unwrap_spec(secs=float)
@@ -404,7 +404,7 @@
         lltype.free(t_ref, flavor='raw')
         if not pbuf:
             raise OperationError(space.w_ValueError,
-                space.wrap(_get_error_msg()))
+                                 space.wrap(_get_error_msg()))
         return pbuf
 
     tup_w = space.fixedview(w_tup)

From noreply at buildbot.pypy.org  Sat Jun 13 16:47:10 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 13 Jun 2015 16:47:10 +0200 (CEST)
Subject: [pypy-commit] stmgc c8-gil-like: Minor tweaks to make life easier
	for the pypy jit
Message-ID: <20150613144710.617031C0823@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: c8-gil-like
Changeset: r1821:8009f12c327b
Date: 2015-06-13 16:47 +0200
http://bitbucket.org/pypy/stmgc/changeset/8009f12c327b/

Log:	Minor tweaks to make life easier for the pypy jit

diff --git a/c8/demo/demo_random2.c b/c8/demo/demo_random2.c
--- a/c8/demo/demo_random2.c
+++ b/c8/demo/demo_random2.c
@@ -365,7 +365,7 @@
     } else if (get_rand(20) == 1) {
         long pushed = push_roots();
         stm_become_inevitable(&stm_thread_local, "please");
-        assert(stm_is_inevitable());
+        assert(stm_is_inevitable(&stm_thread_local));
         pop_roots(pushed);
         p= NULL;
     } else if (get_rand(20) == 1) {
diff --git a/c8/stm/core.c b/c8/stm/core.c
--- a/c8/stm/core.c
+++ b/c8/stm/core.c
@@ -1220,8 +1220,9 @@
 
 #ifdef STM_NO_AUTOMATIC_SETJMP
 void _test_run_abort(stm_thread_local_t *tl) __attribute__((noreturn));
-int stm_is_inevitable(void)
+int stm_is_inevitable(stm_thread_local_t *tl)
 {
+    assert(STM_SEGMENT->running_thread == tl);
     switch (STM_PSEGMENT->transaction_state) {
     case TS_REGULAR: return 0;
     case TS_INEVITABLE: return 1;
@@ -1593,7 +1594,7 @@
 
 void stm_stop_all_other_threads(void)
 {
-    if (!stm_is_inevitable())         /* may still abort */
+    if (!stm_is_inevitable(STM_SEGMENT->running_thread))  /* may still abort */
         _stm_become_inevitable("stop_all_other_threads");
 
     s_mutex_lock();
diff --git a/c8/stm/setup.c b/c8/stm/setup.c
--- a/c8/stm/setup.c
+++ b/c8/stm/setup.c
@@ -233,6 +233,8 @@
 {
     int num;
     s_mutex_lock();
+    tl->self = tl;    /* for faster access to &stm_thread_local (and easier
+                         from the PyPy JIT, too) */
     if (stm_all_thread_locals == NULL) {
         stm_all_thread_locals = tl->next = tl->prev = tl;
         num = 0;
diff --git a/c8/stmgc.h b/c8/stmgc.h
--- a/c8/stmgc.h
+++ b/c8/stmgc.h
@@ -71,7 +71,7 @@
     /* the next fields are handled internally by the library */
     int last_associated_segment_num;   /* always a valid seg num */
     int thread_local_counter;
-    struct stm_thread_local_s *prev, *next;
+    struct stm_thread_local_s *self, *prev, *next;
     void *creating_pthread[2];
 } stm_thread_local_t;
 
@@ -87,10 +87,10 @@
 long _stm_start_transaction(stm_thread_local_t *tl);
 void _stm_commit_transaction(void);
 void _stm_leave_noninevitable_transactional_zone(void);
-#define _stm_detach_inevitable_transaction(tl)  do {            \
-    write_fence();                                              \
-    assert(_stm_detached_inevitable_from_thread == 0);          \
-    _stm_detached_inevitable_from_thread = (intptr_t)(tl);      \
+#define _stm_detach_inevitable_transaction(tl)  do {                    \
+    write_fence();                                                      \
+    assert(_stm_detached_inevitable_from_thread == 0);                  \
+    _stm_detached_inevitable_from_thread = (intptr_t)(tl->self);        \
 } while (0)
 void _stm_reattach_transaction(stm_thread_local_t *tl);
 void _stm_become_inevitable(const char*);
@@ -391,10 +391,10 @@
 
 
 #ifdef STM_NO_AUTOMATIC_SETJMP
-int stm_is_inevitable(void);
+int stm_is_inevitable(stm_thread_local_t *tl);
 #else
-static inline int stm_is_inevitable(void) {
-    return !rewind_jmp_armed(&STM_SEGMENT->running_thread->rjthread);
+static inline int stm_is_inevitable(stm_thread_local_t *tl) {
+    return !rewind_jmp_armed(&tl->rjthread);
 }
 #endif
 
@@ -441,7 +441,7 @@
 }
 static inline void stm_leave_transactional_zone(stm_thread_local_t *tl) {
     assert(STM_SEGMENT->running_thread == tl);
-    if (stm_is_inevitable()) {
+    if (stm_is_inevitable(tl)) {
 #ifdef STM_DEBUGPRINT
         fprintf(stderr, "stm_leave_transactional_zone fast path\n");
 #endif
@@ -472,7 +472,7 @@
 static inline void stm_become_inevitable(stm_thread_local_t *tl,
                                          const char* msg) {
     assert(STM_SEGMENT->running_thread == tl);
-    if (!stm_is_inevitable())
+    if (!stm_is_inevitable(tl))
         _stm_become_inevitable(msg);
     /* now, we're running the inevitable transaction, so this var should be 0 */
     assert(_stm_detached_inevitable_from_thread == 0);
diff --git a/c8/test/support.py b/c8/test/support.py
--- a/c8/test/support.py
+++ b/c8/test/support.py
@@ -88,7 +88,7 @@
 //bool _check_become_globally_unique_transaction(stm_thread_local_t *tl);
 bool _check_stop_all_other_threads(void);
 void stm_resume_all_other_threads(void);
-int stm_is_inevitable(void);
+int stm_is_inevitable(stm_thread_local_t *);
 long current_segment_num(void);
 
 void _set_type_id(object_t *obj, uint32_t h);
@@ -770,14 +770,14 @@
         lib.stmcb_expand_marker = ffi.NULL
         lib.stmcb_debug_print = ffi.NULL
         tl = self.tls[self.current_thread]
-        if lib._stm_in_transaction(tl) and lib.stm_is_inevitable():
+        if lib._stm_in_transaction(tl) and self.is_inevitable():
             self.commit_transaction()      # must succeed!
         #
         for n, tl in enumerate(self.tls):
             if lib._stm_in_transaction(tl):
                 if self.current_thread != n:
                     self.switch(n)
-                if lib.stm_is_inevitable():
+                if self.is_inevitable():
                     self.commit_transaction()   # must succeed!
                 else:
                     self.abort_transaction()
@@ -786,6 +786,11 @@
             lib.stm_unregister_thread_local(tl)
         lib.stm_teardown()
 
+    def is_inevitable(self):
+        tl = self.tls[self.current_thread]
+        assert lib._stm_in_transaction(tl)
+        return lib.stm_is_inevitable(tl)
+
     def get_stm_thread_local(self):
         return self.tls[self.current_thread]
 
diff --git a/c8/test/test_basic.py b/c8/test/test_basic.py
--- a/c8/test/test_basic.py
+++ b/c8/test/test_basic.py
@@ -640,7 +640,7 @@
 
     def test_inevitable_transaction_has_priority(self):
         self.start_transaction()
-        assert lib.stm_is_inevitable() == 0
+        assert self.is_inevitable() == 0
         lp1 = stm_allocate(16)
         stm_set_char(lp1, 'a')
         self.push_root(lp1)
@@ -654,9 +654,9 @@
         self.start_transaction()
         stm_write(lp1)
         stm_set_char(lp1, 'b')
-        assert lib.stm_is_inevitable() == 0
+        assert self.is_inevitable() == 0
         self.become_inevitable()
-        assert lib.stm_is_inevitable() == 1
+        assert self.is_inevitable() == 1
         self.commit_transaction()
         #
         py.test.raises(Conflict, self.switch, 0)
diff --git a/c8/test/test_extra.py b/c8/test/test_extra.py
--- a/c8/test/test_extra.py
+++ b/c8/test/test_extra.py
@@ -189,7 +189,7 @@
         self.switch(1)
         self.start_transaction()
         self.become_globally_unique_transaction()
-        assert lib.stm_is_inevitable()
+        assert self.is_inevitable()
         #
         py.test.raises(Conflict, self.switch, 0)
 
@@ -199,7 +199,7 @@
         self.switch(1)
         self.start_transaction()
         self.stop_all_other_threads()
-        assert lib.stm_is_inevitable()
+        assert self.is_inevitable()
         #
         py.test.raises(Conflict, self.switch, 0)
         #
@@ -213,6 +213,6 @@
         self.start_transaction()
         self.stop_all_other_threads()
         self.resume_all_other_threads()
-        assert lib.stm_is_inevitable()
+        assert self.is_inevitable()
         #
         self.switch(0)   # no conflict

From noreply at buildbot.pypy.org  Sat Jun 13 17:59:06 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 13 Jun 2015 17:59:06 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c8-gil-like: in-progress: pypy jit
Message-ID: <20150613155906.E4DFF1C11B5@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c8-gil-like
Changeset: r78084:720b5a8e744f
Date: 2015-06-13 17:59 +0200
http://bitbucket.org/pypy/pypy/changeset/720b5a8e744f/

Log:	in-progress: pypy jit

diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -78,6 +78,8 @@
         if self.cpu.supports_floats:
             support.ensure_sse2_floats()
             self._build_float_constants()
+        if self.cpu.gc_ll_descr.stm:
+            self._build_stm_enter_leave_transactional_zone_helpers()
 
     def setup(self, looptoken):
         assert self.memcpy_addr != 0, "setup_once() not called?"
@@ -125,6 +127,36 @@
         self.float_const_neg_addr = float_constants
         self.float_const_abs_addr = float_constants + 16
 
+    def _build_stm_enter_leave_transactional_zone_helpers(self):
+        assert IS_X86_64 and self.cpu.supports_floats
+        # a helper to call _stm_leave_noninevitable_transactional_zone(),
+        # preserving all registers that are used to pass arguments.
+        # (Push an odd total number of registers, to align the stack.)
+        mc = codebuf.MachineCodeBlockWrapper()
+        self._push_all_regs_to_frame(mc, [eax], True, callee_only=True)
+        mc.CALL(imm(rstm.adr_stm_leave_noninevitable_transactional_zone))
+        self._pop_all_regs_from_frame(mc, [eax], True, callee_only=True)
+        mc.RET()
+        self._stm_leave_noninevitable_tr_slowpath = mc.materialize(
+            self.cpu.asmmemmgr, [])
+        #
+        # a second helper to call _stm_reattach_transaction(tl),
+        # preserving only registers that might store the result of a call
+        mc = codebuf.MachineCodeBlockWrapper()
+        mc.SUB_ri(esp.value, 3 * WORD)     # 3 instead of 2 to align the stack
+        mc.MOV_sr(0, eax.value)     # not edx, we're not running 32-bit
+        mc.MOVSD_sx(1, xmm0.value)
+        # load the value of tl (== tl->self) into edi as argument
+        mc.MOV(edi, self.heap_stm_thread_local_self())
+        mc.CALL(imm(rstm.adr_stm_reattach_transaction))
+        # pop
+        mc.MOVSD_xs(xmm0.value, 1)
+        mc.MOV_rs(eax.value, 0)
+        mc.ADD_ri(esp.value, 3 * WORD)
+        mc.RET()
+        self._stm_reattach_tr_slowpath = mc.materialize(self.cpu.asmmemmgr, [])
+
+
     def set_extra_stack_depth(self, mc, value):
         if self._is_asmgcc():
             extra_ofs = self.cpu.get_ofs_of_frame_field('jf_extra_stack_depth')
@@ -898,6 +930,16 @@
         """STM: AddressLoc for '&stm_thread_local.rjthread.moved_off_base'."""
         return self.heap_tl(rstm.adr_rjthread_moved_off_base)
 
+    def heap_stm_thread_local_self(self):
+        """STM: AddressLoc for '&stm_thread_local.self', i.e. such that
+        reading it returns the (absolute) address of 'stm_thread_local'."""
+        return self.heap_tl(rstm.adr_stm_thread_local_self)
+
+    def heap_stm_detached_inevitable_from_thread(self):
+        """STM: AddressLoc for '&stm_detached_inevitable_from_thread'."""
+        return heap(self.SEGMENT_NO,
+                    rstm.adr_stm_detached_inevitable_from_thread)
+
     def _call_header_shadowstack(self):
         # put the frame in ebp on the shadowstack for the GC to find
         # (ebp is a writeable object and does not need a write-barrier
diff --git a/rpython/jit/backend/x86/callbuilder.py b/rpython/jit/backend/x86/callbuilder.py
--- a/rpython/jit/backend/x86/callbuilder.py
+++ b/rpython/jit/backend/x86/callbuilder.py
@@ -684,41 +684,70 @@
             self.mc.MOV_rs(eax.value, 0)
 
     def call_stm_before_ex_call(self):
+        from rpython.jit.backend.x86 import rx86
         from rpython.rlib import rstm
-        # XXX slowish: before any CALL_RELEASE_GIL, invoke the
-        # pypy_stm_commit_if_not_atomic() function.  Messy because
-        # we need to save the register arguments first.
+        # Generate the same logic as stm_leave_transactional_zone()
         #
-        n = min(self.next_arg_gpr, len(self.ARGUMENTS_GPR))
-        for i in range(n):
-            self.mc.PUSH_r(self.ARGUMENTS_GPR[i].value)    # PUSH gpr arg
-        m = min(self.next_arg_xmm, len(self.ARGUMENTS_XMM))
-        extra = m + ((n + m) & 1)
-        # in total the stack is moved down by (n + extra) words,
-        # which needs to be an even value for alignment:
-        assert ((n + extra) & 1) == 0
-        if extra > 0:
-            self.mc.SUB_ri(esp.value, extra * WORD)        # SUB rsp, extra
-            for i in range(m):
-                self.mc.MOVSD_sx(i * WORD, self.ARGUMENTS_XMM[i].value)
-                                                           # MOVSD [rsp+..], xmm
+        # First, stm_is_inevitable(), which is '!rewind_jmp_armed()',
+        # which is 'moved_off_base == 0':
+        rjmovd_o_b = self.asm.heap_rjthread_moved_off_base()
+        mc = self.mc
+        mc.CMP(rjmovd_o_b, imm(0))
+        mc.J_il8(rx86.Conditions['E'], 0)
+        je_location = mc.get_relative_pos()
         #
-        self.mc.CALL(imm(rstm.adr_pypy_stm_commit_if_not_atomic))
+        # Slow path: call a helper that will save all registers and
+        # call _stm_leave_noninevitable_transactional_zone()
+        mc.CALL(imm(self.asm._stm_leave_noninevitable_tr_slowpath))
+        mc.JMP_l8(0)      # jump to done, patched later
+        jmp_location = mc.get_relative_pos()
         #
-        if extra > 0:
-            for i in range(m):
-                self.mc.MOVSD_xs(self.ARGUMENTS_XMM[i].value, i * WORD)
-            self.mc.ADD_ri(esp.value, extra * WORD)
-        for i in range(n-1, -1, -1):
-            self.mc.POP_r(self.ARGUMENTS_GPR[i].value)
+        offset = jmp_location - je_location
+        assert 0 < offset <= 127
+        mc.overwrite(je_location - 1, chr(offset))
+        #
+        # Fast path: inline _stm_detach_inevitable_transaction()
+        # <- Here comes the write_fence(), which is not needed in x86 assembler
+        # assert(_stm_detached_inevitable_from_thread == 0): dropped
+        # _stm_detached_inevitable_from_thread = tl (== tl->self):
+        mc.MOV(eax, self.asm.heap_stm_thread_local_self())
+        mc.MOV(self.asm.heap_stm_detached_inevitable_from_thread(), eax)
+        #
+        offset = mc.get_relative_pos() - jmp_location
+        assert 0 < offset <= 127
+        mc.overwrite(jmp_location - 1, chr(offset))
 
     def call_stm_after_ex_call(self):
+        from rpython.jit.backend.x86 import rx86
         from rpython.rlib import rstm
-        # after any CALL_RELEASE_GIL, invoke the
-        # pypy_stm_start_if_not_atomic() function
-        self.save_result_value(True)
-        self.mc.CALL(imm(rstm.adr_pypy_stm_start_if_not_atomic))
-        self.restore_result_value(True)
+        # Generate the same logic as stm_enter_transactional_zone()
+        #
+        # Need to save away the result value, which is (likely) in eax
+        assert not self.result_value_saved_early
+        mc = self.mc
+        mc.MOV(edi, eax)
+        #
+        # compare_and_swap(&_stm_detached_inevitable_from_thread, tl, 0)
+        mc.MOV(eax, self.asm.heap_stm_thread_local_self())
+        mc.XOR(esi, esi)
+        adr = self.asm.heap_stm_detached_inevitable_from_thread()
+        m_address = mc._addr_as_reg_offset(adr.value_j())
+        mc.LOCK()
+        mc.CMPXCHG_mr(m_address, esi.value)
+        #
+        # restore the result value, back to eax
+        mc.MOV(eax, edi)
+        #
+        # if successful, jump over the next CALL
+        mc.J_il8(rx86.Conditions['Z'], 0)
+        jz_location = mc.get_relative_pos()
+        #
+        # if unsuccessful, invoke _stm_reattach_transaction()
+        mc.CALL(imm(self.asm._stm_reattach_tr_slowpath))
+        #
+        offset = mc.get_relative_pos() - jz_location
+        assert 0 < offset <= 127
+        mc.overwrite(jz_location - 1, chr(offset))
 
 
 if IS_X86_32:
diff --git a/rpython/jit/backend/x86/rx86.py b/rpython/jit/backend/x86/rx86.py
--- a/rpython/jit/backend/x86/rx86.py
+++ b/rpython/jit/backend/x86/rx86.py
@@ -630,6 +630,11 @@
     FLDL_s  = insn('\xDD', orbyte(0<<3), stack_sp(1))
     FLDS_s  = insn('\xD9', orbyte(0<<3), stack_sp(1))
 
+    # the 'lock' and 'cmpxchg' instructions
+    LOCK = insn('\xF0')
+    CMPXCHG_mr = insn(rex_w, '\x0F\xB1', register(2,8), mem_reg_plus_const(1))
+    CMPXCHG_jr = insn(rex_w, '\x0F\xB1', register(2,8), abs_(1))
+
     # ------------------------------ Random mess -----------------------
     RDTSC = insn('\x0F\x31')
 
diff --git a/rpython/rlib/rstm.py b/rpython/rlib/rstm.py
--- a/rpython/rlib/rstm.py
+++ b/rpython/rlib/rstm.py
@@ -43,10 +43,14 @@
 
 adr_pypy__rewind_jmp_copy_stack_slice = (
     CFlexSymbolic('((long)&pypy__rewind_jmp_copy_stack_slice)'))
-#adr_pypy_stm_commit_if_not_atomic = (
-#    CFlexSymbolic('((long)&pypy_stm_commit_if_not_atomic)'))
-#adr_pypy_stm_start_if_not_atomic = (
-#    CFlexSymbolic('((long)&pypy_stm_start_if_not_atomic)'))
+adr_stm_detached_inevitable_from_thread = (
+    CFlexSymbolic('((long)&_stm_detach_inevitable_transaction)'))
+adr_stm_thread_local_self = (
+    CFlexSymbolic('((long)&stm_thread_local.self)'))
+adr_stm_leave_noninevitable_transactional_zone = (
+    CFlexSymbolic('((long)&_stm_leave_noninevitable_transactional_zone)'))
+adr_stm_reattach_transaction = (
+    CFlexSymbolic('((long)&_stm_reattach_transaction)'))
 
 
 def rewind_jmp_frame():

From noreply at buildbot.pypy.org  Sat Jun 13 18:03:47 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 13 Jun 2015 18:03:47 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c8-gil-like: import stmgc/8009f12c327b
Message-ID: <20150613160347.67BBA1C11B5@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c8-gil-like
Changeset: r78085:259c750c401c
Date: 2015-06-13 18:03 +0200
http://bitbucket.org/pypy/pypy/changeset/259c750c401c/

Log:	import stmgc/8009f12c327b

diff --git a/rpython/translator/stm/src_stm/revision b/rpython/translator/stm/src_stm/revision
--- a/rpython/translator/stm/src_stm/revision
+++ b/rpython/translator/stm/src_stm/revision
@@ -1,1 +1,1 @@
-5af967809206
+8009f12c327b
diff --git a/rpython/translator/stm/src_stm/stm/core.c b/rpython/translator/stm/src_stm/stm/core.c
--- a/rpython/translator/stm/src_stm/stm/core.c
+++ b/rpython/translator/stm/src_stm/stm/core.c
@@ -1220,8 +1220,9 @@
 
 #ifdef STM_NO_AUTOMATIC_SETJMP
 void _test_run_abort(stm_thread_local_t *tl) __attribute__((noreturn));
-int stm_is_inevitable(void)
+int stm_is_inevitable(stm_thread_local_t *tl)
 {
+    assert(STM_SEGMENT->running_thread == tl);
     switch (STM_PSEGMENT->transaction_state) {
     case TS_REGULAR: return 0;
     case TS_INEVITABLE: return 1;
@@ -1593,7 +1594,7 @@
 
 void stm_stop_all_other_threads(void)
 {
-    if (!stm_is_inevitable())         /* may still abort */
+    if (!stm_is_inevitable(STM_SEGMENT->running_thread))  /* may still abort */
         _stm_become_inevitable("stop_all_other_threads");
 
     s_mutex_lock();
diff --git a/rpython/translator/stm/src_stm/stm/setup.c b/rpython/translator/stm/src_stm/stm/setup.c
--- a/rpython/translator/stm/src_stm/stm/setup.c
+++ b/rpython/translator/stm/src_stm/stm/setup.c
@@ -233,6 +233,8 @@
 {
     int num;
     s_mutex_lock();
+    tl->self = tl;    /* for faster access to &stm_thread_local (and easier
+                         from the PyPy JIT, too) */
     if (stm_all_thread_locals == NULL) {
         stm_all_thread_locals = tl->next = tl->prev = tl;
         num = 0;
diff --git a/rpython/translator/stm/src_stm/stmgc.h b/rpython/translator/stm/src_stm/stmgc.h
--- a/rpython/translator/stm/src_stm/stmgc.h
+++ b/rpython/translator/stm/src_stm/stmgc.h
@@ -71,7 +71,7 @@
     /* the next fields are handled internally by the library */
     int last_associated_segment_num;   /* always a valid seg num */
     int thread_local_counter;
-    struct stm_thread_local_s *prev, *next;
+    struct stm_thread_local_s *self, *prev, *next;
     void *creating_pthread[2];
 } stm_thread_local_t;
 
@@ -87,10 +87,10 @@
 long _stm_start_transaction(stm_thread_local_t *tl);
 void _stm_commit_transaction(void);
 void _stm_leave_noninevitable_transactional_zone(void);
-#define _stm_detach_inevitable_transaction(tl)  do {            \
-    write_fence();                                              \
-    assert(_stm_detached_inevitable_from_thread == 0);          \
-    _stm_detached_inevitable_from_thread = (intptr_t)(tl);      \
+#define _stm_detach_inevitable_transaction(tl)  do {                    \
+    write_fence();                                                      \
+    assert(_stm_detached_inevitable_from_thread == 0);                  \
+    _stm_detached_inevitable_from_thread = (intptr_t)(tl->self);        \
 } while (0)
 void _stm_reattach_transaction(stm_thread_local_t *tl);
 void _stm_become_inevitable(const char*);
@@ -391,10 +391,10 @@
 
 
 #ifdef STM_NO_AUTOMATIC_SETJMP
-int stm_is_inevitable(void);
+int stm_is_inevitable(stm_thread_local_t *tl);
 #else
-static inline int stm_is_inevitable(void) {
-    return !rewind_jmp_armed(&STM_SEGMENT->running_thread->rjthread);
+static inline int stm_is_inevitable(stm_thread_local_t *tl) {
+    return !rewind_jmp_armed(&tl->rjthread);
 }
 #endif
 
@@ -441,7 +441,7 @@
 }
 static inline void stm_leave_transactional_zone(stm_thread_local_t *tl) {
     assert(STM_SEGMENT->running_thread == tl);
-    if (stm_is_inevitable()) {
+    if (stm_is_inevitable(tl)) {
 #ifdef STM_DEBUGPRINT
         fprintf(stderr, "stm_leave_transactional_zone fast path\n");
 #endif
@@ -472,7 +472,7 @@
 static inline void stm_become_inevitable(stm_thread_local_t *tl,
                                          const char* msg) {
     assert(STM_SEGMENT->running_thread == tl);
-    if (!stm_is_inevitable())
+    if (!stm_is_inevitable(tl))
         _stm_become_inevitable(msg);
     /* now, we're running the inevitable transaction, so this var should be 0 */
     assert(_stm_detached_inevitable_from_thread == 0);

From noreply at buildbot.pypy.org  Sat Jun 13 18:18:35 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 13 Jun 2015 18:18:35 +0200 (CEST)
Subject: [pypy-commit] stmgc c8-gil-like: Kill 'self' again,
	found a better way anyway
Message-ID: <20150613161835.E17551C033F@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: c8-gil-like
Changeset: r1822:a8a04abfa22e
Date: 2015-06-13 18:19 +0200
http://bitbucket.org/pypy/stmgc/changeset/a8a04abfa22e/

Log:	Kill 'self' again, found a better way anyway

diff --git a/c8/stm/setup.c b/c8/stm/setup.c
--- a/c8/stm/setup.c
+++ b/c8/stm/setup.c
@@ -233,8 +233,6 @@
 {
     int num;
     s_mutex_lock();
-    tl->self = tl;    /* for faster access to &stm_thread_local (and easier
-                         from the PyPy JIT, too) */
     if (stm_all_thread_locals == NULL) {
         stm_all_thread_locals = tl->next = tl->prev = tl;
         num = 0;
diff --git a/c8/stmgc.h b/c8/stmgc.h
--- a/c8/stmgc.h
+++ b/c8/stmgc.h
@@ -71,7 +71,7 @@
     /* the next fields are handled internally by the library */
     int last_associated_segment_num;   /* always a valid seg num */
     int thread_local_counter;
-    struct stm_thread_local_s *self, *prev, *next;
+    struct stm_thread_local_s *prev, *next;
     void *creating_pthread[2];
 } stm_thread_local_t;
 
@@ -87,10 +87,10 @@
 long _stm_start_transaction(stm_thread_local_t *tl);
 void _stm_commit_transaction(void);
 void _stm_leave_noninevitable_transactional_zone(void);
-#define _stm_detach_inevitable_transaction(tl)  do {                    \
-    write_fence();                                                      \
-    assert(_stm_detached_inevitable_from_thread == 0);                  \
-    _stm_detached_inevitable_from_thread = (intptr_t)(tl->self);        \
+#define _stm_detach_inevitable_transaction(tl)  do {            \
+    write_fence();                                              \
+    assert(_stm_detached_inevitable_from_thread == 0);          \
+    _stm_detached_inevitable_from_thread = (intptr_t)(tl);      \
 } while (0)
 void _stm_reattach_transaction(stm_thread_local_t *tl);
 void _stm_become_inevitable(const char*);

From noreply at buildbot.pypy.org  Sat Jun 13 18:27:27 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 13 Jun 2015 18:27:27 +0200 (CEST)
Subject: [pypy-commit] stmgc c8-gil-like: Backed out changeset a8a04abfa22e
Message-ID: <20150613162727.2F0801C033F@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: c8-gil-like
Changeset: r1823:0fdbe7318e9f
Date: 2015-06-13 18:27 +0200
http://bitbucket.org/pypy/stmgc/changeset/0fdbe7318e9f/

Log:	Backed out changeset a8a04abfa22e

diff --git a/c8/stm/setup.c b/c8/stm/setup.c
--- a/c8/stm/setup.c
+++ b/c8/stm/setup.c
@@ -233,6 +233,8 @@
 {
     int num;
     s_mutex_lock();
+    tl->self = tl;    /* for faster access to &stm_thread_local (and easier
+                         from the PyPy JIT, too) */
     if (stm_all_thread_locals == NULL) {
         stm_all_thread_locals = tl->next = tl->prev = tl;
         num = 0;
diff --git a/c8/stmgc.h b/c8/stmgc.h
--- a/c8/stmgc.h
+++ b/c8/stmgc.h
@@ -71,7 +71,7 @@
     /* the next fields are handled internally by the library */
     int last_associated_segment_num;   /* always a valid seg num */
     int thread_local_counter;
-    struct stm_thread_local_s *prev, *next;
+    struct stm_thread_local_s *self, *prev, *next;
     void *creating_pthread[2];
 } stm_thread_local_t;
 
@@ -87,10 +87,10 @@
 long _stm_start_transaction(stm_thread_local_t *tl);
 void _stm_commit_transaction(void);
 void _stm_leave_noninevitable_transactional_zone(void);
-#define _stm_detach_inevitable_transaction(tl)  do {            \
-    write_fence();                                              \
-    assert(_stm_detached_inevitable_from_thread == 0);          \
-    _stm_detached_inevitable_from_thread = (intptr_t)(tl);      \
+#define _stm_detach_inevitable_transaction(tl)  do {                    \
+    write_fence();                                                      \
+    assert(_stm_detached_inevitable_from_thread == 0);                  \
+    _stm_detached_inevitable_from_thread = (intptr_t)(tl->self);        \
 } while (0)
 void _stm_reattach_transaction(stm_thread_local_t *tl);
 void _stm_become_inevitable(const char*);

From noreply at buildbot.pypy.org  Sat Jun 13 18:27:55 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 13 Jun 2015 18:27:55 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c8-gil-like: Fix
	_generate_cmp_break_transaction
Message-ID: <20150613162755.06AAB1C033F@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c8-gil-like
Changeset: r78086:1d9bc828649e
Date: 2015-06-13 18:17 +0200
http://bitbucket.org/pypy/pypy/changeset/1d9bc828649e/

Log:	Fix _generate_cmp_break_transaction

diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -1154,8 +1154,7 @@
     def convert_addresses_to_linear(self, reg1, reg2=None):
         if not self.cpu.gc_ll_descr.stm:   # stm-only
             return
-        if not IS_X86_64:
-            todo()   # "needed for X86_64_SCRATCH_REG"
+        assert IS_X86_64
         sb_adr = rstm.adr_segment_base
         assert rx86.fits_in_32bits(sb_adr)    # because it is in the 2nd page
         self.mc.MOV_rj(X86_64_SCRATCH_REG.value, (self.SEGMENT_GC, sb_adr))
@@ -2833,24 +2832,24 @@
         self.mc.MOV_rr(reg.value, ebp.value)
 
     def _generate_cmp_break_transaction(self):
-        # emits the check with a CMP instruction:
-        #    pypy_stm_nursery_low_fill_mark < STM_SEGMENT->nursery_current
-        # so if it is followed with a JB, it will follow the jump if
+        # emits the check with a CMP instruction (as signed integers):
+        #    STM_SEGMENT->nursery_current >= STM_SEGMENT->nursery_mark
+        # so if it is followed by a JGE, it will follow the jump if
         # we should break the transaction now.
         #
         assert self.cpu.gc_ll_descr.stm
-        if not IS_X86_64:
-            todo()   # "needed for X86_64_SCRATCH_REG"
-        psnlfm_adr = rstm.adr_pypy_stm_nursery_low_fill_mark
-        self.mc.MOV(X86_64_SCRATCH_REG, self.heap_tl(psnlfm_adr))
-        nf_adr = rstm.adr_nursery_free
-        assert rx86.fits_in_32bits(nf_adr)    # because it is in the 2nd page
-        self.mc.CMP_rj(X86_64_SCRATCH_REG.value, (self.SEGMENT_GC, nf_adr))
+        assert IS_X86_64
+        nc_adr = rstm.adr_nursery_free
+        nm_adr = rstm.adr_nursery_mark
+        assert rx86.fits_in_32bits(nc_adr)    # because it is in the 2nd page
+        assert rx86.fits_in_32bits(nm_adr)    # because it is in the 2nd page
+        self.mc.MOV_rj(X86_64_SCRATCH_REG.value, (self.SEGMENT_GC, nc_adr))
+        self.mc.CMP_rj(X86_64_SCRATCH_REG.value, (self.SEGMENT_GC, nm_adr))
 
     def genop_stm_should_break_transaction(self, op, arglocs, result_loc):
         self._generate_cmp_break_transaction()
         rl = result_loc.lowest8bits()
-        self.mc.SET_ir(rx86.Conditions['B'], rl.value)
+        self.mc.SET_ir(rx86.Conditions['GE'], rl.value)
         self.mc.MOVZX8_rr(result_loc.value, rl.value)
 
     def genop_guard_stm_should_break_transaction(self, op, guard_op,
@@ -2858,14 +2857,13 @@
                                                  result_loc):
         self._generate_cmp_break_transaction()
         if guard_op.getopnum() == rop.GUARD_FALSE:
-            self.implement_guard(guard_token, 'B')   # JB goes to "yes, break"
+            self.implement_guard(guard_token, 'GE')  # JGE goes to "yes, break"
         else:
-            self.implement_guard(guard_token, 'AE')  # JAE goes to "no, don't"
+            self.implement_guard(guard_token, 'L')   # JL goes to "no, don't"
 
     def genop_discard_stm_read(self, op, arglocs):
         assert self.cpu.gc_ll_descr.stm
-        if not IS_X86_64:
-            todo()   # "needed for X86_64_SCRATCH_REG"
+        assert IS_X86_64
         mc = self.mc
         rmreg = X86_64_SCRATCH_REG.value
         mc.MOVZX8_rj(rmreg, (self.SEGMENT_GC,
@@ -3008,9 +3006,6 @@
 
 cond_call_register_arguments = [edi, esi, edx, ecx]
 
-def todo():
-    CRASH   # not done yet
-
 class BridgeAlreadyCompiled(Exception):
     pass
 
diff --git a/rpython/rlib/rstm.py b/rpython/rlib/rstm.py
--- a/rpython/rlib/rstm.py
+++ b/rpython/rlib/rstm.py
@@ -19,6 +19,7 @@
 stm_nb_segments = CFlexSymbolic('STM_NB_SEGMENTS')
 adr_nursery_free = CFlexSymbolic('((long)&STM_SEGMENT->nursery_current)')
 adr_nursery_top  = CFlexSymbolic('((long)&STM_SEGMENT->nursery_end)')
+adr_nursery_mark = CFlexSymbolic('((long)&STM_SEGMENT->nursery_mark)')
 adr_pypy_stm_nursery_low_fill_mark = (
     CFlexSymbolic('((long)&pypy_stm_nursery_low_fill_mark)'))
 adr_rjthread_head = (
@@ -44,7 +45,7 @@
 adr_pypy__rewind_jmp_copy_stack_slice = (
     CFlexSymbolic('((long)&pypy__rewind_jmp_copy_stack_slice)'))
 adr_stm_detached_inevitable_from_thread = (
-    CFlexSymbolic('((long)&_stm_detach_inevitable_transaction)'))
+    CFlexSymbolic('((long)&_stm_detached_inevitable_from_thread)'))
 adr_stm_thread_local_self = (
     CFlexSymbolic('((long)&stm_thread_local.self)'))
 adr_stm_leave_noninevitable_transactional_zone = (

From noreply at buildbot.pypy.org  Sat Jun 13 22:55:25 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Sat, 13 Jun 2015 22:55:25 +0200 (CEST)
Subject: [pypy-commit] benchmarks default: disable cffi-dependent benchmark
 until translation runs cffi-import
Message-ID: <20150613205525.3A3691C033F@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: 
Changeset: r331:45bdb6aca4bb
Date: 2015-06-13 23:54 +0300
http://bitbucket.org/pypy/benchmarks/changeset/45bdb6aca4bb/

Log:	disable cffi-dependent benchmark until translation runs cffi-import

diff --git a/benchmarks.py b/benchmarks.py
--- a/benchmarks.py
+++ b/benchmarks.py
@@ -83,7 +83,8 @@
              'raytrace-simple', 'crypto_pyaes', 'bm_mako', 'bm_chameleon',
              'json_bench', 'pidigits', 'hexiom2', 'eparse', 'deltablue',
              'bm_dulwich_log', 'bm_krakatau', 'bm_mdp', 'pypy_interp',
-             'sqlitesynth']:
+             #'sqlitesynth',
+             ]:
     _register_new_bm(name, name, globals(), **opts.get(name, {}))
 
 for name in ['names', 'iteration', 'tcp', 'pb', ]:#'web']:#, 'accepts']:

From noreply at buildbot.pypy.org  Sun Jun 14 02:25:48 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Sun, 14 Jun 2015 02:25:48 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Port virtualenv workaround for shared
 builds from the 2.7 stdlib to the 3.2 stdlib.
Message-ID: <20150614002548.480221C146D@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob <me at manueljacob.de>
Branch: py3k
Changeset: r78087:a8e964090e6c
Date: 2015-06-14 02:25 +0200
http://bitbucket.org/pypy/pypy/changeset/a8e964090e6c/

Log:	Port virtualenv workaround for shared builds from the 2.7 stdlib to
	the 3.2 stdlib.

diff --git a/lib-python/3/subprocess.py b/lib-python/3/subprocess.py
--- a/lib-python/3/subprocess.py
+++ b/lib-python/3/subprocess.py
@@ -648,6 +648,21 @@
         """Create new Popen instance."""
         _cleanup()
 
+        # --- PyPy hack, see _pypy_install_libs_after_virtualenv() ---
+        # match arguments passed by different versions of virtualenv
+        if args[1:] in (
+            ['-c', 'import sys; print(sys.prefix)'],        # 1.6 10ba3f3c
+            ['-c', "\nimport sys\nprefix = sys.prefix\n"    # 1.7 0e9342ce
+             "if sys.version_info[0] == 3:\n"
+             "    prefix = prefix.encode('utf8')\n"
+             "if hasattr(sys.stdout, 'detach'):\n"
+             "    sys.stdout = sys.stdout.detach()\n"
+             "elif hasattr(sys.stdout, 'buffer'):\n"
+             "    sys.stdout = sys.stdout.buffer\nsys.stdout.write(prefix)\n"],
+            ['-c', 'import sys;out=sys.stdout;getattr(out, "buffer"'
+             ', out).write(sys.prefix.encode("utf-8"))']):  # 1.7.2 a9454bce
+            _pypy_install_libs_after_virtualenv(args[0])
+
         self._child_created = False
         if bufsize is None:
             bufsize = -1  # Restore default
@@ -1638,6 +1653,27 @@
             self.send_signal(signal.SIGKILL)
 
 
+def _pypy_install_libs_after_virtualenv(target_executable):
+    # https://bitbucket.org/pypy/pypy/issue/1922/future-proofing-virtualenv
+    #
+    # PyPy 2.4.1 turned --shared on by default.  This means the pypy binary
+    # depends on the 'libpypy-c.so' shared library to be able to run.
+    # The virtualenv code existing at the time did not account for this
+    # and would break.  Try to detect that we're running under such a
+    # virtualenv in the "Testing executable with" phase and copy the
+    # library ourselves.
+    caller = sys._getframe(2)
+    if ('virtualenv_version' in caller.f_globals and
+                  'copyfile' in caller.f_globals):
+        dest_dir = sys.pypy_resolvedirof(target_executable)
+        src_dir = sys.pypy_resolvedirof(sys.executable)
+        for libname in ['libpypy-c.so', 'libpypy-c.dylib']:
+            dest_library = os.path.join(dest_dir, libname)
+            src_library = os.path.join(src_dir, libname)
+            if os.path.exists(src_library):
+                caller.f_globals['copyfile'](src_library, dest_library)
+
+
 def _demo_posix():
     #
     # Example 1: Simple redirection: Get process list

From noreply at buildbot.pypy.org  Sun Jun 14 11:35:36 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 14 Jun 2015 11:35:36 +0200 (CEST)
Subject: [pypy-commit] stmgc c8-gil-like: compiling with
 _STM_NURSERY_ZEROED=1 in non-debug mode would not
Message-ID: <20150614093536.D41611C2083@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: c8-gil-like
Changeset: r1824:15af88362d03
Date: 2015-06-14 11:36 +0200
http://bitbucket.org/pypy/stmgc/changeset/15af88362d03/

Log:	compiling with _STM_NURSERY_ZEROED=1 in non-debug mode would not
	actually clean up the nursery. Crash. Took a while to figure this
	out

diff --git a/c8/stm/nursery.c b/c8/stm/nursery.c
--- a/c8/stm/nursery.c
+++ b/c8/stm/nursery.c
@@ -471,7 +471,6 @@
     }
     OPT_ASSERT((nursery_used & 7) == 0);
 
-#ifndef NDEBUG
     /* reset the nursery by zeroing it */
     char *realnursery;
     realnursery = REAL_ADDRESS(pseg->pub.segment_base, _stm_nursery_start);
@@ -483,8 +482,9 @@
                        (NURSERY_END - _stm_nursery_start) - nursery_used);
 
 #else
+# ifndef NDEBUG
     memset(realnursery, 0xa0, nursery_used);
-#endif
+# endif
 #endif
 
     pseg->total_throw_away_nursery += nursery_used;

From noreply at buildbot.pypy.org  Sun Jun 14 11:35:38 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 14 Jun 2015 11:35:38 +0200 (CEST)
Subject: [pypy-commit] stmgc c8-gil-like: Finish porting duhton-c8
Message-ID: <20150614093538.06B6B1C2083@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: c8-gil-like
Changeset: r1825:1621b474ec2b
Date: 2015-06-14 11:36 +0200
http://bitbucket.org/pypy/stmgc/changeset/1621b474ec2b/

Log:	Finish porting duhton-c8

diff --git a/duhton-c8/duhton.c b/duhton-c8/duhton.c
--- a/duhton-c8/duhton.c
+++ b/duhton-c8/duhton.c
@@ -4,6 +4,7 @@
 
 int main(int argc, char **argv)
 {
+    rewind_jmp_buf rjbuf;
     char *filename = NULL;
     int interactive = 1;
 	int i;
@@ -35,6 +36,7 @@
 	}
 
     Du_Initialize(num_threads);
+    stm_rewind_jmp_enterframe(&stm_thread_local, &rjbuf);
 
     while (1) {
         if (interactive) {
@@ -42,7 +44,7 @@
             fflush(stdout);
         }
         stm_enter_transactional_zone(&stm_thread_local);
-        stm_become_inevitable(&stm_thread_local, "starting point");
+        //stm_become_inevitable(&stm_thread_local, "starting point");
         DuObject *code = Du_Compile(filename, interactive);
 
         if (code == NULL) {
@@ -66,6 +68,7 @@
             break;
     }
 
+    stm_rewind_jmp_leaveframe(&stm_thread_local, &rjbuf);
     Du_Finalize();
     return 0;
 }
diff --git a/duhton-c8/glob.c b/duhton-c8/glob.c
--- a/duhton-c8/glob.c
+++ b/duhton-c8/glob.c
@@ -718,7 +718,7 @@
     Du_TransactionRun();
 
     stm_enter_transactional_zone(&stm_thread_local);
-    stm_become_inevitable(&stm_thread_local, "run-transactions finished");
+    //stm_become_inevitable(&stm_thread_local, "run-transactions finished");
     return Du_None;
 }
 
@@ -797,6 +797,8 @@
 
 void Du_Initialize(int num_threads)
 {
+    rewind_jmp_buf rjbuf;
+
     stm_setup();
 
     //stm_start_inevitable_transaction(&stm_thread_local);
@@ -810,8 +812,8 @@
     /* prebuilt objs stay on the shadowstack forever */
 
     stm_register_thread_local(&stm_thread_local);
+    stm_rewind_jmp_enterframe(&stm_thread_local, &rjbuf);
     stm_enter_transactional_zone(&stm_thread_local);
-    stm_become_inevitable(&stm_thread_local, "initialization");
 
     all_threads_count = num_threads;
     all_threads = (pthread_t*)malloc(sizeof(pthread_t) * num_threads);
@@ -860,6 +862,7 @@
     DuFrame_SetBuiltinMacro(Du_Globals, "assert", du_assert);
     DuFrame_SetSymbolStr(Du_Globals, "None", Du_None);
     stm_leave_transactional_zone(&stm_thread_local);
+    stm_rewind_jmp_leaveframe(&stm_thread_local, &rjbuf);
 }
 
 void Du_Finalize(void)
diff --git a/duhton-c8/object.c b/duhton-c8/object.c
--- a/duhton-c8/object.c
+++ b/duhton-c8/object.c
@@ -38,6 +38,8 @@
                                   uintptr_t offset_itemsize[2])
 {
     DuType *tp = Du_Types[((struct DuObject_s *)obj)->type_id];
+    if (tp->dt_cards_itemsize == 0)
+        Du_FatalError("object of type '%s' has no cards", tp->dt_name);
     offset_itemsize[0] = tp->dt_cards_offset;
     offset_itemsize[1] = tp->dt_cards_itemsize;
 }
diff --git a/duhton-c8/transaction.c b/duhton-c8/transaction.c
--- a/duhton-c8/transaction.c
+++ b/duhton-c8/transaction.c
@@ -74,17 +74,18 @@
 
 static DuObject *next_cell(void)
 {
-    DuObject *pending = TLOBJ;
+    DuObject *pending;
 
+    /* this code is critical enough so that we want it to
+       be serialized perfectly using inevitable transactions */
+    stm_become_inevitable(&stm_thread_local, "next_cell");
+
+    pending = TLOBJ;
     if (pending == NULL) {
         /* fish from the global list of pending transactions */
         DuConsObject *root;
 
       restart:
-        /* this code is critical enough so that we want it to
-           be serialized perfectly using inevitable transactions */
-        stm_start_inevitable_transaction(&stm_thread_local);
-
         root = du_pending_transactions;
         _du_read1(root);        /* not immutable... */
 
@@ -99,7 +100,7 @@
             return result;
         }
         else {
-            stm_commit_transaction();
+            _stm_commit_transaction();
 
             /* nothing to do, wait */
             int ts = __sync_add_and_fetch(&thread_sleeping, 1);
@@ -118,6 +119,7 @@
                 if (__sync_bool_compare_and_swap(&thread_sleeping, ts, ts - 1))
                     break;
             }
+            _stm_start_transaction(&stm_thread_local);
             goto restart;
         }
     }
@@ -125,8 +127,6 @@
     /* we have at least one thread-local transaction pending */
     TLOBJ = NULL;
 
-    stm_start_inevitable_transaction(&stm_thread_local);
-
     /* _du_read1(pending); IMMUTABLE */
     DuObject *result = _DuCons_CAR(pending);
     DuObject *next = _DuCons_NEXT(pending);
@@ -165,6 +165,7 @@
     rewind_jmp_buf rjbuf;
     stm_register_thread_local(&stm_thread_local);
     stm_rewind_jmp_enterframe(&stm_thread_local, &rjbuf);
+    stm_enter_transactional_zone(&stm_thread_local);
 
     TLOBJ = NULL;
 
@@ -176,15 +177,12 @@
         assert(TLOBJ == NULL);
 
         TLOBJ = cell;
-        stm_commit_transaction(); /* inevitable */
-        stm_start_transaction(&stm_thread_local);
+        stm_force_transaction_break(&stm_thread_local);
+
         cell = TLOBJ;
         TLOBJ = NULL;
 
         run_transaction(cell);
-
-        stm_commit_transaction();
-
     }
 
     stm_rewind_jmp_leaveframe(&stm_thread_local, &rjbuf);

From noreply at buildbot.pypy.org  Sun Jun 14 11:37:34 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 14 Jun 2015 11:37:34 +0200 (CEST)
Subject: [pypy-commit] stmgc default: compiling with _STM_NURSERY_ZEROED=1
 in non-debug mode would not
Message-ID: <20150614093734.617071C2083@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r1826:ea1bad49a23b
Date: 2015-06-14 11:36 +0200
http://bitbucket.org/pypy/stmgc/changeset/ea1bad49a23b/

Log:	compiling with _STM_NURSERY_ZEROED=1 in non-debug mode would not
	actually clean up the nursery. Crash. Took a while to figure this
	out

diff --git a/c8/stm/nursery.c b/c8/stm/nursery.c
--- a/c8/stm/nursery.c
+++ b/c8/stm/nursery.c
@@ -464,7 +464,6 @@
     }
     OPT_ASSERT((nursery_used & 7) == 0);
 
-#ifndef NDEBUG
     /* reset the nursery by zeroing it */
     char *realnursery;
     realnursery = REAL_ADDRESS(pseg->pub.segment_base, _stm_nursery_start);
@@ -476,8 +475,9 @@
                        (NURSERY_END - _stm_nursery_start) - nursery_used);
 
 #else
+# ifndef NDEBUG
     memset(realnursery, 0xa0, nursery_used);
-#endif
+# endif
 #endif
 
     pseg->pub.nursery_current = (stm_char *)_stm_nursery_start;

From noreply at buildbot.pypy.org  Sun Jun 14 11:37:35 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 14 Jun 2015 11:37:35 +0200 (CEST)
Subject: [pypy-commit] stmgc default: improve
Message-ID: <20150614093735.75D0C1C2083@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r1827:664fb39b6441
Date: 2015-06-12 12:54 +0200
http://bitbucket.org/pypy/stmgc/changeset/664fb39b6441/

Log:	improve

diff --git a/c7/gdb/gdb_stm.py b/c7/gdb/gdb_stm.py
--- a/c7/gdb/gdb_stm.py
+++ b/c7/gdb/gdb_stm.py
@@ -74,11 +74,13 @@
 def thread_to_segment_id(thread_id):
     base = int(gdb.parse_and_eval('stm_object_pages'))
     for j in range(1, get_nb_segments() + 1):
-        ts = get_psegment(j, '->transaction_state')
-        if int(ts) != 0:
-            ti = get_psegment(j, '->pub.running_thread->creating_pthread[0]')
-            if int(ti) == thread_id:
-                return j
+        #ti = get_psegment(j, '->pub.running_thread->creating_pthread[0]')
+        ti = get_psegment(j, '->running_pthread')
+        if int(ti) == thread_id:
+            ts = get_psegment(j, '->transaction_state')
+            if int(ts) == 0:
+                print >> sys.stderr, "note: transaction_state == 0"
+            return j
     raise Exception("thread not found: %r" % (thread_id,))
 
 def interactive_segment_base(thread=None):
@@ -106,11 +108,13 @@
     sb = interactive_segment_base(thread)
     if p is not None and p.type.code == gdb.TYPE_CODE_PTR:
         return gdb.Value(sb + int(p)).cast(p.type).dereference()
-    elif p is None or int(p) == 0:
+    else:
+        if p is None:
+            p = 0
+        else:
+            p = int(p)
         T = gdb.lookup_type('char').pointer()
-        return gdb.Value(sb).cast(T)
-    else:
-        raise TypeError("gc() first argument must be a GC pointer or 0")
+        return gdb.Value(sb + p).cast(T)
 
 @gdb_function
 def psegment(thread=None):

From noreply at buildbot.pypy.org  Sun Jun 14 11:46:52 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 14 Jun 2015 11:46:52 +0200 (CEST)
Subject: [pypy-commit] stmgc c8-gil-like: Why build duhton-c8 with zeroed
	nursery?
Message-ID: <20150614094652.5E1701C0683@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: c8-gil-like
Changeset: r1828:367bbbec6707
Date: 2015-06-14 11:47 +0200
http://bitbucket.org/pypy/stmgc/changeset/367bbbec6707/

Log:	Why build duhton-c8 with zeroed nursery?

diff --git a/duhton-c8/Makefile b/duhton-c8/Makefile
--- a/duhton-c8/Makefile
+++ b/duhton-c8/Makefile
@@ -3,7 +3,7 @@
 
 C8HEADERS = ../c8/stmgc.h ../c8/stm/*.h
 
-COMMON = -pthread -lrt -g -Wall -D_STM_NURSERY_ZEROED=1
+COMMON = -pthread -lrt -g -Wall #-D_STM_NURSERY_ZEROED=1
 
 
 all: duhton_debug duhton duhton_release

From noreply at buildbot.pypy.org  Sun Jun 14 11:52:22 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 14 Jun 2015 11:52:22 +0200 (CEST)
Subject: [pypy-commit] stmgc c8-gil-like: Close ready-to-merge branch
Message-ID: <20150614095222.5669F1C088E@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: c8-gil-like
Changeset: r1829:6fc9b0ffaee0
Date: 2015-06-14 11:51 +0200
http://bitbucket.org/pypy/stmgc/changeset/6fc9b0ffaee0/

Log:	Close ready-to-merge branch


From noreply at buildbot.pypy.org  Sun Jun 14 11:52:24 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 14 Jun 2015 11:52:24 +0200 (CEST)
Subject: [pypy-commit] stmgc default: hg merge c8-gil-like
Message-ID: <20150614095224.23F531C088E@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r1830:ab54aa35b24a
Date: 2015-06-14 11:53 +0200
http://bitbucket.org/pypy/stmgc/changeset/ab54aa35b24a/

Log:	hg merge c8-gil-like

	Fixes the bad timings of a program that does many tiny external
	calls. Previously, it would cause many tiny transactions. Now a
	single larger inevitable transaction covers the series of calls.

diff too long, truncating to 2000 out of 2448 lines

diff --git a/c8/CALL_RELEASE_GIL b/c8/CALL_RELEASE_GIL
new file mode 100644
--- /dev/null
+++ b/c8/CALL_RELEASE_GIL
@@ -0,0 +1,120 @@
+
+c8-gil-like
+===========
+
+A branch to have "GIL-like" behavior for inevitable transactions: one
+not-too-short inevitable transaction that is passed around multiple
+threads.
+
+The goal is to have good fast-case behavior with the PyPy JIT around
+CALL_RELEASE_GIL.  This is how it works in default (with shadowstack):
+
+
+- "rpy_fastgil" is a global variable.  The value 0 means the GIL is
+  definitely unlocked; the value 1 means it is probably locked (it is
+  actually locked only if some mutex object is acquired too).
+
+- before CALL_RELEASE_GIL, we know that we have the GIL and we need to
+  release it.  So we know that "rpy_fastgil" is 1, and we just write 0
+  there.
+
+- then we do the external call.
+
+- after CALL_RELEASE_GIL, two cases:
+
+  - if "rpy_fastgil" has been changed to 1 by some other thread *or*
+    if the (non-thread-local) shadowstack pointer changed, then we
+    call reacqgil_addr();
+
+  - otherwise, we swap rpy_fastgil back to 1 and we're done.
+
+- if the external call is long enough, a different thread will notice
+  that rpy_fastgil == 0 by regular polling, and grab the GIL for
+  itself by swapping it back to 1.  (The changes from 0 to 1 are done
+  with atomic instructions.)
+
+- a different mechanism is used when we voluntarily release the GIL,
+  based on the mutex mentioned above.  The mutex is also used by the
+  the reacqgil_addr() function if it actually needs to wait.
+
+
+Plan for porting this idea to stmgc:
+
+- we add a few macros to stmgc.h which can be used by C code, around
+  external calls; and we also inline these macros manually around
+  CALL_RELEASE_GIL in PyPy's JIT.
+
+- we add the "detached" mode to inevitable transactions: it means that
+  no thread is actively running this inevitable transaction for now,
+  but it was not committed yet.  It is meant to be reattached, by the
+  same or a different thread.
+
+- we add a global variable, "stm_detached_inevitable_from_thread".  It
+  is equal to the stm_thread_local pointer of the thread that detached
+  inevitable transaction (like rpy_fastgil == 0), or NULL if there is
+  no detached inevitable transaction (like rpy_fastgil == 1).
+
+- the macro stm_detach_inevitable_transaction() simply writes the
+  current thread's stm_thread_local pointer into the global variable
+  stm_detached_inevitable_from_thread.  It can only be used if the
+  current transaction is inevitable (and in particular the inevitable
+  transaction was not detached already, because we're running it).
+  After the macro is called, the current thread is assumed not to be
+  running in a transaction any more (no more object or shadowstack
+  access).
+
+- the macro stm_reattach_transaction() does an atomic swap on
+  stm_detached_inevitable_from_thread to change it to NULL.  If the
+  old value was equal to our own stm_thread_local pointer, we are done.  If
+  not, we call a helper, _stm_reattach_transaction().
+
+- we also add the macro stm_detach_transation().  If the current
+  thread is inevitable it calls stm_detach_inevitable_transaction().
+  Otherwise it calls a helper, _stm_detach_noninevitable_transaction().
+
+- _stm_reattach_transaction(old): called with the old value from
+  stm_detached_inevitable_from_thread (which was swapped to be NULL just
+  now).  If old != NULL, this swap had the effect that we took over
+  the inevitable transaction originally detached from a different
+  thread; we need to fix a few things like the stm_thread_local and %gs but
+  then we can continue running this reattached inevitable transaction.
+  If old == NULL, we need to fall back to the current
+  stm_start_transaction().  (A priori, there is no need to wait at
+  this point.  The waiting point is later, in the optional
+  stm_become_inevitable()).
+
+- _stm_detach_noninevitable_transaction(): we try to make the
+  transaction inevitable.  If it works we can then use
+  stm_detach_inevitable_transaction().  On the other hand, if we can't
+  make it inevitable without waiting, then instead we just commit it
+  and continue.  In the latter case,
+  stm_detached_inevitable_from_thread is still NULL.
+
+- other place to fix: major collections.  Maybe simply look inside
+  stm_detached_inevitable_from_thread, and if not NULL, grab the
+  inevitable transaction and commit it now.  Or maybe not.  The point
+  is that we need to prevent a thread from asynchronously grabbing it
+  by an atomic swap of stm_detached_inevitable_from_thread; instead,
+  the parallel threads that finish their external calls should all
+  find NULL in this variable and call _stm_reattach_transaction()
+  which will wait for the major GC to end.
+
+- stm_become_inevitable(): if it finds a detached inevitable
+  transaction, it should attach and commit it as a way to get rid of
+  it.  This is why it might be better to call directly
+  stm_start_inevitable_transaction() when possible: that one is
+  allowed to attach to a detached inevitable transaction and simply
+  return, unlike stm_become_inevitable() which must continue running
+  the existing transaction.
+
+- commit logic of a non-inevitable transaction: we wait if there is
+  an inevitable transaction.  Here too, if the inevitable transaction
+  is found to be detached, we could just commit it now.  Or, a better
+  approach: if we find a detached inevitable transaction we grab it
+  temporarily, and commit only the *non-inevitable* transaction if it
+  doesn't conflict.  The inevitable transaction is then detached
+  again.  (Note that the conflict detection is: we don't commit any
+  write to any of the objects in the inevitable transaction's
+  read-set.  This relies on inevitable threads maintaining their
+  read-set correctly, which should be the case in PyPy, but needs to
+  be checked.)
diff --git a/c8/demo/demo_random.c b/c8/demo/demo_random.c
--- a/c8/demo/demo_random.c
+++ b/c8/demo/demo_random.c
@@ -8,6 +8,8 @@
 #include <sys/wait.h>
 
 #include "stmgc.h"
+#include "stm/fprintcolor.h"
+#include "stm/fprintcolor.c"
 
 #define NUMTHREADS 2
 #define STEPS_PER_THREAD 500
@@ -48,8 +50,10 @@
     int num_roots;
     int num_roots_at_transaction_start;
     int steps_left;
+    long globally_unique;
 };
 __thread struct thread_data td;
+static long progress = 1;
 
 struct thread_data *_get_td(void)
 {
@@ -57,9 +61,16 @@
 }
 
 
+long check_size(long size)
+{
+    assert(size >= sizeof(struct node_s));
+    assert(size <= sizeof(struct node_s) + 4096*70);
+    return size;
+}
+
 ssize_t stmcb_size_rounded_up(struct object_s *ob)
 {
-    return ((struct node_s*)ob)->my_size;
+    return check_size(((struct node_s*)ob)->my_size);
 }
 
 void stmcb_trace(struct object_s *obj, void visit(object_t **))
@@ -69,7 +80,8 @@
 
     /* and the same value at the end: */
     /* note, ->next may be the same as last_next */
-    nodeptr_t *last_next = (nodeptr_t*)((char*)n + n->my_size - sizeof(void*));
+    nodeptr_t *last_next = (nodeptr_t*)((char*)n + check_size(n->my_size)
+                                        - sizeof(void*));
 
     assert(n->next == *last_next);
 
@@ -113,36 +125,36 @@
     }
 }
 
-void reload_roots()
-{
-    int i;
-    assert(td.num_roots == td.num_roots_at_transaction_start);
-    for (i = td.num_roots_at_transaction_start - 1; i >= 0; i--) {
-        if (td.roots[i])
-            STM_POP_ROOT(stm_thread_local, td.roots[i]);
-    }
-
-    for (i = 0; i < td.num_roots_at_transaction_start; i++) {
-        if (td.roots[i])
-            STM_PUSH_ROOT(stm_thread_local, td.roots[i]);
-    }
-}
-
 void push_roots()
 {
     int i;
+    assert(td.num_roots_at_transaction_start <= td.num_roots);
     for (i = td.num_roots_at_transaction_start; i < td.num_roots; i++) {
         if (td.roots[i])
             STM_PUSH_ROOT(stm_thread_local, td.roots[i]);
     }
+    STM_SEGMENT->no_safe_point_here = 0;
 }
 
 void pop_roots()
 {
     int i;
-    for (i = td.num_roots - 1; i >= td.num_roots_at_transaction_start; i--) {
-        if (td.roots[i])
+    STM_SEGMENT->no_safe_point_here = 1;
+
+    assert(td.num_roots_at_transaction_start <= td.num_roots);
+    for (i = td.num_roots - 1; i >= 0; i--) {
+        if (td.roots[i]) {
             STM_POP_ROOT(stm_thread_local, td.roots[i]);
+            assert(td.roots[i]);
+        }
+    }
+
+    dprintf(("stm_is_inevitable() = %d\n", (int)stm_is_inevitable()));
+    for (i = 0; i < td.num_roots_at_transaction_start; i++) {
+        if (td.roots[i]) {
+            dprintf(("root %d: %p\n", i, td.roots[i]));
+            STM_PUSH_ROOT(stm_thread_local, td.roots[i]);
+        }
     }
 }
 
@@ -150,6 +162,7 @@
 {
     int i;
     assert(idx >= td.num_roots_at_transaction_start);
+    assert(idx < td.num_roots);
 
     for (i = idx; i < td.num_roots - 1; i++)
         td.roots[i] = td.roots[i + 1];
@@ -158,6 +171,7 @@
 
 void add_root(objptr_t r)
 {
+    assert(td.num_roots_at_transaction_start <= td.num_roots);
     if (r && td.num_roots < MAXROOTS) {
         td.roots[td.num_roots++] = r;
     }
@@ -184,7 +198,8 @@
         nodeptr_t n = (nodeptr_t)p;
 
         /* and the same value at the end: */
-        nodeptr_t TLPREFIX *last_next = (nodeptr_t TLPREFIX *)((stm_char*)n + n->my_size - sizeof(void*));
+        nodeptr_t TLPREFIX *last_next = (nodeptr_t TLPREFIX *)((stm_char*)n +
+                                       check_size(n->my_size) - sizeof(void*));
         assert(n->next == *last_next);
         n->next = (nodeptr_t)v;
         *last_next = (nodeptr_t)v;
@@ -196,7 +211,8 @@
     nodeptr_t n = (nodeptr_t)p;
 
     /* and the same value at the end: */
-    nodeptr_t TLPREFIX *last_next = (nodeptr_t TLPREFIX *)((stm_char*)n + n->my_size - sizeof(void*));
+    nodeptr_t TLPREFIX *last_next = (nodeptr_t TLPREFIX *)((stm_char*)n +
+                                       check_size(n->my_size) - sizeof(void*));
     OPT_ASSERT(n->next == *last_next);
 
     return n->next;
@@ -229,7 +245,7 @@
                            sizeof(struct node_s) + (get_rand(100000) & ~15),
                            sizeof(struct node_s) + 4096,
                            sizeof(struct node_s) + 4096*70};
-        size_t size = sizes[get_rand(4)];
+        size_t size = check_size(sizes[get_rand(4)]);
         p = stm_allocate(size);
         nodeptr_t n = (nodeptr_t)p;
         n->sig = SIGNATURE;
@@ -240,7 +256,6 @@
         n->next = NULL;
         *last_next = NULL;
         pop_roots();
-        /* reload_roots not necessary, all are old after start_transaction */
         break;
     case 4:  // read and validate 'p'
         read_barrier(p);
@@ -288,6 +303,15 @@
     return p;
 }
 
+static void end_gut(void)
+{
+    if (td.globally_unique != 0) {
+        fprintf(stderr, "[GUT END]");
+        assert(progress == td.globally_unique);
+        td.globally_unique = 0;
+        stm_resume_all_other_threads();
+    }
+}
 
 objptr_t do_step(objptr_t p)
 {
@@ -308,8 +332,14 @@
         return NULL;
     } else if (get_rand(240) == 1) {
         push_roots();
-        stm_become_globally_unique_transaction(&stm_thread_local, "really");
-        fprintf(stderr, "[GUT/%d]", (int)STM_SEGMENT->segment_num);
+        if (td.globally_unique == 0) {
+            stm_stop_all_other_threads();
+            td.globally_unique = progress;
+            fprintf(stderr, "[GUT/%d]", (int)STM_SEGMENT->segment_num);
+        }
+        else {
+            end_gut();
+        }
         pop_roots();
         return NULL;
     }
@@ -347,37 +377,53 @@
 
     objptr_t p;
 
-    stm_start_transaction(&stm_thread_local);
+    stm_enter_transactional_zone(&stm_thread_local);
     assert(td.num_roots >= td.num_roots_at_transaction_start);
     td.num_roots = td.num_roots_at_transaction_start;
     p = NULL;
     pop_roots();                /* does nothing.. */
-    reload_roots();
 
     while (td.steps_left-->0) {
         if (td.steps_left % 8 == 0)
             fprintf(stdout, "#");
 
-        assert(p == NULL || ((nodeptr_t)p)->sig == SIGNATURE);
+        int local_seg = STM_SEGMENT->segment_num;
+        int p_sig = p == NULL ? 0 : ((nodeptr_t)p)->sig;
+
+        assert(p == NULL || p_sig == SIGNATURE);
+        (void)local_seg;
+        (void)p_sig;
+
+        if (!td.globally_unique)
+            ++progress;   /* racy, but good enough */
 
         p = do_step(p);
 
         if (p == (objptr_t)-1) {
             push_roots();
+            end_gut();
 
             long call_fork = (arg != NULL && *(long *)arg);
             if (call_fork == 0) {   /* common case */
-                stm_commit_transaction();
-                td.num_roots_at_transaction_start = td.num_roots;
-                if (get_rand(100) < 98) {
-                    stm_start_transaction(&stm_thread_local);
-                } else {
-                    stm_start_inevitable_transaction(&stm_thread_local);
+                if (get_rand(100) < 50) {
+                    stm_leave_transactional_zone(&stm_thread_local);
+                    /* Nothing here; it's unlikely that a different thread
+                       manages to steal the detached inev transaction.
+                       Give them a little chance with a usleep(). */
+                    dprintf(("sleep...\n"));
+                    usleep(1);
+                    dprintf(("sleep done\n"));
+                    td.num_roots_at_transaction_start = td.num_roots;
+                    stm_enter_transactional_zone(&stm_thread_local);
+                }
+                else {
+                    _stm_commit_transaction();
+                    td.num_roots_at_transaction_start = td.num_roots;
+                    _stm_start_transaction(&stm_thread_local);
                 }
                 td.num_roots = td.num_roots_at_transaction_start;
                 p = NULL;
                 pop_roots();
-                reload_roots();
             }
             else {
                 /* run a fork() inside the transaction */
@@ -401,16 +447,17 @@
         }
     }
     push_roots();
-    stm_commit_transaction();
+    end_gut();
+    stm_force_transaction_break(&stm_thread_local);
 
     /* even out the shadow stack before leaveframe: */
-    stm_start_inevitable_transaction(&stm_thread_local);
+    stm_become_inevitable(&stm_thread_local, "before leaveframe");
     while (td.num_roots > 0) {
         td.num_roots--;
         objptr_t t;
         STM_POP_ROOT(stm_thread_local, t);
     }
-    stm_commit_transaction();
+    stm_leave_transactional_zone(&stm_thread_local);
 
     stm_rewind_jmp_leaveframe(&stm_thread_local, &rjbuf);
     stm_unregister_thread_local(&stm_thread_local);
diff --git a/c8/demo/demo_random2.c b/c8/demo/demo_random2.c
--- a/c8/demo/demo_random2.c
+++ b/c8/demo/demo_random2.c
@@ -8,6 +8,8 @@
 #include <sys/wait.h>
 
 #include "stmgc.h"
+#include "stm/fprintcolor.h"
+#include "stm/fprintcolor.c"
 
 #define NUMTHREADS 3
 #define STEPS_PER_THREAD 50000
@@ -52,8 +54,10 @@
     int active_roots_num;
     long roots_on_ss;
     long roots_on_ss_at_tr_start;
+    long globally_unique;
 };
 __thread struct thread_data td;
+static long progress = 1;
 
 struct thread_data *_get_td(void)
 {
@@ -61,9 +65,16 @@
 }
 
 
+long check_size(long size)
+{
+    assert(size >= sizeof(struct node_s));
+    assert(size <= sizeof(struct node_s) + 4096*70);
+    return size;
+}
+
 ssize_t stmcb_size_rounded_up(struct object_s *ob)
 {
-    return ((struct node_s*)ob)->my_size;
+    return check_size(((struct node_s*)ob)->my_size);
 }
 
 void stmcb_trace(struct object_s *obj, void visit(object_t **))
@@ -73,7 +84,8 @@
 
     /* and the same value at the end: */
     /* note, ->next may be the same as last_next */
-    nodeptr_t *last_next = (nodeptr_t*)((char*)n + n->my_size - sizeof(void*));
+    nodeptr_t *last_next = (nodeptr_t*)((char*)n + check_size(n->my_size)
+                                        - sizeof(void*));
 
     assert(n->next == *last_next);
 
@@ -193,7 +205,8 @@
         nodeptr_t n = (nodeptr_t)p;
 
         /* and the same value at the end: */
-        nodeptr_t TLPREFIX *last_next = (nodeptr_t TLPREFIX *)((stm_char*)n + n->my_size - sizeof(void*));
+        nodeptr_t TLPREFIX *last_next = (nodeptr_t TLPREFIX *)((stm_char*)n +
+                                       check_size(n->my_size) - sizeof(void*));
         assert(n->next == *last_next);
         n->next = (nodeptr_t)v;
         *last_next = (nodeptr_t)v;
@@ -205,7 +218,8 @@
     nodeptr_t n = (nodeptr_t)p;
 
     /* and the same value at the end: */
-    nodeptr_t TLPREFIX *last_next = (nodeptr_t TLPREFIX *)((stm_char*)n + n->my_size - sizeof(void*));
+    nodeptr_t TLPREFIX *last_next = (nodeptr_t TLPREFIX *)((stm_char*)n +
+                                       check_size(n->my_size) - sizeof(void*));
     OPT_ASSERT(n->next == *last_next);
 
     return n->next;
@@ -239,6 +253,7 @@
             sizeof(struct node_s)+32, sizeof(struct node_s)+48,
             sizeof(struct node_s) + (get_rand(100000) & ~15)};
         size_t size = sizes[get_rand(sizeof(sizes) / sizeof(size_t))];
+        size = check_size(size);
         p = stm_allocate(size);
         nodeptr_t n = (nodeptr_t)p;
         n->sig = SIGNATURE;
@@ -296,6 +311,16 @@
     return p;
 }
 
+static void end_gut(void)
+{
+    if (td.globally_unique != 0) {
+        fprintf(stderr, "[GUT END]");
+        assert(progress == td.globally_unique);
+        td.globally_unique = 0;
+        stm_resume_all_other_threads();
+    }
+}
+
 void frame_loop();
 objptr_t do_step(objptr_t p)
 {
@@ -309,13 +334,22 @@
         p = simple_events(p, _r);
     } else if (get_rand(20) == 1) {
         long pushed = push_roots();
-        stm_commit_transaction();
-        td.roots_on_ss_at_tr_start = td.roots_on_ss;
-
-        if (get_rand(100) < 98) {
-            stm_start_transaction(&stm_thread_local);
-        } else {
-            stm_start_inevitable_transaction(&stm_thread_local);
+        end_gut();
+        if (get_rand(100) < 95) {
+            stm_leave_transactional_zone(&stm_thread_local);
+            /* Nothing here; it's unlikely that a different thread
+               manages to steal the detached inev transaction.
+               Give them a little chance with a usleep(). */
+            dprintf(("sleep...\n"));
+            usleep(1);
+            dprintf(("sleep done\n"));
+            td.roots_on_ss_at_tr_start = td.roots_on_ss;
+            stm_enter_transactional_zone(&stm_thread_local);
+        }
+        else {
+            _stm_commit_transaction();
+            td.roots_on_ss_at_tr_start = td.roots_on_ss;
+            _stm_start_transaction(&stm_thread_local);
         }
         td.roots_on_ss = td.roots_on_ss_at_tr_start;
         td.active_roots_num = 0;
@@ -331,15 +365,21 @@
     } else if (get_rand(20) == 1) {
         long pushed = push_roots();
         stm_become_inevitable(&stm_thread_local, "please");
-        assert(stm_is_inevitable());
+        assert(stm_is_inevitable(&stm_thread_local));
         pop_roots(pushed);
         p= NULL;
     } else if (get_rand(20) == 1) {
         p = (objptr_t)-1; // possibly fork
-    } else if (get_rand(20) == 1) {
+    } else if (get_rand(100) == 1) {
         long pushed = push_roots();
-        stm_become_globally_unique_transaction(&stm_thread_local, "really");
-        fprintf(stderr, "[GUT/%d]", (int)STM_SEGMENT->segment_num);
+        if (td.globally_unique == 0) {
+            stm_stop_all_other_threads();
+            td.globally_unique = progress;
+            fprintf(stderr, "[GUT/%d]", (int)STM_SEGMENT->segment_num);
+        }
+        else {
+            end_gut();
+        }
         pop_roots(pushed);
         p = NULL;
     }
@@ -364,6 +404,8 @@
 
         p = do_step(p);
 
+        if (!td.globally_unique)
+            ++progress;   /* racy, but good enough */
 
         if (p == (objptr_t)-1) {
             p = NULL;
@@ -371,6 +413,7 @@
             long call_fork = (thread_may_fork != NULL && *(long *)thread_may_fork);
             if (call_fork) {   /* common case */
                 long pushed = push_roots();
+                end_gut();
                 /* run a fork() inside the transaction */
                 printf("==========   FORK  =========\n");
                 *(long*)thread_may_fork = 0;
@@ -426,7 +469,7 @@
     setup_thread();
 
     td.roots_on_ss_at_tr_start = 0;
-    stm_start_transaction(&stm_thread_local);
+    stm_enter_transactional_zone(&stm_thread_local);
     td.roots_on_ss = td.roots_on_ss_at_tr_start;
     td.active_roots_num = 0;
 
@@ -435,7 +478,8 @@
         frame_loop();
     }
 
-    stm_commit_transaction();
+    end_gut();
+    stm_leave_transactional_zone(&stm_thread_local);
 
     stm_rewind_jmp_leaveframe(&stm_thread_local, &rjbuf);
     stm_unregister_thread_local(&stm_thread_local);
diff --git a/c8/demo/demo_simple.c b/c8/demo/demo_simple.c
--- a/c8/demo/demo_simple.c
+++ b/c8/demo/demo_simple.c
@@ -70,18 +70,20 @@
 
     object_t *tmp;
     int i = 0;
+
+    stm_enter_transactional_zone(&stm_thread_local);
     while (i < ITERS) {
-        stm_start_transaction(&stm_thread_local);
         tl_counter++;
         if (i % 500 < 250)
             STM_PUSH_ROOT(stm_thread_local, stm_allocate(16));//gl_counter++;
         else
             STM_POP_ROOT(stm_thread_local, tmp);
-        stm_commit_transaction();
+        stm_force_transaction_break(&stm_thread_local);
         i++;
     }
 
     OPT_ASSERT(org == (char *)stm_thread_local.shadowstack);
+    stm_leave_transactional_zone(&stm_thread_local);
 
     stm_rewind_jmp_leaveframe(&stm_thread_local, &rjbuf);
     stm_unregister_thread_local(&stm_thread_local);
diff --git a/c8/demo/test_shadowstack.c b/c8/demo/test_shadowstack.c
--- a/c8/demo/test_shadowstack.c
+++ b/c8/demo/test_shadowstack.c
@@ -43,17 +43,16 @@
     stm_register_thread_local(&stm_thread_local);
     stm_rewind_jmp_enterframe(&stm_thread_local, &rjbuf);
 
-    stm_start_transaction(&stm_thread_local);
+    stm_enter_transactional_zone(&stm_thread_local);
     node_t *node = (node_t *)stm_allocate(sizeof(struct node_s));
     node->value = 129821;
     STM_PUSH_ROOT(stm_thread_local, node);
     STM_PUSH_ROOT(stm_thread_local, 333);  /* odd value */
-    stm_commit_transaction();
 
     /* now in a new transaction, pop the node off the shadowstack, but
        then do a major collection.  It should still be found by the
        tracing logic. */
-    stm_start_transaction(&stm_thread_local);
+    stm_force_transaction_break(&stm_thread_local);
     STM_POP_ROOT_RET(stm_thread_local);
     STM_POP_ROOT(stm_thread_local, node);
     assert(node->value == 129821);
diff --git a/c8/stm/atomic.h b/c8/stm/atomic.h
--- a/c8/stm/atomic.h
+++ b/c8/stm/atomic.h
@@ -24,15 +24,21 @@
 
 #if defined(__i386__) || defined(__amd64__)
 
-# define HAVE_FULL_EXCHANGE_INSN
   static inline void spin_loop(void) { asm("pause" : : : "memory"); }
   static inline void write_fence(void) { asm("" : : : "memory"); }
+/*# define atomic_exchange(ptr, old, new)  do {         \
+          (old) = __sync_lock_test_and_set(ptr, new);   \
+      } while (0)*/
 
 #else
 
   static inline void spin_loop(void) { asm("" : : : "memory"); }
   static inline void write_fence(void) { __sync_synchronize(); }
 
+/*# define atomic_exchange(ptr, old, new)  do {           \
+          (old) = *(ptr);                                 \
+      } while (UNLIKELY(!__sync_bool_compare_and_swap(ptr, old, new))); */
+
 #endif
 
 
diff --git a/c8/stm/core.c b/c8/stm/core.c
--- a/c8/stm/core.c
+++ b/c8/stm/core.c
@@ -324,10 +324,7 @@
     /* Don't check this 'cl'. This entry is already checked */
 
     if (STM_PSEGMENT->transaction_state == TS_INEVITABLE) {
-        //assert(first_cl->next == INEV_RUNNING);
-        /* the above assert may fail when running a major collection
-           while the commit of the inevitable transaction is in progress
-           and the element is already attached */
+        assert(first_cl->next == INEV_RUNNING);
         return true;
     }
 
@@ -496,11 +493,23 @@
 
 static void wait_for_other_inevitable(struct stm_commit_log_entry_s *old)
 {
+    intptr_t detached = fetch_detached_transaction();
+    if (detached != 0) {
+        commit_fetched_detached_transaction(detached);
+        return;
+    }
+
     timing_event(STM_SEGMENT->running_thread, STM_WAIT_OTHER_INEVITABLE);
 
     while (old->next == INEV_RUNNING && !safe_point_requested()) {
         spin_loop();
         usleep(10);    /* XXXXXX */
+
+        detached = fetch_detached_transaction();
+        if (detached != 0) {
+            commit_fetched_detached_transaction(detached);
+            break;
+        }
     }
     timing_event(STM_SEGMENT->running_thread, STM_WAIT_DONE);
 }
@@ -509,7 +518,8 @@
 static void readd_wb_executed_flags(void);
 static void check_all_write_barrier_flags(char *segbase, struct list_s *list);
 
-static void _validate_and_attach(struct stm_commit_log_entry_s *new)
+static bool _validate_and_attach(struct stm_commit_log_entry_s *new,
+                                 bool can_sleep)
 {
     struct stm_commit_log_entry_s *old;
 
@@ -571,6 +581,8 @@
             /* XXXXXX for now just sleep.  We should really ask to inev
                transaction to do the commit for us, and then we can
                continue running. */
+            if (!can_sleep)
+                return false;
             dprintf(("_validate_and_attach(%p) failed, "
                      "waiting for inevitable\n", new));
             wait_for_other_inevitable(old);
@@ -591,18 +603,17 @@
 
     if (is_commit) {
         /* compare with _validate_and_add_to_commit_log */
-        STM_PSEGMENT->transaction_state = TS_NONE;
-        STM_PSEGMENT->safe_point = SP_NO_TRANSACTION;
-
         list_clear(STM_PSEGMENT->modified_old_objects);
         STM_PSEGMENT->last_commit_log_entry = new;
         release_modification_lock_wr(STM_SEGMENT->segment_num);
     }
+    return true;
 }
 
-static void _validate_and_turn_inevitable(void)
+static bool _validate_and_turn_inevitable(bool can_sleep)
 {
-    _validate_and_attach((struct stm_commit_log_entry_s *)INEV_RUNNING);
+    return _validate_and_attach((struct stm_commit_log_entry_s *)INEV_RUNNING,
+                                can_sleep);
 }
 
 static void _validate_and_add_to_commit_log(void)
@@ -611,6 +622,8 @@
 
     new = _create_commit_log_entry();
     if (STM_PSEGMENT->transaction_state == TS_INEVITABLE) {
+        assert(_stm_detached_inevitable_from_thread == 0);  /* running it */
+
         old = STM_PSEGMENT->last_commit_log_entry;
         new->rev_num = old->rev_num + 1;
         OPT_ASSERT(old->next == INEV_RUNNING);
@@ -621,17 +634,18 @@
                                       STM_PSEGMENT->modified_old_objects);
 
         /* compare with _validate_and_attach: */
-        STM_PSEGMENT->transaction_state = TS_NONE;
-        STM_PSEGMENT->safe_point = SP_NO_TRANSACTION;
+        acquire_modification_lock_wr(STM_SEGMENT->segment_num);
         list_clear(STM_PSEGMENT->modified_old_objects);
         STM_PSEGMENT->last_commit_log_entry = new;
 
         /* do it: */
         bool yes = __sync_bool_compare_and_swap(&old->next, INEV_RUNNING, new);
         OPT_ASSERT(yes);
+
+        release_modification_lock_wr(STM_SEGMENT->segment_num);
     }
     else {
-        _validate_and_attach(new);
+        _validate_and_attach(new, /*can_sleep=*/true);
     }
 }
 
@@ -1123,7 +1137,7 @@
 
 
 
-static void _stm_start_transaction(stm_thread_local_t *tl)
+static void _do_start_transaction(stm_thread_local_t *tl)
 {
     assert(!_stm_in_transaction(tl));
 
@@ -1140,7 +1154,7 @@
 #endif
     STM_PSEGMENT->shadowstack_at_start_of_transaction = tl->shadowstack;
     STM_PSEGMENT->threadlocal_at_start_of_transaction = tl->thread_local_obj;
-
+    STM_PSEGMENT->total_throw_away_nursery = 0;
 
     assert(list_is_empty(STM_PSEGMENT->modified_old_objects));
     assert(list_is_empty(STM_PSEGMENT->large_overflow_objects));
@@ -1181,35 +1195,34 @@
     stm_validate();
 }
 
-long stm_start_transaction(stm_thread_local_t *tl)
+#ifdef STM_NO_AUTOMATIC_SETJMP
+static int did_abort = 0;
+#endif
+
+long _stm_start_transaction(stm_thread_local_t *tl)
 {
     s_mutex_lock();
 #ifdef STM_NO_AUTOMATIC_SETJMP
-    long repeat_count = 0;    /* test/support.py */
+    long repeat_count = did_abort;    /* test/support.py */
+    did_abort = 0;
 #else
     long repeat_count = stm_rewind_jmp_setjmp(tl);
 #endif
-    _stm_start_transaction(tl);
+    _do_start_transaction(tl);
+
+    if (repeat_count == 0) {  /* else, 'nursery_mark' was already set
+                                 in abort_data_structures_from_segment_num() */
+        STM_SEGMENT->nursery_mark = ((stm_char *)_stm_nursery_start +
+                                     stm_fill_mark_nursery_bytes);
+    }
     return repeat_count;
 }
 
-void stm_start_inevitable_transaction(stm_thread_local_t *tl)
-{
-    /* used to be more efficient, starting directly an inevitable transaction,
-       but there is no real point any more, I believe */
-    rewind_jmp_buf rjbuf;
-    stm_rewind_jmp_enterframe(tl, &rjbuf);
-
-    stm_start_transaction(tl);
-    stm_become_inevitable(tl, "start_inevitable_transaction");
-
-    stm_rewind_jmp_leaveframe(tl, &rjbuf);
-}
-
 #ifdef STM_NO_AUTOMATIC_SETJMP
 void _test_run_abort(stm_thread_local_t *tl) __attribute__((noreturn));
-int stm_is_inevitable(void)
+int stm_is_inevitable(stm_thread_local_t *tl)
 {
+    assert(STM_SEGMENT->running_thread == tl);
     switch (STM_PSEGMENT->transaction_state) {
     case TS_REGULAR: return 0;
     case TS_INEVITABLE: return 1;
@@ -1224,6 +1237,7 @@
 {
     stm_thread_local_t *tl = STM_SEGMENT->running_thread;
 
+    assert(_has_mutex());
     STM_PSEGMENT->safe_point = SP_NO_TRANSACTION;
     STM_PSEGMENT->transaction_state = TS_NONE;
 
@@ -1231,7 +1245,15 @@
     list_clear(STM_PSEGMENT->objects_pointing_to_nursery);
     list_clear(STM_PSEGMENT->old_objects_with_cards_set);
     list_clear(STM_PSEGMENT->large_overflow_objects);
-    timing_event(tl, event);
+    if (tl != NULL)
+        timing_event(tl, event);
+
+    /* If somebody is waiting for us to reach a safe point, we simply
+       signal it now and leave this transaction.  This should be enough
+       for synchronize_all_threads() to retry and notice that we are
+       no longer SP_RUNNING. */
+    if (STM_SEGMENT->nursery_end != NURSERY_END)
+        cond_signal(C_AT_SAFE_POINT);
 
     release_thread_segment(tl);
     /* cannot access STM_SEGMENT or STM_PSEGMENT from here ! */
@@ -1280,24 +1302,55 @@
 }
 
 
-void stm_commit_transaction(void)
+void _stm_commit_transaction(void)
+{
+    assert(STM_PSEGMENT->running_pthread == pthread_self());
+    _core_commit_transaction(/*external=*/ false);
+}
+
+static void _core_commit_transaction(bool external)
 {
     exec_local_finalizers();
 
     assert(!_has_mutex());
     assert(STM_PSEGMENT->safe_point == SP_RUNNING);
-    assert(STM_PSEGMENT->running_pthread == pthread_self());
+    assert(STM_PSEGMENT->transaction_state != TS_NONE);
+    if (globally_unique_transaction) {
+        stm_fatalerror("cannot commit between stm_stop_all_other_threads "
+                       "and stm_resume_all_other_threads");
+    }
 
-    dprintf(("> stm_commit_transaction()\n"));
-    minor_collection(1);
+    dprintf(("> stm_commit_transaction(external=%d)\n", (int)external));
+    minor_collection(/*commit=*/ true, external);
+    if (!external && is_major_collection_requested()) {
+        s_mutex_lock();
+        if (is_major_collection_requested()) {   /* if still true */
+            major_collection_with_mutex();
+        }
+        s_mutex_unlock();
+    }
 
     push_large_overflow_objects_to_other_segments();
     /* push before validate. otherwise they are reachable too early */
 
+    if (external) {
+        /* from this point on, unlink the original 'stm_thread_local_t *'
+           from its segment.  Better do it as soon as possible, because
+           other threads might be spin-looping, waiting for the -1 to
+           disappear. */
+        STM_SEGMENT->running_thread = NULL;
+        write_fence();
+        assert(_stm_detached_inevitable_from_thread == -1);
+        _stm_detached_inevitable_from_thread = 0;
+    }
+
     bool was_inev = STM_PSEGMENT->transaction_state == TS_INEVITABLE;
     _validate_and_add_to_commit_log();
 
-    stm_rewind_jmp_forget(STM_SEGMENT->running_thread);
+    if (!was_inev) {
+        assert(!external);
+        stm_rewind_jmp_forget(STM_SEGMENT->running_thread);
+    }
 
     /* XXX do we still need a s_mutex_lock() section here? */
     s_mutex_lock();
@@ -1314,23 +1367,9 @@
 
     invoke_and_clear_user_callbacks(0);   /* for commit */
 
-    /* >>>>> there may be a FORK() happening in the safepoint below <<<<<*/
-    enter_safe_point_if_requested();
-    assert(STM_SEGMENT->nursery_end == NURSERY_END);
-
-    /* if a major collection is required, do it here */
-    if (is_major_collection_requested()) {
-        major_collection_with_mutex();
-    }
-
-    _verify_cards_cleared_in_all_lists(get_priv_segment(STM_SEGMENT->segment_num));
-
-    if (globally_unique_transaction && was_inev) {
-        committed_globally_unique_transaction();
-    }
-
     /* done */
     stm_thread_local_t *tl = STM_SEGMENT->running_thread;
+    assert(external == (tl == NULL));
     _finish_transaction(STM_TRANSACTION_COMMIT);
     /* cannot access STM_SEGMENT or STM_PSEGMENT from here ! */
 
@@ -1338,7 +1377,8 @@
 
     /* between transactions, call finalizers. this will execute
        a transaction itself */
-    invoke_general_finalizers(tl);
+    if (tl != NULL)
+        invoke_general_finalizers(tl);
 }
 
 static void reset_modified_from_backup_copies(int segment_num)
@@ -1399,7 +1439,7 @@
 
     abort_finalizers(pseg);
 
-    long bytes_in_nursery = throw_away_nursery(pseg);
+    throw_away_nursery(pseg);
 
     /* clear CARD_MARKED on objs (don't care about CARD_MARKED_OLD) */
     LIST_FOREACH_R(pseg->old_objects_with_cards_set, object_t * /*item*/,
@@ -1433,7 +1473,26 @@
     assert(tl->shadowstack == pseg->shadowstack_at_start_of_transaction);
 #endif
     tl->thread_local_obj = pseg->threadlocal_at_start_of_transaction;
-    tl->last_abort__bytes_in_nursery = bytes_in_nursery;
+
+
+    /* Set the next nursery_mark: first compute the value that
+       nursery_mark must have had at the start of the aborted transaction */
+    stm_char *old_mark =pseg->pub.nursery_mark + pseg->total_throw_away_nursery;
+
+    /* This means that the limit, in term of bytes, was: */
+    uintptr_t old_limit = old_mark - (stm_char *)_stm_nursery_start;
+
+    /* If 'total_throw_away_nursery' is smaller than old_limit, use that */
+    if (pseg->total_throw_away_nursery < old_limit)
+        old_limit = pseg->total_throw_away_nursery;
+
+    /* Now set the new limit to 90% of the old limit */
+    pseg->pub.nursery_mark = ((stm_char *)_stm_nursery_start +
+                              (uintptr_t)(old_limit * 0.9));
+
+#ifdef STM_NO_AUTOMATIC_SETJMP
+    did_abort = 1;
+#endif
 
     list_clear(pseg->objects_pointing_to_nursery);
     list_clear(pseg->old_objects_with_cards_set);
@@ -1502,36 +1561,40 @@
 
 void _stm_become_inevitable(const char *msg)
 {
-    if (STM_PSEGMENT->transaction_state == TS_REGULAR) {
+    assert(STM_PSEGMENT->transaction_state == TS_REGULAR);
+    _stm_collectable_safe_point();
+
+    if (msg != MSG_INEV_DONT_SLEEP) {
         dprintf(("become_inevitable: %s\n", msg));
-        _stm_collectable_safe_point();
         timing_become_inevitable();
-
-        _validate_and_turn_inevitable();
-        STM_PSEGMENT->transaction_state = TS_INEVITABLE;
-
-        stm_rewind_jmp_forget(STM_SEGMENT->running_thread);
-        invoke_and_clear_user_callbacks(0);   /* for commit */
+        _validate_and_turn_inevitable(/*can_sleep=*/true);
     }
     else {
-        assert(STM_PSEGMENT->transaction_state == TS_INEVITABLE);
+        if (!_validate_and_turn_inevitable(/*can_sleep=*/false))
+            return;
+        timing_become_inevitable();
     }
+    STM_PSEGMENT->transaction_state = TS_INEVITABLE;
+
+    stm_rewind_jmp_forget(STM_SEGMENT->running_thread);
+    invoke_and_clear_user_callbacks(0);   /* for commit */
 }
 
+#if 0
 void stm_become_globally_unique_transaction(stm_thread_local_t *tl,
                                             const char *msg)
 {
-    stm_become_inevitable(tl, msg);   /* may still abort */
+    stm_become_inevitable(tl, msg);
 
     s_mutex_lock();
     synchronize_all_threads(STOP_OTHERS_AND_BECOME_GLOBALLY_UNIQUE);
     s_mutex_unlock();
 }
-
+#endif
 
 void stm_stop_all_other_threads(void)
 {
-    if (!stm_is_inevitable())         /* may still abort */
+    if (!stm_is_inevitable(STM_SEGMENT->running_thread))  /* may still abort */
         _stm_become_inevitable("stop_all_other_threads");
 
     s_mutex_lock();
diff --git a/c8/stm/core.h b/c8/stm/core.h
--- a/c8/stm/core.h
+++ b/c8/stm/core.h
@@ -152,6 +152,9 @@
     stm_char *sq_fragments[SYNC_QUEUE_SIZE];
     int sq_fragsizes[SYNC_QUEUE_SIZE];
     int sq_len;
+
+    /* For nursery_mark */
+    uintptr_t total_throw_away_nursery;
 };
 
 enum /* safe_point */ {
@@ -170,6 +173,8 @@
     TS_INEVITABLE,
 };
 
+#define MSG_INEV_DONT_SLEEP  ((const char *)1)
+
 #define in_transaction(tl)                                              \
     (get_segment((tl)->last_associated_segment_num)->running_thread == (tl))
 
@@ -297,6 +302,7 @@
 
 static void _signal_handler(int sig, siginfo_t *siginfo, void *context);
 static bool _stm_validate(void);
+static void _core_commit_transaction(bool external);
 
 static inline bool was_read_remote(char *base, object_t *obj)
 {
diff --git a/c8/stm/detach.c b/c8/stm/detach.c
new file mode 100644
--- /dev/null
+++ b/c8/stm/detach.c
@@ -0,0 +1,175 @@
+#ifndef _STM_CORE_H_
+# error "must be compiled via stmgc.c"
+#endif
+
+#include <errno.h>
+
+
+/* Idea: if stm_leave_transactional_zone() is quickly followed by
+   stm_enter_transactional_zone() in the same thread, then we should
+   simply try to have one inevitable transaction that does both sides.
+   This is useful if there are many such small interruptions.
+
+   stm_leave_transactional_zone() tries to make sure the transaction
+   is inevitable, and then sticks the current 'stm_thread_local_t *'
+   into _stm_detached_inevitable_from_thread.
+   stm_enter_transactional_zone() has a fast-path if the same
+   'stm_thread_local_t *' is still there.
+
+   If a different thread grabs it, it atomically replaces the value in
+   _stm_detached_inevitable_from_thread with -1, commits it (this part
+   involves reading for example the shadowstack of the thread that
+   originally detached), and at the point where we know the original
+   stm_thread_local_t is no longer relevant, we reset
+   _stm_detached_inevitable_from_thread to 0.
+*/
+
+volatile intptr_t _stm_detached_inevitable_from_thread;
+
+
+static void setup_detach(void)
+{
+    _stm_detached_inevitable_from_thread = 0;
+}
+
+
+void _stm_leave_noninevitable_transactional_zone(void)
+{
+    int saved_errno = errno;
+    dprintf(("leave_noninevitable_transactional_zone\n"));
+    _stm_become_inevitable(MSG_INEV_DONT_SLEEP);
+
+    /* did it work? */
+    if (STM_PSEGMENT->transaction_state == TS_INEVITABLE) {   /* yes */
+        dprintf(("leave_noninevitable_transactional_zone: now inevitable\n"));
+        stm_thread_local_t *tl = STM_SEGMENT->running_thread;
+        _stm_detach_inevitable_transaction(tl);
+    }
+    else {   /* no */
+        dprintf(("leave_noninevitable_transactional_zone: commit\n"));
+        _stm_commit_transaction();
+    }
+    errno = saved_errno;
+}
+
+static void commit_external_inevitable_transaction(void)
+{
+    assert(STM_PSEGMENT->transaction_state == TS_INEVITABLE); /* can't abort */
+    _core_commit_transaction(/*external=*/ true);
+}
+
+void _stm_reattach_transaction(stm_thread_local_t *tl)
+{
+    intptr_t old;
+    int saved_errno = errno;
+ restart:
+    old = _stm_detached_inevitable_from_thread;
+    if (old != 0) {
+        if (old == -1) {
+            /* busy-loop: wait until _stm_detached_inevitable_from_thread
+               is reset to a value different from -1 */
+            dprintf(("reattach_transaction: busy wait...\n"));
+            while (_stm_detached_inevitable_from_thread == -1)
+                spin_loop();
+
+            /* then retry */
+            goto restart;
+        }
+
+        if (!__sync_bool_compare_and_swap(&_stm_detached_inevitable_from_thread,
+                                          old, -1))
+            goto restart;
+
+        stm_thread_local_t *old_tl = (stm_thread_local_t *)old;
+        int remote_seg_num = old_tl->last_associated_segment_num;
+        dprintf(("reattach_transaction: commit detached from seg %d\n",
+                 remote_seg_num));
+
+        tl->last_associated_segment_num = remote_seg_num;
+        ensure_gs_register(remote_seg_num);
+        commit_external_inevitable_transaction();
+    }
+    dprintf(("reattach_transaction: start a new transaction\n"));
+    _stm_start_transaction(tl);
+    errno = saved_errno;
+}
+
+void stm_force_transaction_break(stm_thread_local_t *tl)
+{
+    dprintf(("> stm_force_transaction_break()\n"));
+    assert(STM_SEGMENT->running_thread == tl);
+    _stm_commit_transaction();
+    _stm_start_transaction(tl);
+}
+
+static intptr_t fetch_detached_transaction(void)
+{
+    intptr_t cur;
+ restart:
+    cur = _stm_detached_inevitable_from_thread;
+    if (cur == 0) {    /* fast-path */
+        return 0;   /* _stm_detached_inevitable_from_thread not changed */
+    }
+    if (cur == -1) {
+        /* busy-loop: wait until _stm_detached_inevitable_from_thread
+           is reset to a value different from -1 */
+        while (_stm_detached_inevitable_from_thread == -1)
+            spin_loop();
+        goto restart;
+    }
+    if (!__sync_bool_compare_and_swap(&_stm_detached_inevitable_from_thread,
+                                      cur, -1))
+        goto restart;
+
+    /* this is the only case where we grabbed a detached transaction.
+       _stm_detached_inevitable_from_thread is still -1, until
+       commit_fetched_detached_transaction() is called. */
+    assert(_stm_detached_inevitable_from_thread == -1);
+    return cur;
+}
+
+static void commit_fetched_detached_transaction(intptr_t old)
+{
+    /* Here, 'seg_num' is the segment that contains the detached
+       inevitable transaction from fetch_detached_transaction(),
+       probably belonging to an unrelated thread.  We fetched it,
+       which means that nobody else can concurrently fetch it now, but
+       everybody will see that there is still a concurrent inevitable
+       transaction.  This should guarantee there are no race
+       conditions.
+    */
+    int mysegnum = STM_SEGMENT->segment_num;
+    int segnum = ((stm_thread_local_t *)old)->last_associated_segment_num;
+    dprintf(("commit_fetched_detached_transaction from seg %d\n", segnum));
+    assert(segnum > 0);
+
+    if (segnum != mysegnum) {
+        set_gs_register(get_segment_base(segnum));
+    }
+    commit_external_inevitable_transaction();
+
+    if (segnum != mysegnum) {
+        set_gs_register(get_segment_base(mysegnum));
+    }
+}
+
+static void commit_detached_transaction_if_from(stm_thread_local_t *tl)
+{
+    intptr_t old;
+ restart:
+    old = _stm_detached_inevitable_from_thread;
+    if (old == (intptr_t)tl) {
+        if (!__sync_bool_compare_and_swap(&_stm_detached_inevitable_from_thread,
+                                          old, -1))
+            goto restart;
+        commit_fetched_detached_transaction(old);
+        return;
+    }
+    if (old == -1) {
+        /* busy-loop: wait until _stm_detached_inevitable_from_thread
+           is reset to a value different from -1 */
+        while (_stm_detached_inevitable_from_thread == -1)
+            spin_loop();
+        goto restart;
+    }
+}
diff --git a/c8/stm/detach.h b/c8/stm/detach.h
new file mode 100644
--- /dev/null
+++ b/c8/stm/detach.h
@@ -0,0 +1,5 @@
+
+static void setup_detach(void);
+static intptr_t fetch_detached_transaction(void);
+static void commit_fetched_detached_transaction(intptr_t old);
+static void commit_detached_transaction_if_from(stm_thread_local_t *tl);
diff --git a/c8/stm/finalizer.c b/c8/stm/finalizer.c
--- a/c8/stm/finalizer.c
+++ b/c8/stm/finalizer.c
@@ -494,11 +494,11 @@
 
     rewind_jmp_buf rjbuf;
     stm_rewind_jmp_enterframe(tl, &rjbuf);
-    stm_start_transaction(tl);
+    _stm_start_transaction(tl);
 
     _execute_finalizers(&g_finalizers);
 
-    stm_commit_transaction();
+    _stm_commit_transaction();
     stm_rewind_jmp_leaveframe(tl, &rjbuf);
 
     __sync_lock_release(&lock);
diff --git a/c8/stm/forksupport.c b/c8/stm/forksupport.c
--- a/c8/stm/forksupport.c
+++ b/c8/stm/forksupport.c
@@ -40,7 +40,7 @@
 
     bool was_in_transaction = _stm_in_transaction(this_tl);
     if (!was_in_transaction)
-        stm_start_transaction(this_tl);
+        _stm_start_transaction(this_tl);
     assert(in_transaction(this_tl));
 
     stm_become_inevitable(this_tl, "fork");
@@ -73,7 +73,7 @@
     s_mutex_unlock();
 
     if (!was_in_transaction) {
-        stm_commit_transaction();
+        _stm_commit_transaction();
     }
 
     dprintf(("forksupport_parent: continuing to run\n"));
@@ -159,7 +159,7 @@
     assert(STM_SEGMENT->segment_num == segnum);
 
     if (!fork_was_in_transaction) {
-        stm_commit_transaction();
+        _stm_commit_transaction();
     }
 
     /* Done */
diff --git a/c8/stm/fprintcolor.h b/c8/stm/fprintcolor.h
--- a/c8/stm/fprintcolor.h
+++ b/c8/stm/fprintcolor.h
@@ -37,5 +37,6 @@
 /* ------------------------------------------------------------ */
 
 
+__attribute__((unused))
 static void stm_fatalerror(const char *format, ...)
      __attribute__((format (printf, 1, 2), noreturn));
diff --git a/c8/stm/nursery.c b/c8/stm/nursery.c
--- a/c8/stm/nursery.c
+++ b/c8/stm/nursery.c
@@ -11,8 +11,13 @@
 static uintptr_t _stm_nursery_start;
 
 
+#define DEFAULT_FILL_MARK_NURSERY_BYTES   (NURSERY_SIZE / 4)
+
+uintptr_t stm_fill_mark_nursery_bytes = DEFAULT_FILL_MARK_NURSERY_BYTES;
+
 /************************************************************/
 
+
 static void setup_nursery(void)
 {
     assert(_STM_FAST_ALLOC <= NURSERY_SIZE);
@@ -309,6 +314,7 @@
     else
         assert(finalbase <= ssbase && ssbase <= current);
 
+    dprintf(("collect_roots_in_nursery:\n"));
     while (current > ssbase) {
         --current;
         uintptr_t x = (uintptr_t)current->ss;
@@ -320,6 +326,7 @@
         else {
             /* it is an odd-valued marker, ignore */
         }
+        dprintf(("    %p: %p -> %p\n", current, (void *)x, current->ss));
     }
 
     minor_trace_if_young(&tl->thread_local_obj);
@@ -447,7 +454,7 @@
 }
 
 
-static size_t throw_away_nursery(struct stm_priv_segment_info_s *pseg)
+static void throw_away_nursery(struct stm_priv_segment_info_s *pseg)
 {
 #pragma push_macro("STM_PSEGMENT")
 #pragma push_macro("STM_SEGMENT")
@@ -480,7 +487,9 @@
 # endif
 #endif
 
+    pseg->total_throw_away_nursery += nursery_used;
     pseg->pub.nursery_current = (stm_char *)_stm_nursery_start;
+    pseg->pub.nursery_mark -= nursery_used;
 
     /* free any object left from 'young_outside_nursery' */
     if (!tree_is_cleared(pseg->young_outside_nursery)) {
@@ -505,8 +514,6 @@
     }
 
     tree_clear(pseg->nursery_objects_shadows);
-
-    return nursery_used;
 #pragma pop_macro("STM_SEGMENT")
 #pragma pop_macro("STM_PSEGMENT")
 }
@@ -519,6 +526,7 @@
 static void _do_minor_collection(bool commit)
 {
     dprintf(("minor_collection commit=%d\n", (int)commit));
+    assert(!STM_SEGMENT->no_safe_point_here);
 
     STM_PSEGMENT->minor_collect_will_commit_now = commit;
 
@@ -561,11 +569,12 @@
     assert(MINOR_NOTHING_TO_DO(STM_PSEGMENT));
 }
 
-static void minor_collection(bool commit)
+static void minor_collection(bool commit, bool external)
 {
     assert(!_has_mutex());
 
-    stm_safe_point();
+    if (!external)
+        stm_safe_point();
 
     timing_event(STM_SEGMENT->running_thread, STM_GC_MINOR_START);
 
@@ -579,7 +588,7 @@
     if (level > 0)
         force_major_collection_request();
 
-    minor_collection(/*commit=*/ false);
+    minor_collection(/*commit=*/ false, /*external=*/ false);
 
 #ifdef STM_TESTS
     /* tests don't want aborts in stm_allocate, thus
diff --git a/c8/stm/nursery.h b/c8/stm/nursery.h
--- a/c8/stm/nursery.h
+++ b/c8/stm/nursery.h
@@ -10,9 +10,9 @@
                                 object_t *obj, uint8_t mark_value,
                                 bool mark_all, bool really_clear);
 
-static void minor_collection(bool commit);
+static void minor_collection(bool commit, bool external);
 static void check_nursery_at_transaction_start(void);
-static size_t throw_away_nursery(struct stm_priv_segment_info_s *pseg);
+static void throw_away_nursery(struct stm_priv_segment_info_s *pseg);
 static void major_do_validation_and_minor_collections(void);
 
 static void assert_memset_zero(void *s, size_t n);
diff --git a/c8/stm/setup.c b/c8/stm/setup.c
--- a/c8/stm/setup.c
+++ b/c8/stm/setup.c
@@ -134,8 +134,12 @@
     setup_pages();
     setup_forksupport();
     setup_finalizer();
+    setup_detach();
 
     set_gs_register(get_segment_base(0));
+
+    dprintf(("nursery: %p -> %p\n", (void *)NURSERY_START,
+                                    (void *)NURSERY_END));
 }
 
 void stm_teardown(void)
@@ -229,6 +233,8 @@
 {
     int num;
     s_mutex_lock();
+    tl->self = tl;    /* for faster access to &stm_thread_local (and easier
+                         from the PyPy JIT, too) */
     if (stm_all_thread_locals == NULL) {
         stm_all_thread_locals = tl->next = tl->prev = tl;
         num = 0;
@@ -263,6 +269,8 @@
 
 void stm_unregister_thread_local(stm_thread_local_t *tl)
 {
+    commit_detached_transaction_if_from(tl);
+
     s_mutex_lock();
     assert(tl->prev != NULL);
     assert(tl->next != NULL);
diff --git a/c8/stm/sync.c b/c8/stm/sync.c
--- a/c8/stm/sync.c
+++ b/c8/stm/sync.c
@@ -1,6 +1,7 @@
 #include <sys/syscall.h>
 #include <sys/prctl.h>
 #include <asm/prctl.h>
+#include <time.h>
 
 #ifndef _STM_CORE_H_
 # error "must be compiled via stmgc.c"
@@ -21,25 +22,29 @@
 
 static void setup_sync(void)
 {
-    if (pthread_mutex_init(&sync_ctl.global_mutex, NULL) != 0)
-        stm_fatalerror("mutex initialization: %m");
+    int err = pthread_mutex_init(&sync_ctl.global_mutex, NULL);
+    if (err != 0)
+        stm_fatalerror("mutex initialization: %d", err);
 
     long i;
     for (i = 0; i < _C_TOTAL; i++) {
-        if (pthread_cond_init(&sync_ctl.cond[i], NULL) != 0)
-            stm_fatalerror("cond initialization: %m");
+        err = pthread_cond_init(&sync_ctl.cond[i], NULL);
+        if (err != 0)
+            stm_fatalerror("cond initialization: %d", err);
     }
 }
 
 static void teardown_sync(void)
 {
-    if (pthread_mutex_destroy(&sync_ctl.global_mutex) != 0)
-        stm_fatalerror("mutex destroy: %m");
+    int err = pthread_mutex_destroy(&sync_ctl.global_mutex);
+    if (err != 0)
+        stm_fatalerror("mutex destroy: %d", err);
 
     long i;
     for (i = 0; i < _C_TOTAL; i++) {
-        if (pthread_cond_destroy(&sync_ctl.cond[i]) != 0)
-            stm_fatalerror("cond destroy: %m");
+        err = pthread_cond_destroy(&sync_ctl.cond[i]);
+        if (err != 0)
+            stm_fatalerror("cond destroy: %d", err);
     }
 
     memset(&sync_ctl, 0, sizeof(sync_ctl));
@@ -59,19 +64,30 @@
         stm_fatalerror("syscall(arch_prctl, ARCH_SET_GS): %m");
 }
 
+static void ensure_gs_register(long segnum)
+{
+    /* XXX use this instead of set_gs_register() in many places */
+    if (STM_SEGMENT->segment_num != segnum) {
+        set_gs_register(get_segment_base(segnum));
+        assert(STM_SEGMENT->segment_num == segnum);
+    }
+}
+
 static inline void s_mutex_lock(void)
 {
     assert(!_has_mutex_here);
-    if (UNLIKELY(pthread_mutex_lock(&sync_ctl.global_mutex) != 0))
-        stm_fatalerror("pthread_mutex_lock: %m");
+    int err = pthread_mutex_lock(&sync_ctl.global_mutex);
+    if (UNLIKELY(err != 0))
+        stm_fatalerror("pthread_mutex_lock: %d", err);
     assert((_has_mutex_here = true, 1));
 }
 
 static inline void s_mutex_unlock(void)
 {
     assert(_has_mutex_here);
-    if (UNLIKELY(pthread_mutex_unlock(&sync_ctl.global_mutex) != 0))
-        stm_fatalerror("pthread_mutex_unlock: %m");
+    int err = pthread_mutex_unlock(&sync_ctl.global_mutex);
+    if (UNLIKELY(err != 0))
+        stm_fatalerror("pthread_mutex_unlock: %d", err);
     assert((_has_mutex_here = false, 1));
 }
 
@@ -83,26 +99,70 @@
 #endif
 
     assert(_has_mutex_here);
-    if (UNLIKELY(pthread_cond_wait(&sync_ctl.cond[ctype],
-                                   &sync_ctl.global_mutex) != 0))
-        stm_fatalerror("pthread_cond_wait/%d: %m", (int)ctype);
+    int err = pthread_cond_wait(&sync_ctl.cond[ctype],
+                                &sync_ctl.global_mutex);
+    if (UNLIKELY(err != 0))
+        stm_fatalerror("pthread_cond_wait/%d: %d", (int)ctype, err);
+}
+
+static inline void timespec_delay(struct timespec *t, double incr)
+{
+#ifdef CLOCK_REALTIME
+    clock_gettime(CLOCK_REALTIME, t);
+#else
+    struct timeval tv;
+    RPY_GETTIMEOFDAY(&tv);
+    t->tv_sec = tv.tv_sec;
+    t->tv_nsec = tv.tv_usec * 1000 + 999;
+#endif
+    /* assumes that "incr" is not too large, less than 1 second */
+    long nsec = t->tv_nsec + (long)(incr * 1000000000.0);
+    if (nsec >= 1000000000) {
+        t->tv_sec += 1;
+        nsec -= 1000000000;
+        assert(nsec < 1000000000);
+    }
+    t->tv_nsec = nsec;
+}
+
+static inline bool cond_wait_timeout(enum cond_type_e ctype, double delay)
+{
+#ifdef STM_NO_COND_WAIT
+    stm_fatalerror("*** cond_wait/%d called!", (int)ctype);
+#endif
+
+    assert(_has_mutex_here);
+
+    struct timespec t;
+    timespec_delay(&t, delay);
+
+    int err = pthread_cond_timedwait(&sync_ctl.cond[ctype],
+                                     &sync_ctl.global_mutex, &t);
+    if (err == 0)
+        return true;     /* success */
+    if (LIKELY(err == ETIMEDOUT))
+        return false;    /* timeout */
+    stm_fatalerror("pthread_cond_timedwait/%d: %d", (int)ctype, err);
 }
 
 static inline void cond_signal(enum cond_type_e ctype)
 {
-    if (UNLIKELY(pthread_cond_signal(&sync_ctl.cond[ctype]) != 0))
-        stm_fatalerror("pthread_cond_signal/%d: %m", (int)ctype);
+    int err = pthread_cond_signal(&sync_ctl.cond[ctype]);
+    if (UNLIKELY(err != 0))
+        stm_fatalerror("pthread_cond_signal/%d: %d", (int)ctype, err);
 }
 
 static inline void cond_broadcast(enum cond_type_e ctype)
 {
-    if (UNLIKELY(pthread_cond_broadcast(&sync_ctl.cond[ctype]) != 0))
-        stm_fatalerror("pthread_cond_broadcast/%d: %m", (int)ctype);
+    int err = pthread_cond_broadcast(&sync_ctl.cond[ctype]);
+    if (UNLIKELY(err != 0))
+        stm_fatalerror("pthread_cond_broadcast/%d: %d", (int)ctype, err);
 }
 
 /************************************************************/
 
 
+#if 0
 void stm_wait_for_current_inevitable_transaction(void)
 {
  restart:
@@ -125,7 +185,7 @@
     }
     s_mutex_unlock();
 }
-
+#endif
 
 
 static bool acquire_thread_segment(stm_thread_local_t *tl)
@@ -155,10 +215,12 @@
         num = (num+1) % (NB_SEGMENTS-1);
         if (sync_ctl.in_use1[num+1] == 0) {
             /* we're getting 'num', a different number. */
-            dprintf(("acquired different segment: %d->%d\n",
-                     tl->last_associated_segment_num, num+1));
+            int old_num = tl->last_associated_segment_num;
+            dprintf(("acquired different segment: %d->%d\n", old_num, num+1));
             tl->last_associated_segment_num = num+1;
             set_gs_register(get_segment_base(num+1));
+            dprintf(("                            %d->%d\n", old_num, num+1));
+            (void)old_num;
             goto got_num;
         }
     }
@@ -185,18 +247,22 @@
 
 static void release_thread_segment(stm_thread_local_t *tl)
 {
+    int segnum;
     assert(_has_mutex());
 
     cond_signal(C_SEGMENT_FREE);
 
     assert(STM_SEGMENT->running_thread == tl);
-    assert(tl->last_associated_segment_num == STM_SEGMENT->segment_num);
-    assert(in_transaction(tl));
-    STM_SEGMENT->running_thread = NULL;
-    assert(!in_transaction(tl));
+    segnum = STM_SEGMENT->segment_num;
+    if (tl != NULL) {
+        assert(tl->last_associated_segment_num == segnum);
+        assert(in_transaction(tl));
+        STM_SEGMENT->running_thread = NULL;
+        assert(!in_transaction(tl));
+    }
 
-    assert(sync_ctl.in_use1[tl->last_associated_segment_num] == 1);
-    sync_ctl.in_use1[tl->last_associated_segment_num] = 0;
+    assert(sync_ctl.in_use1[segnum] == 1);
+    sync_ctl.in_use1[segnum] = 0;
 }
 
 __attribute__((unused))
@@ -263,16 +329,19 @@
     }
     assert(!pause_signalled);
     pause_signalled = true;
+    dprintf(("request to pause\n"));
 }
 
 static inline long count_other_threads_sp_running(void)
 {
     /* Return the number of other threads in SP_RUNNING.
-       Asserts that SP_RUNNING threads still have the NSE_SIGxxx. */
+       Asserts that SP_RUNNING threads still have the NSE_SIGxxx.
+       (A detached inevitable transaction is still SP_RUNNING.) */
     long i;
     long result = 0;
-    int my_num = STM_SEGMENT->segment_num;
+    int my_num;
 
+    my_num = STM_SEGMENT->segment_num;
     for (i = 1; i < NB_SEGMENTS; i++) {
         if (i != my_num && get_priv_segment(i)->safe_point == SP_RUNNING) {
             assert(get_segment(i)->nursery_end <= _STM_NSE_SIGNAL_MAX);
@@ -295,6 +364,7 @@
         if (get_segment(i)->nursery_end == NSE_SIGPAUSE)
             get_segment(i)->nursery_end = NURSERY_END;
     }
+    dprintf(("request removed\n"));
     cond_broadcast(C_REQUEST_REMOVED);
 }
 
@@ -312,6 +382,8 @@
         if (STM_SEGMENT->nursery_end == NURSERY_END)
             break;    /* no safe point requested */
 
+        dprintf(("enter safe point\n"));
+        assert(!STM_SEGMENT->no_safe_point_here);
         assert(STM_SEGMENT->nursery_end == NSE_SIGPAUSE);
         assert(pause_signalled);
 
@@ -326,11 +398,15 @@
         cond_wait(C_REQUEST_REMOVED);
         STM_PSEGMENT->safe_point = SP_RUNNING;
         timing_event(STM_SEGMENT->running_thread, STM_WAIT_DONE);
+        assert(!STM_SEGMENT->no_safe_point_here);
+        dprintf(("left safe point\n"));
     }
 }
 
 static void synchronize_all_threads(enum sync_type_e sync_type)
 {
+ restart:
+    assert(_has_mutex());
     enter_safe_point_if_requested();
 
     /* Only one thread should reach this point concurrently.  This is
@@ -349,8 +425,19 @@
     /* If some other threads are SP_RUNNING, we cannot proceed now.
        Wait until all other threads are suspended. */
     while (count_other_threads_sp_running() > 0) {
+
+        intptr_t detached = fetch_detached_transaction();
+        if (detached != 0) {
+            remove_requests_for_safe_point();    /* => C_REQUEST_REMOVED */
+            s_mutex_unlock();
+            commit_fetched_detached_transaction(detached);
+            s_mutex_lock();
+            goto restart;
+        }
+
         STM_PSEGMENT->safe_point = SP_WAIT_FOR_C_AT_SAFE_POINT;
-        cond_wait(C_AT_SAFE_POINT);
+        cond_wait_timeout(C_AT_SAFE_POINT, 0.00001);
+        /* every 10 microsec, try again fetch_detached_transaction() */
         STM_PSEGMENT->safe_point = SP_RUNNING;
 
         if (must_abort()) {
diff --git a/c8/stm/sync.h b/c8/stm/sync.h
--- a/c8/stm/sync.h
+++ b/c8/stm/sync.h
@@ -17,6 +17,7 @@
 static bool _has_mutex(void);
 #endif
 static void set_gs_register(char *value);
+static void ensure_gs_register(long segnum);
 
 
 /* acquire and release one of the segments for running the given thread
diff --git a/c8/stmgc.c b/c8/stmgc.c
--- a/c8/stmgc.c
+++ b/c8/stmgc.c
@@ -18,6 +18,7 @@
 #include "stm/rewind_setjmp.h"
 #include "stm/finalizer.h"
 #include "stm/locks.h"
+#include "stm/detach.h"
 
 #include "stm/misc.c"
 #include "stm/list.c"
@@ -41,3 +42,4 @@
 #include "stm/rewind_setjmp.c"
 #include "stm/finalizer.c"
 #include "stm/hashtable.c"
+#include "stm/detach.c"
diff --git a/c8/stmgc.h b/c8/stmgc.h
--- a/c8/stmgc.h
+++ b/c8/stmgc.h
@@ -13,6 +13,7 @@
 #include <limits.h>
 #include <unistd.h>
 
+#include "stm/atomic.h"
 #include "stm/rewind_setjmp.h"
 
 #if LONG_MAX == 2147483647
@@ -39,9 +40,11 @@
 
 struct stm_segment_info_s {
     uint8_t transaction_read_version;
+    uint8_t no_safe_point_here;    /* set from outside, triggers an assert */
     int segment_num;
     char *segment_base;
     stm_char *nursery_current;
+    stm_char *nursery_mark;
     uintptr_t nursery_end;
     struct stm_thread_local_s *running_thread;
 };
@@ -65,13 +68,10 @@
        the following raw region of memory is cleared. */
     char *mem_clear_on_abort;
     size_t mem_bytes_to_clear_on_abort;
-    /* after an abort, some details about the abort are stored there.
-       (this field is not modified on a successful commit) */
-    long last_abort__bytes_in_nursery;
     /* the next fields are handled internally by the library */
     int last_associated_segment_num;   /* always a valid seg num */
     int thread_local_counter;
-    struct stm_thread_local_s *prev, *next;
+    struct stm_thread_local_s *self, *prev, *next;
     void *creating_pthread[2];
 } stm_thread_local_t;
 
@@ -82,6 +82,17 @@
 void _stm_write_slowpath_card(object_t *, uintptr_t);
 object_t *_stm_allocate_slowpath(ssize_t);
 object_t *_stm_allocate_external(ssize_t);
+
+extern volatile intptr_t _stm_detached_inevitable_from_thread;
+long _stm_start_transaction(stm_thread_local_t *tl);
+void _stm_commit_transaction(void);
+void _stm_leave_noninevitable_transactional_zone(void);
+#define _stm_detach_inevitable_transaction(tl)  do {                    \
+    write_fence();                                                      \
+    assert(_stm_detached_inevitable_from_thread == 0);                  \
+    _stm_detached_inevitable_from_thread = (intptr_t)(tl->self);        \
+} while (0)
+void _stm_reattach_transaction(stm_thread_local_t *tl);
 void _stm_become_inevitable(const char*);
 void _stm_collectable_safe_point(void);
 
@@ -379,39 +390,92 @@
     rewind_jmp_enum_shadowstack(&(tl)->rjthread, callback)
 
 
-/* Starting and ending transactions.  stm_read(), stm_write() and
-   stm_allocate() should only be called from within a transaction.
-   The stm_start_transaction() call returns the number of times it
-   returned, starting at 0.  If it is > 0, then the transaction was
-   aborted and restarted this number of times. */
-long stm_start_transaction(stm_thread_local_t *tl);
-void stm_start_inevitable_transaction(stm_thread_local_t *tl);
-void stm_commit_transaction(void);
+#ifdef STM_NO_AUTOMATIC_SETJMP
+int stm_is_inevitable(stm_thread_local_t *tl);
+#else
+static inline int stm_is_inevitable(stm_thread_local_t *tl) {
+    return !rewind_jmp_armed(&tl->rjthread);
+}
+#endif
 
-/* Temporary fix?  Call this outside a transaction.  If there is an
-   inevitable transaction running somewhere else, wait until it finishes. */
-void stm_wait_for_current_inevitable_transaction(void);
+
+/* Entering and leaving a "transactional code zone": a (typically very
+   large) section in the code where we are running a transaction.
+   This is the STM equivalent to "acquire the GIL" and "release the
+   GIL", respectively.  stm_read(), stm_write(), stm_allocate(), and
+   other functions should only be called from within a transaction.
+
+   Note that transactions, in the STM sense, cover _at least_ one
+   transactional code zone.  They may be longer; for example, if one
+   thread does a lot of stm_enter_transactional_zone() +
+   stm_become_inevitable() + stm_leave_transactional_zone(), as is
+   typical in a thread that does a lot of C function calls, then we
+   get only a few bigger inevitable transactions that cover the many
+   short transactional zones.  This is done by having
+   stm_leave_transactional_zone() turn the current transaction
+   inevitable and detach it from the running thread (if there is no
+   other inevitable transaction running so far).  Then
+   stm_enter_transactional_zone() will try to reattach to it.  This is
+   far more efficient than constantly starting and committing
+   transactions.
+
+   stm_enter_transactional_zone() and stm_leave_transactional_zone()
+   preserve the value of errno.
+*/
+#ifdef STM_DEBUGPRINT
+#include <stdio.h>
+#endif
+static inline void stm_enter_transactional_zone(stm_thread_local_t *tl) {
+    if (__sync_bool_compare_and_swap(&_stm_detached_inevitable_from_thread,
+                                     (intptr_t)tl, 0)) {
+#ifdef STM_DEBUGPRINT
+        fprintf(stderr, "stm_enter_transactional_zone fast path\n");
+#endif
+    }
+    else {
+        _stm_reattach_transaction(tl);
+        /* _stm_detached_inevitable_from_thread should be 0 here, but
+           it can already have been changed from a parallel thread
+           (assuming we're not inevitable ourselves) */
+    }
+}
+static inline void stm_leave_transactional_zone(stm_thread_local_t *tl) {
+    assert(STM_SEGMENT->running_thread == tl);
+    if (stm_is_inevitable(tl)) {
+#ifdef STM_DEBUGPRINT
+        fprintf(stderr, "stm_leave_transactional_zone fast path\n");
+#endif
+        _stm_detach_inevitable_transaction(tl);
+    }
+    else {
+        _stm_leave_noninevitable_transactional_zone();
+    }
+}
+
+/* stm_force_transaction_break() is in theory equivalent to
+   stm_leave_transactional_zone() immediately followed by
+   stm_enter_transactional_zone(); however, it is supposed to be
+   called in CPU-heavy threads that had a transaction run for a while,
+   and so it *always* forces a commit and starts the next transaction.
+   The new transaction is never inevitable.  See also
+   stm_should_break_transaction(). */
+void stm_force_transaction_break(stm_thread_local_t *tl);
 
 /* Abort the currently running transaction.  This function never
-   returns: it jumps back to the stm_start_transaction(). */
+   returns: it jumps back to the start of the transaction (which must
+   not be inevitable). */
 void stm_abort_transaction(void) __attribute__((noreturn));
 
-#ifdef STM_NO_AUTOMATIC_SETJMP
-int stm_is_inevitable(void);
-#else
-static inline int stm_is_inevitable(void) {
-    return !rewind_jmp_armed(&STM_SEGMENT->running_thread->rjthread);
-}
-#endif
-
 /* Turn the current transaction inevitable.
    stm_become_inevitable() itself may still abort the transaction instead
    of returning. */
 static inline void stm_become_inevitable(stm_thread_local_t *tl,
                                          const char* msg) {
     assert(STM_SEGMENT->running_thread == tl);
-    if (!stm_is_inevitable())
+    if (!stm_is_inevitable(tl))
         _stm_become_inevitable(msg);
+    /* now, we're running the inevitable transaction, so this var should be 0 */
+    assert(_stm_detached_inevitable_from_thread == 0);
 }
 
 /* Forces a safe-point if needed.  Normally not needed: this is
@@ -425,6 +489,23 @@
 void stm_collect(long level);
 
 
+/* A way to detect that we've run for a while and should call
+   stm_force_transaction_break() */
+static inline int stm_should_break_transaction(void)
+{
+    return ((intptr_t)STM_SEGMENT->nursery_current >=
+            (intptr_t)STM_SEGMENT->nursery_mark);
+}
+extern uintptr_t stm_fill_mark_nursery_bytes;
+/* ^^^ at the start of a transaction, 'nursery_mark' is initialized to
+   'stm_fill_mark_nursery_bytes' inside the nursery.  This value can
+   be larger than the nursery; every minor collection shifts the
+   current 'nursery_mark' down by one nursery-size.  After an abort
+   and restart, 'nursery_mark' is set to ~90% of the value it reached
+   in the last attempt.
+*/
+
+
 /* Prepare an immortal "prebuilt" object managed by the GC.  Takes a
    pointer to an 'object_t', which should not actually be a GC-managed
    structure but a real static structure.  Returns the equivalent
@@ -466,8 +547,8 @@
    other threads.  A very heavy-handed way to make sure that no other
    transaction is running concurrently.  Avoid as much as possible.
    Other transactions will continue running only after this transaction
-   commits.  (xxx deprecated and may be removed) */
-void stm_become_globally_unique_transaction(stm_thread_local_t *tl, const char *msg);
+   commits.  (deprecated, not working any more according to demo_random2) */
+//void stm_become_globally_unique_transaction(stm_thread_local_t *tl, const char *msg);
 
 /* Moves the transaction forward in time by validating the read and
    write set with all commits that happened since the last validation
diff --git a/c8/test/support.py b/c8/test/support.py

From noreply at buildbot.pypy.org  Sun Jun 14 16:38:06 2015
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sun, 14 Jun 2015 16:38:06 +0200 (CEST)
Subject: [pypy-commit] pypy default: More operations are allowed when a
	stream is detached.
Message-ID: <20150614143806.5F8CE1C0962@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: 
Changeset: r78088:30f7b6f0a467
Date: 2015-06-13 23:08 +0200
http://bitbucket.org/pypy/pypy/changeset/30f7b6f0a467/

Log:	More operations are allowed when a stream is detached.

diff --git a/pypy/module/_io/interp_textio.py b/pypy/module/_io/interp_textio.py
--- a/pypy/module/_io/interp_textio.py
+++ b/pypy/module/_io/interp_textio.py
@@ -431,9 +431,12 @@
         if self.state == STATE_ZERO:
             raise OperationError(space.w_ValueError, space.wrap(
                 "I/O operation on uninitialized object"))
-        elif self.state == STATE_DETACHED:
+
+    def _check_attached(self, space):
+        if self.state == STATE_DETACHED:
             raise OperationError(space.w_ValueError, space.wrap(
                 "underlying buffer has been detached"))
+        self._check_init(space)
 
     def _check_closed(self, space, message=None):
         self._check_init(space)
@@ -452,40 +455,41 @@
         )
 
     def readable_w(self, space):
-        self._check_init(space)
+        self._check_attached(space)
         return space.call_method(self.w_buffer, "readable")
 
     def writable_w(self, space):
-        self._check_init(space)
+        self._check_attached(space)
         return space.call_method(self.w_buffer, "writable")
 
     def seekable_w(self, space):
-        self._check_init(space)
+        self._check_attached(space)
         return space.call_method(self.w_buffer, "seekable")
 
     def isatty_w(self, space):
-        self._check_init(space)
+        self._check_attached(space)
         return space.call_method(self.w_buffer, "isatty")
 
     def fileno_w(self, space):
-        self._check_init(space)
+        self._check_attached(space)
         return space.call_method(self.w_buffer, "fileno")
 
     def closed_get_w(self, space):
-        self._check_init(space)
+        self._check_attached(space)
         return space.getattr(self.w_buffer, space.wrap("closed"))
 
     def newlines_get_w(self, space):
-        self._check_init(space)
+        self._check_attached(space)
         if self.w_decoder is None:
             return space.w_None
         return space.findattr(self.w_decoder, space.wrap("newlines"))
 
     def name_get_w(self, space):
-        self._check_init(space)
+        self._check_attached(space)
         return space.getattr(self.w_buffer, space.wrap("name"))
 
     def flush_w(self, space):
+        self._check_attached(space)
         self._check_closed(space)
         self.telling = self.seekable
         self._writeflush(space)
@@ -493,13 +497,13 @@
 
     @unwrap_spec(w_pos = WrappedDefault(None))
     def truncate_w(self, space, w_pos=None):
-        self._check_init(space)
+        self._check_attached(space)
 
         space.call_method(self, "flush")
         return space.call_method(self.w_buffer, "truncate", w_pos)
 
     def close_w(self, space):
-        self._check_init(space)
+        self._check_attached(space)
         if not space.is_true(space.getattr(self.w_buffer,
                                            space.wrap("closed"))):
             try:
@@ -585,6 +589,7 @@
         return not eof
 
     def next_w(self, space):
+        self._check_attached(space)
         self.telling = False
         try:
             return W_TextIOBase.next_w(self, space)
@@ -594,7 +599,7 @@
             raise
 
     def read_w(self, space, w_size=None):
-        self._check_closed(space)
+        self._check_attached(space)
         if not self.w_decoder:
             raise OperationError(space.w_IOError, space.wrap("not readable"))
 
@@ -635,7 +640,7 @@
         return space.wrap(builder.build())
 
     def readline_w(self, space, w_limit=None):
-        self._check_closed(space)
+        self._check_attached(space)
         self._writeflush(space)
 
         limit = convert_size(space, w_limit)
@@ -730,8 +735,8 @@
     # write methods
 
     def write_w(self, space, w_text):
-        self._check_init(space)
-        self._check_closed(space)
+        self._check_attached(space)
+        # self._check_closed(space)
 
         if not self.w_encoder:
             raise OperationError(space.w_IOError, space.wrap("not writable"))
@@ -802,7 +807,7 @@
                 break
 
     def detach_w(self, space):
-        self._check_init(space)
+        self._check_attached(space)
         space.call_method(self, "flush")
         w_buffer = self.w_buffer
         self.w_buffer = None
@@ -835,7 +840,7 @@
 
     @unwrap_spec(whence=int)
     def seek_w(self, space, w_pos, whence=0):
-        self._check_closed(space)
+        self._check_attached(space)
 
         if not self.seekable:
             raise OperationError(space.w_IOError, space.wrap(
@@ -1011,11 +1016,11 @@
         return space.newlong_from_rbigint(cookie.pack())
 
     def chunk_size_get_w(self, space):
-        self._check_init(space)
+        self._check_attached(space)
         return space.wrap(self.chunk_size)
 
     def chunk_size_set_w(self, space, w_size):
-        self._check_init(space)
+        self._check_attached(space)
         size = space.int_w(w_size)
         if size <= 0:
             raise OperationError(space.w_ValueError,
diff --git a/pypy/module/_io/test/test_textio.py b/pypy/module/_io/test/test_textio.py
--- a/pypy/module/_io/test/test_textio.py
+++ b/pypy/module/_io/test/test_textio.py
@@ -57,6 +57,13 @@
         raises(ValueError, f.close)
         raises(ValueError, f.detach)
         raises(ValueError, f.flush)
+
+        # Operations independent of the detached stream should still work
+        repr(f)
+        assert f.encoding == "UTF-8"
+        assert f.errors == "strict"
+        assert not f.line_buffering
+
         assert not b.closed
         b.close()
 

From noreply at buildbot.pypy.org  Sun Jun 14 16:38:07 2015
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sun, 14 Jun 2015 16:38:07 +0200 (CEST)
Subject: [pypy-commit] pypy default: CPython issue #5700: flush() was not
 called in close() if closefd=False.
Message-ID: <20150614143807.8786D1C0962@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: 
Changeset: r78089:c7298ea71c4d
Date: 2015-06-14 16:01 +0200
http://bitbucket.org/pypy/pypy/changeset/c7298ea71c4d/

Log:	CPython issue #5700: flush() was not called in close() if
	closefd=False.

diff --git a/pypy/module/_io/interp_fileio.py b/pypy/module/_io/interp_fileio.py
--- a/pypy/module/_io/interp_fileio.py
+++ b/pypy/module/_io/interp_fileio.py
@@ -242,11 +242,18 @@
                                exception_name='w_IOError')
 
     def close_w(self, space):
+        try:
+            W_RawIOBase.close_w(self, space)
+        except OperationError:
+            if not self.closefd:
+                self.fd = -1
+                raise
+            self._close(space)
+            raise
         if not self.closefd:
             self.fd = -1
             return
         self._close(space)
-        W_RawIOBase.close_w(self, space)
 
     def _dircheck(self, space, w_filename):
         # On Unix, fopen will succeed for directories.
diff --git a/pypy/module/_io/test/test_fileio.py b/pypy/module/_io/test/test_fileio.py
--- a/pypy/module/_io/test/test_fileio.py
+++ b/pypy/module/_io/test/test_fileio.py
@@ -209,6 +209,22 @@
             if os.path.exists(self.tmpfile):
                 os.unlink(self.tmpfile)
 
+    def test_flush_error_on_close(self):
+        # Test that the file is closed despite failed flush
+        # and that flush() is called before file closed.
+        import _io, os
+        fd = os.open(self.tmpfile, os.O_RDONLY, 0666)
+        f = _io.FileIO(fd, 'r', closefd=False)
+        closed = []
+        def bad_flush():
+            closed[:] = [f.closed]
+            raise IOError()
+        f.flush = bad_flush
+        raises(IOError, f.close) # exception not swallowed
+        assert f.closed
+        assert closed         # flush() called
+        assert not closed[0]  # flush() called before file closed
+        os.close(fd)
 
 def test_flush_at_exit():
     from pypy import conftest

From noreply at buildbot.pypy.org  Sun Jun 14 16:38:08 2015
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sun, 14 Jun 2015 16:38:08 +0200 (CEST)
Subject: [pypy-commit] pypy default: CPython Issue #21802: The reader in
 BufferedRWPair now is closed even when closing
Message-ID: <20150614143808.A84D31C0962@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: 
Changeset: r78090:0117c2d4c183
Date: 2015-06-14 16:38 +0200
http://bitbucket.org/pypy/pypy/changeset/0117c2d4c183/

Log:	CPython Issue #21802: The reader in BufferedRWPair now is closed
	even when closing writer failed in BufferedRWPair.close()

diff --git a/pypy/module/_io/interp_bufferedio.py b/pypy/module/_io/interp_bufferedio.py
--- a/pypy/module/_io/interp_bufferedio.py
+++ b/pypy/module/_io/interp_bufferedio.py
@@ -972,9 +972,26 @@
             method, writer=True)
 
     # forward to both
-    for method in ['close']:
-        locals()[method + '_w'] = make_forwarding_method(
-            method, writer=True, reader=True)
+    def close_w(self, space, __args__):
+        if self.w_writer is None:
+            raise oefmt(space.w_ValueError,
+                        "I/O operation on uninitialized object")
+        w_meth = space.getattr(self.w_writer, space.wrap("close"))
+        try:
+            space.call_args(w_meth, __args__)
+        except OperationError as e:
+            pass
+        else:
+            e = None
+
+        if self.w_reader is None:
+            raise oefmt(space.w_ValueError,
+                        "I/O operation on uninitialized object")
+        w_meth = space.getattr(self.w_reader, space.wrap("close"))
+        space.call_args(w_meth, __args__)
+
+        if e:
+            raise e
 
     def isatty_w(self, space):
         if space.is_true(space.call_method(self.w_writer, "isatty")):
diff --git a/pypy/module/_io/test/test_bufferedio.py b/pypy/module/_io/test/test_bufferedio.py
--- a/pypy/module/_io/test/test_bufferedio.py
+++ b/pypy/module/_io/test/test_bufferedio.py
@@ -592,6 +592,47 @@
 
         raises(IOError, _io.BufferedRWPair, _io.BytesIO(), NotWritable())
 
+    def test_writer_close_error_on_close(self):
+        import _io
+        class MockRawIO(_io._IOBase):
+            def readable(self):
+                return True
+            def writable(self):
+                return True
+        def writer_close():
+            writer_non_existing
+        reader = MockRawIO()
+        writer = MockRawIO()
+        writer.close = writer_close
+        pair = _io.BufferedRWPair(reader, writer)
+        err = raises(NameError, pair.close)
+        assert 'writer_non_existing' in str(err.value)
+        assert not pair.closed
+        assert reader.closed
+        assert not writer.closed
+
+    def test_reader_writer_close_error_on_close(self):
+        import _io
+        class MockRawIO(_io._IOBase):
+            def readable(self):
+                return True
+            def writable(self):
+                return True
+        def reader_close():
+            reader_non_existing
+        def writer_close():
+            writer_non_existing
+        reader = MockRawIO()
+        reader.close = reader_close
+        writer = MockRawIO()
+        writer.close = writer_close
+        pair = _io.BufferedRWPair(reader, writer)
+        err = raises(NameError, pair.close)
+        assert 'reader_non_existing' in str(err.value)
+        assert not pair.closed
+        assert not reader.closed
+        assert not writer.closed
+
 class AppTestBufferedRandom:
     spaceconfig = dict(usemodules=['_io'])
 
diff --git a/pypy/module/_io/test/test_fileio.py b/pypy/module/_io/test/test_fileio.py
--- a/pypy/module/_io/test/test_fileio.py
+++ b/pypy/module/_io/test/test_fileio.py
@@ -197,17 +197,13 @@
     def test_mode_strings(self):
         import _io
         import os
-        try:
-            for modes in [('w', 'wb'), ('wb', 'wb'), ('wb+', 'rb+'),
-                          ('w+b', 'rb+'), ('a', 'ab'), ('ab', 'ab'),
-                          ('ab+', 'ab+'), ('a+b', 'ab+'), ('r', 'rb'),
-                          ('rb', 'rb'), ('rb+', 'rb+'), ('r+b', 'rb+')]:
-                # read modes are last so that TESTFN will exist first
-                with _io.FileIO(self.tmpfile, modes[0]) as f:
-                    assert f.mode == modes[1]
-        finally:
-            if os.path.exists(self.tmpfile):
-                os.unlink(self.tmpfile)
+        for modes in [('w', 'wb'), ('wb', 'wb'), ('wb+', 'rb+'),
+                      ('w+b', 'rb+'), ('a', 'ab'), ('ab', 'ab'),
+                      ('ab+', 'ab+'), ('a+b', 'ab+'), ('r', 'rb'),
+                      ('rb', 'rb'), ('rb+', 'rb+'), ('r+b', 'rb+')]:
+            # read modes are last so that TESTFN will exist first
+            with _io.FileIO(self.tmpfile, modes[0]) as f:
+                assert f.mode == modes[1]
 
     def test_flush_error_on_close(self):
         # Test that the file is closed despite failed flush

From noreply at buildbot.pypy.org  Sun Jun 14 17:58:40 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 14 Jun 2015 17:58:40 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Mostly untested: add here the support
 for atomic transactions
Message-ID: <20150614155840.577BD1C0FB8@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r1831:9c72d7f52305
Date: 2015-06-14 17:59 +0200
http://bitbucket.org/pypy/stmgc/changeset/9c72d7f52305/

Log:	Mostly untested: add here the support for atomic transactions

diff --git a/c8/stm/core.c b/c8/stm/core.c
--- a/c8/stm/core.c
+++ b/c8/stm/core.c
@@ -1155,6 +1155,8 @@
     STM_PSEGMENT->shadowstack_at_start_of_transaction = tl->shadowstack;
     STM_PSEGMENT->threadlocal_at_start_of_transaction = tl->thread_local_obj;
     STM_PSEGMENT->total_throw_away_nursery = 0;
+    assert(tl->self_or_0_if_atomic == (intptr_t)tl);   /* not atomic */
+    assert(STM_PSEGMENT->atomic_nesting_levels == 0);
 
     assert(list_is_empty(STM_PSEGMENT->modified_old_objects));
     assert(list_is_empty(STM_PSEGMENT->large_overflow_objects));
@@ -1319,6 +1321,12 @@
         stm_fatalerror("cannot commit between stm_stop_all_other_threads "
                        "and stm_resume_all_other_threads");
     }
+    if (STM_PSEGMENT->atomic_nesting_levels > 0) {
+        stm_fatalerror("cannot commit between stm_enable_atomic "
+                       "and stm_disable_atomic");
+    }
+    assert(STM_SEGMENT->running_thread->self_or_0_if_atomic ==
+           (intptr_t)(STM_SEGMENT->running_thread));
 
     dprintf(("> stm_commit_transaction(external=%d)\n", (int)external));
     minor_collection(/*commit=*/ true, external);
@@ -1513,6 +1521,8 @@
     abort_data_structures_from_segment_num(STM_SEGMENT->segment_num);
 
     stm_thread_local_t *tl = STM_SEGMENT->running_thread;
+    tl->self_or_0_if_atomic = (intptr_t)tl;   /* clear the 'atomic' flag */
+    STM_PSEGMENT->atomic_nesting_levels = 0;
 
     if (tl->mem_clear_on_abort)
         memset(tl->mem_clear_on_abort, 0, tl->mem_bytes_to_clear_on_abort);
diff --git a/c8/stm/core.h b/c8/stm/core.h
--- a/c8/stm/core.h
+++ b/c8/stm/core.h
@@ -155,6 +155,9 @@
 
     /* For nursery_mark */
     uintptr_t total_throw_away_nursery;
+
+    /* For stm_enable_atomic() */
+    uintptr_t atomic_nesting_levels;
 };
 
 enum /* safe_point */ {
diff --git a/c8/stm/detach.c b/c8/stm/detach.c
--- a/c8/stm/detach.c
+++ b/c8/stm/detach.c
@@ -22,6 +22,14 @@
    originally detached), and at the point where we know the original
    stm_thread_local_t is no longer relevant, we reset
    _stm_detached_inevitable_from_thread to 0.
+
+   The value that stm_leave_transactional_zone() sticks inside
+   _stm_detached_inevitable_from_thread is actually
+   'tl->self_or_0_if_atomic'.  This value is 0 if and only if 'tl' is
+   current running a transaction *and* this transaction is atomic.  So
+   if we're running an atomic transaction, then
+   _stm_detached_inevitable_from_thread remains 0 across
+   leave/enter_transactional.
 */
 
 volatile intptr_t _stm_detached_inevitable_from_thread;
@@ -36,19 +44,35 @@
 void _stm_leave_noninevitable_transactional_zone(void)
 {
     int saved_errno = errno;
-    dprintf(("leave_noninevitable_transactional_zone\n"));
-    _stm_become_inevitable(MSG_INEV_DONT_SLEEP);
 
-    /* did it work? */
-    if (STM_PSEGMENT->transaction_state == TS_INEVITABLE) {   /* yes */
-        dprintf(("leave_noninevitable_transactional_zone: now inevitable\n"));
-        stm_thread_local_t *tl = STM_SEGMENT->running_thread;
-        _stm_detach_inevitable_transaction(tl);
+    if (STM_PSEGMENT->atomic_nesting_levels == 0) {
+        dprintf(("leave_noninevitable_transactional_zone\n"));
+        _stm_become_inevitable(MSG_INEV_DONT_SLEEP);
+
+        /* did it work? */
+        if (STM_PSEGMENT->transaction_state == TS_INEVITABLE) {   /* yes */
+            dprintf((
+                "leave_noninevitable_transactional_zone: now inevitable\n"));
+            stm_thread_local_t *tl = STM_SEGMENT->running_thread;
+            _stm_detach_inevitable_transaction(tl);
+        }
+        else {   /* no */
+            dprintf(("leave_noninevitable_transactional_zone: commit\n"));
+            _stm_commit_transaction();
+        }
     }
-    else {   /* no */
-        dprintf(("leave_noninevitable_transactional_zone: commit\n"));
-        _stm_commit_transaction();
+    else {
+        /* we're atomic, so we can't commit at all */
+        dprintf(("leave_noninevitable_transactional_zone atomic\n"));
+        _stm_become_inevitable("leave_noninevitable_transactional_zone atomic");
+        assert(STM_PSEGMENT->transaction_state == TS_INEVITABLE);
+        assert(_stm_detached_inevitable_from_thread == 0);
+        assert(STM_SEGMENT->running_thread->self_or_0_if_atomic == 0);
+        /* no point in calling _stm_detach_inevitable_transaction()
+           because it would store 0 into a place that is already 0, as
+           checked by the asserts above */
     }
+
     errno = saved_errno;
 }
 
@@ -173,3 +197,54 @@
         goto restart;
     }
 }
+
+uintptr_t stm_is_atomic(stm_thread_local_t *tl)
+{
+    assert(STM_SEGMENT->running_thread == tl);
+    if (tl->self_or_0_if_atomic != 0) {
+        assert(tl->self_or_0_if_atomic == (intptr_t)tl);
+        assert(STM_PSEGMENT->atomic_nesting_levels == 0);
+    }
+    else {
+        assert(STM_PSEGMENT->atomic_nesting_levels > 0);
+    }
+    return STM_PSEGMENT->atomic_nesting_levels;
+}
+
+#define HUGE_INTPTR_VALUE  0x3000000000000000L
+
+void stm_enable_atomic(stm_thread_local_t *tl)
+{
+    if (!stm_is_atomic(tl)) {
+        tl->self_or_0_if_atomic = 0;
+        /* increment 'nursery_mark' by HUGE_INTPTR_VALUE, so that
+           stm_should_break_transaction() returns always false */
+        intptr_t mark = (intptr_t)STM_SEGMENT->nursery_mark;
+        if (mark < 0)
+            mark = 0;
+        if (mark >= HUGE_INTPTR_VALUE)
+            mark = HUGE_INTPTR_VALUE - 1;
+        mark += HUGE_INTPTR_VALUE;
+        STM_SEGMENT->nursery_mark = (stm_char *)mark;
+    }
+    STM_PSEGMENT->atomic_nesting_levels++;
+}
+
+void stm_disable_atomic(stm_thread_local_t *tl)
+{
+    if (!stm_is_atomic(tl))
+        stm_fatalerror("stm_disable_atomic(): already not atomic");
+
+    STM_PSEGMENT->atomic_nesting_levels--;
+
+    if (STM_PSEGMENT->atomic_nesting_levels == 0) {
+        tl->self_or_0_if_atomic = (intptr_t)tl;
+        /* decrement 'nursery_mark' by HUGE_INTPTR_VALUE, to cancel
+           what was done in stm_enable_atomic() */
+        intptr_t mark = (intptr_t)STM_SEGMENT->nursery_mark;
+        mark -= HUGE_INTPTR_VALUE;
+        if (mark < 0)
+            mark = 0;
+        STM_SEGMENT->nursery_mark = (stm_char *)mark;
+    }
+}
diff --git a/c8/stm/setup.c b/c8/stm/setup.c
--- a/c8/stm/setup.c
+++ b/c8/stm/setup.c
@@ -233,8 +233,7 @@
 {
     int num;
     s_mutex_lock();
-    tl->self = tl;    /* for faster access to &stm_thread_local (and easier
-                         from the PyPy JIT, too) */
+    tl->self_or_0_if_atomic = (intptr_t)tl;    /* 'not atomic' */
     if (stm_all_thread_locals == NULL) {
         stm_all_thread_locals = tl->next = tl->prev = tl;
         num = 0;
diff --git a/c8/stmgc.h b/c8/stmgc.h
--- a/c8/stmgc.h
+++ b/c8/stmgc.h
@@ -71,7 +71,8 @@
     /* the next fields are handled internally by the library */
     int last_associated_segment_num;   /* always a valid seg num */
     int thread_local_counter;
-    struct stm_thread_local_s *self, *prev, *next;
+    struct stm_thread_local_s *prev, *next;
+    intptr_t self_or_0_if_atomic;
     void *creating_pthread[2];
 } stm_thread_local_t;
 
@@ -90,7 +91,7 @@
 #define _stm_detach_inevitable_transaction(tl)  do {                    \
     write_fence();                                                      \
     assert(_stm_detached_inevitable_from_thread == 0);                  \
-    _stm_detached_inevitable_from_thread = (intptr_t)(tl->self);        \
+    _stm_detached_inevitable_from_thread = tl->self_or_0_if_atomic;     \
 } while (0)
 void _stm_reattach_transaction(stm_thread_local_t *tl);
 void _stm_become_inevitable(const char*);
@@ -427,7 +428,7 @@
 #endif
 static inline void stm_enter_transactional_zone(stm_thread_local_t *tl) {
     if (__sync_bool_compare_and_swap(&_stm_detached_inevitable_from_thread,
-                                     (intptr_t)tl, 0)) {
+                                     tl->self_or_0_if_atomic, 0)) {
 #ifdef STM_DEBUGPRINT
         fprintf(stderr, "stm_enter_transactional_zone fast path\n");
 #endif
@@ -505,6 +506,15 @@
    in the last attempt.
 */
 
+/* "atomic" transaction: a transaction where stm_should_break_transaction()
+   always returns false, and where stm_leave_transactional_zone() never
+   detach nor terminates the transaction.  (stm_force_transaction_break()
+   crashes if called with an atomic transaction.)
+*/
+uintptr_t stm_is_atomic(stm_thread_local_t *tl);
+void stm_enable_atomic(stm_thread_local_t *tl);
+void stm_disable_atomic(stm_thread_local_t *tl);
+
 
 /* Prepare an immortal "prebuilt" object managed by the GC.  Takes a
    pointer to an 'object_t', which should not actually be a GC-managed

From noreply at buildbot.pypy.org  Sun Jun 14 18:05:21 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 14 Jun 2015 18:05:21 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c8-gil-like: import stmgc/9c72d7f52305
Message-ID: <20150614160521.15C461C0FB8@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c8-gil-like
Changeset: r78091:3be5cbbb7313
Date: 2015-06-14 18:00 +0200
http://bitbucket.org/pypy/pypy/changeset/3be5cbbb7313/

Log:	import stmgc/9c72d7f52305

diff --git a/rpython/translator/stm/src_stm/revision b/rpython/translator/stm/src_stm/revision
--- a/rpython/translator/stm/src_stm/revision
+++ b/rpython/translator/stm/src_stm/revision
@@ -1,1 +1,1 @@
-8009f12c327b
+9c72d7f52305
diff --git a/rpython/translator/stm/src_stm/stm/core.c b/rpython/translator/stm/src_stm/stm/core.c
--- a/rpython/translator/stm/src_stm/stm/core.c
+++ b/rpython/translator/stm/src_stm/stm/core.c
@@ -1155,6 +1155,8 @@
     STM_PSEGMENT->shadowstack_at_start_of_transaction = tl->shadowstack;
     STM_PSEGMENT->threadlocal_at_start_of_transaction = tl->thread_local_obj;
     STM_PSEGMENT->total_throw_away_nursery = 0;
+    assert(tl->self_or_0_if_atomic == (intptr_t)tl);   /* not atomic */
+    assert(STM_PSEGMENT->atomic_nesting_levels == 0);
 
     assert(list_is_empty(STM_PSEGMENT->modified_old_objects));
     assert(list_is_empty(STM_PSEGMENT->large_overflow_objects));
@@ -1319,6 +1321,12 @@
         stm_fatalerror("cannot commit between stm_stop_all_other_threads "
                        "and stm_resume_all_other_threads");
     }
+    if (STM_PSEGMENT->atomic_nesting_levels > 0) {
+        stm_fatalerror("cannot commit between stm_enable_atomic "
+                       "and stm_disable_atomic");
+    }
+    assert(STM_SEGMENT->running_thread->self_or_0_if_atomic ==
+           (intptr_t)(STM_SEGMENT->running_thread));
 
     dprintf(("> stm_commit_transaction(external=%d)\n", (int)external));
     minor_collection(/*commit=*/ true, external);
@@ -1513,6 +1521,8 @@
     abort_data_structures_from_segment_num(STM_SEGMENT->segment_num);
 
     stm_thread_local_t *tl = STM_SEGMENT->running_thread;
+    tl->self_or_0_if_atomic = (intptr_t)tl;   /* clear the 'atomic' flag */
+    STM_PSEGMENT->atomic_nesting_levels = 0;
 
     if (tl->mem_clear_on_abort)
         memset(tl->mem_clear_on_abort, 0, tl->mem_bytes_to_clear_on_abort);
diff --git a/rpython/translator/stm/src_stm/stm/core.h b/rpython/translator/stm/src_stm/stm/core.h
--- a/rpython/translator/stm/src_stm/stm/core.h
+++ b/rpython/translator/stm/src_stm/stm/core.h
@@ -155,6 +155,9 @@
 
     /* For nursery_mark */
     uintptr_t total_throw_away_nursery;
+
+    /* For stm_enable_atomic() */
+    uintptr_t atomic_nesting_levels;
 };
 
 enum /* safe_point */ {
diff --git a/rpython/translator/stm/src_stm/stm/detach.c b/rpython/translator/stm/src_stm/stm/detach.c
--- a/rpython/translator/stm/src_stm/stm/detach.c
+++ b/rpython/translator/stm/src_stm/stm/detach.c
@@ -22,6 +22,14 @@
    originally detached), and at the point where we know the original
    stm_thread_local_t is no longer relevant, we reset
    _stm_detached_inevitable_from_thread to 0.
+
+   The value that stm_leave_transactional_zone() sticks inside
+   _stm_detached_inevitable_from_thread is actually
+   'tl->self_or_0_if_atomic'.  This value is 0 if and only if 'tl' is
+   current running a transaction *and* this transaction is atomic.  So
+   if we're running an atomic transaction, then
+   _stm_detached_inevitable_from_thread remains 0 across
+   leave/enter_transactional.
 */
 
 volatile intptr_t _stm_detached_inevitable_from_thread;
@@ -36,19 +44,35 @@
 void _stm_leave_noninevitable_transactional_zone(void)
 {
     int saved_errno = errno;
-    dprintf(("leave_noninevitable_transactional_zone\n"));
-    _stm_become_inevitable(MSG_INEV_DONT_SLEEP);
 
-    /* did it work? */
-    if (STM_PSEGMENT->transaction_state == TS_INEVITABLE) {   /* yes */
-        dprintf(("leave_noninevitable_transactional_zone: now inevitable\n"));
-        stm_thread_local_t *tl = STM_SEGMENT->running_thread;
-        _stm_detach_inevitable_transaction(tl);
+    if (STM_PSEGMENT->atomic_nesting_levels == 0) {
+        dprintf(("leave_noninevitable_transactional_zone\n"));
+        _stm_become_inevitable(MSG_INEV_DONT_SLEEP);
+
+        /* did it work? */
+        if (STM_PSEGMENT->transaction_state == TS_INEVITABLE) {   /* yes */
+            dprintf((
+                "leave_noninevitable_transactional_zone: now inevitable\n"));
+            stm_thread_local_t *tl = STM_SEGMENT->running_thread;
+            _stm_detach_inevitable_transaction(tl);
+        }
+        else {   /* no */
+            dprintf(("leave_noninevitable_transactional_zone: commit\n"));
+            _stm_commit_transaction();
+        }
     }
-    else {   /* no */
-        dprintf(("leave_noninevitable_transactional_zone: commit\n"));
-        _stm_commit_transaction();
+    else {
+        /* we're atomic, so we can't commit at all */
+        dprintf(("leave_noninevitable_transactional_zone atomic\n"));
+        _stm_become_inevitable("leave_noninevitable_transactional_zone atomic");
+        assert(STM_PSEGMENT->transaction_state == TS_INEVITABLE);
+        assert(_stm_detached_inevitable_from_thread == 0);
+        assert(STM_SEGMENT->running_thread->self_or_0_if_atomic == 0);
+        /* no point in calling _stm_detach_inevitable_transaction()
+           because it would store 0 into a place that is already 0, as
+           checked by the asserts above */
     }
+
     errno = saved_errno;
 }
 
@@ -173,3 +197,54 @@
         goto restart;
     }
 }
+
+uintptr_t stm_is_atomic(stm_thread_local_t *tl)
+{
+    assert(STM_SEGMENT->running_thread == tl);
+    if (tl->self_or_0_if_atomic != 0) {
+        assert(tl->self_or_0_if_atomic == (intptr_t)tl);
+        assert(STM_PSEGMENT->atomic_nesting_levels == 0);
+    }
+    else {
+        assert(STM_PSEGMENT->atomic_nesting_levels > 0);
+    }
+    return STM_PSEGMENT->atomic_nesting_levels;
+}
+
+#define HUGE_INTPTR_VALUE  0x3000000000000000L
+
+void stm_enable_atomic(stm_thread_local_t *tl)
+{
+    if (!stm_is_atomic(tl)) {
+        tl->self_or_0_if_atomic = 0;
+        /* increment 'nursery_mark' by HUGE_INTPTR_VALUE, so that
+           stm_should_break_transaction() returns always false */
+        intptr_t mark = (intptr_t)STM_SEGMENT->nursery_mark;
+        if (mark < 0)
+            mark = 0;
+        if (mark >= HUGE_INTPTR_VALUE)
+            mark = HUGE_INTPTR_VALUE - 1;
+        mark += HUGE_INTPTR_VALUE;
+        STM_SEGMENT->nursery_mark = (stm_char *)mark;
+    }
+    STM_PSEGMENT->atomic_nesting_levels++;
+}
+
+void stm_disable_atomic(stm_thread_local_t *tl)
+{
+    if (!stm_is_atomic(tl))
+        stm_fatalerror("stm_disable_atomic(): already not atomic");
+
+    STM_PSEGMENT->atomic_nesting_levels--;
+
+    if (STM_PSEGMENT->atomic_nesting_levels == 0) {
+        tl->self_or_0_if_atomic = (intptr_t)tl;
+        /* decrement 'nursery_mark' by HUGE_INTPTR_VALUE, to cancel
+           what was done in stm_enable_atomic() */
+        intptr_t mark = (intptr_t)STM_SEGMENT->nursery_mark;
+        mark -= HUGE_INTPTR_VALUE;
+        if (mark < 0)
+            mark = 0;
+        STM_SEGMENT->nursery_mark = (stm_char *)mark;
+    }
+}
diff --git a/rpython/translator/stm/src_stm/stm/nursery.c b/rpython/translator/stm/src_stm/stm/nursery.c
--- a/rpython/translator/stm/src_stm/stm/nursery.c
+++ b/rpython/translator/stm/src_stm/stm/nursery.c
@@ -471,7 +471,6 @@
     }
     OPT_ASSERT((nursery_used & 7) == 0);
 
-#ifndef NDEBUG
     /* reset the nursery by zeroing it */
     char *realnursery;
     realnursery = REAL_ADDRESS(pseg->pub.segment_base, _stm_nursery_start);
@@ -483,8 +482,9 @@
                        (NURSERY_END - _stm_nursery_start) - nursery_used);
 
 #else
+# ifndef NDEBUG
     memset(realnursery, 0xa0, nursery_used);
-#endif
+# endif
 #endif
 
     pseg->total_throw_away_nursery += nursery_used;
diff --git a/rpython/translator/stm/src_stm/stm/setup.c b/rpython/translator/stm/src_stm/stm/setup.c
--- a/rpython/translator/stm/src_stm/stm/setup.c
+++ b/rpython/translator/stm/src_stm/stm/setup.c
@@ -233,8 +233,7 @@
 {
     int num;
     s_mutex_lock();
-    tl->self = tl;    /* for faster access to &stm_thread_local (and easier
-                         from the PyPy JIT, too) */
+    tl->self_or_0_if_atomic = (intptr_t)tl;    /* 'not atomic' */
     if (stm_all_thread_locals == NULL) {
         stm_all_thread_locals = tl->next = tl->prev = tl;
         num = 0;
diff --git a/rpython/translator/stm/src_stm/stmgc.h b/rpython/translator/stm/src_stm/stmgc.h
--- a/rpython/translator/stm/src_stm/stmgc.h
+++ b/rpython/translator/stm/src_stm/stmgc.h
@@ -71,7 +71,8 @@
     /* the next fields are handled internally by the library */
     int last_associated_segment_num;   /* always a valid seg num */
     int thread_local_counter;
-    struct stm_thread_local_s *self, *prev, *next;
+    struct stm_thread_local_s *prev, *next;
+    intptr_t self_or_0_if_atomic;
     void *creating_pthread[2];
 } stm_thread_local_t;
 
@@ -90,7 +91,7 @@
 #define _stm_detach_inevitable_transaction(tl)  do {                    \
     write_fence();                                                      \
     assert(_stm_detached_inevitable_from_thread == 0);                  \
-    _stm_detached_inevitable_from_thread = (intptr_t)(tl->self);        \
+    _stm_detached_inevitable_from_thread = tl->self_or_0_if_atomic;     \
 } while (0)
 void _stm_reattach_transaction(stm_thread_local_t *tl);
 void _stm_become_inevitable(const char*);
@@ -427,7 +428,7 @@
 #endif
 static inline void stm_enter_transactional_zone(stm_thread_local_t *tl) {
     if (__sync_bool_compare_and_swap(&_stm_detached_inevitable_from_thread,
-                                     (intptr_t)tl, 0)) {
+                                     tl->self_or_0_if_atomic, 0)) {
 #ifdef STM_DEBUGPRINT
         fprintf(stderr, "stm_enter_transactional_zone fast path\n");
 #endif
@@ -505,6 +506,15 @@
    in the last attempt.
 */
 
+/* "atomic" transaction: a transaction where stm_should_break_transaction()
+   always returns false, and where stm_leave_transactional_zone() never
+   detach nor terminates the transaction.  (stm_force_transaction_break()
+   crashes if called with an atomic transaction.)
+*/
+uintptr_t stm_is_atomic(stm_thread_local_t *tl);
+void stm_enable_atomic(stm_thread_local_t *tl);
+void stm_disable_atomic(stm_thread_local_t *tl);
+
 
 /* Prepare an immortal "prebuilt" object managed by the GC.  Takes a
    pointer to an 'object_t', which should not actually be a GC-managed

From noreply at buildbot.pypy.org  Sun Jun 14 18:05:22 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 14 Jun 2015 18:05:22 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c8-gil-like: Use the new built-in atomic
	support
Message-ID: <20150614160522.35ECE1C0FB8@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c8-gil-like
Changeset: r78092:55cbeb8df665
Date: 2015-06-14 18:05 +0200
http://bitbucket.org/pypy/pypy/changeset/55cbeb8df665/

Log:	Use the new built-in atomic support

diff --git a/rpython/translator/stm/funcgen.py b/rpython/translator/stm/funcgen.py
--- a/rpython/translator/stm/funcgen.py
+++ b/rpython/translator/stm/funcgen.py
@@ -212,14 +212,14 @@
     return 'stm_force_transaction_break(&stm_thread_local);'
 
 def stm_increment_atomic(funcgen, op):
-    return r'fprintf(stderr, "stm_increment_atomic: reimplement\n"); abort();'
+    return 'stm_enable_atomic(&stm_thread_local);'
 
 def stm_decrement_atomic(funcgen, op):
-    return r'fprintf(stderr, "stm_decrement_atomic: reimplement\n"); abort();'
+    return 'stm_disable_atomic(&stm_thread_local);'
 
 def stm_get_atomic(funcgen, op):
     result = funcgen.expr(op.result)
-    return '%s = 0;  // XXX stm_get_atomic' % (result,)
+    return '%s = stm_is_atomic(&stm_thread_local);' % (result,)
 
 def stm_is_inevitable(funcgen, op):
     result = funcgen.expr(op.result)

From noreply at buildbot.pypy.org  Sun Jun 14 18:06:14 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 14 Jun 2015 18:06:14 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c8-gil-like: Bump the maximum memory from
	2.5 to 7.5 GB
Message-ID: <20150614160614.BD1A51C0FB8@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c8-gil-like
Changeset: r78093:119c613c7c94
Date: 2015-06-14 17:07 +0100
http://bitbucket.org/pypy/pypy/changeset/119c613c7c94/

Log:	Bump the maximum memory from 2.5 to 7.5 GB

diff --git a/rpython/translator/stm/src_stm/stm/core.h b/rpython/translator/stm/src_stm/stm/core.h
--- a/rpython/translator/stm/src_stm/stm/core.h
+++ b/rpython/translator/stm/src_stm/stm/core.h
@@ -16,7 +16,7 @@
 #endif
 
 
-#define NB_PAGES            (2500*256)    // 2500MB
+#define NB_PAGES            (7500*256)    // 7500MB
 #define NB_SEGMENTS         (STM_NB_SEGMENTS+1) /* +1 for sharing seg 0 */
 #define NB_SEGMENTS_MAX     240    /* don't increase NB_SEGMENTS past this */
 #define NB_NURSERY_PAGES    (STM_GC_NURSERY/4)

From noreply at buildbot.pypy.org  Sun Jun 14 18:17:52 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 14 Jun 2015 18:17:52 +0200 (CEST)
Subject: [pypy-commit] stmgc default: This should be a no-op unless I'm
	missing something.
Message-ID: <20150614161752.71DF11C0683@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r1832:7a87c63be4d2
Date: 2015-06-14 18:18 +0200
http://bitbucket.org/pypy/stmgc/changeset/7a87c63be4d2/

Log:	This should be a no-op unless I'm missing something.

diff --git a/c8/stm/detach.c b/c8/stm/detach.c
--- a/c8/stm/detach.c
+++ b/c8/stm/detach.c
@@ -82,10 +82,23 @@
     _core_commit_transaction(/*external=*/ true);
 }
 
-void _stm_reattach_transaction(stm_thread_local_t *tl)
+void _stm_reattach_transaction(intptr_t self)
 {
     intptr_t old;
     int saved_errno = errno;
+    stm_thread_local_t *tl = (stm_thread_local_t *)self;
+
+    /* if 'self_or_0_if_atomic == 0', it means that we are trying to
+       reattach in a thread that is currently running a transaction
+       that is atomic.  That should only be possible if we're
+       inevitable too.  And in that case,
+       '_stm_detached_inevitable_from_thread' must always be 0, and
+       the previous call to compare_and_swap(0, 0) should have worked
+       (and done nothing), so we should not end here.
+    */
+    if (self == 0)
+        stm_fatalerror("atomic inconsistency");
+
  restart:
     old = _stm_detached_inevitable_from_thread;
     if (old != 0) {
diff --git a/c8/stmgc.h b/c8/stmgc.h
--- a/c8/stmgc.h
+++ b/c8/stmgc.h
@@ -93,7 +93,7 @@
     assert(_stm_detached_inevitable_from_thread == 0);                  \
     _stm_detached_inevitable_from_thread = tl->self_or_0_if_atomic;     \
 } while (0)
-void _stm_reattach_transaction(stm_thread_local_t *tl);
+void _stm_reattach_transaction(intptr_t);
 void _stm_become_inevitable(const char*);
 void _stm_collectable_safe_point(void);
 
@@ -427,14 +427,15 @@
 #include <stdio.h>
 #endif
 static inline void stm_enter_transactional_zone(stm_thread_local_t *tl) {
+    intptr_t self = tl->self_or_0_if_atomic;
     if (__sync_bool_compare_and_swap(&_stm_detached_inevitable_from_thread,
-                                     tl->self_or_0_if_atomic, 0)) {
+                                     self, 0)) {
 #ifdef STM_DEBUGPRINT
         fprintf(stderr, "stm_enter_transactional_zone fast path\n");
 #endif
     }
     else {
-        _stm_reattach_transaction(tl);
+        _stm_reattach_transaction(self);
         /* _stm_detached_inevitable_from_thread should be 0 here, but
            it can already have been changed from a parallel thread
            (assuming we're not inevitable ourselves) */

From noreply at buildbot.pypy.org  Sun Jun 14 18:28:08 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 14 Jun 2015 18:28:08 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c8-gil-like: import stmgc/7a87c63be4d2.
	adapt the JIT
Message-ID: <20150614162808.74B5D1C0683@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c8-gil-like
Changeset: r78094:c6856f2622dd
Date: 2015-06-14 18:21 +0200
http://bitbucket.org/pypy/pypy/changeset/c6856f2622dd/

Log:	import stmgc/7a87c63be4d2. adapt the JIT

diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -146,8 +146,8 @@
         mc.SUB_ri(esp.value, 3 * WORD)     # 3 instead of 2 to align the stack
         mc.MOV_sr(0, eax.value)     # not edx, we're not running 32-bit
         mc.MOVSD_sx(1, xmm0.value)
-        # load the value of tl (== tl->self) into edi as argument
-        mc.MOV(edi, self.heap_stm_thread_local_self())
+        # load the value of 'tl->self_or_0_if_atomic' into edi as argument
+        mc.MOV(edi, self.heap_stm_thread_local_self_or_0_if_atomic())
         mc.CALL(imm(rstm.adr_stm_reattach_transaction))
         # pop
         mc.MOVSD_xs(xmm0.value, 1)
@@ -930,10 +930,10 @@
         """STM: AddressLoc for '&stm_thread_local.rjthread.moved_off_base'."""
         return self.heap_tl(rstm.adr_rjthread_moved_off_base)
 
-    def heap_stm_thread_local_self(self):
+    def heap_stm_thread_local_self_or_0_if_atomic(self):
         """STM: AddressLoc for '&stm_thread_local.self', i.e. such that
         reading it returns the (absolute) address of 'stm_thread_local'."""
-        return self.heap_tl(rstm.adr_stm_thread_local_self)
+        return self.heap_tl(rstm.adr_stm_thread_local_self_or_0_if_atomic)
 
     def heap_stm_detached_inevitable_from_thread(self):
         """STM: AddressLoc for '&stm_detached_inevitable_from_thread'."""
diff --git a/rpython/jit/backend/x86/callbuilder.py b/rpython/jit/backend/x86/callbuilder.py
--- a/rpython/jit/backend/x86/callbuilder.py
+++ b/rpython/jit/backend/x86/callbuilder.py
@@ -709,8 +709,8 @@
         # Fast path: inline _stm_detach_inevitable_transaction()
         # <- Here comes the write_fence(), which is not needed in x86 assembler
         # assert(_stm_detached_inevitable_from_thread == 0): dropped
-        # _stm_detached_inevitable_from_thread = tl (== tl->self):
-        mc.MOV(eax, self.asm.heap_stm_thread_local_self())
+        # _stm_detached_inevitable_from_thread = tl->self_or_0_if_atomic:
+        mc.MOV(eax, self.asm.heap_stm_thread_local_self_or_0_if_atomic())
         mc.MOV(self.asm.heap_stm_detached_inevitable_from_thread(), eax)
         #
         offset = mc.get_relative_pos() - jmp_location
@@ -727,8 +727,8 @@
         mc = self.mc
         mc.MOV(edi, eax)
         #
-        # compare_and_swap(&_stm_detached_inevitable_from_thread, tl, 0)
-        mc.MOV(eax, self.asm.heap_stm_thread_local_self())
+        # compare_and_swap(&_stm_detached_inevitable_from_thread, self_or_0, 0)
+        mc.MOV(eax, self.asm.heap_stm_thread_local_self_or_0_if_atomic())
         mc.XOR(esi, esi)
         adr = self.asm.heap_stm_detached_inevitable_from_thread()
         m_address = mc._addr_as_reg_offset(adr.value_j())
diff --git a/rpython/rlib/rstm.py b/rpython/rlib/rstm.py
--- a/rpython/rlib/rstm.py
+++ b/rpython/rlib/rstm.py
@@ -46,8 +46,8 @@
     CFlexSymbolic('((long)&pypy__rewind_jmp_copy_stack_slice)'))
 adr_stm_detached_inevitable_from_thread = (
     CFlexSymbolic('((long)&_stm_detached_inevitable_from_thread)'))
-adr_stm_thread_local_self = (
-    CFlexSymbolic('((long)&stm_thread_local.self)'))
+adr_stm_thread_local_self_or_0_if_atomic = (
+    CFlexSymbolic('((long)&stm_thread_local.self_or_0_if_atomic)'))
 adr_stm_leave_noninevitable_transactional_zone = (
     CFlexSymbolic('((long)&_stm_leave_noninevitable_transactional_zone)'))
 adr_stm_reattach_transaction = (
diff --git a/rpython/translator/stm/src_stm/revision b/rpython/translator/stm/src_stm/revision
--- a/rpython/translator/stm/src_stm/revision
+++ b/rpython/translator/stm/src_stm/revision
@@ -1,1 +1,1 @@
-9c72d7f52305
+7a87c63be4d2
diff --git a/rpython/translator/stm/src_stm/stm/core.h b/rpython/translator/stm/src_stm/stm/core.h
--- a/rpython/translator/stm/src_stm/stm/core.h
+++ b/rpython/translator/stm/src_stm/stm/core.h
@@ -16,7 +16,7 @@
 #endif
 
 
-#define NB_PAGES            (7500*256)    // 7500MB
+#define NB_PAGES            (2500*256)    // 2500MB
 #define NB_SEGMENTS         (STM_NB_SEGMENTS+1) /* +1 for sharing seg 0 */
 #define NB_SEGMENTS_MAX     240    /* don't increase NB_SEGMENTS past this */
 #define NB_NURSERY_PAGES    (STM_GC_NURSERY/4)
diff --git a/rpython/translator/stm/src_stm/stm/detach.c b/rpython/translator/stm/src_stm/stm/detach.c
--- a/rpython/translator/stm/src_stm/stm/detach.c
+++ b/rpython/translator/stm/src_stm/stm/detach.c
@@ -82,10 +82,23 @@
     _core_commit_transaction(/*external=*/ true);
 }
 
-void _stm_reattach_transaction(stm_thread_local_t *tl)
+void _stm_reattach_transaction(intptr_t self)
 {
     intptr_t old;
     int saved_errno = errno;
+    stm_thread_local_t *tl = (stm_thread_local_t *)self;
+
+    /* if 'self_or_0_if_atomic == 0', it means that we are trying to
+       reattach in a thread that is currently running a transaction
+       that is atomic.  That should only be possible if we're
+       inevitable too.  And in that case,
+       '_stm_detached_inevitable_from_thread' must always be 0, and
+       the previous call to compare_and_swap(0, 0) should have worked
+       (and done nothing), so we should not end here.
+    */
+    if (self == 0)
+        stm_fatalerror("atomic inconsistency");
+
  restart:
     old = _stm_detached_inevitable_from_thread;
     if (old != 0) {
diff --git a/rpython/translator/stm/src_stm/stmgc.h b/rpython/translator/stm/src_stm/stmgc.h
--- a/rpython/translator/stm/src_stm/stmgc.h
+++ b/rpython/translator/stm/src_stm/stmgc.h
@@ -93,7 +93,7 @@
     assert(_stm_detached_inevitable_from_thread == 0);                  \
     _stm_detached_inevitable_from_thread = tl->self_or_0_if_atomic;     \
 } while (0)
-void _stm_reattach_transaction(stm_thread_local_t *tl);
+void _stm_reattach_transaction(intptr_t);
 void _stm_become_inevitable(const char*);
 void _stm_collectable_safe_point(void);
 
@@ -427,14 +427,15 @@
 #include <stdio.h>
 #endif
 static inline void stm_enter_transactional_zone(stm_thread_local_t *tl) {
+    intptr_t self = tl->self_or_0_if_atomic;
     if (__sync_bool_compare_and_swap(&_stm_detached_inevitable_from_thread,
-                                     tl->self_or_0_if_atomic, 0)) {
+                                     self, 0)) {
 #ifdef STM_DEBUGPRINT
         fprintf(stderr, "stm_enter_transactional_zone fast path\n");
 #endif
     }
     else {
-        _stm_reattach_transaction(tl);
+        _stm_reattach_transaction(self);
         /* _stm_detached_inevitable_from_thread should be 0 here, but
            it can already have been changed from a parallel thread
            (assuming we're not inevitable ourselves) */

From noreply at buildbot.pypy.org  Sun Jun 14 21:44:45 2015
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sun, 14 Jun 2015 21:44:45 +0200 (CEST)
Subject: [pypy-commit] pypy default: multiprocessing: Move the imports to
 module initialization, 
Message-ID: <20150614194445.6538B1C1FE3@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: 
Changeset: r78095:2acc241fc909
Date: 2015-06-14 21:44 +0200
http://bitbucket.org/pypy/pypy/changeset/2acc241fc909/

Log:	multiprocessing: Move the imports to module initialization, to avoid
	a deadlock with the import lock.

diff --git a/pypy/module/_multiprocessing/__init__.py b/pypy/module/_multiprocessing/__init__.py
--- a/pypy/module/_multiprocessing/__init__.py
+++ b/pypy/module/_multiprocessing/__init__.py
@@ -18,3 +18,8 @@
         interpleveldefs['PipeConnection'] = \
             'interp_connection.W_PipeConnection'
         interpleveldefs['win32'] = 'interp_win32.win32_namespace(space)'
+
+    def init(self, space):
+        MixedModule.init(self, space)
+        from pypy.module._multiprocessing.interp_connection import State
+        space.fromcache(State).init(space)
diff --git a/pypy/module/_multiprocessing/interp_connection.py b/pypy/module/_multiprocessing/interp_connection.py
--- a/pypy/module/_multiprocessing/interp_connection.py
+++ b/pypy/module/_multiprocessing/interp_connection.py
@@ -15,11 +15,21 @@
 PY_SSIZE_T_MAX = sys.maxint
 PY_SSIZE_T_MIN = -sys.maxint - 1
 
+class State(object):
+    def __init__(self, space):
+        pass
+
+    def init(self, space):
+        w_builtins = space.getbuiltinmodule('__builtin__')
+        w_module = space.call_method(
+            w_builtins, '__import__', space.wrap("multiprocessing"))
+        self.w_BufferTooShort = space.getattr(w_module, space.wrap("BufferTooShort"))
+
+        self.w_picklemodule = space.call_method(
+            w_builtins, '__import__', space.wrap("pickle"))
+
 def BufferTooShort(space, w_data):
-    w_builtins = space.getbuiltinmodule('__builtin__')
-    w_module = space.call_method(
-        w_builtins, '__import__', space.wrap("multiprocessing"))
-    w_BufferTooShort = space.getattr(w_module, space.wrap("BufferTooShort"))
+    w_BufferTooShort = space.fromcache(State).w_BufferTooShort
     return OperationError(w_BufferTooShort, w_data)
 
 def w_handle(space, handle):
@@ -144,9 +154,7 @@
     def send(self, space, w_obj):
         self._check_writable(space)
 
-        w_builtins = space.getbuiltinmodule('__builtin__')
-        w_picklemodule = space.call_method(
-            w_builtins, '__import__', space.wrap("pickle"))
+        w_picklemodule = space.fromcache(State).w_picklemodule
         w_protocol = space.getattr(
             w_picklemodule, space.wrap("HIGHEST_PROTOCOL"))
         w_pickled = space.call_method(
@@ -170,8 +178,7 @@
                 rffi.free_charp(newbuf)
 
         w_builtins = space.getbuiltinmodule('__builtin__')
-        w_picklemodule = space.call_method(
-            w_builtins, '__import__', space.wrap("pickle"))
+        w_picklemodule = space.fromcache(State).w_picklemodule
         w_unpickled = space.call_method(
             w_picklemodule, "loads", w_received)
 
diff --git a/pypy/module/_multiprocessing/test/test_memory.py b/pypy/module/_multiprocessing/test/test_memory.py
--- a/pypy/module/_multiprocessing/test/test_memory.py
+++ b/pypy/module/_multiprocessing/test/test_memory.py
@@ -1,6 +1,8 @@
 class AppTestMemory:
     spaceconfig = dict(usemodules=('_multiprocessing', 'mmap',
-                                   '_rawffi', 'itertools'))
+                                   '_rawffi', 'itertools',
+                                   'signal', 'select', 'fcntl',
+                                   'binascii'))
 
     def test_address_of(self):
         import _multiprocessing
diff --git a/pypy/module/_multiprocessing/test/test_semaphore.py b/pypy/module/_multiprocessing/test/test_semaphore.py
--- a/pypy/module/_multiprocessing/test/test_semaphore.py
+++ b/pypy/module/_multiprocessing/test/test_semaphore.py
@@ -3,7 +3,9 @@
 
 
 class AppTestSemaphore:
-    spaceconfig = dict(usemodules=('_multiprocessing', 'thread'))
+    spaceconfig = dict(usemodules=('_multiprocessing', 'thread',
+                                   'signal', 'select', 'fcntl',
+                                   'binascii', 'struct'))
 
     def setup_class(cls):
         cls.w_SEMAPHORE = cls.space.wrap(SEMAPHORE)
diff --git a/pypy/objspace/fake/checkmodule.py b/pypy/objspace/fake/checkmodule.py
--- a/pypy/objspace/fake/checkmodule.py
+++ b/pypy/objspace/fake/checkmodule.py
@@ -14,6 +14,7 @@
         # force computation and record what we wrap
         module = mod.Module(space, W_Root())
         module.setup_after_space_initialization()
+        module.init(space)
         modules.append(module)
         for name in module.loaders:
             seeobj_w.append(module._load_lazily(space, name))

From noreply at buildbot.pypy.org  Sun Jun 14 22:25:43 2015
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sun, 14 Jun 2015 22:25:43 +0200 (CEST)
Subject: [pypy-commit] pypy default: Cffi Tweaks in tklib_build until we get
 access to TK_HEX_VERSION, 
Message-ID: <20150614202543.BF0F31C0683@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: 
Changeset: r78096:b31cf2e51d1b
Date: 2015-06-14 22:25 +0200
http://bitbucket.org/pypy/pypy/changeset/b31cf2e51d1b/

Log:	Cffi Tweaks in tklib_build until we get access to TK_HEX_VERSION,
	before we build the real library.

diff --git a/lib_pypy/_tkinter/tklib_build.py b/lib_pypy/_tkinter/tklib_build.py
--- a/lib_pypy/_tkinter/tklib_build.py
+++ b/lib_pypy/_tkinter/tklib_build.py
@@ -3,6 +3,54 @@
 from cffi import FFI
 import sys, os
 
+# XXX find a better way to detect paths
+# XXX pick up CPPFLAGS and LDFLAGS and add to these paths?
+if sys.platform.startswith("openbsd"):
+    incdirs = ['/usr/local/include/tcl8.5', '/usr/local/include/tk8.5', '/usr/X11R6/include']
+    linklibs = ['tk85', 'tcl85']
+    libdirs = ['/usr/local/lib', '/usr/X11R6/lib']
+elif sys.platform.startswith("freebsd"):
+    incdirs = ['/usr/local/include/tcl8.6', '/usr/local/include/tk8.6', '/usr/local/include/X11', '/usr/local/include']
+    linklibs = ['tk86', 'tcl86']
+    libdirs = ['/usr/local/lib']
+elif sys.platform == 'win32':
+    incdirs = []
+    linklibs = ['tcl85', 'tk85']
+    libdirs = []
+elif sys.platform == 'darwin':
+    incdirs = ['/System/Library/Frameworks/Tk.framework/Versions/Current/Headers/']
+    linklibs = ['tcl', 'tk']
+    libdirs = []
+else:
+    for _ver in ['', '8.6', '8.5', '']:
+        incdirs = ['/usr/include/tcl' + _ver]
+        linklibs = ['tcl' + _ver, 'tk' + _ver]
+        libdirs = []
+        if os.path.isdir(incdirs[0]):
+            break
+
+config_ffi = FFI()
+config_ffi.cdef(
+"#define TK_HEX_VERSION ...")
+config_lib = config_ffi.set_source("_tkinter.config_cffi", """
+#include <tk.h>
+#define TK_HEX_VERSION ((TK_MAJOR_VERSION << 24) | \
+                        (TK_MINOR_VERSION << 16) | \
+                        (TK_RELEASE_LEVEL << 8) | \
+                        (TK_RELEASE_SERIAL << 0))
+""",
+include_dirs=incdirs,
+libraries=linklibs,
+library_dirs = libdirs
+)
+
+config_ffi.compile(os.path.dirname(os.path.dirname(sys.argv[0])))
+from _tkinter.config_cffi import lib as config_lib
+TK_HEX_VERSION = config_lib.TK_HEX_VERSION
+
+HAVE_LIBTOMMATH = ((0x08050208 <= TK_HEX_VERSION < 0x08060000) or
+                   (0x08060200 <= TK_HEX_VERSION))
+
 tkffi = FFI()
 
 tkffi.cdef("""
@@ -116,32 +164,6 @@
 void Tcl_FindExecutable(char *argv0);
 """)
 
-# XXX find a better way to detect paths
-# XXX pick up CPPFLAGS and LDFLAGS and add to these paths?
-if sys.platform.startswith("openbsd"):
-    incdirs = ['/usr/local/include/tcl8.5', '/usr/local/include/tk8.5', '/usr/X11R6/include']
-    linklibs = ['tk85', 'tcl85']
-    libdirs = ['/usr/local/lib', '/usr/X11R6/lib']
-elif sys.platform.startswith("freebsd"):
-    incdirs = ['/usr/local/include/tcl8.6', '/usr/local/include/tk8.6', '/usr/local/include/X11', '/usr/local/include']
-    linklibs = ['tk86', 'tcl86']
-    libdirs = ['/usr/local/lib']
-elif sys.platform == 'win32':
-    incdirs = []
-    linklibs = ['tcl85', 'tk85']
-    libdirs = []
-elif sys.platform == 'darwin':
-    incdirs = ['/System/Library/Frameworks/Tk.framework/Versions/Current/Headers/']
-    linklibs = ['tcl', 'tk']
-    libdirs = []
-else:
-    for _ver in ['', '8.6', '8.5', '']:
-        incdirs = ['/usr/include/tcl' + _ver]
-        linklibs = ['tcl' + _ver, 'tk' + _ver]
-        libdirs = []
-        if os.path.isdir(incdirs[0]):
-            break
-
 tkffi.set_source("_tkinter.tklib_cffi", """
 #include <tcl.h>
 #include <tk.h>
@@ -155,4 +177,4 @@
 )
 
 if __name__ == "__main__":
-    tkffi.compile()
+    print tkffi.compile(os.path.dirname(os.path.dirname(sys.argv[0])))

From noreply at buildbot.pypy.org  Sun Jun 14 22:27:52 2015
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sun, 14 Jun 2015 22:27:52 +0200 (CEST)
Subject: [pypy-commit] pypy default: Better add '..',
	for the case when dirname() is empty.
Message-ID: <20150614202752.2E8981C088E@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: 
Changeset: r78097:b1de7a251090
Date: 2015-06-14 22:28 +0200
http://bitbucket.org/pypy/pypy/changeset/b1de7a251090/

Log:	Better add '..', for the case when dirname() is empty.

diff --git a/lib_pypy/_tkinter/tklib_build.py b/lib_pypy/_tkinter/tklib_build.py
--- a/lib_pypy/_tkinter/tklib_build.py
+++ b/lib_pypy/_tkinter/tklib_build.py
@@ -44,7 +44,7 @@
 library_dirs = libdirs
 )
 
-config_ffi.compile(os.path.dirname(os.path.dirname(sys.argv[0])))
+config_ffi.compile(os.path.join(os.path.dirname(sys.argv[0]), '..'))
 from _tkinter.config_cffi import lib as config_lib
 TK_HEX_VERSION = config_lib.TK_HEX_VERSION
 
@@ -177,4 +177,4 @@
 )
 
 if __name__ == "__main__":
-    print tkffi.compile(os.path.dirname(os.path.dirname(sys.argv[0])))
+    tkffi.compile(os.path.join(os.path.dirname(sys.argv[0]), '..'))

From noreply at buildbot.pypy.org  Sun Jun 14 22:29:58 2015
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sun, 14 Jun 2015 22:29:58 +0200 (CEST)
Subject: [pypy-commit] pypy default: _tkinter: "const" some struct members
 to silence gcc warnings.
Message-ID: <20150614202958.B50111C088E@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: 
Changeset: r78098:7f1fa7382673
Date: 2015-06-14 22:30 +0200
http://bitbucket.org/pypy/pypy/changeset/7f1fa7382673/

Log:	_tkinter: "const" some struct members to silence gcc warnings.

diff --git a/lib_pypy/_tkinter/tklib_build.py b/lib_pypy/_tkinter/tklib_build.py
--- a/lib_pypy/_tkinter/tklib_build.py
+++ b/lib_pypy/_tkinter/tklib_build.py
@@ -75,13 +75,13 @@
 typedef ...* Tcl_Command;
 
 typedef struct Tcl_ObjType {
-    char *name;
+    const char *name;
     ...;
 } Tcl_ObjType;
 typedef struct Tcl_Obj {
     char *bytes;
     int length;
-    Tcl_ObjType *typePtr;
+    const Tcl_ObjType *typePtr;
     union {                     /* The internal representation: */
         long longValue;         /*   - an long integer value. */
         double doubleValue;     /*   - a double-precision floating value. */

From noreply at buildbot.pypy.org  Mon Jun 15 04:07:54 2015
From: noreply at buildbot.pypy.org (stefanor)
Date: Mon, 15 Jun 2015 04:07:54 +0200 (CEST)
Subject: [pypy-commit] cffi default: Correctly locate py2.7 debug build
	extensions
Message-ID: <20150615020754.4C6401C0FE0@cobra.cs.uni-duesseldorf.de>

Author: Stefano Rivera <stefano at rivera.za.net>
Branch: 
Changeset: r2185:78ba2e4ab0ff
Date: 2015-06-14 19:08 -0700
http://bitbucket.org/cffi/cffi/changeset/78ba2e4ab0ff/

Log:	Correctly locate py2.7 debug build extensions

diff --git a/testing/cffi1/test_zdist.py b/testing/cffi1/test_zdist.py
--- a/testing/cffi1/test_zdist.py
+++ b/testing/cffi1/test_zdist.py
@@ -59,7 +59,12 @@
             if (name.endswith('.so') or name.endswith('.pyd') or
                 name.endswith('.dylib')):
                 found_so = os.path.join(curdir, name)
-                name = name.split('.')[0] + '.SO' # foo.cpython-34m.so => foo.SO
+                # foo.cpython-34m.so => foo
+                name = name.split('.')[0]
+                # foo_d.so => foo (Python 2 debug builds)
+                if name.endswith('_d') and hasattr(sys, 'gettotalrefcount'):
+                    name = name.rsplit('_', 1)[0]
+                name += '.SO'
             if name.startswith('pycparser') and name.endswith('.egg'):
                 continue    # no clue why this shows up sometimes and not others
             assert name in content, "found unexpected file %r" % (

From noreply at buildbot.pypy.org  Mon Jun 15 09:03:37 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 15 Jun 2015 09:03:37 +0200 (CEST)
Subject: [pypy-commit] cffi default: Explicitly complain if we find 'typedef
 int... t; ' in a call to verify()
Message-ID: <20150615070337.D525C1C1013@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2186:17be464b073d
Date: 2015-06-15 09:04 +0200
http://bitbucket.org/cffi/cffi/changeset/17be464b073d/

Log:	Explicitly complain if we find 'typedef int... t;' in a call to
	verify()

diff --git a/cffi/cparser.py b/cffi/cparser.py
--- a/cffi/cparser.py
+++ b/cffi/cparser.py
@@ -102,6 +102,7 @@
         self._packed = False
         self._int_constants = {}
         self._recomplete = []
+        self._uses_new_feature = None
 
     def _parse(self, csource):
         csource, macros = _preprocess(csource)
@@ -648,4 +649,7 @@
         for t in typenames[:-1]:
             if t not in ['int', 'short', 'long', 'signed', 'unsigned', 'char']:
                 raise api.FFIError(':%d: bad usage of "..."' % decl.coord.line)
+        if self._uses_new_feature is None:
+            self._uses_new_feature = "'typedef %s... %s'" % (
+                ' '.join(typenames[:-1]), decl.name)
         return model.UnknownIntegerType(decl.name)
diff --git a/cffi/verifier.py b/cffi/verifier.py
--- a/cffi/verifier.py
+++ b/cffi/verifier.py
@@ -28,6 +28,10 @@
     def __init__(self, ffi, preamble, tmpdir=None, modulename=None,
                  ext_package=None, tag='', force_generic_engine=False,
                  source_extension='.c', flags=None, relative_to=None, **kwds):
+        if ffi._parser._uses_new_feature:
+            raise ffiplatform.VerificationError(
+                "feature not supported with ffi.verify(), but only "
+                "with ffi.set_source(): %s" % (ffi._parser._uses_new_feature,))
         self.ffi = ffi
         self.preamble = preamble
         if not modulename:
diff --git a/testing/cffi0/test_verify.py b/testing/cffi0/test_verify.py
--- a/testing/cffi0/test_verify.py
+++ b/testing/cffi0/test_verify.py
@@ -2235,3 +2235,15 @@
                      "const T myglob = { 0.1, 42 };")
     assert ffi.typeof(lib.myglob) == ffi.typeof("T")
     assert lib.myglob.x == 42
+
+def test_dont_support_int_dotdotdot():
+    ffi = FFI()
+    ffi.cdef("typedef int... t1;")
+    e = py.test.raises(VerificationError, ffi.verify, "")
+    assert str(e.value) == ("feature not supported with ffi.verify(), but only "
+                            "with ffi.set_source(): 'typedef int... t1'")
+    ffi = FFI()
+    ffi.cdef("typedef unsigned long... t1;")
+    e = py.test.raises(VerificationError, ffi.verify, "")
+    assert str(e.value) == ("feature not supported with ffi.verify(), but only "
+                         "with ffi.set_source(): 'typedef unsigned long... t1'")

From noreply at buildbot.pypy.org  Mon Jun 15 09:20:31 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Mon, 15 Jun 2015 09:20:31 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: added test case to sum on llgraph
Message-ID: <20150615072031.CE1CD1C1013@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r78099:2e80beb1f2b4
Date: 2015-06-15 09:20 +0200
http://bitbucket.org/pypy/pypy/changeset/2e80beb1f2b4/

Log:	added test case to sum on llgraph

diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -354,6 +354,20 @@
         assert result == sum(range(30))
         self.check_vectorized(1, 1)
 
+    def define_sum_multi():
+        return """
+        a = |30|
+        b = sum(a)
+        c = |60|
+        d = sum(c)
+        b + c
+        """
+
+    def test_sum_multi(self):
+        result = self.run("sum_multi")
+        assert result == sum(range(30)) + sum(range(0,60))
+        self.check_vectorized(1, 1)
+
     def define_sum_float_to_int16():
         return """
         a = |30|
diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -31,7 +31,11 @@
             try:
                 newbox = _cache[box]
             except KeyError:
-                newbox = _cache[box] = box.__class__()
+                if isinstance(box, BoxVectorAccum):
+                    newbox = _cache[box] = \
+                        box.__class__(box, box.scalar_var, box.operator)
+                else:
+                    newbox = _cache[box] = box.__class__()
             return newbox
         #
         self.inputargs = map(mapping, inputargs)
@@ -696,6 +700,17 @@
         assert len(vx) == len(vy)
         return [_vx == _vy for _vx,_vy in zip(vx,vy)]
 
+    def bh_vec_int_xor(self, vx, vy):
+        return [int(x) ^ int(y) for x,y in zip(vx,vy)]
+
+    def bh_vec_float_pack(self, vector, value, index, count):
+        if isinstance(value, list):
+            for i in range(count):
+                vector[index + i] = value[i]
+        else:
+            vector[index] = value
+        return vector
+
     def bh_vec_cast_float_to_singlefloat(self, vx):
         return vx
 
@@ -872,7 +887,7 @@
                     def prod(acc, x): return acc * x
                     value = reduce(prod, value, 1)
                 else:
-                    raise NotImplementedError
+                    raise NotImplementedError("accum operator in fail guard")
             values.append(value)
         if hasattr(descr, '_llgraph_bridge'):
             target = (descr._llgraph_bridge, -1)
diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/vectorize.py
@@ -484,10 +484,10 @@
         self.threshold = threshold
 
     def unpack_cost(self, index, op):
-        raise NotImplementedError
+        raise NotImplementedError("unpack cost")
 
     def savings_for_pack(self, pack, times):
-        raise NotImplementedError
+        raise NotImplementedError("savings for pack")
 
     def savings_for_unpacking(self, node, index):
         savings = 0
@@ -691,13 +691,15 @@
         return None, -1
 
     def accumulate_prepare(self, sched_data, renamer):
+        vec_reg_size = sched_data.vec_reg_size
         for pack in self.packs:
             if not pack.is_accumulating():
                 continue
             accum = pack.accum
             # create a new vector box for the parameters
             box = pack.input_type.new_vector_box()
-            op = ResOperation(rop.VEC_BOX, [ConstInt(0)], box)
+            size = vec_reg_size // pack.input_type.getsize()
+            op = ResOperation(rop.VEC_BOX, [ConstInt(size)], box)
             sched_data.invariant_oplist.append(op)
             result = box.clonebox()
             # clear the box to zero TODO might not be zero for every reduction?
diff --git a/rpython/jit/metainterp/test/test_vectorize.py b/rpython/jit/metainterp/test/test_vectorize.py
--- a/rpython/jit/metainterp/test/test_vectorize.py
+++ b/rpython/jit/metainterp/test/test_vectorize.py
@@ -153,6 +153,24 @@
         res = self.meta_interp(f, [30])
         assert res == f(30) == 128
 
+    def test_sum(self):
+        myjitdriver = JitDriver(greens = [], reds = 'auto', vectorize=True)
+        myjitdriver2 = JitDriver(greens = [], reds = 'auto', vectorize=True)
+        T = lltype.Array(rffi.DOUBLE, hints={'nolength': True})
+        def f(d):
+            va = lltype.malloc(T, d, flavor='raw', zero=True)
+            for j in range(d):
+                va[j] = float(j)
+            i = 0
+            accum = 0
+            while i < d:
+                myjitdriver.jit_merge_point()
+                accum += va[i]
+                i += 1
+            lltype.free(va, flavor='raw')
+            return accum
+        res = self.meta_interp(f, [60])
+        assert res == f(60) == sum(range(60))
 
 class VectorizeLLtypeTests(VectorizeTests):
     pass

From noreply at buildbot.pypy.org  Mon Jun 15 09:33:16 2015
From: noreply at buildbot.pypy.org (amauryfa)
Date: Mon, 15 Jun 2015 09:33:16 +0200 (CEST)
Subject: [pypy-commit] pypy default: Use verify(),
 it's simpler because it also imports the library without having to
 give it a real name.
Message-ID: <20150615073316.ACB2D1C1013@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: 
Changeset: r78100:d68cc00b94d0
Date: 2015-06-14 22:40 +0200
http://bitbucket.org/pypy/pypy/changeset/d68cc00b94d0/

Log:	Use verify(), it's simpler because it also imports the library
	without having to give it a real name.

diff --git a/lib_pypy/_tkinter/tklib_build.py b/lib_pypy/_tkinter/tklib_build.py
--- a/lib_pypy/_tkinter/tklib_build.py
+++ b/lib_pypy/_tkinter/tklib_build.py
@@ -32,7 +32,7 @@
 config_ffi = FFI()
 config_ffi.cdef(
 "#define TK_HEX_VERSION ...")
-config_lib = config_ffi.set_source("_tkinter.config_cffi", """
+config_lib = config_ffi.verify("""
 #include <tk.h>
 #define TK_HEX_VERSION ((TK_MAJOR_VERSION << 24) | \
                         (TK_MINOR_VERSION << 16) | \
@@ -44,8 +44,6 @@
 library_dirs = libdirs
 )
 
-config_ffi.compile(os.path.join(os.path.dirname(sys.argv[0]), '..'))
-from _tkinter.config_cffi import lib as config_lib
 TK_HEX_VERSION = config_lib.TK_HEX_VERSION
 
 HAVE_LIBTOMMATH = ((0x08050208 <= TK_HEX_VERSION < 0x08060000) or

From noreply at buildbot.pypy.org  Mon Jun 15 09:33:17 2015
From: noreply at buildbot.pypy.org (amauryfa)
Date: Mon, 15 Jun 2015 09:33:17 +0200 (CEST)
Subject: [pypy-commit] pypy default: Tkinter: Add support for bignum numbers.
Message-ID: <20150615073317.D24C81C1013@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: 
Changeset: r78101:dad26a9f1b29
Date: 2015-06-14 23:48 +0200
http://bitbucket.org/pypy/pypy/changeset/dad26a9f1b29/

Log:	Tkinter: Add support for bignum numbers.

diff --git a/lib_pypy/_tkinter/app.py b/lib_pypy/_tkinter/app.py
--- a/lib_pypy/_tkinter/app.py
+++ b/lib_pypy/_tkinter/app.py
@@ -141,6 +141,7 @@
 
         Tcl_AppInit(self)
         # EnableEventHook()
+        self._typeCache.add_extra_types(self)
         return self
 
     def __del__(self):
diff --git a/lib_pypy/_tkinter/tclobj.py b/lib_pypy/_tkinter/tclobj.py
--- a/lib_pypy/_tkinter/tclobj.py
+++ b/lib_pypy/_tkinter/tclobj.py
@@ -1,6 +1,7 @@
 # TclObject, conversions with Python objects
 
 from .tklib_cffi import ffi as tkffi, lib as tklib
+import binascii
 
 class TypeCache(object):
     def __init__(self):
@@ -8,10 +9,18 @@
         self.ByteArrayType = tklib.Tcl_GetObjType("bytearray")
         self.DoubleType = tklib.Tcl_GetObjType("double")
         self.IntType = tklib.Tcl_GetObjType("int")
+        self.BigNumType = None
         self.ListType = tklib.Tcl_GetObjType("list")
         self.ProcBodyType = tklib.Tcl_GetObjType("procbody")
         self.StringType = tklib.Tcl_GetObjType("string")
 
+    def add_extra_types(self, app):
+        # Some types are not registered in Tcl.
+        result = app.call('expr', '2**63')
+        typePtr = AsObj(result).typePtr
+        if tkffi.string(typePtr.name) == "bignum":
+            self.BigNumType = typePtr
+
 
 def FromTclString(s):
     # If the result contains any bytes with the top bit set, it's
@@ -30,6 +39,24 @@
     return s
 
 
+# Only when tklib.HAVE_LIBTOMMATH!
+def FromBignumObj(app, value):
+    bigValue = tkffi.new("mp_int*")
+    if tklib.Tcl_GetBignumFromObj(app.interp, value, bigValue) != tklib.TCL_OK:
+        app.raiseTclError()
+    try:
+        numBytes = tklib.mp_unsigned_bin_size(bigValue)
+        buf = tkffi.new("unsigned char[]", numBytes)
+        bufSize_ptr = tkffi.new("unsigned long*", numBytes)
+        if tklib.mp_to_unsigned_bin_n(
+                bigValue, buf, bufSize_ptr) != tklib.MP_OKAY:
+            raise MemoryError
+        bytes = tkffi.buffer(buf)[0:bufSize_ptr[0]]
+        sign = -1 if bigValue.sign == tklib.MP_NEG else 1
+        return sign * int(binascii.hexlify(bytes), 16)
+    finally:
+        tklib.mp_clear(bigValue)
+
 def FromObj(app, value):
     """Convert a TclObj pointer into a Python object."""
     typeCache = app._typeCache
@@ -37,17 +64,19 @@
         buf = tkffi.buffer(value.bytes, value.length)
         return FromTclString(buf[:])
 
-    elif value.typePtr == typeCache.BooleanType:
+    if value.typePtr == typeCache.BooleanType:
         return bool(value.internalRep.longValue)
-    elif value.typePtr == typeCache.ByteArrayType:
+    if value.typePtr == typeCache.ByteArrayType:
         size = tkffi.new('int*')
         data = tklib.Tcl_GetByteArrayFromObj(value, size)
         return tkffi.buffer(data, size[0])[:]
-    elif value.typePtr == typeCache.DoubleType:
+    if value.typePtr == typeCache.DoubleType:
         return value.internalRep.doubleValue
-    elif value.typePtr == typeCache.IntType:
+    if value.typePtr == typeCache.IntType:
         return value.internalRep.longValue
-    elif value.typePtr == typeCache.ListType:
+    if value.typePtr == typeCache.BigNumType and tklib.HAVE_LIBTOMMATH:
+        return FromBignumObj(app, value)
+    if value.typePtr == typeCache.ListType:
         size = tkffi.new('int*')
         status = tklib.Tcl_ListObjLength(app.interp, value, size)
         if status == tklib.TCL_ERROR:
@@ -61,9 +90,9 @@
                 app.raiseTclError()
             result.append(FromObj(app, tcl_elem[0]))
         return tuple(result)
-    elif value.typePtr == typeCache.ProcBodyType:
+    if value.typePtr == typeCache.ProcBodyType:
         pass  # fall through and return tcl object.
-    elif value.typePtr == typeCache.StringType:
+    if value.typePtr == typeCache.StringType:
         buf = tklib.Tcl_GetUnicode(value)
         length = tklib.Tcl_GetCharLength(value)
         buf = tkffi.buffer(tkffi.cast("char*", buf), length*2)[:]
diff --git a/lib_pypy/_tkinter/tklib_build.py b/lib_pypy/_tkinter/tklib_build.py
--- a/lib_pypy/_tkinter/tklib_build.py
+++ b/lib_pypy/_tkinter/tklib_build.py
@@ -46,14 +46,16 @@
 
 TK_HEX_VERSION = config_lib.TK_HEX_VERSION
 
-HAVE_LIBTOMMATH = ((0x08050208 <= TK_HEX_VERSION < 0x08060000) or
-                   (0x08060200 <= TK_HEX_VERSION))
+HAVE_LIBTOMMATH = int((0x08050208 <= TK_HEX_VERSION < 0x08060000) or
+                      (0x08060200 <= TK_HEX_VERSION))
 
 tkffi = FFI()
 
 tkffi.cdef("""
 char *get_tk_version();
 char *get_tcl_version();
+#define HAVE_LIBTOMMATH ...
+
 #define TCL_READABLE ...
 #define TCL_WRITABLE ...
 #define TCL_EXCEPTION ...
@@ -162,13 +164,34 @@
 void Tcl_FindExecutable(char *argv0);
 """)
 
+if HAVE_LIBTOMMATH:
+    tkffi.cdef("""
+#define MP_OKAY ...
+#define MP_NEG ...
+typedef struct {
+    int sign;
+    ...;
+} mp_int;
+
+int Tcl_GetBignumFromObj(Tcl_Interp *interp, Tcl_Obj *obj, mp_int *value);
+
+int mp_unsigned_bin_size(mp_int *a);
+int mp_to_unsigned_bin_n(mp_int * a, unsigned char *b, unsigned long *outlen);
+void mp_clear(mp_int *a);
+""")
+
 tkffi.set_source("_tkinter.tklib_cffi", """
+#define HAVE_LIBTOMMATH %(HAVE_LIBTOMMATH)s
 #include <tcl.h>
 #include <tk.h>
 
+#if HAVE_LIBTOMMATH
+#include <tclTomMath.h>
+#endif 
+
 char *get_tk_version() { return TK_VERSION; }
 char *get_tcl_version() { return TCL_VERSION; }
-""",
+""" % globals(),
 include_dirs=incdirs,
 libraries=linklibs,
 library_dirs = libdirs

From noreply at buildbot.pypy.org  Mon Jun 15 09:33:19 2015
From: noreply at buildbot.pypy.org (amauryfa)
Date: Mon, 15 Jun 2015 09:33:19 +0200 (CEST)
Subject: [pypy-commit] pypy default: Fix one boolean test
Message-ID: <20150615073319.052ED1C1013@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: 
Changeset: r78102:a2f74ab95bf7
Date: 2015-06-14 23:51 +0200
http://bitbucket.org/pypy/pypy/changeset/a2f74ab95bf7/

Log:	Fix one boolean test

diff --git a/lib_pypy/_tkinter/app.py b/lib_pypy/_tkinter/app.py
--- a/lib_pypy/_tkinter/app.py
+++ b/lib_pypy/_tkinter/app.py
@@ -439,8 +439,8 @@
             tklib.Tcl_Free(argv[0])
 
     def getboolean(self, s):
-        if isinstance(s, int):
-            return s
+        if isinstance(s, (int, long)):
+            return bool(s)
         if isinstance(s, unicode):
             s = str(s)
         if '\x00' in s:
@@ -452,7 +452,7 @@
         return bool(v[0])
 
     def getint(self, s):
-        if isinstance(s, int):
+        if isinstance(s, (int, long)):
             return s
         if isinstance(s, unicode):
             s = str(s)

From noreply at buildbot.pypy.org  Mon Jun 15 09:33:20 2015
From: noreply at buildbot.pypy.org (amauryfa)
Date: Mon, 15 Jun 2015 09:33:20 +0200 (CEST)
Subject: [pypy-commit] pypy default: Another fix for boolean values
Message-ID: <20150615073320.36AAE1C1013@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: 
Changeset: r78103:62653cade172
Date: 2015-06-14 23:57 +0200
http://bitbucket.org/pypy/pypy/changeset/62653cade172/

Log:	Another fix for boolean values

diff --git a/lib_pypy/_tkinter/tclobj.py b/lib_pypy/_tkinter/tclobj.py
--- a/lib_pypy/_tkinter/tclobj.py
+++ b/lib_pypy/_tkinter/tclobj.py
@@ -5,7 +5,8 @@
 
 class TypeCache(object):
     def __init__(self):
-        self.BooleanType = tklib.Tcl_GetObjType("boolean")
+        self.OldBooleanType = tklib.Tcl_GetObjType("boolean")
+        self.BooleanType = None
         self.ByteArrayType = tklib.Tcl_GetObjType("bytearray")
         self.DoubleType = tklib.Tcl_GetObjType("double")
         self.IntType = tklib.Tcl_GetObjType("int")
@@ -16,6 +17,11 @@
 
     def add_extra_types(self, app):
         # Some types are not registered in Tcl.
+        result = app.call('expr', 'true')
+        typePtr = AsObj(result).typePtr
+        if tkffi.string(typePtr.name) == "booleanString":
+            self.BooleanType = typePtr
+
         result = app.call('expr', '2**63')
         typePtr = AsObj(result).typePtr
         if tkffi.string(typePtr.name) == "bignum":
@@ -64,8 +70,12 @@
         buf = tkffi.buffer(value.bytes, value.length)
         return FromTclString(buf[:])
 
-    if value.typePtr == typeCache.BooleanType:
-        return bool(value.internalRep.longValue)
+    if value.typePtr in (typeCache.BooleanType, typeCache.OldBooleanType):
+        value_ptr = tkffi.new("int*")
+        if tklib.Tcl_GetBooleanFromObj(
+                app.interp, value, value_ptr) == tklib.TCL_ERROR:
+            app.raiseTclError()
+        return bool(value_ptr[0])
     if value.typePtr == typeCache.ByteArrayType:
         size = tkffi.new('int*')
         data = tklib.Tcl_GetByteArrayFromObj(value, size)
diff --git a/lib_pypy/_tkinter/tklib_build.py b/lib_pypy/_tkinter/tklib_build.py
--- a/lib_pypy/_tkinter/tklib_build.py
+++ b/lib_pypy/_tkinter/tklib_build.py
@@ -120,6 +120,7 @@
 int Tcl_GetBoolean(Tcl_Interp* interp, const char* src, int* boolPtr);
 int Tcl_GetInt(Tcl_Interp* interp, const char* src, int* intPtr);
 int Tcl_GetDouble(Tcl_Interp* interp, const char* src, double* doublePtr);
+int Tcl_GetBooleanFromObj(Tcl_Interp* interp, Tcl_Obj* objPtr, int* valuePtr);
 char *Tcl_GetString(Tcl_Obj* objPtr);
 char *Tcl_GetStringFromObj(Tcl_Obj* objPtr, int* lengthPtr);
 unsigned char *Tcl_GetByteArrayFromObj(Tcl_Obj* objPtr, int* lengthPtr);

From noreply at buildbot.pypy.org  Mon Jun 15 09:33:21 2015
From: noreply at buildbot.pypy.org (amauryfa)
Date: Mon, 15 Jun 2015 09:33:21 +0200 (CEST)
Subject: [pypy-commit] pypy default: tkinter: add "WideInt" type.
Message-ID: <20150615073321.5B51D1C1013@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: 
Changeset: r78104:fd331e4bf733
Date: 2015-06-15 09:12 +0200
http://bitbucket.org/pypy/pypy/changeset/fd331e4bf733/

Log:	tkinter: add "WideInt" type.

diff --git a/lib_pypy/_tkinter/app.py b/lib_pypy/_tkinter/app.py
--- a/lib_pypy/_tkinter/app.py
+++ b/lib_pypy/_tkinter/app.py
@@ -2,7 +2,8 @@
 
 from .tklib_cffi import ffi as tkffi, lib as tklib
 from . import TclError
-from .tclobj import TclObject, FromObj, FromTclString, AsObj, TypeCache
+from .tclobj import (TclObject, FromObj, FromTclString, AsObj, TypeCache,
+                     FromBignumObj, FromWideIntObj)
 
 import contextlib
 import sys
@@ -458,11 +459,23 @@
             s = str(s)
         if '\x00' in s:
             raise TypeError
-        v = tkffi.new("int*")
-        res = tklib.Tcl_GetInt(self.interp, s, v)
-        if res == tklib.TCL_ERROR:
-            self.raiseTclError()
-        return v[0]
+        if tklib.HAVE_LIBTOMMATH or tklib.HAVE_WIDE_INT_TYPE:
+            value = tklib.Tcl_NewStringObj(s, -1)
+            if not value:
+                self.raiseTclError()
+            try:
+                if tklib.HAVE_LIBTOMMATH:
+                    return FromBignumObj(self, value)
+                else:
+                    return FromWideIntObj(self, value)
+            finally:
+                tklib.Tcl_DecrRefCount(value)
+        else:
+            v = tkffi.new("int*")
+            res = tklib.Tcl_GetInt(self.interp, s, v)
+            if res == tklib.TCL_ERROR:
+                self.raiseTclError()
+            return v[0]
 
     def getdouble(self, s):
         if isinstance(s, float):
diff --git a/lib_pypy/_tkinter/tclobj.py b/lib_pypy/_tkinter/tclobj.py
--- a/lib_pypy/_tkinter/tclobj.py
+++ b/lib_pypy/_tkinter/tclobj.py
@@ -10,6 +10,7 @@
         self.ByteArrayType = tklib.Tcl_GetObjType("bytearray")
         self.DoubleType = tklib.Tcl_GetObjType("double")
         self.IntType = tklib.Tcl_GetObjType("int")
+        self.WideIntType = tklib.Tcl_GetObjType("wideInt")
         self.BigNumType = None
         self.ListType = tklib.Tcl_GetObjType("list")
         self.ProcBodyType = tklib.Tcl_GetObjType("procbody")
@@ -45,6 +46,13 @@
     return s
 
 
+# Only when tklib.HAVE_WIDE_INT_TYPE.
+def FromWideIntObj(app, value):
+    wide = tkffi.new("Tcl_WideInt*")
+    if tklib.Tcl_GetWideIntFromObj(app.interp, value, wide) != tklib.TCL_OK:
+        app.raiseTclError()
+    return wide[0]
+
 # Only when tklib.HAVE_LIBTOMMATH!
 def FromBignumObj(app, value):
     bigValue = tkffi.new("mp_int*")
@@ -57,9 +65,11 @@
         if tklib.mp_to_unsigned_bin_n(
                 bigValue, buf, bufSize_ptr) != tklib.MP_OKAY:
             raise MemoryError
+        if bufSize_ptr[0] == 0:
+            return 0
         bytes = tkffi.buffer(buf)[0:bufSize_ptr[0]]
         sign = -1 if bigValue.sign == tklib.MP_NEG else 1
-        return sign * int(binascii.hexlify(bytes), 16)
+        return int(sign * int(binascii.hexlify(bytes), 16))
     finally:
         tklib.mp_clear(bigValue)
 
@@ -113,27 +123,40 @@
 def AsObj(value):
     if isinstance(value, str):
         return tklib.Tcl_NewStringObj(value, len(value))
-    elif isinstance(value, bool):
+    if isinstance(value, bool):
         return tklib.Tcl_NewBooleanObj(value)
-    elif isinstance(value, int):
+    if isinstance(value, int):
         return tklib.Tcl_NewLongObj(value)
-    elif isinstance(value, float):
+    if isinstance(value, long):
+        try:
+            tkffi.new("long[]", [value])
+        except OverflowError:
+            try:
+                tkffi.new("Tcl_WideInt[]", [value])
+            except OverflowError:
+                pass
+                # Too wide, fall through defaut object handling.
+            else:
+                return tklib.Tcl_NewWideIntObj(value)
+        else:
+            return tklib.Tcl_NewLongObj(value)
+    if isinstance(value, float):
         return tklib.Tcl_NewDoubleObj(value)
-    elif isinstance(value, tuple):
+    if isinstance(value, tuple):
         argv = tkffi.new("Tcl_Obj*[]", len(value))
         for i in range(len(value)):
             argv[i] = AsObj(value[i])
         return tklib.Tcl_NewListObj(len(value), argv)
-    elif isinstance(value, unicode):
+    if isinstance(value, unicode):
         encoded = value.encode('utf-16')[2:]
         buf = tkffi.new("char[]", encoded)
         inbuf = tkffi.cast("Tcl_UniChar*", buf)
         return tklib.Tcl_NewUnicodeObj(buf, len(encoded)/2)
-    elif isinstance(value, TclObject):
+    if isinstance(value, TclObject):
         tklib.Tcl_IncrRefCount(value._value)
         return value._value
-    else:
-        return AsObj(str(value))
+
+    return AsObj(str(value))
 
 class TclObject(object):
     def __new__(cls, value):
diff --git a/lib_pypy/_tkinter/tklib_build.py b/lib_pypy/_tkinter/tklib_build.py
--- a/lib_pypy/_tkinter/tklib_build.py
+++ b/lib_pypy/_tkinter/tklib_build.py
@@ -30,14 +30,21 @@
             break
 
 config_ffi = FFI()
-config_ffi.cdef(
-"#define TK_HEX_VERSION ...")
+config_ffi.cdef("""
+#define TK_HEX_VERSION ...
+#define HAVE_WIDE_INT_TYPE ...
+""")
 config_lib = config_ffi.verify("""
 #include <tk.h>
 #define TK_HEX_VERSION ((TK_MAJOR_VERSION << 24) | \
                         (TK_MINOR_VERSION << 16) | \
                         (TK_RELEASE_LEVEL << 8) | \
                         (TK_RELEASE_SERIAL << 0))
+#ifdef TCL_WIDE_INT_TYPE
+#define HAVE_WIDE_INT_TYPE 1
+#else
+#define HAVE_WIDE_INT_TYPE 0
+#endif
 """,
 include_dirs=incdirs,
 libraries=linklibs,
@@ -48,6 +55,7 @@
 
 HAVE_LIBTOMMATH = int((0x08050208 <= TK_HEX_VERSION < 0x08060000) or
                       (0x08060200 <= TK_HEX_VERSION))
+HAVE_WIDE_INT_TYPE = config_lib.HAVE_WIDE_INT_TYPE
 
 tkffi = FFI()
 
@@ -55,6 +63,7 @@
 char *get_tk_version();
 char *get_tcl_version();
 #define HAVE_LIBTOMMATH ...
+#define HAVE_WIDE_INT_TYPE ...
 
 #define TCL_READABLE ...
 #define TCL_WRITABLE ...
@@ -165,6 +174,13 @@
 void Tcl_FindExecutable(char *argv0);
 """)
 
+if HAVE_WIDE_INT_TYPE:
+    tkffi.cdef("""
+typedef int... Tcl_WideInt;
+
+int Tcl_GetWideIntFromObj(Tcl_Interp *interp, Tcl_Obj *obj, Tcl_WideInt *value);
+""")
+
 if HAVE_LIBTOMMATH:
     tkffi.cdef("""
 #define MP_OKAY ...
@@ -183,6 +199,7 @@
 
 tkffi.set_source("_tkinter.tklib_cffi", """
 #define HAVE_LIBTOMMATH %(HAVE_LIBTOMMATH)s
+#define HAVE_WIDE_INT_TYPE %(HAVE_WIDE_INT_TYPE)s
 #include <tcl.h>
 #include <tk.h>
 

From noreply at buildbot.pypy.org  Mon Jun 15 09:33:22 2015
From: noreply at buildbot.pypy.org (amauryfa)
Date: Mon, 15 Jun 2015 09:33:22 +0200 (CEST)
Subject: [pypy-commit] pypy default: tcl: Conversion from long to Bignum.
Message-ID: <20150615073322.7161E1C1013@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: 
Changeset: r78105:83e70cd0540a
Date: 2015-06-15 09:32 +0200
http://bitbucket.org/pypy/pypy/changeset/83e70cd0540a/

Log:	tcl: Conversion from long to Bignum.

diff --git a/lib_pypy/_tkinter/tclobj.py b/lib_pypy/_tkinter/tclobj.py
--- a/lib_pypy/_tkinter/tclobj.py
+++ b/lib_pypy/_tkinter/tclobj.py
@@ -73,6 +73,20 @@
     finally:
         tklib.mp_clear(bigValue)
 
+def AsBignumObj(value):
+    sign = -1 if value < 0 else 1
+    hexstr = '%x' % abs(value)
+    bigValue = tkffi.new("mp_int*")
+    tklib.mp_init(bigValue)
+    try:
+        if tklib.mp_read_radix(bigValue, hexstr, 16) != tklib.MP_OKAY:
+            raise MemoryError
+        bigValue.sign = tklib.MP_NEG if value < 0 else tklib.MP_ZPOS
+        return tklib.Tcl_NewBignumObj(bigValue)
+    finally:
+        tklib.mp_clear(bigValue)
+
+
 def FromObj(app, value):
     """Convert a TclObj pointer into a Python object."""
     typeCache = app._typeCache
@@ -131,15 +145,19 @@
         try:
             tkffi.new("long[]", [value])
         except OverflowError:
+            pass 
+        else:
+            return tklib.Tcl_NewLongObj(value)
+        if tklib.HAVE_WIDE_INT_TYPE:
             try:
                 tkffi.new("Tcl_WideInt[]", [value])
             except OverflowError:
                 pass
-                # Too wide, fall through defaut object handling.
             else:
                 return tklib.Tcl_NewWideIntObj(value)
-        else:
-            return tklib.Tcl_NewLongObj(value)
+        if tklib.HAVE_LIBTOMMATH:
+            return AsBignumObj(value)
+            
     if isinstance(value, float):
         return tklib.Tcl_NewDoubleObj(value)
     if isinstance(value, tuple):
diff --git a/lib_pypy/_tkinter/tklib_build.py b/lib_pypy/_tkinter/tklib_build.py
--- a/lib_pypy/_tkinter/tklib_build.py
+++ b/lib_pypy/_tkinter/tklib_build.py
@@ -184,6 +184,7 @@
 if HAVE_LIBTOMMATH:
     tkffi.cdef("""
 #define MP_OKAY ...
+#define MP_ZPOS ...
 #define MP_NEG ...
 typedef struct {
     int sign;
@@ -191,9 +192,12 @@
 } mp_int;
 
 int Tcl_GetBignumFromObj(Tcl_Interp *interp, Tcl_Obj *obj, mp_int *value);
+Tcl_Obj *Tcl_NewBignumObj(mp_int *value);
 
 int mp_unsigned_bin_size(mp_int *a);
 int mp_to_unsigned_bin_n(mp_int * a, unsigned char *b, unsigned long *outlen);
+int mp_read_radix(mp_int *a, const char *str, int radix);
+int mp_init(mp_int *a);
 void mp_clear(mp_int *a);
 """)
 

From noreply at buildbot.pypy.org  Mon Jun 15 09:41:10 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Mon, 15 Jun 2015 09:41:10 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: error in the test case added array
 instead of summed scalar
Message-ID: <20150615074110.841D31C00BD@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r78106:1d9b7639a828
Date: 2015-06-15 09:40 +0200
http://bitbucket.org/pypy/pypy/changeset/1d9b7639a828/

Log:	error in the test case added array instead of summed scalar

diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -360,12 +360,12 @@
         b = sum(a)
         c = |60|
         d = sum(c)
-        b + c
+        b + d
         """
 
     def test_sum_multi(self):
         result = self.run("sum_multi")
-        assert result == sum(range(30)) + sum(range(0,60))
+        assert result == sum(range(30)) + sum(range(60))
         self.check_vectorized(1, 1)
 
     def define_sum_float_to_int16():

From noreply at buildbot.pypy.org  Mon Jun 15 10:49:38 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Mon, 15 Jun 2015 10:49:38 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: jit.promote in the numpy internal
 iterator generates slightly different traces,
 adjusted tests with the new constant
Message-ID: <20150615084938.752C11C00BD@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r78107:3038636d0684
Date: 2015-06-15 10:49 +0200
http://bitbucket.org/pypy/pypy/changeset/3038636d0684/

Log:	jit.promote in the numpy internal iterator generates slightly
	different traces, adjusted tests with the new constant

diff --git a/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py b/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
--- a/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
@@ -14,6 +14,7 @@
         assert len(log.loops) == 1
         loop = log._filter(log.loops[0])
         assert loop.match("""
+            ...
             guard_class(p0, #, descr=...)
             p4 = getfield_gc_pure(p0, descr=<FieldP pypy.module.micronumpy.iterators.ArrayIter.inst_array \d+>)
             i5 = getfield_gc(p2, descr=<FieldS pypy.module.micronumpy.iterators.IterState.inst_offset \d+>)
@@ -40,7 +41,8 @@
             i26 = int_is_true(i25)
             guard_true(i26, descr=...)
             i27 = getfield_gc_pure(p6, descr=<FieldS pypy.module.micronumpy.descriptor.W_Dtype.inst_elsize \d+>)
-            i28 = int_add(i5, i27)
+            guard_value(i27, 8, descr=...)
+            i28 = int_add(i5, 8)
             i29 = getfield_gc_pure(p0, descr=<FieldS pypy.module.micronumpy.iterators.ArrayIter.inst_size \d+>)
             i30 = int_ge(i23, i29)
             guard_false(i30, descr=...)
@@ -65,6 +67,7 @@
         assert len(log.loops) == 1
         loop = log._filter(log.loops[0])
         assert loop.match("""
+            ...
             f31 = raw_load(i9, i29, descr=<ArrayF 8>)
             guard_not_invalidated(descr=...)
             i34 = getarrayitem_raw(#, #, descr=<ArrayU 1>)  # XXX what are these?
@@ -73,7 +76,7 @@
             guard_true(i32, descr=...)
             i35 = getarrayitem_raw(#, #, descr=<ArrayU 1>)  # XXX equiv test_zjit
             i36 = int_add(i24, 1)
-            i37 = int_add(i29, i28)
+            i37 = int_add(i29, 8)
             i38 = int_ge(i36, i30)
             guard_false(i38, descr=...)
             guard_value(i35, #, descr=...)                  # XXX
@@ -175,7 +178,7 @@
             guard_false(i88, descr=...)
             f90 = raw_load(i67, i89, descr=<ArrayF 8>)
             i91 = int_add(i87, 1)
-            i93 = int_add(i89, i76)
+            i93 = int_add(i89, 8)
             i94 = int_add(i79, 1)
             i95 = getfield_raw(#, descr=<FieldS pypysig_long_struct.c_value 0>)
             setfield_gc(p97, i91, descr=<FieldS pypy.module.micronumpy.iterators.IterState.inst_index .+>)
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py b/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
@@ -1375,8 +1375,22 @@
 
 
     def test_abc(self):
-        py.test.skip()
         trace ="""
+        [p0, p1, p5, p6, p7, p17, p19, i46, i37, i41]
+        guard_not_invalidated() [p1, p0, p5, p6, p7, p17, p19]
+        i59 = int_lt(i46, i37)
+        guard_true(i59) [p1, p0, i46, p5, p6, p7, p17, p19]
+        f60 = getarrayitem_raw(i41, i46, descr=floatarraydescr)
+        f61 = float_add(f60, 1.000000)
+        setarrayitem_raw(i41, i46, f61, descr=floatarraydescr)
+        i62 = int_add(i46, 1)
+        setfield_gc(50, i62, descr=<FieldS pypy.objspace.std.typeobject.IntMutableCell.inst_intvalue 8>)
+        i63 = int_ge(i62, 2024)
+        guard_false(i63) [p1, p0, p5, p6, p7, p17, p19, i62]
+        i64 = getfield_raw(140099887568000, descr=<FieldS pypysig_long_struct.c_value 0>)
+        i65 = int_lt(i64, 0)
+        guard_false(i65) [p1, p0, p5, p6, p7, p17, p19, None]
+        jump(p0, p1, p5, p6, p7, p17, p19, i62, i37, i41)
         """
         opt = self.vectorize(self.parse_loop(trace))
         self.debug_print_operations(opt.loop)
diff --git a/rpython/jit/tool/jitoutput.py b/rpython/jit/tool/jitoutput.py
--- a/rpython/jit/tool/jitoutput.py
+++ b/rpython/jit/tool/jitoutput.py
@@ -26,6 +26,8 @@
     (('nvirtuals',), '^nvirtuals:\s+(\d+)$'),
     (('nvholes',), '^nvholes:\s+(\d+)$'),
     (('nvreused',), '^nvreused:\s+(\d+)$'),
+    (('vecopt_tried',), '^vecopt tried:\s+(\d+)$'),
+    (('vecopt_success',), '^vecopt success:\s+(\d+)$'),
     (('total_compiled_loops',),   '^Total # of loops:\s+(\d+)$'),
     (('total_compiled_bridges',), '^Total # of bridges:\s+(\d+)$'),
     (('total_freed_loops',),      '^Freed # of loops:\s+(\d+)$'),
@@ -57,6 +59,8 @@
     nvirtuals = 0
     nvholes = 0
     nvreused = 0
+    vecopt_tried = 0
+    vecopt_success = 0
 
     def __init__(self):
         self.ops = Ops()

From noreply at buildbot.pypy.org  Mon Jun 15 12:04:39 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Mon, 15 Jun 2015 12:04:39 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: added test case to ensure correctness of
 transformation and find all missing assembler implementations
Message-ID: <20150615100439.E04C21C0683@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r78108:04d9da20e80b
Date: 2015-06-15 12:04 +0200
http://bitbucket.org/pypy/pypy/changeset/04d9da20e80b/

Log:	added test case to ensure correctness of transformation and find all
	missing assembler implementations

diff --git a/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py b/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
--- a/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
@@ -1,8 +1,56 @@
+import py
+
 from pypy.module.pypyjit.test_pypy_c.test_00_model import BaseTestPyPyC
 from rpython.rlib.rawstorage import misaligned_is_fine
 
 
 class TestMicroNumPy(BaseTestPyPyC):
+
+    arith_comb = [('+','float','float', 4*3427,   3427, 1.0,3.0),
+             ('+','float','int',   9*7834,   7843, 4.0,5.0),
+             ('+','int','float',   8*2571,   2571, 9.0,-1.0),
+             ('+','float','int',   -18*2653,   2653, 4.0,-22.0),
+             ('+','int','int',     -1*1499,   1499, 24.0,-25.0),
+             ('-','float','float', -2*5523,  5523, 1.0,3.0),
+             ('*','float','float', 3*2999,   2999, 1.0,3.0),
+             ('/','float','float', 3*7632,   7632, 3.0,1.0),
+             ('/','float','float', 1.5*7632, 7632, 3.0,2.0),
+             ('&','int','int',     0,        1500, 1,0),
+             ('&','int','int',     1500,     1500, 1,1),
+             ('|','int','int',     1500,     1500, 0,1),
+             ('|','int','int',     0,        1500, 0,0),
+            ]
+    type_permuated = []
+    types = { 'int': ['int8','int16','int32','int64'],
+              'float': ['float32', 'float64']
+            }
+    for arith in arith_comb:
+        t1 = arith[1]
+        t2 = arith[2]
+        possible_t1 = types[t1]
+        possible_t2 = types[t2]
+        for ta in possible_t1:
+            for tb in possible_t2:
+                op, _, _, r, c, a, b = arith
+                t = (op, ta, tb, r, c, a, b)
+                type_permuated.append(t)
+
+    @py.test.mark.parametrize("op,adtype,bdtype,result,count,a,b", type_permuated)
+    def test_vector_call2(self, op, adtype, bdtype, result, count, a, b):
+        source = """
+        def main():
+            import _numpypy.multiarray as np
+            a = np.array([{a}]*{count}, dtype='{adtype}')
+            b = np.array([{b}]*{count}, dtype='{bdtype}')
+            c = a {op} b
+            return c.sum()
+        """.format(op=op, adtype=adtype, bdtype=bdtype, count=count, a=a, b=b)
+        exec py.code.Source(source).compile()
+        vlog = self.run(main, [], vectorize=1)
+        log = self.run(main, [], vectorize=0)
+        assert log.result == vlog.result
+        assert log.result == result
+
     def test_reduce_logical_xor(self):
         def main():
             import _numpypy.multiarray as np

From noreply at buildbot.pypy.org  Mon Jun 15 13:09:03 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 15 Jun 2015 13:09:03 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: an attempt to sort out vtable mess
Message-ID: <20150615110903.74DCF1C0962@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r78109:db9ae212ed3c
Date: 2015-06-15 13:00 +0200
http://bitbucket.org/pypy/pypy/changeset/db9ae212ed3c/

Log:	an attempt to sort out vtable mess

diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -89,9 +89,9 @@
         return getkind(self.RESULT)[0]
 
 class SizeDescr(AbstractDescr):
-    def __init__(self, S, is_object, runner):
+    def __init__(self, S, vtable, runner):
         self.S = S
-        self._is_object = is_object
+        self._is_object = vtable is not None
         self.all_fielddescrs = heaptracker.all_fielddescrs(runner, S,
                                     get_field_descr=LLGraphCPU.fielddescrof)
 
@@ -398,12 +398,12 @@
             self.descrs[key] = descr
             return descr
 
-    def sizeof(self, S, is_object):
+    def sizeof(self, S, vtable):
         key = ('size', S)
         try:
             return self.descrs[key]
         except KeyError:
-            descr = SizeDescr(S, is_object, self)
+            descr = SizeDescr(S, vtable, self)
             self.descrs[key] = descr
             return descr
 
diff --git a/rpython/jit/backend/llsupport/descr.py b/rpython/jit/backend/llsupport/descr.py
--- a/rpython/jit/backend/llsupport/descr.py
+++ b/rpython/jit/backend/llsupport/descr.py
@@ -70,7 +70,7 @@
 
 BaseSizeDescr = SizeDescr
 
-def get_size_descr(gccache, STRUCT, is_object):
+def get_size_descr(gccache, STRUCT, vtable):
     cache = gccache._cache_size
     try:
         return cache[STRUCT]
@@ -78,11 +78,10 @@
         size = symbolic.get_size(STRUCT, gccache.translate_support_code)
         count_fields_if_immut = heaptracker.count_fields_if_immutable(STRUCT)
         gc_fielddescrs = heaptracker.gc_fielddescrs(gccache, STRUCT)
-        if is_object:
+        if vtable:
             assert heaptracker.has_gcstruct_a_vtable(STRUCT)
             sizedescr = SizeDescrWithVTable(size, count_fields_if_immut,
-                                            gc_fielddescrs, None,
-                heaptracker.get_vtable_for_gcstruct(gccache, STRUCT))
+                                            gc_fielddescrs, None, vtable)
         else:
             assert not heaptracker.has_gcstruct_a_vtable(STRUCT)
             sizedescr = SizeDescr(size, count_fields_if_immut,
@@ -193,7 +192,7 @@
         cachedict = cache.setdefault(STRUCT, {})
         cachedict[fieldname] = fielddescr
         fielddescr.parent_descr = get_size_descr(gccache, STRUCT,
-                                  heaptracker.has_gcstruct_a_vtable(STRUCT))
+                                  heaptracker.get_vtable_for_gcstruct(STRUCT))
         return fielddescr
 
 def get_type_flag(TYPE):
diff --git a/rpython/jit/backend/llsupport/llmodel.py b/rpython/jit/backend/llsupport/llmodel.py
--- a/rpython/jit/backend/llsupport/llmodel.py
+++ b/rpython/jit/backend/llsupport/llmodel.py
@@ -298,8 +298,8 @@
     def cast_int_to_ptr(self, x, TYPE):
         return rffi.cast(TYPE, x)
 
-    def sizeof(self, S, is_object):
-        return get_size_descr(self.gc_ll_descr, S, is_object)
+    def sizeof(self, S, vtable):
+        return get_size_descr(self.gc_ll_descr, S, vtable)
 
     def fielddescrof(self, STRUCT, fieldname):
         return get_field_descr(self.gc_ll_descr, STRUCT, fieldname)
diff --git a/rpython/jit/codewriter/heaptracker.py b/rpython/jit/codewriter/heaptracker.py
--- a/rpython/jit/codewriter/heaptracker.py
+++ b/rpython/jit/codewriter/heaptracker.py
@@ -64,8 +64,6 @@
     if not has_gcstruct_a_vtable(GCSTRUCT):
         return None
     setup_cache_gcstruct2vtable(gccache)
-    if not hasattr(gccache, '_cache_gcstruct2vtable'):
-        return lltype.malloc(GCSTRUCT.typeptr.TO, flavor='raw', immortal=True)
     return gccache._cache_gcstruct2vtable[GCSTRUCT]
 
 def setup_cache_gcstruct2vtable(gccache):
diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py
--- a/rpython/jit/codewriter/jtransform.py
+++ b/rpython/jit/codewriter/jtransform.py
@@ -933,7 +933,8 @@
             opname = 'new_with_vtable'
         else:
             opname = 'new'
-        sizedescr = self.cpu.sizeof(STRUCT, opname == 'new_with_vtable')
+            vtable = None
+        sizedescr = self.cpu.sizeof(STRUCT, vtable)
         op1 = SpaceOperation(opname, [sizedescr], op.result)
         if zero:
             return self.zero_contents([op1], op.result, STRUCT)
diff --git a/rpython/jit/metainterp/virtualref.py b/rpython/jit/metainterp/virtualref.py
--- a/rpython/jit/metainterp/virtualref.py
+++ b/rpython/jit/metainterp/virtualref.py
@@ -20,6 +20,8 @@
         self.jit_virtual_ref_vtable = lltype.malloc(rclass.OBJECT_VTABLE,
                                                     zero=True, flavor='raw',
                                                     immortal=True)
+        self.descr = self.cpu.sizeof(self.JIT_VIRTUAL_REF,
+                                     vtable=self.jit_virtual_ref_vtable)
         self.jit_virtual_ref_vtable.name = rclass.alloc_array_name(
             'jit_virtual_ref')
         # build some constants
@@ -33,7 +35,6 @@
         self.descr_virtual_token = fielddescrof(self.JIT_VIRTUAL_REF,
                                                 'virtual_token')
         self.descr_forced = fielddescrof(self.JIT_VIRTUAL_REF, 'forced')
-        self.descr = self.cpu.sizeof(self.JIT_VIRTUAL_REF, False)
         #
         # record the type JIT_VIRTUAL_REF explicitly in the rtyper, too
         if hasattr(self.warmrunnerdesc, 'rtyper'):    # <-- for tests

From noreply at buildbot.pypy.org  Mon Jun 15 13:09:04 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 15 Jun 2015 13:09:04 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: add a fast path to avoid issues with
	JitVirtualRef
Message-ID: <20150615110904.B5D571C0962@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r78110:69624489a478
Date: 2015-06-15 13:02 +0200
http://bitbucket.org/pypy/pypy/changeset/69624489a478/

Log:	add a fast path to avoid issues with JitVirtualRef

diff --git a/rpython/jit/backend/llsupport/descr.py b/rpython/jit/backend/llsupport/descr.py
--- a/rpython/jit/backend/llsupport/descr.py
+++ b/rpython/jit/backend/llsupport/descr.py
@@ -130,8 +130,10 @@
             cls = llmemory.cast_adr_to_ptr(
                 heaptracker.int2adr(self.parent_descr.get_vtable()),
                 lltype.Ptr(rclass.OBJECT_VTABLE))
-            assert rclass.ll_isinstance(lltype.cast_opaque_ptr(
-                rclass.OBJECTPTR, struct), cls)
+            tpptr = lltype.cast_opaque_ptr(rclass.OBJECTPTR, struct).typeptr
+            if tpptr != cls:
+                assert rclass.ll_isinstance(lltype.cast_opaque_ptr(
+                    rclass.OBJECTPTR, struct), cls)
         else:
             pass
 

From noreply at buildbot.pypy.org  Mon Jun 15 13:09:05 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 15 Jun 2015 13:09:05 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: add a comment
Message-ID: <20150615110905.D70A91C0962@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r78111:d8fba7cf087d
Date: 2015-06-15 13:02 +0200
http://bitbucket.org/pypy/pypy/changeset/d8fba7cf087d/

Log:	add a comment

diff --git a/rpython/jit/backend/llsupport/descr.py b/rpython/jit/backend/llsupport/descr.py
--- a/rpython/jit/backend/llsupport/descr.py
+++ b/rpython/jit/backend/llsupport/descr.py
@@ -131,6 +131,9 @@
                 heaptracker.int2adr(self.parent_descr.get_vtable()),
                 lltype.Ptr(rclass.OBJECT_VTABLE))
             tpptr = lltype.cast_opaque_ptr(rclass.OBJECTPTR, struct).typeptr
+            # this comparison is necessary, since we want to make sure
+            # that vtable for JitVirtualRef is the same without actually reading
+            # fields
             if tpptr != cls:
                 assert rclass.ll_isinstance(lltype.cast_opaque_ptr(
                     rclass.OBJECTPTR, struct), cls)

From noreply at buildbot.pypy.org  Mon Jun 15 13:09:07 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 15 Jun 2015 13:09:07 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: fixes
Message-ID: <20150615110907.046C91C0962@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r78112:efd889daffae
Date: 2015-06-15 13:07 +0200
http://bitbucket.org/pypy/pypy/changeset/efd889daffae/

Log:	fixes

diff --git a/rpython/jit/backend/llsupport/descr.py b/rpython/jit/backend/llsupport/descr.py
--- a/rpython/jit/backend/llsupport/descr.py
+++ b/rpython/jit/backend/llsupport/descr.py
@@ -72,6 +72,7 @@
 
 def get_size_descr(gccache, STRUCT, vtable):
     cache = gccache._cache_size
+    assert not isinstance(vtable, bool)
     try:
         return cache[STRUCT]
     except KeyError:
@@ -197,7 +198,7 @@
         cachedict = cache.setdefault(STRUCT, {})
         cachedict[fieldname] = fielddescr
         fielddescr.parent_descr = get_size_descr(gccache, STRUCT,
-                                  heaptracker.get_vtable_for_gcstruct(STRUCT))
+                        heaptracker.get_vtable_for_gcstruct(gccache, STRUCT))
         return fielddescr
 
 def get_type_flag(TYPE):
diff --git a/rpython/jit/codewriter/heaptracker.py b/rpython/jit/codewriter/heaptracker.py
--- a/rpython/jit/codewriter/heaptracker.py
+++ b/rpython/jit/codewriter/heaptracker.py
@@ -60,10 +60,14 @@
     # xxx hack: from a GcStruct representing an instance's
     # lowleveltype, return the corresponding vtable pointer.
     # Returns None if the GcStruct does not belong to an instance.
-    assert isinstance(GCSTRUCT, lltype.GcStruct)
+    if not isinstance(GCSTRUCT, lltype.GcStruct):
+        return None
     if not has_gcstruct_a_vtable(GCSTRUCT):
         return None
     setup_cache_gcstruct2vtable(gccache)
+    if not hasattr(gccache, '_cache_gcstruct2vtable'):
+        # boehm and stuff
+        return lltype.malloc(GCSTRUCT.typeptr.TO, flavor='raw', immortal=True)
     return gccache._cache_gcstruct2vtable[GCSTRUCT]
 
 def setup_cache_gcstruct2vtable(gccache):
@@ -89,7 +93,7 @@
 
 def register_known_gctype(cpu, vtable, STRUCT):
     # register the correspondance 'vtable' <-> 'STRUCT' in the cpu
-    sizedescr = cpu.sizeof(STRUCT, has_gcstruct_a_vtable(STRUCT))
+    sizedescr = cpu.sizeof(STRUCT, vtable)
     assert sizedescr.as_vtable_size_descr() is sizedescr
     if getattr(sizedescr, '_corresponding_vtable', None):
         assert sizedescr._corresponding_vtable == vtable
diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py
--- a/rpython/jit/codewriter/jtransform.py
+++ b/rpython/jit/codewriter/jtransform.py
@@ -1601,7 +1601,7 @@
                 descrs = (self.cpu.arraydescrof(ARRAY),
                           self.cpu.fielddescrof(LIST, 'length'),
                           self.cpu.fielddescrof(LIST, 'items'),
-                          self.cpu.sizeof(LIST, False))
+                          self.cpu.sizeof(LIST, None))
         else:
             prefix = 'do_fixed_'
             if self._array_of_voids(LIST):
diff --git a/rpython/jit/metainterp/optimizeopt/TODO b/rpython/jit/metainterp/optimizeopt/TODO
--- a/rpython/jit/metainterp/optimizeopt/TODO
+++ b/rpython/jit/metainterp/optimizeopt/TODO
@@ -1,3 +1,5 @@
 * arraylen_gc is not handling length bound optimization at all (we need to
   wait till unrolling for tests)
 * mark_opaque_pointer is ignored (which is fine until unrolling)
+* clean up and unify _corresponding_vtable and vtable fields on SizeDescr
+  and check if all the usecases are necessary
\ No newline at end of file
diff --git a/rpython/jit/metainterp/virtualref.py b/rpython/jit/metainterp/virtualref.py
--- a/rpython/jit/metainterp/virtualref.py
+++ b/rpython/jit/metainterp/virtualref.py
@@ -20,6 +20,9 @@
         self.jit_virtual_ref_vtable = lltype.malloc(rclass.OBJECT_VTABLE,
                                                     zero=True, flavor='raw',
                                                     immortal=True)
+        if hasattr(self.cpu, 'gc_ll_descr'):
+            heaptracker.setup_cache_gcstruct2vtable(self.cpu.gc_ll_descr)
+            self.cpu.gc_ll_descr._cache_gcstruct2vtable[self.JIT_VIRTUAL_REF] = self.jit_virtual_ref_vtable
         self.descr = self.cpu.sizeof(self.JIT_VIRTUAL_REF,
                                      vtable=self.jit_virtual_ref_vtable)
         self.jit_virtual_ref_vtable.name = rclass.alloc_array_name(
@@ -29,9 +32,6 @@
         adr = heaptracker.adr2int(adr)
         self.jit_virtual_ref_const_class = history.ConstInt(adr)
         fielddescrof = self.cpu.fielddescrof
-        if hasattr(self.cpu, 'gc_ll_descr'):
-            heaptracker.setup_cache_gcstruct2vtable(self.cpu.gc_ll_descr)
-            self.cpu.gc_ll_descr._cache_gcstruct2vtable[self.JIT_VIRTUAL_REF] = self.jit_virtual_ref_vtable
         self.descr_virtual_token = fielddescrof(self.JIT_VIRTUAL_REF,
                                                 'virtual_token')
         self.descr_forced = fielddescrof(self.JIT_VIRTUAL_REF, 'forced')

From noreply at buildbot.pypy.org  Mon Jun 15 13:09:08 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 15 Jun 2015 13:09:08 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: merge
Message-ID: <20150615110908.2DB0A1C0962@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r78113:c81b38d01c8e
Date: 2015-06-15 13:09 +0200
http://bitbucket.org/pypy/pypy/changeset/c81b38d01c8e/

Log:	merge

diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py
--- a/rpython/jit/backend/llsupport/rewrite.py
+++ b/rpython/jit/backend/llsupport/rewrite.py
@@ -119,6 +119,7 @@
         #
         for i in range(len(operations)):
             op = operations[i]
+            assert op.get_forwarded() is None
             if op.getopnum() == rop.DEBUG_MERGE_POINT:
                 continue
             # ---------- turn NEWxxx into CALL_MALLOC_xxx ----------

From noreply at buildbot.pypy.org  Thu Jun 18 15:12:50 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 18 Jun 2015 15:12:50 +0200 (CEST)
Subject: [pypy-commit] pypy default: issue #2067: fix the error message
Message-ID: <20150618131250.65B351C1FC9@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78179:8324e92a5b49
Date: 2015-06-18 15:12 +0200
http://bitbucket.org/pypy/pypy/changeset/8324e92a5b49/

Log:	issue #2067: fix the error message

diff --git a/pypy/module/_socket/interp_func.py b/pypy/module/_socket/interp_func.py
--- a/pypy/module/_socket/interp_func.py
+++ b/pypy/module/_socket/interp_func.py
@@ -2,7 +2,7 @@
 from rpython.rlib.rsocket import SocketError, INVALID_SOCKET
 from rpython.rlib.rarithmetic import intmask
 
-from pypy.interpreter.error import OperationError
+from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.gateway import unwrap_spec, WrappedDefault
 from pypy.module._socket.interp_socket import (
     converted_error, W_Socket, addr_as_object, ipaddr_from_object
@@ -247,9 +247,9 @@
         ip = rsocket.inet_ntop(family, packed)
     except SocketError, e:
         raise converted_error(space, e)
-    except ValueError, e:     # XXX the message is lost in RPython
-        raise OperationError(space.w_ValueError,
-                  space.wrap(str(e)))
+    except ValueError:
+        raise oefmt(space.w_ValueError,
+                    "invalid length of packed IP address string")
     return space.wrap(ip)
 
 @unwrap_spec(family=int, socktype=int, proto=int, flags=int)

From noreply at buildbot.pypy.org  Tue Jun 23 09:42:06 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 23 Jun 2015 09:42:06 +0200 (CEST)
Subject: [pypy-commit] pypy default: Give up,
	copy the whole test here and fix it
Message-ID: <20150623074206.DE48D1C02A3@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78251:92a888903431
Date: 2015-06-23 09:42 +0200
http://bitbucket.org/pypy/pypy/changeset/92a888903431/

Log:	Give up, copy the whole test here and fix it

diff --git a/pypy/module/_io/interp_textio.py b/pypy/module/_io/interp_textio.py
--- a/pypy/module/_io/interp_textio.py
+++ b/pypy/module/_io/interp_textio.py
@@ -548,10 +548,6 @@
         remain buffered in the decoder, yet to be converted."""
 
         if not self.w_decoder:
-            # very unsure about the following check, but some tests seem
-            # to expect a ValueError instead of an IOError in case the
-            # file was already closed.
-            self._check_closed(space)
             raise OperationError(space.w_IOError, space.wrap("not readable"))
 
         if self.telling:
@@ -604,11 +600,8 @@
 
     def read_w(self, space, w_size=None):
         self._check_attached(space)
+        self._check_closed(space)
         if not self.w_decoder:
-            # very unsure about the following check, but some tests seem
-            # to expect a ValueError instead of an IOError in case the
-            # file was already closed.
-            self._check_closed(space)
             raise OperationError(space.w_IOError, space.wrap("not readable"))
 
         size = convert_size(space, w_size)
@@ -649,6 +642,7 @@
 
     def readline_w(self, space, w_limit=None):
         self._check_attached(space)
+        self._check_closed(space)
         self._writeflush(space)
 
         limit = convert_size(space, w_limit)
@@ -744,7 +738,7 @@
 
     def write_w(self, space, w_text):
         self._check_attached(space)
-        # self._check_closed(space)
+        self._check_closed(space)
 
         if not self.w_encoder:
             raise OperationError(space.w_IOError, space.wrap("not writable"))
diff --git a/pypy/module/_io/test/test_io.py b/pypy/module/_io/test/test_io.py
--- a/pypy/module/_io/test/test_io.py
+++ b/pypy/module/_io/test/test_io.py
@@ -391,3 +391,55 @@
             f.seek(1, 0)
             f.read(buffer_size * 2)
             assert f.tell() == 1 + buffer_size * 2
+
+
+class AppTestIoAferClose:
+    spaceconfig = dict(usemodules=['_io'])
+
+    def setup_class(cls):
+        tmpfile = udir.join('tmpfile').ensure()
+        cls.w_tmpfile = cls.space.wrap(str(tmpfile))
+
+    def test_io_after_close(self):
+        import _io
+        for kwargs in [
+                {"mode": "w"},
+                {"mode": "wb"},
+                {"mode": "w", "buffering": 1},
+                {"mode": "w", "buffering": 2},
+                {"mode": "wb", "buffering": 0},
+                {"mode": "r"},
+                {"mode": "rb"},
+                {"mode": "r", "buffering": 1},
+                {"mode": "r", "buffering": 2},
+                {"mode": "rb", "buffering": 0},
+                {"mode": "w+"},
+                {"mode": "w+b"},
+                {"mode": "w+", "buffering": 1},
+                {"mode": "w+", "buffering": 2},
+                {"mode": "w+b", "buffering": 0},
+            ]:
+            print kwargs
+            f = _io.open(self.tmpfile, **kwargs)
+            f.close()
+            raises(ValueError, f.flush)
+            raises(ValueError, f.fileno)
+            raises(ValueError, f.isatty)
+            raises(ValueError, f.__iter__)
+            if hasattr(f, "peek"):
+                raises(ValueError, f.peek, 1)
+            raises(ValueError, f.read)
+            if hasattr(f, "read1"):
+                raises(ValueError, f.read1, 1024)
+            if hasattr(f, "readall"):
+                raises(ValueError, f.readall)
+            if hasattr(f, "readinto"):
+                raises(ValueError, f.readinto, bytearray(1024))
+            raises(ValueError, f.readline)
+            raises(ValueError, f.readlines)
+            raises(ValueError, f.seek, 0)
+            raises(ValueError, f.tell)
+            raises(ValueError, f.truncate)
+            raises(ValueError, f.write, b"" if "b" in kwargs['mode'] else u"")
+            raises(ValueError, f.writelines, [])
+            raises(ValueError, next, f)

From noreply at buildbot.pypy.org  Tue Jun 23 09:20:31 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 23 Jun 2015 09:20:31 +0200 (CEST)
Subject: [pypy-commit] cffi default: Add a passing test
Message-ID: <20150623072031.F3E9C1C1FA8@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2191:ed37c9e07e1b
Date: 2015-06-23 09:21 +0200
http://bitbucket.org/cffi/cffi/changeset/ed37c9e07e1b/

Log:	Add a passing test

diff --git a/testing/cffi1/test_recompiler.py b/testing/cffi1/test_recompiler.py
--- a/testing/cffi1/test_recompiler.py
+++ b/testing/cffi1/test_recompiler.py
@@ -838,6 +838,22 @@
     assert isinstance(addr, ffi.CData)
     assert ffi.typeof(addr) == ffi.typeof("long(*)(long)")
 
+def test_address_of_function_with_struct():
+    ffi = FFI()
+    ffi.cdef("struct foo_s { int x; }; long myfunc(struct foo_s);")
+    lib = verify(ffi, "test_addressof_function_with_struct", """
+        struct foo_s { int x; };
+        char myfunc(struct foo_s input) { return (char)(input.x + 42); }
+    """)
+    s = ffi.new("struct foo_s *", [5])[0]
+    assert lib.myfunc(s) == 47
+    assert not isinstance(lib.myfunc, ffi.CData)
+    assert ffi.typeof(lib.myfunc) == ffi.typeof("long(*)(struct foo_s)")
+    addr = ffi.addressof(lib, 'myfunc')
+    assert addr(s) == 47
+    assert isinstance(addr, ffi.CData)
+    assert ffi.typeof(addr) == ffi.typeof("long(*)(struct foo_s)")
+
 def test_issue198():
     ffi = FFI()
     ffi.cdef("""

From noreply at buildbot.pypy.org  Tue Jun 23 09:28:21 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 23 Jun 2015 09:28:21 +0200 (CEST)
Subject: [pypy-commit] pypy default: Test and fix: ffi.addressof(lib,
 'func-with-struct-args') would return
Message-ID: <20150623072821.F2C511C1FCE@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78249:e8bd7b84971b
Date: 2015-06-23 09:28 +0200
http://bitbucket.org/pypy/pypy/changeset/e8bd7b84971b/

Log:	Test and fix: ffi.addressof(lib, 'func-with-struct-args') would
	return a badly typed cdata object

diff --git a/pypy/module/_cffi_backend/lib_obj.py b/pypy/module/_cffi_backend/lib_obj.py
--- a/pypy/module/_cffi_backend/lib_obj.py
+++ b/pypy/module/_cffi_backend/lib_obj.py
@@ -237,7 +237,7 @@
         if isinstance(w_value, W_FunctionWrapper):
             # '&func' returns a regular cdata pointer-to-function
             if w_value.directfnptr:
-                ctype = w_value.rawfunctype.nostruct_ctype
+                ctype = w_value.typeof(self.ffi)
                 return W_CData(space, w_value.directfnptr, ctype)
             else:
                 return w_value    # backward compatibility
diff --git a/pypy/module/_cffi_backend/test/test_recompiler.py b/pypy/module/_cffi_backend/test/test_recompiler.py
--- a/pypy/module/_cffi_backend/test/test_recompiler.py
+++ b/pypy/module/_cffi_backend/test/test_recompiler.py
@@ -828,6 +828,22 @@
         assert isinstance(addr, ffi.CData)
         assert ffi.typeof(addr) == ffi.typeof("long(*)(long)")
 
+    def test_address_of_function_with_struct(self):
+        ffi, lib = self.prepare(
+            "struct foo_s { int x; }; long myfunc(struct foo_s);",
+            "test_addressof_function_with_struct", """
+                struct foo_s { int x; };
+                char myfunc(struct foo_s input) { return (char)(input.x + 42); }
+            """)
+        s = ffi.new("struct foo_s *", [5])[0]
+        assert lib.myfunc(s) == 47
+        assert not isinstance(lib.myfunc, ffi.CData)
+        assert ffi.typeof(lib.myfunc) == ffi.typeof("long(*)(struct foo_s)")
+        addr = ffi.addressof(lib, 'myfunc')
+        assert addr(s) == 47
+        assert isinstance(addr, ffi.CData)
+        assert ffi.typeof(addr) == ffi.typeof("long(*)(struct foo_s)")
+
     def test_issue198(self):
         ffi, lib = self.prepare("""
             typedef struct{...;} opaque_t;

From noreply at buildbot.pypy.org  Thu Jun 18 23:24:54 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 18 Jun 2015 23:24:54 +0200 (CEST)
Subject: [pypy-commit] stmgc queue: tweaks
Message-ID: <20150618212454.9EC601C1FE9@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: queue
Changeset: r1869:72471a4e7d9e
Date: 2015-06-18 21:20 +0200
http://bitbucket.org/pypy/stmgc/changeset/72471a4e7d9e/

Log:	tweaks

diff --git a/c8/stm/queue.c b/c8/stm/queue.c
--- a/c8/stm/queue.c
+++ b/c8/stm/queue.c
@@ -109,9 +109,9 @@
     spinlock_release(get_priv_segment(num)->active_queues_lock);
 }
 
-static void queue_activate(stm_queue_t *queue)
+static void queue_activate(stm_queue_t *queue, stm_queue_segment_t *seg)
 {
-    stm_queue_segment_t *seg = &queue->segs[STM_SEGMENT->segment_num - 1];
+    assert(seg == &queue->segs[STM_SEGMENT->segment_num - 1]);
 
     if (!seg->active) {
         queue_lock_acquire();
@@ -202,13 +202,13 @@
        delays or transaction breaks.  you need to push roots!
     */
     stm_queue_segment_t *seg = &queue->segs[STM_SEGMENT->segment_num - 1];
+    queue_activate(queue, seg);
+
     queue_entry_t *entry = malloc(sizeof(queue_entry_t));
     assert(entry);
     entry->object = newitem;
     entry->next = seg->added_in_this_transaction;
     seg->added_in_this_transaction = entry;
-
-    queue_activate(queue);
     seg->unfinished_tasks_in_this_transaction++;
 
     /* add qobj to 'objects_pointing_to_nursery' if it has the
@@ -270,11 +270,11 @@
            'old_objects_popped' list for now.  From now on, this entry
            "belongs" to this segment and should never be read by
            another segment. */
+        queue_activate(queue, seg);
+
         queue_check_entry(entry);
         entry->next = seg->old_objects_popped;
         seg->old_objects_popped = entry;
-
-        queue_activate(queue);
         return entry->object;
     }
     else {
@@ -322,8 +322,8 @@
 
 void stm_queue_task_done(stm_queue_t *queue)
 {
-    queue_activate(queue);
     stm_queue_segment_t *seg = &queue->segs[STM_SEGMENT->segment_num - 1];
+    queue_activate(queue, seg);
     seg->unfinished_tasks_in_this_transaction--;
 }
 

From noreply at buildbot.pypy.org  Thu Jun 18 23:24:55 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 18 Jun 2015 23:24:55 +0200 (CEST)
Subject: [pypy-commit] stmgc queue: Aaaaaaaa ok found the next bug
Message-ID: <20150618212455.99D8B1C1FEA@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: queue
Changeset: r1870:10bb05551db5
Date: 2015-06-18 21:23 +0200
http://bitbucket.org/pypy/stmgc/changeset/10bb05551db5/

Log:	Aaaaaaaa ok found the next bug

diff --git a/c8/stm/queue.c b/c8/stm/queue.c
--- a/c8/stm/queue.c
+++ b/c8/stm/queue.c
@@ -250,6 +250,11 @@
     }
 
  retry:
+    /* careful, STM_SEGMENT->segment_num may change here because
+       we're starting new transactions below! */
+    seg = &queue->segs[STM_SEGMENT->segment_num - 1];
+    assert(!seg->added_in_this_transaction);
+
     /* can't easily use compare_and_swap here.  The issue is that
        if we do "compare_and_swap(&old_entry, entry, entry->next)",
        then we need to read entry->next, but a parallel thread

From noreply at buildbot.pypy.org  Thu Jun 18 11:45:40 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 18 Jun 2015 11:45:40 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c8: import stmgc/7592a0f11ac2
Message-ID: <20150618094540.900081C1F90@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c8
Changeset: r78171:eeb45349ea15
Date: 2015-06-18 10:45 +0100
http://bitbucket.org/pypy/pypy/changeset/eeb45349ea15/

Log:	import stmgc/7592a0f11ac2

diff --git a/rpython/translator/stm/src_stm/revision b/rpython/translator/stm/src_stm/revision
--- a/rpython/translator/stm/src_stm/revision
+++ b/rpython/translator/stm/src_stm/revision
@@ -1,1 +1,1 @@
-3ca830828468
+7592a0f11ac2
diff --git a/rpython/translator/stm/src_stm/stm/sync.c b/rpython/translator/stm/src_stm/stm/sync.c
--- a/rpython/translator/stm/src_stm/stm/sync.c
+++ b/rpython/translator/stm/src_stm/stm/sync.c
@@ -115,7 +115,12 @@
     t->tv_sec = tv.tv_sec;
     t->tv_nsec = tv.tv_usec * 1000 + 999;
 #endif
-    /* assumes that "incr" is not too large, less than 1 second */
+
+    long integral_part = (long)incr;
+    t->tv_sec += integral_part;
+    incr -= integral_part;
+    assert(incr >= 0.0 && incr <= 1.0);
+
     long nsec = t->tv_nsec + (long)(incr * 1000000000.0);
     if (nsec >= 1000000000) {
         t->tv_sec += 1;
@@ -131,15 +136,21 @@
     stm_fatalerror("*** cond_wait/%d called!", (int)ctype);
 #endif
 
+ retry:
     assert(_has_mutex_here);
 
     int err = pthread_cond_timedwait(&sync_ctl.cond[ctype],
                                      &sync_ctl.global_mutex, pt);
-    if (err == 0)
+    switch (err) {
+    case 0:
         return true;     /* success */
-    if (LIKELY(err == ETIMEDOUT))
+    case ETIMEDOUT:
         return false;    /* timeout */
-    stm_fatalerror("pthread_cond_timedwait/%d: %d", (int)ctype, err);
+    case EINTR:
+        goto retry;
+    default:
+        stm_fatalerror("pthread_cond_timedwait/%d: %d", (int)ctype, err);
+    }
 }
 
 static bool cond_wait_timeout(enum cond_type_e ctype, double delay)

From noreply at buildbot.pypy.org  Thu Jun 18 18:47:02 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 18 Jun 2015 18:47:02 +0200 (CEST)
Subject: [pypy-commit] stmgc default: After cond_wait_timeout() returns with
 "timeout", we need to check
Message-ID: <20150618164702.AD6EA1C1FDE@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r1863:b28eec420b62
Date: 2015-06-18 18:47 +0200
http://bitbucket.org/pypy/stmgc/changeset/b28eec420b62/

Log:	After cond_wait_timeout() returns with "timeout", we need to check
	again if a safe-point was requested.

diff --git a/c8/stm/core.c b/c8/stm/core.c
--- a/c8/stm/core.c
+++ b/c8/stm/core.c
@@ -500,6 +500,7 @@
     intptr_t detached = 0;
 
     s_mutex_lock();
+ wait_some_more:
     if (safe_point_requested()) {
         /* XXXXXX if the safe point below aborts, in
            _validate_and_attach(), 'new' leaks */
@@ -508,12 +509,10 @@
     else if (STM_PSEGMENT->last_commit_log_entry->next == INEV_RUNNING) {
         /* loop until C_SEGMENT_FREE_OR_SAFE_POINT_REQ is signalled, but
            try to detach an inevitable transaction regularly */
-        while (1) {
-            detached = fetch_detached_transaction();
-            if (detached != 0)
-                break;
-            if (cond_wait_timeout(C_SEGMENT_FREE_OR_SAFE_POINT_REQ, 0.00001))
-                break;
+        detached = fetch_detached_transaction();
+        if (detached == 0) {
+            if (!cond_wait_timeout(C_SEGMENT_FREE_OR_SAFE_POINT_REQ, 0.00001))
+                goto wait_some_more;
         }
     }
     s_mutex_unlock();
@@ -1578,31 +1577,32 @@
             stm_abort_transaction();    /* is already inevitable, abort   */
 #endif
 
-            intptr_t detached = 0;
+            bool timed_out = false;
 
             s_mutex_lock();
             if (any_soon_finished_or_inevitable_thread_segment() &&
                     !safe_point_requested()) {
 
                 /* wait until C_SEGMENT_FREE_OR_SAFE_POINT_REQ is signalled */
-                while (!cond_wait_timeout(C_SEGMENT_FREE_OR_SAFE_POINT_REQ,
-                                          0.000054321)) {
-                    /* try to detach another inevitable transaction, but
-                       only after waiting a bit.  This is necessary to avoid
-                       deadlocks in some situations, which are hopefully
-                       not too common.  We don't want two threads constantly
-                       detaching each other. */
-                    detached = fetch_detached_transaction();
-                    if (detached != 0)
-                        break;
-                }
+                if (!cond_wait_timeout(C_SEGMENT_FREE_OR_SAFE_POINT_REQ,
+                                       0.000054321))
+                    timed_out = true;
             }
             s_mutex_unlock();
 
-            if (detached != 0)
-                commit_fetched_detached_transaction(detached);
-
-            num_waits++;
+            if (timed_out) {
+                /* try to detach another inevitable transaction, but
+                   only after waiting a bit.  This is necessary to avoid
+                   deadlocks in some situations, which are hopefully
+                   not too common.  We don't want two threads constantly
+                   detaching each other. */
+                intptr_t detached = fetch_detached_transaction();
+                if (detached != 0)
+                    commit_fetched_detached_transaction(detached);
+            }
+            else {
+                num_waits++;
+            }
             goto retry_from_start;
         }
         if (!_validate_and_turn_inevitable())

From noreply at buildbot.pypy.org  Thu Jun 18 23:24:56 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 18 Jun 2015 23:24:56 +0200 (CEST)
Subject: [pypy-commit] stmgc queue: fix fix fix minor collections
Message-ID: <20150618212456.975BF1C1FEB@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: queue
Changeset: r1871:0a5991d79ba3
Date: 2015-06-18 23:25 +0200
http://bitbucket.org/pypy/stmgc/changeset/0a5991d79ba3/

Log:	fix fix fix minor collections

diff --git a/c8/stm/nursery.c b/c8/stm/nursery.c
--- a/c8/stm/nursery.c
+++ b/c8/stm/nursery.c
@@ -553,6 +553,9 @@
     if (STM_PSEGMENT->finalizers != NULL)
         collect_objs_still_young_but_with_finalizers();
 
+    if (STM_PSEGMENT->active_queues != NULL)
+        collect_active_queues();
+
     collect_oldrefs_to_nursery();
     assert(list_is_empty(STM_PSEGMENT->old_objects_with_cards_set));
 
diff --git a/c8/stm/queue.c b/c8/stm/queue.c
--- a/c8/stm/queue.c
+++ b/c8/stm/queue.c
@@ -210,13 +210,6 @@
     entry->next = seg->added_in_this_transaction;
     seg->added_in_this_transaction = entry;
     seg->unfinished_tasks_in_this_transaction++;
-
-    /* add qobj to 'objects_pointing_to_nursery' if it has the
-       WRITE_BARRIER flag */
-    if (qobj->stm_flags & GCFLAG_WRITE_BARRIER) {
-        qobj->stm_flags &= ~GCFLAG_WRITE_BARRIER;
-        LIST_APPEND(STM_PSEGMENT->objects_pointing_to_nursery, qobj);
-    }
 }
 
 static void queue_check_entry(queue_entry_t *entry)
@@ -381,14 +374,30 @@
         }
         queue_trace_list(queue->old_entries, trace, NULL);
     }
-    else {
-        /* for minor collections: it is enough to trace the objects
-           added in the current transaction.  All other objects are
-           old (or, worse, belong to a parallel thread and must not
-           be traced). */
+    /* for minor collections, done differently.
+       see collect_active_queues() below */
+}
+
+static void collect_active_queues(void)
+{
+    wlog_t *item;
+    TREE_LOOP_FORWARD(STM_PSEGMENT->active_queues, item) {
+        /* it is enough to trace the objects added in the current
+           transaction.  All other objects reachable from the queue
+           are old (or, worse, belong to a parallel thread and must
+           not be traced).  Performance note: this is linear in the
+           total number of active queues, but at least each queue that
+           has not been touched for a while in a long transaction is
+           handled very cheaply.
+        */
+        stm_queue_t *queue = (stm_queue_t *)item->addr;
         stm_queue_segment_t *seg = &queue->segs[STM_SEGMENT->segment_num - 1];
-        queue_trace_list(seg->added_in_this_transaction, trace,
-                         seg->added_young_limit);
-        seg->added_young_limit = seg->added_in_this_transaction;
-    }
+        if (seg->added_young_limit != seg->added_in_this_transaction) {
+            dprintf(("minor collection trace queue %p\n", queue));
+            queue_trace_list(seg->added_in_this_transaction,
+                             &minor_trace_if_young,
+                             seg->added_young_limit);
+            seg->added_young_limit = seg->added_in_this_transaction;
+        }
+    } TREE_LOOP_END;
 }
diff --git a/c8/stm/queue.h b/c8/stm/queue.h
--- a/c8/stm/queue.h
+++ b/c8/stm/queue.h
@@ -1,1 +1,2 @@
 static void queues_deactivate_all(bool at_commit);
+static void collect_active_queues(void);
diff --git a/c8/test/test_queue.py b/c8/test/test_queue.py
--- a/c8/test/test_queue.py
+++ b/c8/test/test_queue.py
@@ -168,6 +168,7 @@
         #
         obj2 = stm_allocate(32)
         stm_set_char(obj2, 'H')
+        print 'put2'
         self.put(qobj, obj2)
         stm_minor_collect()
         obj2 = self.get(qobj)

From noreply at buildbot.pypy.org  Thu Jun 18 23:24:53 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 18 Jun 2015 23:24:53 +0200 (CEST)
Subject: [pypy-commit] stmgc queue: Missing push/pop root
Message-ID: <20150618212453.A704E1C1FE8@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: queue
Changeset: r1868:159fbfd040f9
Date: 2015-06-18 21:16 +0200
http://bitbucket.org/pypy/stmgc/changeset/159fbfd040f9/

Log:	Missing push/pop root

diff --git a/c8/stm/queue.c b/c8/stm/queue.c
--- a/c8/stm/queue.c
+++ b/c8/stm/queue.c
@@ -284,7 +284,9 @@
 #endif
         if (timeout == 0.0) {
             if (!stm_is_inevitable(tl)) {
+                STM_PUSH_ROOT(*tl, qobj);
                 stm_become_inevitable(tl, "stm_queue_get");
+                STM_POP_ROOT(*tl, qobj);
                 goto retry;
             }
             else

From noreply at buildbot.pypy.org  Tue Jun 23 09:33:35 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Tue, 23 Jun 2015 09:33:35 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: add a new doc file to describe vecopt,
 describe what the current opt. is capable of and added some
 limitations
Message-ID: <20150623073335.5D6CF1C2018@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r78250:77f58b1dcf7f
Date: 2015-06-23 09:33 +0200
http://bitbucket.org/pypy/pypy/changeset/77f58b1dcf7f/

Log:	add a new doc file to describe vecopt, describe what the current
	opt. is capable of and added some limitations

diff --git a/rpython/doc/jit/vectorization.rst b/rpython/doc/jit/vectorization.rst
new file mode 100644
--- /dev/null
+++ b/rpython/doc/jit/vectorization.rst
@@ -0,0 +1,45 @@
+
+Vectorization
+=============
+
+TBA
+
+Features
+--------
+
+Currently the following operations can be vectorized if the trace contains parallelism:
+
+* float32/float64: add, substract, multiply, divide, negate, absolute
+* int8/int16/int32/int64 arithmetic: add, substract, multiply, negate, absolute
+* int8/int16/int32/int64 logical: and, or, xor
+
+Reduction is implemented:
+
+* sum
+
+Planned reductions:
+
+* all, any, prod, min, max
+
+To find parallel instructions the tracer must provide enough information about
+memory load/store operations. They must be adjacent in memory. The requirement for
+that is that they use the same index variable and offset can be expressed as a
+a linear or affine combination.
+
+Unrolled guards are strengthend on a arithmetical level (See GuardStrengthenOpt).
+The resulting vector trace will only have one guard that checks the index.
+
+Calculations on the index variable that are redundant (because of the merged
+load/store instructions) are not removed. The backend removes these instructions
+while assembling the trace.
+
+
+Future Work and Limitations
+---------------------------
+
+* The only SIMD instruction architecture currently supported is SSE4.1
+* Loop that convert types from int(8|16|32|64) to int(8|16) are not supported in
+  the current SSE4.1 assembler implementation.
+  The opcode needed spans over multiple instructions. In terms of performance
+  there might only be little to non advantage to use SIMD instructions for this
+  conversions.
diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/vectorize.py
@@ -1,3 +1,10 @@
+"""
+This is the core of the vec. optimization. It combines dependency.py and schedule.py
+to rewrite a loop in vectorized form.
+
+See the rpython doc for more high level details.
+"""
+
 import py
 
 from rpython.jit.metainterp.resume import Snapshot

From noreply at buildbot.pypy.org  Sat Jun 20 17:25:22 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 20 Jun 2015 17:25:22 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c8: update to stmgc/9ffba4fe03df and tweak
	the stmlog
Message-ID: <20150620152522.5576F1C1F78@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c8
Changeset: r78217:2cba5fa41bf3
Date: 2015-06-20 17:25 +0200
http://bitbucket.org/pypy/pypy/changeset/2cba5fa41bf3/

Log:	update to stmgc/9ffba4fe03df and tweak the stmlog

diff --git a/rpython/translator/stm/src_stm/extracode.h b/rpython/translator/stm/src_stm/extracode.h
--- a/rpython/translator/stm/src_stm/extracode.h
+++ b/rpython/translator/stm/src_stm/extracode.h
@@ -133,12 +133,13 @@
         }
     }
 
+    uintptr_t next_instr = marker->odd_number >> 1;
+
     ll = _fetch_strlen(segment_base, co_lnotab);
     if (ll > 0) {
         long lnotablen = ll;
         unsigned char *lnotab = (unsigned char *)_fetch_stritems(segment_base,
                                                                  co_lnotab);
-        uintptr_t next_instr = marker->odd_number >> 1;
         line = co_firstlineno;
         uintptr_t ii, curaddr = 0;
         for (ii = 0; ii < lnotablen; ii += 2) {
@@ -151,8 +152,9 @@
 
     int result;
     result = snprintf(outputbuf, outputbufsize,
-                      "File \"%s%.*s\", line %ld, in %.*s%s",
-                      fntrunc, (int)fnlen, fn, line, (int)nlen, name, ntrunc);
+                      "File \"%s%.*s\", line %ld, in %.*s%s (#%ld)",
+                      fntrunc, (int)fnlen, fn, line, (int)nlen,
+                      name, ntrunc, next_instr);
     if (result >= outputbufsize)
         result = outputbufsize - 1;
     if (result < 0)
diff --git a/rpython/translator/stm/src_stm/revision b/rpython/translator/stm/src_stm/revision
--- a/rpython/translator/stm/src_stm/revision
+++ b/rpython/translator/stm/src_stm/revision
@@ -1,1 +1,1 @@
-9cb167448d92
+9ffba4fe03df
diff --git a/rpython/translator/stm/src_stm/stm/core.c b/rpython/translator/stm/src_stm/stm/core.c
--- a/rpython/translator/stm/src_stm/stm/core.c
+++ b/rpython/translator/stm/src_stm/stm/core.c
@@ -1365,7 +1365,8 @@
     }
 
     if (STM_PSEGMENT->active_queues)
-        queues_deactivate_all(/*at_commit=*/true);
+        queues_deactivate_all(get_priv_segment(STM_SEGMENT->segment_num),
+                              /*at_commit=*/true);
 
     invoke_and_clear_user_callbacks(0);   /* for commit */
 
@@ -1476,6 +1477,9 @@
 #endif
     tl->thread_local_obj = pseg->threadlocal_at_start_of_transaction;
 
+    if (pseg->active_queues)
+        queues_deactivate_all(pseg, /*at_commit=*/false);
+
 
     /* Set the next nursery_mark: first compute the value that
        nursery_mark must have had at the start of the aborted transaction */
@@ -1521,9 +1525,6 @@
     if (tl->mem_clear_on_abort)
         memset(tl->mem_clear_on_abort, 0, tl->mem_bytes_to_clear_on_abort);
 
-    if (STM_PSEGMENT->active_queues)
-        queues_deactivate_all(/*at_commit=*/false);
-
     invoke_and_clear_user_callbacks(1);   /* for abort */
 
     if (is_abort(STM_SEGMENT->nursery_end)) {
@@ -1570,6 +1571,8 @@
 {
     int num_waits = 0;
 
+    timing_become_inevitable();
+
  retry_from_start:
     assert(STM_PSEGMENT->transaction_state == TS_REGULAR);
     _stm_collectable_safe_point();
@@ -1619,7 +1622,6 @@
         if (!_validate_and_turn_inevitable())
             return;
     }
-    timing_become_inevitable();
     soon_finished_or_inevitable_thread_segment();
     STM_PSEGMENT->transaction_state = TS_INEVITABLE;
 
diff --git a/rpython/translator/stm/src_stm/stm/queue.c b/rpython/translator/stm/src_stm/stm/queue.c
--- a/rpython/translator/stm/src_stm/stm/queue.c
+++ b/rpython/translator/stm/src_stm/stm/queue.c
@@ -126,16 +126,21 @@
     }
 }
 
-static void queues_deactivate_all(bool at_commit)
+static void queues_deactivate_all(struct stm_priv_segment_info_s *pseg,
+                                  bool at_commit)
 {
-    queue_lock_acquire();
+#pragma push_macro("STM_PSEGMENT")
+#pragma push_macro("STM_SEGMENT")
+#undef STM_PSEGMENT
+#undef STM_SEGMENT
+    spinlock_acquire(pseg->active_queues_lock);
 
     bool added_any_old_entries = false;
     bool finished_more_tasks = false;
     wlog_t *item;
-    TREE_LOOP_FORWARD(STM_PSEGMENT->active_queues, item) {
+    TREE_LOOP_FORWARD(pseg->active_queues, item) {
         stm_queue_t *queue = (stm_queue_t *)item->addr;
-        stm_queue_segment_t *seg = &queue->segs[STM_SEGMENT->segment_num - 1];
+        stm_queue_segment_t *seg = &queue->segs[pseg->pub.segment_num - 1];
         queue_entry_t *head, *freehead;
 
         if (at_commit) {
@@ -188,16 +193,17 @@
 
     } TREE_LOOP_END;
 
-    tree_free(STM_PSEGMENT->active_queues);
-    STM_PSEGMENT->active_queues = NULL;
+    tree_free(pseg->active_queues);
+    pseg->active_queues = NULL;
 
-    queue_lock_release();
+    spinlock_release(pseg->active_queues_lock);
 
-    assert(_has_mutex());
     if (added_any_old_entries)
         cond_broadcast(C_QUEUE_OLD_ENTRIES);
     if (finished_more_tasks)
         cond_broadcast(C_QUEUE_FINISHED_MORE_TASKS);
+#pragma pop_macro("STM_SEGMENT")
+#pragma pop_macro("STM_PSEGMENT")
 }
 
 void stm_queue_put(object_t *qobj, stm_queue_t *queue, object_t *newitem)
diff --git a/rpython/translator/stm/src_stm/stm/queue.h b/rpython/translator/stm/src_stm/stm/queue.h
--- a/rpython/translator/stm/src_stm/stm/queue.h
+++ b/rpython/translator/stm/src_stm/stm/queue.h
@@ -1,4 +1,5 @@
 /* Imported by rpython/translator/stm/import_stmgc.py */
-static void queues_deactivate_all(bool at_commit);
+static void queues_deactivate_all(struct stm_priv_segment_info_s *pseg,
+                                  bool at_commit);
 static void collect_active_queues(void);           /* minor collections */
 static void mark_visit_from_active_queues(void);   /* major collections */

From noreply at buildbot.pypy.org  Thu Jun 18 18:49:25 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 18 Jun 2015 18:49:25 +0200 (CEST)
Subject: [pypy-commit] stmgc queue: hg merge default
Message-ID: <20150618164925.5E9091C1FDF@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: queue
Changeset: r1864:0a10e04f2119
Date: 2015-06-18 18:50 +0200
http://bitbucket.org/pypy/stmgc/changeset/0a10e04f2119/

Log:	hg merge default

diff --git a/c8/stm/core.c b/c8/stm/core.c
--- a/c8/stm/core.c
+++ b/c8/stm/core.c
@@ -495,6 +495,32 @@
 static void readd_wb_executed_flags(void);
 static void check_all_write_barrier_flags(char *segbase, struct list_s *list);
 
+static void wait_for_inevitable(void)
+{
+    intptr_t detached = 0;
+
+    s_mutex_lock();
+ wait_some_more:
+    if (safe_point_requested()) {
+        /* XXXXXX if the safe point below aborts, in
+           _validate_and_attach(), 'new' leaks */
+        enter_safe_point_if_requested();
+    }
+    else if (STM_PSEGMENT->last_commit_log_entry->next == INEV_RUNNING) {
+        /* loop until C_SEGMENT_FREE_OR_SAFE_POINT_REQ is signalled, but
+           try to detach an inevitable transaction regularly */
+        detached = fetch_detached_transaction();
+        if (detached == 0) {
+            if (!cond_wait_timeout(C_SEGMENT_FREE_OR_SAFE_POINT_REQ, 0.00001))
+                goto wait_some_more;
+        }
+    }
+    s_mutex_unlock();
+
+    if (detached != 0)
+        commit_fetched_detached_transaction(detached);
+}
+
 /* This is called to do stm_validate() and then attach 'new' at the
    head of the 'commit_log_root' chained list.  This function sleeps
    and retries until it succeeds or aborts.
@@ -523,24 +549,10 @@
 #endif
 
     if (STM_PSEGMENT->last_commit_log_entry->next == INEV_RUNNING) {
-        s_mutex_lock();
-        if (safe_point_requested()) {
-            /* XXXXXX if the safe point below aborts, 'new' leaks */
-            enter_safe_point_if_requested();
-        }
-        else if (STM_PSEGMENT->last_commit_log_entry->next == INEV_RUNNING) {
-            cond_wait(C_SEGMENT_FREE_OR_SAFE_POINT);
-        }
-        s_mutex_unlock();
+        wait_for_inevitable();
         goto retry_from_start;   /* redo _stm_validate() now */
     }
 
-    intptr_t detached = fetch_detached_transaction();
-    if (detached != 0) {
-        commit_fetched_detached_transaction(detached);
-        goto retry_from_start;
-    }
-
     /* we must not remove the WB_EXECUTED flags before validation as
        it is part of a condition in import_objects() called by
        copy_bk_objs_in_page_from to not overwrite our modifications.
@@ -1119,7 +1131,7 @@
 {
     assert(!_stm_in_transaction(tl));
 
-    while (!acquire_thread_segment(tl)) {}
+    acquire_thread_segment(tl);
     /* GS invalid before this point! */
 
     assert(STM_PSEGMENT->safe_point == SP_NO_TRANSACTION);
@@ -1571,13 +1583,33 @@
             timing_become_inevitable(); /* for tests: another transaction */
             stm_abort_transaction();    /* is already inevitable, abort   */
 #endif
+
+            bool timed_out = false;
+
             s_mutex_lock();
             if (any_soon_finished_or_inevitable_thread_segment() &&
                     !safe_point_requested()) {
-                cond_wait(C_SEGMENT_FREE_OR_SAFE_POINT);
+
+                /* wait until C_SEGMENT_FREE_OR_SAFE_POINT_REQ is signalled */
+                if (!cond_wait_timeout(C_SEGMENT_FREE_OR_SAFE_POINT_REQ,
+                                       0.000054321))
+                    timed_out = true;
             }
             s_mutex_unlock();
-            num_waits++;
+
+            if (timed_out) {
+                /* try to detach another inevitable transaction, but
+                   only after waiting a bit.  This is necessary to avoid
+                   deadlocks in some situations, which are hopefully
+                   not too common.  We don't want two threads constantly
+                   detaching each other. */
+                intptr_t detached = fetch_detached_transaction();
+                if (detached != 0)
+                    commit_fetched_detached_transaction(detached);
+            }
+            else {
+                num_waits++;
+            }
             goto retry_from_start;
         }
         if (!_validate_and_turn_inevitable())
diff --git a/c8/stm/sync.c b/c8/stm/sync.c
--- a/c8/stm/sync.c
+++ b/c8/stm/sync.c
@@ -203,10 +203,11 @@
 #endif
 
 
-static bool acquire_thread_segment(stm_thread_local_t *tl)
+static void acquire_thread_segment(stm_thread_local_t *tl)
 {
     /* This function acquires a segment for the currently running thread,
        and set up the GS register if it changed. */
+ retry_from_start:
     assert(_has_mutex());
     assert(_is_tl_registered(tl));
 
@@ -240,13 +241,13 @@
         }
     }
     /* No segment available.  Wait until release_thread_segment()
-       signals that one segment has been freed. */
+       signals that one segment has been freed.  Note that we prefer
+       waiting rather than detaching an inevitable transaction, here. */
     timing_event(tl, STM_WAIT_FREE_SEGMENT);
     cond_wait(C_SEGMENT_FREE);
     timing_event(tl, STM_WAIT_DONE);
 
-    /* Return false to the caller, which will call us again */
-    return false;
+    goto retry_from_start;
 
  got_num:
     OPT_ASSERT(num >= 0 && num < NB_SEGMENTS-1);
@@ -257,7 +258,6 @@
     assert(!in_transaction(tl));
     STM_SEGMENT->running_thread = tl;
     assert(in_transaction(tl));
-    return true;
 }
 
 static void release_thread_segment(stm_thread_local_t *tl)
@@ -266,7 +266,7 @@
     assert(_has_mutex());
 
     cond_signal(C_SEGMENT_FREE);
-    cond_broadcast(C_SEGMENT_FREE_OR_SAFE_POINT);   /* often no listener */
+    cond_broadcast(C_SEGMENT_FREE_OR_SAFE_POINT_REQ);  /* often no listener */
 
     assert(STM_SEGMENT->running_thread == tl);
     segnum = STM_SEGMENT->segment_num;
@@ -362,7 +362,7 @@
     assert(!pause_signalled);
     pause_signalled = true;
     dprintf(("request to pause\n"));
-    cond_broadcast(C_SEGMENT_FREE_OR_SAFE_POINT);
+    cond_broadcast(C_SEGMENT_FREE_OR_SAFE_POINT_REQ);
 }
 
 static inline long count_other_threads_sp_running(void)
diff --git a/c8/stm/sync.h b/c8/stm/sync.h
--- a/c8/stm/sync.h
+++ b/c8/stm/sync.h
@@ -5,7 +5,7 @@
     C_AT_SAFE_POINT,
     C_REQUEST_REMOVED,
     C_SEGMENT_FREE,
-    C_SEGMENT_FREE_OR_SAFE_POINT,
+    C_SEGMENT_FREE_OR_SAFE_POINT_REQ,
     C_QUEUE_OLD_ENTRIES,
     C_QUEUE_FINISHED_MORE_TASKS,
     _C_TOTAL
@@ -25,7 +25,7 @@
 
 /* acquire and release one of the segments for running the given thread
    (must have the mutex acquired!) */
-static bool acquire_thread_segment(stm_thread_local_t *tl);
+static void acquire_thread_segment(stm_thread_local_t *tl);
 static void release_thread_segment(stm_thread_local_t *tl);
 static void soon_finished_or_inevitable_thread_segment(void);
 static bool any_soon_finished_or_inevitable_thread_segment(void);
diff --git a/c8/test/test_hashtable.py b/c8/test/test_hashtable.py
--- a/c8/test/test_hashtable.py
+++ b/c8/test/test_hashtable.py
@@ -46,6 +46,8 @@
             try:
                 assert lib._get_type_id(obj) == 421419
                 self.seen_hashtables -= 1
+                h = get_hashtable(obj)
+                lib.stm_hashtable_free(h)
             except:
                 self.errors.append(sys.exc_info()[2])
                 raise

From noreply at buildbot.pypy.org  Tue Jun 23 09:28:20 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 23 Jun 2015 09:28:20 +0200 (CEST)
Subject: [pypy-commit] pypy default: Mostly no-op reorganization. It avoids
 promoting the whole W_FunctionWrapper
Message-ID: <20150623072820.C3D111C1FBA@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78248:53cddb11ed7c
Date: 2015-06-23 09:25 +0200
http://bitbucket.org/pypy/pypy/changeset/53cddb11ed7c/

Log:	Mostly no-op reorganization. It avoids promoting the whole
	W_FunctionWrapper instance, and instead promotes only its
	rawfunctype attribute.

diff --git a/pypy/module/_cffi_backend/lib_obj.py b/pypy/module/_cffi_backend/lib_obj.py
--- a/pypy/module/_cffi_backend/lib_obj.py
+++ b/pypy/module/_cffi_backend/lib_obj.py
@@ -60,12 +60,12 @@
             self.ffi, self.ctx.c_types, getarg(g.c_type_op))
         assert isinstance(rawfunctype, realize_c_type.W_RawFuncType)
         #
-        w_ct, locs = rawfunctype.unwrap_as_nostruct_fnptr(self.ffi)
+        rawfunctype.prepare_nostruct_fnptr(self.ffi)
         #
         ptr = rffi.cast(rffi.CCHARP, g.c_address)
         assert ptr
-        return W_FunctionWrapper(self.space, ptr, g.c_size_or_direct_fn, w_ct,
-                                 locs, rawfunctype, fnname, self.libname)
+        return W_FunctionWrapper(self.space, ptr, g.c_size_or_direct_fn,
+                                 rawfunctype, fnname, self.libname)
 
     @jit.elidable_promote()
     def _get_attr_elidable(self, attr):
@@ -237,7 +237,8 @@
         if isinstance(w_value, W_FunctionWrapper):
             # '&func' returns a regular cdata pointer-to-function
             if w_value.directfnptr:
-                return W_CData(space, w_value.directfnptr, w_value.ctype)
+                ctype = w_value.rawfunctype.nostruct_ctype
+                return W_CData(space, w_value.directfnptr, ctype)
             else:
                 return w_value    # backward compatibility
         #
diff --git a/pypy/module/_cffi_backend/realize_c_type.py b/pypy/module/_cffi_backend/realize_c_type.py
--- a/pypy/module/_cffi_backend/realize_c_type.py
+++ b/pypy/module/_cffi_backend/realize_c_type.py
@@ -1,4 +1,5 @@
 import sys
+from rpython.rlib import jit
 from rpython.rlib.rarithmetic import intmask
 from rpython.rlib.objectmodel import specialize
 from rpython.rtyper.lltypesystem import lltype, rffi
@@ -135,8 +136,12 @@
 
 class W_RawFuncType(W_Root):
     """Temporary: represents a C function type (not a function pointer)"""
+
+    _immutable_fields_ = ['nostruct_ctype', 'nostruct_locs', 'nostruct_nargs']
     _ctfuncptr = None
-    _nostruct_ctfuncptr = (None, None)
+    nostruct_ctype = None
+    nostruct_locs = None
+    nostruct_nargs = 0
 
     def __init__(self, opcodes, base_index):
         self.opcodes = opcodes
@@ -168,14 +173,16 @@
         assert self._ctfuncptr is not None
         return self._ctfuncptr
 
-    def unwrap_as_nostruct_fnptr(self, ffi):
-        # tweaked version: instead of returning the ctfuncptr corresponding
-        # exactly to the OP_FUNCTION ... OP_FUNCTION_END opcodes, return
-        # another one in which the struct args are replaced with ptr-to-
-        # struct, and a struct return value is replaced with a hidden first
-        # arg of type ptr-to-struct.  This is how recompiler.py produces
+    @jit.dont_look_inside
+    def prepare_nostruct_fnptr(self, ffi):
+        # tweaked version: instead of returning the ctfuncptr
+        # corresponding exactly to the OP_FUNCTION ... OP_FUNCTION_END
+        # opcodes, this builds in self.nostruct_ctype another one in
+        # which the struct args are replaced with ptr-to- struct, and
+        # a struct return value is replaced with a hidden first arg of
+        # type ptr-to-struct.  This is how recompiler.py produces
         # trampoline functions for PyPy.
-        if self._nostruct_ctfuncptr[0] is None:
+        if self.nostruct_ctype is None:
             fargs, fret, ellipsis = self._unpack(ffi)
             # 'locs' will be a string of the same length as the final fargs,
             # containing 'A' where a struct argument was detected, and 'R'
@@ -198,8 +205,10 @@
                 locs = None
             else:
                 locs = ''.join(locs)
-            self._nostruct_ctfuncptr = (ctfuncptr, locs)
-        return self._nostruct_ctfuncptr
+            self.nostruct_ctype = ctfuncptr
+            self.nostruct_locs = locs
+            self.nostruct_nargs = len(ctfuncptr.fargs) - (locs is not None and
+                                                          locs[0] == 'R')
 
     def unexpected_fn_type(self, ffi):
         fargs, fret, ellipsis = self._unpack(ffi)
diff --git a/pypy/module/_cffi_backend/wrapper.py b/pypy/module/_cffi_backend/wrapper.py
--- a/pypy/module/_cffi_backend/wrapper.py
+++ b/pypy/module/_cffi_backend/wrapper.py
@@ -25,8 +25,14 @@
     _immutable_ = True
     common_doc_str = 'direct call to the C function of the same name'
 
-    def __init__(self, space, fnptr, directfnptr, ctype,
-                 locs, rawfunctype, fnname, modulename):
+    def __init__(self, space, fnptr, directfnptr,
+                 rawfunctype, fnname, modulename):
+        # everything related to the type of the function is accessed
+        # as immutable attributes of the 'rawfunctype' object, which
+        # is a W_RawFuncType.  This gives us an obvious thing to
+        # promote in order to do the call.
+        ctype = rawfunctype.nostruct_ctype
+        locs = rawfunctype.nostruct_locs
         assert isinstance(ctype, W_CTypeFunc)
         assert ctype.cif_descr is not None     # not for '...' functions
         assert locs is None or len(ctype.fargs) == len(locs)
@@ -34,84 +40,86 @@
         self.space = space
         self.fnptr = fnptr
         self.directfnptr = directfnptr
-        self.ctype = ctype
-        self.locs = locs
         self.rawfunctype = rawfunctype
         self.fnname = fnname
         self.modulename = modulename
-        self.nargs_expected = len(ctype.fargs) - (locs is not None and
-                                                  locs[0] == 'R')
 
     def typeof(self, ffi):
         return self.rawfunctype.unwrap_as_fnptr(ffi)
 
-    @jit.unroll_safe
-    def _prepare(self, args_w, start_index):
-        # replaces struct/union arguments with ptr-to-struct/union arguments
+    def descr_call(self, args_w):
         space = self.space
-        locs = self.locs
-        fargs = self.ctype.fargs
-        for i in range(start_index, len(locs)):
-            if locs[i] != 'A':
-                continue
-            w_arg = args_w[i]
-            farg = fargs[i]      # <ptr to struct/union>
-            assert isinstance(farg, W_CTypePtrOrArray)
-            if isinstance(w_arg, W_CData) and w_arg.ctype is farg.ctitem:
-                # fast way: we are given a W_CData "struct", so just make
-                # a new W_CData "ptr-to-struct" which points to the same
-                # raw memory.  We use unsafe_escaping_ptr(), so we have to
-                # make sure the original 'w_arg' stays alive; the easiest
-                # is to build an instance of W_CDataPtrToStructOrUnion.
-                w_arg = W_CDataPtrToStructOrUnion(
-                    space, w_arg.unsafe_escaping_ptr(), farg, w_arg)
-            else:
-                # slow way: build a new "ptr to struct" W_CData by calling
-                # the equivalent of ffi.new()
-                if space.is_w(w_arg, space.w_None):
-                    continue
-                w_arg = farg.newp(w_arg)
-            args_w[i] = w_arg
-
-    def descr_call(self, args_w):
-        self = jit.promote(self)
-        if len(args_w) != self.nargs_expected:
-            space = self.space
-            if self.nargs_expected == 0:
+        rawfunctype = jit.promote(self.rawfunctype)
+        ctype = rawfunctype.nostruct_ctype
+        locs = rawfunctype.nostruct_locs
+        nargs_expected = rawfunctype.nostruct_nargs
+        #
+        if len(args_w) != nargs_expected:
+            if nargs_expected == 0:
                 raise oefmt(space.w_TypeError,
                             "%s() takes no arguments (%d given)",
                             self.fnname, len(args_w))
-            elif self.nargs_expected == 1:
+            elif nargs_expected == 1:
                 raise oefmt(space.w_TypeError,
                             "%s() takes exactly one argument (%d given)",
                             self.fnname, len(args_w))
             else:
                 raise oefmt(space.w_TypeError,
                             "%s() takes exactly %d arguments (%d given)",
-                            self.fnname, self.nargs_expected, len(args_w))
+                            self.fnname, nargs_expected, len(args_w))
         #
-        if self.locs is not None:
+        if locs is not None:
             # This case is if there are structs as arguments or return values.
             # If the result we want to present to the user is "returns struct",
             # then internally allocate the struct and pass a pointer to it as
             # a first argument.
-            if self.locs[0] == 'R':
-                w_result_cdata = self.ctype.fargs[0].newp(self.space.w_None)
+            if locs[0] == 'R':
+                w_result_cdata = ctype.fargs[0].newp(space.w_None)
                 args_w = [w_result_cdata] + args_w
-                self._prepare(args_w, 1)
-                self.ctype._call(self.fnptr, args_w)    # returns w_None
+                prepare_args(space, rawfunctype, args_w, 1)
+                #
+                ctype._call(self.fnptr, args_w)    # returns w_None
+                #
                 assert isinstance(w_result_cdata, W_CDataPtrToStructOrUnion)
                 return w_result_cdata.structobj
             else:
                 args_w = args_w[:]
-                self._prepare(args_w, 0)
+                prepare_args(space, rawfunctype, args_w, 0)
         #
-        return self.ctype._call(self.fnptr, args_w)
+        return ctype._call(self.fnptr, args_w)
 
     def descr_repr(self, space):
         return space.wrap("<FFIFunctionWrapper for %s()>" % (self.fnname,))
 
 
+ at jit.unroll_safe
+def prepare_args(space, rawfunctype, args_w, start_index):
+    # replaces struct/union arguments with ptr-to-struct/union arguments
+    locs = rawfunctype.nostruct_locs
+    fargs = rawfunctype.nostruct_ctype.fargs
+    for i in range(start_index, len(locs)):
+        if locs[i] != 'A':
+            continue
+        w_arg = args_w[i]
+        farg = fargs[i]      # <ptr to struct/union>
+        assert isinstance(farg, W_CTypePtrOrArray)
+        if isinstance(w_arg, W_CData) and w_arg.ctype is farg.ctitem:
+            # fast way: we are given a W_CData "struct", so just make
+            # a new W_CData "ptr-to-struct" which points to the same
+            # raw memory.  We use unsafe_escaping_ptr(), so we have to
+            # make sure the original 'w_arg' stays alive; the easiest
+            # is to build an instance of W_CDataPtrToStructOrUnion.
+            w_arg = W_CDataPtrToStructOrUnion(
+                space, w_arg.unsafe_escaping_ptr(), farg, w_arg)
+        else:
+            # slow way: build a new "ptr to struct" W_CData by calling
+            # the equivalent of ffi.new()
+            if space.is_w(w_arg, space.w_None):
+                continue
+            w_arg = farg.newp(w_arg)
+        args_w[i] = w_arg
+
+
 W_FunctionWrapper.typedef = TypeDef(
         'FFIFunctionWrapper',
         __repr__ = interp2app(W_FunctionWrapper.descr_repr),

From noreply at buildbot.pypy.org  Sun Jun 21 14:32:52 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 21 Jun 2015 14:32:52 +0200 (CEST)
Subject: [pypy-commit] pypy default: Duplicate 9a1683dd96e2 here
Message-ID: <20150621123252.EC8381C2003@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78230:9f36a6b3d844
Date: 2015-06-21 14:33 +0200
http://bitbucket.org/pypy/pypy/changeset/9f36a6b3d844/

Log:	Duplicate 9a1683dd96e2 here

diff --git a/pypy/module/_io/interp_textio.py b/pypy/module/_io/interp_textio.py
--- a/pypy/module/_io/interp_textio.py
+++ b/pypy/module/_io/interp_textio.py
@@ -605,6 +605,10 @@
     def read_w(self, space, w_size=None):
         self._check_attached(space)
         if not self.w_decoder:
+            # very unsure about the following check, but some tests seem
+            # to expect a ValueError instead of an IOError in case the
+            # file was already closed.
+            self._check_closed(space)
             raise OperationError(space.w_IOError, space.wrap("not readable"))
 
         size = convert_size(space, w_size)

From noreply at buildbot.pypy.org  Sun Jun 21 21:32:22 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Sun, 21 Jun 2015 21:32:22 +0200 (CEST)
Subject: [pypy-commit] pypy run-create_cffi_imports: merge default into
	branch
Message-ID: <20150621193222.3A3291C1FF5@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: run-create_cffi_imports
Changeset: r78233:2217963bdd14
Date: 2015-06-21 22:32 +0300
http://bitbucket.org/pypy/pypy/changeset/2217963bdd14/

Log:	merge default into branch

diff --git a/lib_pypy/_tkinter/tklib_build.py b/lib_pypy/_tkinter/tklib_build.py
--- a/lib_pypy/_tkinter/tklib_build.py
+++ b/lib_pypy/_tkinter/tklib_build.py
@@ -179,6 +179,7 @@
 typedef int... Tcl_WideInt;
 
 int Tcl_GetWideIntFromObj(Tcl_Interp *interp, Tcl_Obj *obj, Tcl_WideInt *value);
+Tcl_Obj *Tcl_NewWideIntObj(Tcl_WideInt value);
 """)
 
 if HAVE_LIBTOMMATH:
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -11,3 +11,8 @@
 .. branch: stdlib-2.7.10
 
 Update stdlib to version 2.7.10
+
+.. branch: issue2062
+
+.. branch: disable-unroll-for-short-loops
+The JIT no longer performs loop unrolling if the loop compiles to too much code.
diff --git a/pypy/interpreter/pytraceback.py b/pypy/interpreter/pytraceback.py
--- a/pypy/interpreter/pytraceback.py
+++ b/pypy/interpreter/pytraceback.py
@@ -60,7 +60,6 @@
 
 
 def check_traceback(space, w_tb, msg):
-    from pypy.interpreter.typedef import PyTraceback
     if w_tb is None or not space.isinstance_w(w_tb, space.gettypeobject(PyTraceback.typedef)):
         raise OperationError(space.w_TypeError, space.wrap(msg))
     return w_tb
diff --git a/pypy/module/_cffi_backend/lib_obj.py b/pypy/module/_cffi_backend/lib_obj.py
--- a/pypy/module/_cffi_backend/lib_obj.py
+++ b/pypy/module/_cffi_backend/lib_obj.py
@@ -173,6 +173,8 @@
             if w_value is None:
                 if is_getattr and attr == '__all__':
                     return self.dir1(ignore_type=cffi_opcode.OP_GLOBAL_VAR)
+                if is_getattr and attr == '__dict__':
+                    return self.full_dict_copy()
                 raise oefmt(self.space.w_AttributeError,
                             "cffi library '%s' has no function, constant "
                             "or global variable named '%s'",
@@ -212,6 +214,17 @@
                 names_w.append(space.wrap(rffi.charp2str(g[i].c_name)))
         return space.newlist(names_w)
 
+    def full_dict_copy(self):
+        space = self.space
+        total = rffi.getintfield(self.ctx, 'c_num_globals')
+        g = self.ctx.c_globals
+        w_result = space.newdict()
+        for i in range(total):
+            w_attr = space.wrap(rffi.charp2str(g[i].c_name))
+            w_value = self._get_attr(w_attr)
+            space.setitem(w_result, w_attr, w_value)
+        return w_result
+
     def address_of_func_or_global_var(self, varname):
         # rebuild a string object from 'varname', to do typechecks and
         # to force a unicode back to a plain string
diff --git a/pypy/module/_cffi_backend/test/test_recompiler.py b/pypy/module/_cffi_backend/test/test_recompiler.py
--- a/pypy/module/_cffi_backend/test/test_recompiler.py
+++ b/pypy/module/_cffi_backend/test/test_recompiler.py
@@ -276,6 +276,15 @@
         """)
         lib.aa = 5
         assert dir(lib) == ['aa', 'ff', 'my_constant']
+        #
+        aaobj = lib.__dict__['aa']
+        assert not isinstance(aaobj, int)    # some internal object instead
+        assert lib.__dict__ == {
+            'ff': lib.ff,
+            'aa': aaobj,
+            'my_constant': -45}
+        lib.__dict__['ff'] = "??"
+        assert lib.ff(10) == 15
 
     def test_verify_opaque_struct(self):
         ffi, lib = self.prepare(
@@ -984,5 +993,5 @@
         assert sys.modules['_CFFI_test_import_from_lib.lib'] is lib
         from _CFFI_test_import_from_lib.lib import MYFOO
         assert MYFOO == 42
-        assert not hasattr(lib, '__dict__')
+        assert hasattr(lib, '__dict__')
         assert lib.__all__ == ['MYFOO', 'mybar']   # but not 'myvar'
diff --git a/pypy/module/_io/interp_textio.py b/pypy/module/_io/interp_textio.py
--- a/pypy/module/_io/interp_textio.py
+++ b/pypy/module/_io/interp_textio.py
@@ -548,6 +548,10 @@
         remain buffered in the decoder, yet to be converted."""
 
         if not self.w_decoder:
+            # very unsure about the following check, but some tests seem
+            # to expect a ValueError instead of an IOError in case the
+            # file was already closed.
+            self._check_closed(space)
             raise OperationError(space.w_IOError, space.wrap("not readable"))
 
         if self.telling:
@@ -601,6 +605,10 @@
     def read_w(self, space, w_size=None):
         self._check_attached(space)
         if not self.w_decoder:
+            # very unsure about the following check, but some tests seem
+            # to expect a ValueError instead of an IOError in case the
+            # file was already closed.
+            self._check_closed(space)
             raise OperationError(space.w_IOError, space.wrap("not readable"))
 
         size = convert_size(space, w_size)
diff --git a/pypy/module/_socket/__init__.py b/pypy/module/_socket/__init__.py
--- a/pypy/module/_socket/__init__.py
+++ b/pypy/module/_socket/__init__.py
@@ -18,6 +18,10 @@
         from rpython.rlib.rsocket import rsocket_startup
         rsocket_startup()
 
+    def shutdown(self, space):
+        from pypy.module._socket.interp_socket import close_all_sockets
+        close_all_sockets(space)
+
     def buildloaders(cls):
         from rpython.rlib import rsocket
         for name in """
diff --git a/pypy/module/_socket/interp_func.py b/pypy/module/_socket/interp_func.py
--- a/pypy/module/_socket/interp_func.py
+++ b/pypy/module/_socket/interp_func.py
@@ -142,7 +142,7 @@
         sock = rsocket.fromfd(fd, family, type, proto)
     except SocketError, e:
         raise converted_error(space, e)
-    return space.wrap(W_Socket(sock))
+    return space.wrap(W_Socket(space, sock))
 
 @unwrap_spec(family=int, type=int, proto=int)
 def socketpair(space, family=rsocket.socketpair_default_family,
@@ -160,8 +160,8 @@
     except SocketError, e:
         raise converted_error(space, e)
     return space.newtuple([
-        space.wrap(W_Socket(sock1)),
-        space.wrap(W_Socket(sock2))
+        space.wrap(W_Socket(space, sock1)),
+        space.wrap(W_Socket(space, sock2))
     ])
 
 # The following 4 functions refuse all negative numbers, like CPython 2.6.
diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py
--- a/pypy/module/_socket/interp_socket.py
+++ b/pypy/module/_socket/interp_socket.py
@@ -1,4 +1,5 @@
-from rpython.rlib import rsocket
+import sys
+from rpython.rlib import rsocket, rweaklist
 from rpython.rlib.rarithmetic import intmask
 from rpython.rlib.rsocket import (
     RSocket, AF_INET, SOCK_STREAM, SocketError, SocketErrorWithErrno,
@@ -153,8 +154,9 @@
 
 
 class W_Socket(W_Root):
-    def __init__(self, sock):
+    def __init__(self, space, sock):
         self.sock = sock
+        register_socket(space, sock)
 
     def get_type_w(self, space):
         return space.wrap(self.sock.type)
@@ -183,7 +185,7 @@
             fd, addr = self.sock.accept()
             sock = rsocket.make_socket(
                 fd, self.sock.family, self.sock.type, self.sock.proto)
-            return space.newtuple([space.wrap(W_Socket(sock)),
+            return space.newtuple([space.wrap(W_Socket(space, sock)),
                                    addr_as_object(addr, sock.fd, space)])
         except SocketError as e:
             raise converted_error(space, e)
@@ -248,7 +250,7 @@
     def dup_w(self, space):
         try:
             sock = self.sock.dup()
-            return W_Socket(sock)
+            return W_Socket(space, sock)
         except SocketError as e:
             raise converted_error(space, e)
 
@@ -592,10 +594,50 @@
         sock = RSocket(family, type, proto)
     except SocketError as e:
         raise converted_error(space, e)
-    W_Socket.__init__(self, sock)
+    W_Socket.__init__(self, space, sock)
     return space.wrap(self)
 descr_socket_new = interp2app(newsocket)
 
+
+# ____________________________________________________________
+# Automatic shutdown()/close()
+
+# On some systems, the C library does not guarantee that when the program
+# finishes, all data sent so far is really sent even if the socket is not
+# explicitly closed.  This behavior has been observed on Windows but not
+# on Linux, so far.
+NEED_EXPLICIT_CLOSE = (sys.platform == 'win32')
+
+class OpenRSockets(rweaklist.RWeakListMixin):
+    pass
+class OpenRSocketsState:
+    def __init__(self, space):
+        self.openrsockets = OpenRSockets()
+        self.openrsockets.initialize()
+
+def getopenrsockets(space):
+    if NEED_EXPLICIT_CLOSE and space.config.translation.rweakref:
+        return space.fromcache(OpenRSocketsState).openrsockets
+    else:
+        return None
+
+def register_socket(space, socket):
+    openrsockets = getopenrsockets(space)
+    if openrsockets is not None:
+        openrsockets.add_handle(socket)
+
+def close_all_sockets(space):
+    openrsockets = getopenrsockets(space)
+    if openrsockets is not None:
+        for sock_wref in openrsockets.get_all_handles():
+            sock = sock_wref()
+            if sock is not None:
+                try:
+                    sock.close()
+                except SocketError:
+                    pass
+
+
 # ____________________________________________________________
 # Error handling
 
diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py
--- a/pypy/module/_socket/test/test_sock_app.py
+++ b/pypy/module/_socket/test/test_sock_app.py
@@ -309,10 +309,16 @@
 
 
 class AppTestSocket:
+    spaceconfig = dict(usemodules=['_socket', '_weakref', 'struct'])
+
     def setup_class(cls):
         cls.space = space
         cls.w_udir = space.wrap(str(udir))
 
+    def teardown_class(cls):
+        if not cls.runappdirect:
+            cls.space.sys.getmodule('_socket').shutdown(cls.space)
+
     def test_module(self):
         import _socket
         assert _socket.socket.__name__ == 'socket'
@@ -614,6 +620,12 @@
         finally:
             os.chdir(oldcwd)
 
+    def test_automatic_shutdown(self):
+        # doesn't really test anything, but at least should not explode
+        # in close_all_sockets()
+        import _socket
+        self.foo = _socket.socket()
+
 
 class AppTestPacket:
     def setup_class(cls):
diff --git a/pypy/module/cpyext/test/test_version.py b/pypy/module/cpyext/test/test_version.py
--- a/pypy/module/cpyext/test/test_version.py
+++ b/pypy/module/cpyext/test/test_version.py
@@ -16,7 +16,7 @@
         }
         """
         module = self.import_module(name='foo', init=init)
-        assert module.py_version == sys.version[:5]
+        assert module.py_version == '%d.%d.%d' % sys.version_info[:3]
         assert module.py_major_version == sys.version_info.major
         assert module.py_minor_version == sys.version_info.minor
         assert module.py_micro_version == sys.version_info.micro
diff --git a/pypy/module/struct/test/test_struct.py b/pypy/module/struct/test/test_struct.py
--- a/pypy/module/struct/test/test_struct.py
+++ b/pypy/module/struct/test/test_struct.py
@@ -390,9 +390,9 @@
                                       self.struct.pack("ii", 17, 42) +
                                       '\x00' * (19-sz-2))
         exc = raises(TypeError, self.struct.pack_into, "ii", buffer(b), 0, 17, 42)
-        assert str(exc.value) == "argument must be read-write buffer, not buffer"
+        assert str(exc.value) == "must be read-write buffer, not buffer"
         exc = raises(TypeError, self.struct.pack_into, "ii", 'test', 0, 17, 42)
-        assert str(exc.value) == "argument must be read-write buffer, not str"
+        assert str(exc.value) == "must be read-write buffer, not str"
         exc = raises(self.struct.error, self.struct.pack_into, "ii", b[0:1], 0, 17, 42)
         assert str(exc.value) == "pack_into requires a buffer of at least 8 bytes"
 
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
@@ -266,6 +266,15 @@
     """)
     lib.aa = 5
     assert dir(lib) == ['aa', 'ff', 'my_constant']
+    #
+    aaobj = lib.__dict__['aa']
+    assert not isinstance(aaobj, int)    # some internal object instead
+    assert lib.__dict__ == {
+        'ff': lib.ff,
+        'aa': aaobj,
+        'my_constant': -45}
+    lib.__dict__['ff'] = "??"
+    assert lib.ff(10) == 15
 
 def test_verify_opaque_struct():
     ffi = FFI()
@@ -1053,5 +1062,5 @@
     assert sys.modules['_CFFI_test_import_from_lib.lib'] is lib
     from _CFFI_test_import_from_lib.lib import MYFOO
     assert MYFOO == 42
-    assert not hasattr(lib, '__dict__')
+    assert hasattr(lib, '__dict__')
     assert lib.__all__ == ['MYFOO', 'mybar']   # but not 'myvar'
diff --git a/rpython/flowspace/flowcontext.py b/rpython/flowspace/flowcontext.py
--- a/rpython/flowspace/flowcontext.py
+++ b/rpython/flowspace/flowcontext.py
@@ -1207,7 +1207,8 @@
     def nomoreblocks(self, ctx):
         w_exc = self.w_exc
         if w_exc.w_type == const(ImportError):
-            msg = 'import statement always raises %s' % self
+            msg = 'ImportError is raised in RPython: %s' % (
+                getattr(w_exc.w_value, 'value', '<not a constant message>'),)
             raise ImportError(msg)
         link = Link([w_exc.w_type, w_exc.w_value], ctx.graph.exceptblock)
         ctx.recorder.crnt_block.closeblock(link)
diff --git a/rpython/flowspace/test/cant_import.py b/rpython/flowspace/test/cant_import.py
new file mode 100644
--- /dev/null
+++ b/rpython/flowspace/test/cant_import.py
@@ -0,0 +1,1 @@
+raise ImportError("some explanation here")
diff --git a/rpython/flowspace/test/test_objspace.py b/rpython/flowspace/test/test_objspace.py
--- a/rpython/flowspace/test/test_objspace.py
+++ b/rpython/flowspace/test/test_objspace.py
@@ -816,6 +816,12 @@
             from rpython import this_does_not_exist
         py.test.raises(ImportError, 'self.codetest(f)')
 
+    def test_importerror_3(self):
+        def f():
+            import rpython.flowspace.test.cant_import
+        e = py.test.raises(ImportError, 'self.codetest(f)')
+        assert "some explanation here" in str(e.value)
+
     def test_relative_import(self):
         def f():
             from ..objspace import build_flow

From noreply at buildbot.pypy.org  Sun Jun 21 12:13:35 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 21 Jun 2015 12:13:35 +0200 (CEST)
Subject: [pypy-commit] pypy default: fix this test
Message-ID: <20150621101335.393C81C1DD7@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78224:743f2cc270b2
Date: 2015-06-21 12:13 +0200
http://bitbucket.org/pypy/pypy/changeset/743f2cc270b2/

Log:	fix this test

diff --git a/pypy/module/cpyext/test/test_version.py b/pypy/module/cpyext/test/test_version.py
--- a/pypy/module/cpyext/test/test_version.py
+++ b/pypy/module/cpyext/test/test_version.py
@@ -16,7 +16,7 @@
         }
         """
         module = self.import_module(name='foo', init=init)
-        assert module.py_version == sys.version[:5]
+        assert module.py_version == '%d.%d.%d' % sys.version_info[:3]
         assert module.py_major_version == sys.version_info.major
         assert module.py_minor_version == sys.version_info.minor
         assert module.py_micro_version == sys.version_info.micro

From noreply at buildbot.pypy.org  Sun Jun 21 21:32:20 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Sun, 21 Jun 2015 21:32:20 +0200 (CEST)
Subject: [pypy-commit] pypy dtypes-compatability: wip - fixing align
Message-ID: <20150621193220.9C4D71C1FF3@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: dtypes-compatability
Changeset: r78232:90f488ee3b1d
Date: 2015-06-21 22:31 +0300
http://bitbucket.org/pypy/pypy/changeset/90f488ee3b1d/

Log:	wip - fixing align

diff --git a/pypy/module/micronumpy/descriptor.py b/pypy/module/micronumpy/descriptor.py
--- a/pypy/module/micronumpy/descriptor.py
+++ b/pypy/module/micronumpy/descriptor.py
@@ -561,17 +561,20 @@
 
 
 @specialize.arg(2)
-def dtype_from_list(space, w_lst, simple, align=False):
+def dtype_from_list(space, w_lst, simple, align=False, offsets=None):
     lst_w = space.listview(w_lst)
     fields = {}
-    offset = 0
+    if offsets is None:
+        offsets = [0] * len(lst_w)
     names = []
     maxalign = 0
+    total = 0
     for i in range(len(lst_w)):
         w_elem = lst_w[i]
         title = None
         if simple:
-            subdtype = descr__new__(space, space.gettypefor(W_Dtype), w_elem)
+            subdtype = descr__new__(space, space.gettypefor(W_Dtype), w_elem,
+                                    align=align)
             fldname = 'f%d' % i
         else:
             w_shape = space.newtuple([])
@@ -581,7 +584,8 @@
                     w_shape = space.newtuple([w_shape])
             else:
                 w_fldname, w_flddesc = space.fixedview(w_elem, 2)
-            subdtype = descr__new__(space, space.gettypefor(W_Dtype), w_flddesc, w_shape=w_shape)
+            subdtype = descr__new__(space, space.gettypefor(W_Dtype),
+                                    w_flddesc, w_shape=w_shape, align=align)
             if space.isinstance_w(w_fldname, space.w_tuple):
                 fldlist = space.listview(w_fldname)
                 fldname = space.str_w(fldlist[0])
@@ -597,17 +601,27 @@
             if fldname in fields:
                 raise oefmt(space.w_ValueError, "two fields with the same name")
         assert isinstance(subdtype, W_Dtype)
-        fields[fldname] = offset, subdtype
+        fields[fldname] = offsets[i], subdtype
         if title is not None:
-            fields[title] = offset, subdtype
-        offset += subdtype.elsize
+            fields[title] = offsets[i], subdtype
         maxalign = max(subdtype.elsize, maxalign)
+        if  i + 1 < len(offsets) and offsets[i + 1] == 0:
+            delta = subdtype.elsize
+            if align:
+                # Set offset to the next power-of-two above delta
+                delta = (delta + maxalign -1) & (-maxalign)
+                if delta > offsets[i]:
+                    for j in range(i):
+                        offsets[j+1] = delta + offsets[j]
+            offsets[i + 1] = offsets[i] + delta
+            print maxalign, delta, subdtype, subdtype.elsize
+        if align:
+            total = len(offsets) * maxalign
+        else:
+            total += subdtype.elsize
         names.append((fldname, title))
-    if align:
-        # Set offset to the next power-of-two above offset
-        offset = (offset + maxalign -1) & (-maxalign)
     retval = W_Dtype(types.RecordType(space), space.gettypefor(boxes.W_VoidBox),
-                   names=names, fields=fields, elsize=offset)
+                   names=names, fields=fields, elsize=total)
     retval.flags |= NPY.NEEDS_PYAPI
     return retval
 
@@ -628,6 +642,28 @@
         return None
     return space.listview(w_val)
 
+def _usefields(space, w_dict, align):
+    # Only for testing, a shortened version of the real _usefields
+    allfields = []
+    for fname in w_dict.iterkeys().iterator:
+        obj = _get_list_or_none(space, w_dict, fname)
+        num = space.int_w(obj[1])
+        format = dtype_from_spec(space, obj[0], align=align)
+        if len(obj) > 2:
+            title = obj[2]
+        else:
+            title = None
+        allfields.append((fname, format, num, title))
+    allfields.sort(key=lambda x: x[2])
+    names   = [x[0] for x in allfields]
+    formats = [x[1] for x in allfields]
+    offsets = [x[2] for x in allfields]
+    titles  = [x[3] for x in allfields]
+    aslist = []
+    for i in range(len(names)):
+        aslist.append(space.newtuple([space.wrap(names[i]), space.wrap(formats[i])]))
+    return dtype_from_list(space, space.newlist(aslist), False, align=align, offsets=offsets)
+    
 def dtype_from_dict(space, w_dict, align):
     from pypy.objspace.std.dictmultiobject import W_DictMultiObject
     assert isinstance(w_dict, W_DictMultiObject)
@@ -638,8 +674,11 @@
     metadata_w = _get_val_or_none(space, w_dict, 'metadata')
     aligned_w = _get_val_or_none(space, w_dict, 'align')
     if names_w is None or formats_w is None:
-        return get_appbridge_cache(space).call_method(space,
-            'numpy.core._internal', '_usefields', Arguments(space, [w_dict, space.wrap(align)]))
+        if we_are_translated():
+            return get_appbridge_cache(space).call_method(space,
+                'numpy.core._internal', '_usefields', Arguments(space, [w_dict, space.wrap(align)]))
+        else:
+            return _usefields(space, w_dict, align)
     n = len(names_w)
     if (n != len(formats_w) or 
         (offsets_w is not None and n != len(offsets_w)) or
@@ -653,9 +692,10 @@
             raise oefmt(space.w_ValueError,
                     "NumPy dtype descriptor includes 'aligned' entry, "
                     "but its value is neither True nor False");
-    if offsets_w is not None:
-        raise oefmt(space.w_NotImplementedError, "'offsets' "
-            "dict entries not implemented yet") 
+    if offsets_w is None:
+        offsets = None
+    else:
+        offsets = [space.int_w(i) for i in offsets_w]
     if titles_w is not None:
         names_w = []
         for i in range(min(len(names_w), len(titles_w))):
@@ -663,34 +703,29 @@
     aslist = []
     for i in range(min(len(names_w), len(formats_w))):
         aslist.append(space.newtuple([names_w[i], formats_w[i]]))
-    retval = dtype_from_list(space, space.newlist(aslist), False, align)
+    retval = dtype_from_list(space, space.newlist(aslist), False, align=align, offsets=offsets)
     if metadata_w is not None:
         retval.descr_set_metadata(space, metadata_w)
     retval.flags |= NPY.NEEDS_PYAPI
     return retval 
 
-def dtype_from_spec(space, w_spec):
+def dtype_from_spec(space, w_spec, align):
 
     if we_are_translated():
         w_lst = get_appbridge_cache(space).call_method(space,
             'numpy.core._internal', '_commastring', Arguments(space, [w_spec]))
     else:
-        # testing, handle manually
-        if space.eq_w(w_spec, space.wrap('u4,u4,u4')):
-            w_lst = space.newlist([space.wrap('u4')]*3)
-        if space.eq_w(w_spec, space.wrap('u4,u4,u4')):
-            w_lst = space.newlist([space.wrap('u4')]*3)
-        else:
-            raise oefmt(space.w_RuntimeError,
-                    "cannot parse w_spec")
+        # handle only simple cases for testing
+        spec = [s.strip() for s in space.str_w(w_spec).split(',')]
+        w_lst = space.newlist([space.wrap(s) for s in spec]) 
     if not space.isinstance_w(w_lst, space.w_list) or space.len_w(w_lst) < 1:
         raise oefmt(space.w_RuntimeError,
                     "_commastring is not returning a list with len >= 1")
     if space.len_w(w_lst) == 1:
         return descr__new__(space, space.gettypefor(W_Dtype),
-                            space.getitem(w_lst, space.wrap(0)))
+                            space.getitem(w_lst, space.wrap(0)), align=align)
     else:
-        return dtype_from_list(space, w_lst, True)
+        return dtype_from_list(space, w_lst, True, align=align)
 
 
 def _check_for_commastring(s):
@@ -769,7 +804,7 @@
     if space.isinstance_w(w_dtype, space.w_str):
         name = space.str_w(w_dtype)
         if _check_for_commastring(name):
-            return _set_metadata_and_copy(dtype_from_spec(space, w_dtype), space, w_metadata)
+            return _set_metadata_and_copy(dtype_from_spec(space, w_dtype, align), space, w_metadata)
         cname = name[1:] if name[0] == NPY.OPPBYTE else name
         try:
             dtype = cache.dtypes_by_name[cname]
diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py
--- a/pypy/module/micronumpy/test/test_dtypes.py
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -1327,6 +1327,7 @@
         dt1 = np.dtype([('f0', 'i4'),
                        ('f1', [('f1', 'i1'), ('f2', 'i4'), ('f3', 'i1')]),
                        ('f2', 'i1')], align=True)
+        assert dt1['f1'].itemsize == 12
         assert dt1.itemsize == 20
         dt2 = np.dtype({'names':['f0', 'f1', 'f2'],
                        'formats':['i4',

From noreply at buildbot.pypy.org  Thu Jun 18 13:03:56 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Thu, 18 Jun 2015 13:03:56 +0200 (CEST)
Subject: [pypy-commit] pypy regalloc: added logic to put every variable into
 the register depending on how far away the next usage is (less distance is
 more likely to end up in a register)
Message-ID: <20150618110356.032121C1E34@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: regalloc
Changeset: r78174:be630ecb8639
Date: 2015-06-18 13:04 +0200
http://bitbucket.org/pypy/pypy/changeset/be630ecb8639/

Log:	added logic to put every variable into the register depending on how
	far away the next usage is (less distance is more likely to end up
	in a register)

diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -9,7 +9,7 @@
 from rpython.jit.backend.llsupport.gcmap import allocate_gcmap
 from rpython.jit.backend.llsupport.regalloc import (FrameManager, BaseRegalloc,
      RegisterManager, TempBox, compute_vars_longevity, is_comparison_or_ovf_op,
-     valid_addressing_size)
+     valid_addressing_size, next_var_usage)
 from rpython.jit.backend.x86 import rx86
 from rpython.jit.backend.x86.arch import (WORD, JITFRAME_FIXED_SIZE, IS_X86_32,
     IS_X86_64, DEFAULT_FRAME_BYTES)
@@ -27,6 +27,7 @@
 from rpython.rlib import rgc
 from rpython.rlib.objectmodel import we_are_translated
 from rpython.rlib.rarithmetic import r_longlong, r_uint
+from rpython.rlib.rbisect import bisect_right
 from rpython.rtyper.annlowlevel import cast_instance_to_gcref
 from rpython.rtyper.lltypesystem import lltype, rffi, rstr
 from rpython.rtyper.lltypesystem.lloperation import llop
@@ -1342,6 +1343,11 @@
         #self.rm.force_allocate_frame_reg(op.result)
         self.assembler.force_token(self.rm.force_allocate_reg(op.result))
 
+    def freecount(self, type):
+        if type == FLOAT:
+            return len(self.xrm.free_regs)
+        return len(self.rm.free_regs)
+
     def consider_label(self, op):
         descr = op.getdescr()
         assert isinstance(descr, TargetToken)
@@ -1365,6 +1371,8 @@
                 self.assembler.mc.MOV(loc2, ebp)
         self.rm.bindings_to_frame_reg.clear()
         #
+        relocate_index = []
+        relocate = []
         for i in range(len(inputargs)):
             arg = inputargs[i]
             assert isinstance(arg, Box)
@@ -1373,6 +1381,41 @@
             arglocs[i] = loc
             if isinstance(loc, RegLoc):
                 self.fm.mark_as_free(arg)
+            else:
+                # on first enter, try to put as many locations into registers
+                # as possible. They are sorted by the next variable use.
+                # Descending and are poped in reverse order later
+                pos = next_var_usage(self.longevity[arg], self.rm.position)
+                i = bisect_right(relocate_index, pos, len(relocate_index))
+                if i >= position:
+                    relocate_index.insert(i, pos)
+                    relocate.insert(i, (arg, argidx))
+        #
+        forbidden = {}
+        relocate_exit = []
+        while len(relocate) > 0:
+            arg, argidx = relocate.pop()
+            if self.freecount(arg.type) <= 0:
+                continue
+
+            if self.last_real_usage.get(arg, -1) >= position:
+                loc = self.make_sure_var_in_reg(arg, forbidden)
+                forbidden[arg] = None
+                arglocs[argidx] = loc
+                self.fm.mark_as_free(arg)
+            else:
+                relocate_exit.insert(0, arg)
+
+        # there might be still registers available
+        while len(relocate_exit) > 0:
+            arg, argidx = relocate_exit.pop()
+            if self.freecount(arg.type) <= 0:
+                continue
+
+            loc = self.make_sure_var_in_reg(arg, forbidden)
+            forbidden[arg] = None
+            arglocs[argidx] = loc
+            self.fm.mark_as_free(arg)
         #
         # if we are too close to the start of the loop, the label's target may
         # get overridden by redirect_call_assembler().  (rare case)

From noreply at buildbot.pypy.org  Sun Jun 21 14:30:20 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 21 Jun 2015 14:30:20 +0200 (CEST)
Subject: [pypy-commit] pypy default: Test fix
Message-ID: <20150621123020.531BE1C2001@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78229:8fca06d46f8c
Date: 2015-06-21 14:30 +0200
http://bitbucket.org/pypy/pypy/changeset/8fca06d46f8c/

Log:	Test fix

diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py
--- a/pypy/module/_socket/test/test_sock_app.py
+++ b/pypy/module/_socket/test/test_sock_app.py
@@ -316,7 +316,8 @@
         cls.w_udir = space.wrap(str(udir))
 
     def teardown_class(cls):
-        cls.space.sys.getmodule('_socket').shutdown(cls.space)
+        if not cls.runappdirect:
+            cls.space.sys.getmodule('_socket').shutdown(cls.space)
 
     def test_module(self):
         import _socket

From noreply at buildbot.pypy.org  Thu Jun 18 11:51:06 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Thu, 18 Jun 2015 11:51:06 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: rpython issue
Message-ID: <20150618095106.5B59D1C1F97@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r78172:f5454aafa6d6
Date: 2015-06-18 11:51 +0200
http://bitbucket.org/pypy/pypy/changeset/f5454aafa6d6/

Log:	rpython issue

diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -2484,8 +2484,8 @@
         for i,arg in enumerate(fail_args):
             if arg is None:
                 continue
-            assert arg.scalar_var is not None
             if isinstance(arg, BoxVectorAccum):
+                assert arg.scalar_var is not None
                 loc = fail_locs[i]
                 assert isinstance(loc, RegLoc)
                 assert loc.is_xmm

From noreply at buildbot.pypy.org  Sun Jun 21 12:11:59 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 21 Jun 2015 12:11:59 +0200 (CEST)
Subject: [pypy-commit] pypy default: fix test_whatsnew
Message-ID: <20150621101159.747EB1C0FD4@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78223:bd2fe61e71b9
Date: 2015-06-21 12:12 +0200
http://bitbucket.org/pypy/pypy/changeset/bd2fe61e71b9/

Log:	fix test_whatsnew

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -11,3 +11,8 @@
 .. branch: stdlib-2.7.10
 
 Update stdlib to version 2.7.10
+
+.. branch: issue2062
+
+.. branch: disable-unroll-for-short-loops
+The JIT no longer performs loop unrolling if the loop compiles to too much code.

From noreply at buildbot.pypy.org  Sun Jun 21 21:32:19 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Sun, 21 Jun 2015 21:32:19 +0200 (CEST)
Subject: [pypy-commit] pypy dtypes-compatability: test align more thoroughly
 (from numpy tests), percolate align argument
Message-ID: <20150621193219.72CA91C1F71@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: dtypes-compatability
Changeset: r78231:e00c9f683a8c
Date: 2015-06-20 21:14 +0300
http://bitbucket.org/pypy/pypy/changeset/e00c9f683a8c/

Log:	test align more thoroughly (from numpy tests), percolate align
	argument

diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py
--- a/pypy/module/micronumpy/test/test_dtypes.py
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -1310,6 +1310,54 @@
         raises(ValueError, np.dtype, [('a', 'f4', -1)])
         raises(ValueError, np.dtype, [('a', 'f4', (-1, -1))])
 
+    def test_aligned_size(self):
+        import numpy as np
+        # Check that structured dtypes get padded to an aligned size
+        dt = np.dtype('i4, i1', align=True)
+        assert dt.itemsize == 8
+        dt = np.dtype([('f0', 'i4'), ('f1', 'i1')], align=True)
+        assert dt.itemsize == 8
+        dt = np.dtype({'names':['f0', 'f1'],
+                       'formats':['i4', 'u1'],
+                       'offsets':[0, 4]}, align=True)
+        assert dt.itemsize == 8
+        dt = np.dtype({'f0': ('i4', 0), 'f1':('u1', 4)}, align=True)
+        assert dt.itemsize == 8
+        # Nesting should preserve that alignment
+        dt1 = np.dtype([('f0', 'i4'),
+                       ('f1', [('f1', 'i1'), ('f2', 'i4'), ('f3', 'i1')]),
+                       ('f2', 'i1')], align=True)
+        assert dt1.itemsize == 20
+        dt2 = np.dtype({'names':['f0', 'f1', 'f2'],
+                       'formats':['i4',
+                                  [('f1', 'i1'), ('f2', 'i4'), ('f3', 'i1')],
+                                  'i1'],
+                       'offsets':[0, 4, 16]}, align=True)
+        assert dt2.itemsize == 20
+        dt3 = np.dtype({'f0': ('i4', 0),
+                       'f1': ([('f1', 'i1'), ('f2', 'i4'), ('f3', 'i1')], 4),
+                       'f2': ('i1', 16)}, align=True)
+        assert dt3.itemsize == 20
+        assert dt1 == dt2
+        assert dt2 == dt3
+        # Nesting should preserve packing
+        dt1 = np.dtype([('f0', 'i4'),
+                       ('f1', [('f1', 'i1'), ('f2', 'i4'), ('f3', 'i1')]),
+                       ('f2', 'i1')], align=False)
+        assert dt1.itemsize == 11
+        dt2 = np.dtype({'names':['f0', 'f1', 'f2'],
+                       'formats':['i4',
+                                  [('f1', 'i1'), ('f2', 'i4'), ('f3', 'i1')],
+                                  'i1'],
+                       'offsets':[0, 4, 10]}, align=False)
+        assert dt2.itemsize == 11
+        dt3 = np.dtype({'f0': ('i4', 0),
+                       'f1': ([('f1', 'i1'), ('f2', 'i4'), ('f3', 'i1')], 4),
+                       'f2': ('i1', 10)}, align=False)
+        assert dt3.itemsize == 11
+        assert dt1 == dt2
+        assert dt2 == dt3
+
 
 class AppTestNotDirect(BaseNumpyAppTest):
     def setup_class(cls):

From noreply at buildbot.pypy.org  Thu Jun 18 18:56:15 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 18 Jun 2015 18:56:15 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c8: import stmgc branch 'queue'
Message-ID: <20150618165615.8AC0B1C1FE1@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c8
Changeset: r78188:c12cb756db2e
Date: 2015-06-18 17:52 +0100
http://bitbucket.org/pypy/pypy/changeset/c12cb756db2e/

Log:	import stmgc branch 'queue'

diff --git a/rpython/translator/stm/src_stm/revision b/rpython/translator/stm/src_stm/revision
--- a/rpython/translator/stm/src_stm/revision
+++ b/rpython/translator/stm/src_stm/revision
@@ -1,1 +1,1 @@
-d083e426a17d
+0a10e04f2119
diff --git a/rpython/translator/stm/src_stm/stm/core.c b/rpython/translator/stm/src_stm/stm/core.c
--- a/rpython/translator/stm/src_stm/stm/core.c
+++ b/rpython/translator/stm/src_stm/stm/core.c
@@ -495,6 +495,32 @@
 static void readd_wb_executed_flags(void);
 static void check_all_write_barrier_flags(char *segbase, struct list_s *list);
 
+static void wait_for_inevitable(void)
+{
+    intptr_t detached = 0;
+
+    s_mutex_lock();
+ wait_some_more:
+    if (safe_point_requested()) {
+        /* XXXXXX if the safe point below aborts, in
+           _validate_and_attach(), 'new' leaks */
+        enter_safe_point_if_requested();
+    }
+    else if (STM_PSEGMENT->last_commit_log_entry->next == INEV_RUNNING) {
+        /* loop until C_SEGMENT_FREE_OR_SAFE_POINT_REQ is signalled, but
+           try to detach an inevitable transaction regularly */
+        detached = fetch_detached_transaction();
+        if (detached == 0) {
+            if (!cond_wait_timeout(C_SEGMENT_FREE_OR_SAFE_POINT_REQ, 0.00001))
+                goto wait_some_more;
+        }
+    }
+    s_mutex_unlock();
+
+    if (detached != 0)
+        commit_fetched_detached_transaction(detached);
+}
+
 /* This is called to do stm_validate() and then attach 'new' at the
    head of the 'commit_log_root' chained list.  This function sleeps
    and retries until it succeeds or aborts.
@@ -523,24 +549,10 @@
 #endif
 
     if (STM_PSEGMENT->last_commit_log_entry->next == INEV_RUNNING) {
-        s_mutex_lock();
-        if (safe_point_requested()) {
-            /* XXXXXX if the safe point below aborts, 'new' leaks */
-            enter_safe_point_if_requested();
-        }
-        else if (STM_PSEGMENT->last_commit_log_entry->next == INEV_RUNNING) {
-            cond_wait(C_SEGMENT_FREE_OR_SAFE_POINT);
-        }
-        s_mutex_unlock();
+        wait_for_inevitable();
         goto retry_from_start;   /* redo _stm_validate() now */
     }
 
-    intptr_t detached = fetch_detached_transaction();
-    if (detached != 0) {
-        commit_fetched_detached_transaction(detached);
-        goto retry_from_start;
-    }
-
     /* we must not remove the WB_EXECUTED flags before validation as
        it is part of a condition in import_objects() called by
        copy_bk_objs_in_page_from to not overwrite our modifications.
@@ -1119,7 +1131,7 @@
 {
     assert(!_stm_in_transaction(tl));
 
-    while (!acquire_thread_segment(tl)) {}
+    acquire_thread_segment(tl);
     /* GS invalid before this point! */
 
     assert(STM_PSEGMENT->safe_point == SP_NO_TRANSACTION);
@@ -1571,13 +1583,33 @@
             timing_become_inevitable(); /* for tests: another transaction */
             stm_abort_transaction();    /* is already inevitable, abort   */
 #endif
+
+            bool timed_out = false;
+
             s_mutex_lock();
             if (any_soon_finished_or_inevitable_thread_segment() &&
                     !safe_point_requested()) {
-                cond_wait(C_SEGMENT_FREE_OR_SAFE_POINT);
+
+                /* wait until C_SEGMENT_FREE_OR_SAFE_POINT_REQ is signalled */
+                if (!cond_wait_timeout(C_SEGMENT_FREE_OR_SAFE_POINT_REQ,
+                                       0.000054321))
+                    timed_out = true;
             }
             s_mutex_unlock();
-            num_waits++;
+
+            if (timed_out) {
+                /* try to detach another inevitable transaction, but
+                   only after waiting a bit.  This is necessary to avoid
+                   deadlocks in some situations, which are hopefully
+                   not too common.  We don't want two threads constantly
+                   detaching each other. */
+                intptr_t detached = fetch_detached_transaction();
+                if (detached != 0)
+                    commit_fetched_detached_transaction(detached);
+            }
+            else {
+                num_waits++;
+            }
             goto retry_from_start;
         }
         if (!_validate_and_turn_inevitable())
diff --git a/rpython/translator/stm/src_stm/stm/sync.c b/rpython/translator/stm/src_stm/stm/sync.c
--- a/rpython/translator/stm/src_stm/stm/sync.c
+++ b/rpython/translator/stm/src_stm/stm/sync.c
@@ -203,10 +203,11 @@
 #endif
 
 
-static bool acquire_thread_segment(stm_thread_local_t *tl)
+static void acquire_thread_segment(stm_thread_local_t *tl)
 {
     /* This function acquires a segment for the currently running thread,
        and set up the GS register if it changed. */
+ retry_from_start:
     assert(_has_mutex());
     assert(_is_tl_registered(tl));
 
@@ -240,13 +241,13 @@
         }
     }
     /* No segment available.  Wait until release_thread_segment()
-       signals that one segment has been freed. */
+       signals that one segment has been freed.  Note that we prefer
+       waiting rather than detaching an inevitable transaction, here. */
     timing_event(tl, STM_WAIT_FREE_SEGMENT);
     cond_wait(C_SEGMENT_FREE);
     timing_event(tl, STM_WAIT_DONE);
 
-    /* Return false to the caller, which will call us again */
-    return false;
+    goto retry_from_start;
 
  got_num:
     OPT_ASSERT(num >= 0 && num < NB_SEGMENTS-1);
@@ -257,7 +258,6 @@
     assert(!in_transaction(tl));
     STM_SEGMENT->running_thread = tl;
     assert(in_transaction(tl));
-    return true;
 }
 
 static void release_thread_segment(stm_thread_local_t *tl)
@@ -266,7 +266,7 @@
     assert(_has_mutex());
 
     cond_signal(C_SEGMENT_FREE);
-    cond_broadcast(C_SEGMENT_FREE_OR_SAFE_POINT);   /* often no listener */
+    cond_broadcast(C_SEGMENT_FREE_OR_SAFE_POINT_REQ);  /* often no listener */
 
     assert(STM_SEGMENT->running_thread == tl);
     segnum = STM_SEGMENT->segment_num;
@@ -362,7 +362,7 @@
     assert(!pause_signalled);
     pause_signalled = true;
     dprintf(("request to pause\n"));
-    cond_broadcast(C_SEGMENT_FREE_OR_SAFE_POINT);
+    cond_broadcast(C_SEGMENT_FREE_OR_SAFE_POINT_REQ);
 }
 
 static inline long count_other_threads_sp_running(void)
diff --git a/rpython/translator/stm/src_stm/stm/sync.h b/rpython/translator/stm/src_stm/stm/sync.h
--- a/rpython/translator/stm/src_stm/stm/sync.h
+++ b/rpython/translator/stm/src_stm/stm/sync.h
@@ -5,7 +5,7 @@
     C_AT_SAFE_POINT,
     C_REQUEST_REMOVED,
     C_SEGMENT_FREE,
-    C_SEGMENT_FREE_OR_SAFE_POINT,
+    C_SEGMENT_FREE_OR_SAFE_POINT_REQ,
     C_QUEUE_OLD_ENTRIES,
     C_QUEUE_FINISHED_MORE_TASKS,
     _C_TOTAL
@@ -25,7 +25,7 @@
 
 /* acquire and release one of the segments for running the given thread
    (must have the mutex acquired!) */
-static bool acquire_thread_segment(stm_thread_local_t *tl);
+static void acquire_thread_segment(stm_thread_local_t *tl);
 static void release_thread_segment(stm_thread_local_t *tl);
 static void soon_finished_or_inevitable_thread_segment(void);
 static bool any_soon_finished_or_inevitable_thread_segment(void);

From noreply at buildbot.pypy.org  Tue Jun 23 08:39:30 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 23 Jun 2015 08:39:30 +0200 (CEST)
Subject: [pypy-commit] pypy default: Issue #2069: comments and one essential
 fix. On one of the call paths, 
Message-ID: <20150623063930.8D5351C02FD@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78247:7000d2a499d2
Date: 2015-06-23 08:39 +0200
http://bitbucket.org/pypy/pypy/changeset/7000d2a499d2/

Log:	Issue #2069: comments and one essential fix. On one of the call
	paths, a promote() was missing

diff --git a/pypy/module/_cffi_backend/ctypefunc.py b/pypy/module/_cffi_backend/ctypefunc.py
--- a/pypy/module/_cffi_backend/ctypefunc.py
+++ b/pypy/module/_cffi_backend/ctypefunc.py
@@ -143,7 +143,7 @@
     @jit.unroll_safe
     def _call(self, funcaddr, args_w):
         space = self.space
-        cif_descr = self.cif_descr
+        cif_descr = self.cif_descr   # 'self' should have been promoted here
         size = cif_descr.exchange_size
         mustfree_max_plus_1 = 0
         buffer = lltype.malloc(rffi.CCHARP.TO, size, flavor='raw')
diff --git a/pypy/module/_cffi_backend/wrapper.py b/pypy/module/_cffi_backend/wrapper.py
--- a/pypy/module/_cffi_backend/wrapper.py
+++ b/pypy/module/_cffi_backend/wrapper.py
@@ -19,6 +19,8 @@
     wrapper is callable, and the arguments it expects and returns
     are directly the struct/union.  Calling ffi.typeof(wrapper)
     also returns the original struct/union signature.
+
+    This class cannot be used for variadic functions.
     """
     _immutable_ = True
     common_doc_str = 'direct call to the C function of the same name'
@@ -72,6 +74,7 @@
             args_w[i] = w_arg
 
     def descr_call(self, args_w):
+        self = jit.promote(self)
         if len(args_w) != self.nargs_expected:
             space = self.space
             if self.nargs_expected == 0:
diff --git a/rpython/rlib/jit_libffi.py b/rpython/rlib/jit_libffi.py
--- a/rpython/rlib/jit_libffi.py
+++ b/rpython/rlib/jit_libffi.py
@@ -109,6 +109,11 @@
 def jit_ffi_call(cif_description, func_addr, exchange_buffer):
     """Wrapper around ffi_call().  Must receive a CIF_DESCRIPTION_P that
     describes the layout of the 'exchange_buffer'.
+
+    Note that this cannot be optimized if 'cif_description' is not
+    a constant for the JIT, so if it is ever possible, consider promoting
+    it.  The promotion of 'cif_description' must be done earlier, before
+    the raw malloc of 'exchange_buffer'.
     """
     reskind = types.getkind(cif_description.rtype)
     if reskind == 'v':

From noreply at buildbot.pypy.org  Sun Jun 21 21:38:27 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 21 Jun 2015 21:38:27 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c8: Allow 'with stm_ignored' to disable
 locally the insertion of
Message-ID: <20150621193827.ED7CC1C1FFC@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c8
Changeset: r78234:df7e39382535
Date: 2015-06-21 21:33 +0200
http://bitbucket.org/pypy/pypy/changeset/df7e39382535/

Log:	Allow 'with stm_ignored' to disable locally the insertion of
	stm_become_inevitable

diff --git a/rpython/translator/stm/inevitable.py b/rpython/translator/stm/inevitable.py
--- a/rpython/translator/stm/inevitable.py
+++ b/rpython/translator/stm/inevitable.py
@@ -169,11 +169,23 @@
 
 def insert_turn_inevitable(graph):
     for block in graph.iterblocks():
+        stm_ignored = False
         for i in range(len(block.operations)-1, -1, -1):
             op = block.operations[i]
             inev = should_turn_inevitable(op, block)
-            if inev:
+            if inev and not stm_ignored:
                 if not isinstance(inev, str):
                     inev = op.opname
                 inev_op = turn_inevitable_op(inev)
                 block.operations.insert(i, inev_op)
+            if op.opname == 'stm_ignored_stop':
+                assert not stm_ignored, "nested stm_ignored_stop"
+                stm_ignored = True      # backward, so "stop" enables it
+            elif op.opname == 'stm_ignored_start':
+                if not stm_ignored:
+                    raise Exception("%r: 'with stm_ignored: stm_ignored start "
+                                    "without end in the same block" % (graph,))
+                stm_ignored = False     # backward, so "start" disables it
+        if stm_ignored:
+            raise Exception("%r: 'with stm_ignored:' code body too complex"
+                            % (graph,))
diff --git a/rpython/translator/stm/test/test_inevitable.py b/rpython/translator/stm/test/test_inevitable.py
--- a/rpython/translator/stm/test/test_inevitable.py
+++ b/rpython/translator/stm/test/test_inevitable.py
@@ -5,6 +5,7 @@
 from rpython.rtyper.test.test_llinterp import get_interpreter, clear_tcache
 from rpython.translator.stm.inevitable import insert_turn_inevitable
 from rpython.translator.stm import inevitable
+from rpython.rlib.rstm import stm_ignored
 from rpython.conftest import option
 import py
 
@@ -410,3 +411,19 @@
 
         res = self.interpret_inevitable(f1, [1])
         assert res == []
+
+
+    def test_stm_ignored(self):
+        X = lltype.Struct('X', ('foo', lltype.Signed))
+        x1 = lltype.malloc(X, flavor='raw', immortal=True)
+
+        def f1():
+            return x1.foo
+        res = self.interpret_inevitable(f1, [])
+        assert res == ['getfield']
+
+        def f2():
+            with stm_ignored:
+                return x1.foo
+        res = self.interpret_inevitable(f2, [])
+        assert res == []

From noreply at buildbot.pypy.org  Sun Jun 21 21:38:29 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 21 Jun 2015 21:38:29 +0200 (CEST)
Subject: [pypy-commit] pypy default: Simplify code
Message-ID: <20150621193829.180771C1FFF@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78235:ba1832c3fd9d
Date: 2015-06-21 21:38 +0200
http://bitbucket.org/pypy/pypy/changeset/ba1832c3fd9d/

Log:	Simplify code

diff --git a/pypy/module/_cffi_backend/ctypeprim.py b/pypy/module/_cffi_backend/ctypeprim.py
--- a/pypy/module/_cffi_backend/ctypeprim.py
+++ b/pypy/module/_cffi_backend/ctypeprim.py
@@ -134,8 +134,7 @@
 
     def convert_to_object(self, cdata):
         unichardata = rffi.cast(rffi.CWCHARP, cdata)
-        s = rffi.wcharpsize2unicode(unichardata, 1)
-        return self.space.wrap(s)
+        return self.space.wrap(unichardata[0])
 
     def string(self, cdataobj, maxlen):
         with cdataobj as ptr:

From noreply at buildbot.pypy.org  Thu Jun 18 22:25:04 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Thu, 18 Jun 2015 22:25:04 +0200 (CEST)
Subject: [pypy-commit] pypy py3.3: Port _lzma to CFFI 1.0.
Message-ID: <20150618202504.182D11C1FE0@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob <me at manueljacob.de>
Branch: py3.3
Changeset: r78192:09b408b01eab
Date: 2015-06-18 22:25 +0200
http://bitbucket.org/pypy/pypy/changeset/09b408b01eab/

Log:	Port _lzma to CFFI 1.0.

diff --git a/lib_pypy/_lzma.py b/lib_pypy/_lzma.py
--- a/lib_pypy/_lzma.py
+++ b/lib_pypy/_lzma.py
@@ -1,11 +1,9 @@
-# This file was copied from lzmaffi version 0.3.0.
-# It is an almost exact copy of lzmaffi/_lzmamodule2.py.
+# This file is based on lzmaffi/_lzmamodule2.py from lzmaffi version 0.3.0.
 
 # PyPy changes:
 # - added __getstate__() methods that raise TypeError on pickling.
-# - in ffi.verify(), changed modulename to '_lzmaffi'.
+# - ported to CFFI 1.0
 
-from cffi import FFI
 import threading
 import functools
 import collections
@@ -13,6 +11,8 @@
 import sys
 import io
 
+from _lzma_cffi import ffi, lib as m
+
 SUPPORTED_STREAM_FLAGS_VERSION = 0
 
 __all__ = ['CHECK_CRC32',
@@ -58,247 +58,6 @@
 
 _owns = weakref.WeakKeyDictionary()
 
-ffi = FFI()
-ffi.cdef("""
-#define UINT64_MAX ...
-#define LZMA_CONCATENATED ...
-#define LZMA_CHECK_NONE ...
-#define LZMA_CHECK_CRC32 ...
-#define LZMA_CHECK_CRC64 ...
-#define LZMA_CHECK_SHA256 ...
-#define LZMA_CHECK_ID_MAX ...
-#define LZMA_DELTA_TYPE_BYTE ...
-#define LZMA_TELL_ANY_CHECK ...
-#define LZMA_TELL_NO_CHECK ...
-#define LZMA_VLI_UNKNOWN ...
-#define LZMA_FILTER_LZMA1 ...
-#define LZMA_FILTER_LZMA2 ...
-#define LZMA_FILTER_DELTA ...
-#define LZMA_FILTER_X86 ...
-#define LZMA_FILTER_IA64 ...
-#define LZMA_FILTER_ARM ...
-#define LZMA_FILTER_ARMTHUMB ...
-#define LZMA_FILTER_SPARC ...
-#define LZMA_FILTER_POWERPC ...
-#define LZMA_FILTERS_MAX ...
-#define LZMA_STREAM_HEADER_SIZE ...
-#define LZMA_MF_HC3 ...
-#define LZMA_MF_HC4 ...
-#define LZMA_MF_BT2 ...
-#define LZMA_MF_BT3 ...
-#define LZMA_MF_BT4 ...
-#define LZMA_MODE_FAST ...
-#define LZMA_MODE_NORMAL ...
-#define LZMA_PRESET_DEFAULT ...
-#define LZMA_PRESET_EXTREME ...
-
-enum lzma_ret { LZMA_OK, LZMA_STREAM_END, LZMA_NO_CHECK,
-    LZMA_UNSUPPORTED_CHECK, LZMA_GET_CHECK,
-    LZMA_MEM_ERROR, LZMA_MEMLIMIT_ERROR,
-    LZMA_FORMAT_ERROR, LZMA_OPTIONS_ERROR,
-    LZMA_DATA_ERROR, LZMA_BUF_ERROR,
-    LZMA_PROG_ERROR, ... };
-
-enum lzma_action { LZMA_RUN, LZMA_FINISH, ...};
-
-enum lzma_check { ... };
-
-typedef uint64_t lzma_vli;
-
-typedef struct {
-    void* (*alloc)(void*, size_t, size_t);
-    void (*free)(void*, void*);
-    void* opaque;
-    ...;
-} lzma_allocator;
-
-typedef struct {
-    const uint8_t *next_in;
-    size_t avail_in;
-    uint64_t total_in;
-
-    uint8_t *next_out;
-    size_t avail_out;
-    uint64_t total_out;
-    lzma_allocator *allocator;
-    ...;
-} lzma_stream;
-
-typedef struct {
-    int type;
-    uint32_t dist;
-    ...;
-} lzma_options_delta;
-
-typedef struct {
-    uint32_t start_offset;
-    ...;
-} lzma_options_bcj;
-
-typedef struct {
-    uint32_t dict_size;
-    uint32_t lc;
-    uint32_t lp;
-    uint32_t pb;
-    int mode;
-    uint32_t nice_len;
-    int mf;
-    uint32_t depth;
-    ...;
-} lzma_options_lzma;
-
-typedef struct {
-    lzma_vli id;
-    void *options;
-    ...;
-} lzma_filter;
-
-typedef struct {
-    uint32_t version;
-    lzma_vli backward_size;
-    int check;
-    ...;
-} lzma_stream_flags;
-
-typedef ... lzma_index;
-
-typedef struct {
-    uint32_t version;
-    uint32_t header_size;
-    int check;
-    lzma_vli compressed_size;
-    lzma_filter* filters;
-    ...;
-} lzma_block;
-
-bool lzma_check_is_supported(int check);
-
-// Encoder/Decoder
-int lzma_auto_decoder(lzma_stream *strm, uint64_t memlimit, uint32_t flags);
-int lzma_stream_decoder(lzma_stream *strm, uint64_t memlimit, uint32_t flags);
-int lzma_alone_decoder(lzma_stream *strm, uint64_t memlimit);
-int lzma_raw_decoder(lzma_stream *strm, const lzma_filter *filters);
-int lzma_block_decoder(lzma_stream *strm, lzma_block *block);
-
-int lzma_easy_encoder(lzma_stream *strm, uint32_t preset, int check);
-int lzma_alone_encoder(lzma_stream *strm, lzma_options_lzma* options);
-int lzma_raw_encoder(lzma_stream *strm, const lzma_filter *filters);
-
-int lzma_get_check(const lzma_stream *strm);
-
-int lzma_code(lzma_stream *strm, int action);
-
-void lzma_end(lzma_stream *strm);
-
-// Extras
-int lzma_stream_header_decode(lzma_stream_flags *options, const uint8_t *in);
-int lzma_stream_footer_decode(lzma_stream_flags *options, const uint8_t *in);
-int lzma_stream_flags_compare(const lzma_stream_flags *a,
-    const lzma_stream_flags *b);
-
-enum lzma_index_iter_mode { LZMA_INDEX_ITER_ANY, LZMA_INDEX_ITER_STREAM,
-    LZMA_INDEX_ITER_BLOCK, LZMA_INDEX_ITER_NONEMPTY_BLOCK, ... };
-
-// Indexes
-lzma_index* lzma_index_init(lzma_allocator *al);
-void lzma_index_end(lzma_index *i, lzma_allocator *al);
-int lzma_index_stream_padding(lzma_index *i, lzma_vli stream_padding);
-lzma_index* lzma_index_dup(const lzma_index *i, lzma_allocator *al);
-int lzma_index_cat(lzma_index *dest, lzma_index *src, lzma_allocator *al);
-int lzma_index_buffer_decode(lzma_index **i, uint64_t *memlimit,
-    lzma_allocator *allocator, const uint8_t *in, size_t *in_pos,
-    size_t in_size);
-lzma_vli lzma_index_block_count(const lzma_index *i);
-lzma_vli lzma_index_stream_size(const lzma_index *i);
-lzma_vli lzma_index_uncompressed_size(const lzma_index *i);
-lzma_vli lzma_index_size(const lzma_index *i);
-lzma_vli lzma_index_total_size(const lzma_index *i);
-
-// Blocks
-int lzma_block_header_decode(lzma_block *block, lzma_allocator *al,
-    const uint8_t *in);
-int lzma_block_compressed_size(lzma_block *block, lzma_vli unpadded_size);
-
-typedef struct {
-    // cffi doesn't support partial anonymous structs
-    // so we write the definition in full
-    struct {
-        const lzma_stream_flags *flags;
-        const void *reserved_ptr1;
-        const void *reserved_ptr2;
-        const void *reserved_ptr3;
-        lzma_vli number;
-        lzma_vli block_count;
-        lzma_vli compressed_offset;
-        lzma_vli uncompressed_offset;
-        lzma_vli compressed_size;
-        lzma_vli uncompressed_size;
-        lzma_vli padding;
-        lzma_vli reserved_vli1;
-        lzma_vli reserved_vli2;
-        lzma_vli reserved_vli3;
-        lzma_vli reserved_vli4;
-    } stream;
-    struct {
-        lzma_vli number_in_file;
-        lzma_vli compressed_file_offset;
-        lzma_vli uncompressed_file_offset;
-        lzma_vli number_in_stream;
-        lzma_vli compressed_stream_offset;
-        lzma_vli uncompressed_stream_offset;
-        lzma_vli uncompressed_size;
-        lzma_vli unpadded_size;
-        lzma_vli total_size;
-        lzma_vli reserved_vli1;
-        lzma_vli reserved_vli2;
-        lzma_vli reserved_vli3;
-        lzma_vli reserved_vli4;
-        const void *reserved_ptr1;
-        const void *reserved_ptr2;
-        const void *reserved_ptr3;
-        const void *reserved_ptr4;
-    } block;
-    ...;
-} lzma_index_iter;
-
-void lzma_index_iter_init(lzma_index_iter *iter, const lzma_index *i);
-int lzma_index_iter_next(lzma_index_iter *iter, int mode);
-int lzma_index_iter_locate(lzma_index_iter *iter, lzma_vli target);
-
-// Properties
-int lzma_properties_size(uint32_t *size, const lzma_filter *filter);
-int lzma_properties_encode(const lzma_filter *filter, uint8_t *props);
-int lzma_properties_decode(lzma_filter *filter, lzma_allocator *allocator,
-    const uint8_t *props, size_t props_size);
-int lzma_lzma_preset(lzma_options_lzma* options, uint32_t preset);
-
-// Special functions
-void _pylzma_stream_init(lzma_stream *strm);
-void _pylzma_block_header_size_decode(uint32_t b);
-
-void *malloc(size_t size);
-void free(void *ptr);
-void *realloc(void *ptr, size_t size);
-""")
-
-m = ffi.verify("""
-#include <lzma.h>
-#include <stdlib.h>
-void _pylzma_stream_init(lzma_stream *strm) {
-    lzma_stream tmp = LZMA_STREAM_INIT; // macro from lzma.h
-    *strm = tmp;
-}
-
-uint32_t _pylzma_block_header_size_decode(uint32_t b) {
-    return lzma_block_header_size_decode(b); // macro from lzma.h
-}
-""",
-    libraries=['lzma'],
-    include_dirs=['/opt/local/include', '/usr/local/include'],
-    library_dirs=['/opt/local/include', '/usr/local/include'],
-    ext_package='_lzmaffi_mods',
-    modulename='_lzmaffi')
-
 def _new_lzma_stream():
     ret = ffi.new('lzma_stream*')
     m._pylzma_stream_init(ret)
diff --git a/lib_pypy/_lzma_build.py b/lib_pypy/_lzma_build.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_lzma_build.py
@@ -0,0 +1,253 @@
+# This file is based on lzmaffi/_lzmamodule2.py from lzmaffi version 0.3.0.
+
+# PyPy changes:
+# - added __getstate__() methods that raise TypeError on pickling.
+# - ported to CFFI 1.0
+
+from cffi import FFI
+
+
+ffi = FFI()
+
+ffi.cdef("""
+#define UINT64_MAX ...
+#define LZMA_CONCATENATED ...
+#define LZMA_CHECK_NONE ...
+#define LZMA_CHECK_CRC32 ...
+#define LZMA_CHECK_CRC64 ...
+#define LZMA_CHECK_SHA256 ...
+#define LZMA_CHECK_ID_MAX ...
+#define LZMA_DELTA_TYPE_BYTE ...
+#define LZMA_TELL_ANY_CHECK ...
+#define LZMA_TELL_NO_CHECK ...
+#define LZMA_VLI_UNKNOWN ...
+#define LZMA_FILTER_LZMA1 ...
+#define LZMA_FILTER_LZMA2 ...
+#define LZMA_FILTER_DELTA ...
+#define LZMA_FILTER_X86 ...
+#define LZMA_FILTER_IA64 ...
+#define LZMA_FILTER_ARM ...
+#define LZMA_FILTER_ARMTHUMB ...
+#define LZMA_FILTER_SPARC ...
+#define LZMA_FILTER_POWERPC ...
+#define LZMA_FILTERS_MAX ...
+#define LZMA_STREAM_HEADER_SIZE ...
+#define LZMA_MF_HC3 ...
+#define LZMA_MF_HC4 ...
+#define LZMA_MF_BT2 ...
+#define LZMA_MF_BT3 ...
+#define LZMA_MF_BT4 ...
+#define LZMA_MODE_FAST ...
+#define LZMA_MODE_NORMAL ...
+#define LZMA_PRESET_DEFAULT ...
+#define LZMA_PRESET_EXTREME ...
+
+typedef enum { LZMA_OK, LZMA_STREAM_END, LZMA_NO_CHECK,
+    LZMA_UNSUPPORTED_CHECK, LZMA_GET_CHECK,
+    LZMA_MEM_ERROR, LZMA_MEMLIMIT_ERROR,
+    LZMA_FORMAT_ERROR, LZMA_OPTIONS_ERROR,
+    LZMA_DATA_ERROR, LZMA_BUF_ERROR,
+    LZMA_PROG_ERROR, ...
+} lzma_ret;
+
+typedef enum { LZMA_RUN, LZMA_FINISH, ...} lzma_action;
+
+typedef enum { ... } lzma_check;
+
+typedef uint64_t lzma_vli;
+
+typedef struct {
+    void* (*alloc)(void*, size_t, size_t);
+    void (*free)(void*, void*);
+    void* opaque;
+    ...;
+} lzma_allocator;
+
+typedef struct {
+    const uint8_t *next_in;
+    size_t avail_in;
+    uint64_t total_in;
+
+    uint8_t *next_out;
+    size_t avail_out;
+    uint64_t total_out;
+    lzma_allocator *allocator;
+    ...;
+} lzma_stream;
+
+typedef struct {
+    int type;
+    uint32_t dist;
+    ...;
+} lzma_options_delta;
+
+typedef struct {
+    uint32_t start_offset;
+    ...;
+} lzma_options_bcj;
+
+typedef struct {
+    uint32_t dict_size;
+    uint32_t lc;
+    uint32_t lp;
+    uint32_t pb;
+    int mode;
+    uint32_t nice_len;
+    int mf;
+    uint32_t depth;
+    ...;
+} lzma_options_lzma;
+
+typedef struct {
+    lzma_vli id;
+    void *options;
+    ...;
+} lzma_filter;
+
+typedef struct {
+    uint32_t version;
+    lzma_vli backward_size;
+    int check;
+    ...;
+} lzma_stream_flags;
+
+typedef ... lzma_index;
+
+typedef struct {
+    uint32_t version;
+    uint32_t header_size;
+    int check;
+    lzma_vli compressed_size;
+    lzma_filter* filters;
+    ...;
+} lzma_block;
+
+bool lzma_check_is_supported(int check);
+
+// Encoder/Decoder
+int lzma_auto_decoder(lzma_stream *strm, uint64_t memlimit, uint32_t flags);
+int lzma_stream_decoder(lzma_stream *strm, uint64_t memlimit, uint32_t flags);
+int lzma_alone_decoder(lzma_stream *strm, uint64_t memlimit);
+int lzma_raw_decoder(lzma_stream *strm, const lzma_filter *filters);
+int lzma_block_decoder(lzma_stream *strm, lzma_block *block);
+
+int lzma_easy_encoder(lzma_stream *strm, uint32_t preset, int check);
+int lzma_alone_encoder(lzma_stream *strm, lzma_options_lzma* options);
+int lzma_raw_encoder(lzma_stream *strm, const lzma_filter *filters);
+
+int lzma_get_check(const lzma_stream *strm);
+
+int lzma_code(lzma_stream *strm, int action);
+
+void lzma_end(lzma_stream *strm);
+
+// Extras
+int lzma_stream_header_decode(lzma_stream_flags *options, const uint8_t *in);
+int lzma_stream_footer_decode(lzma_stream_flags *options, const uint8_t *in);
+int lzma_stream_flags_compare(const lzma_stream_flags *a,
+    const lzma_stream_flags *b);
+
+typedef enum {
+    LZMA_INDEX_ITER_ANY, LZMA_INDEX_ITER_STREAM, LZMA_INDEX_ITER_BLOCK,
+    LZMA_INDEX_ITER_NONEMPTY_BLOCK, ...
+} lzma_index_iter_mode;
+
+// Indexes
+lzma_index* lzma_index_init(lzma_allocator *al);
+void lzma_index_end(lzma_index *i, lzma_allocator *al);
+int lzma_index_stream_padding(lzma_index *i, lzma_vli stream_padding);
+lzma_index* lzma_index_dup(const lzma_index *i, lzma_allocator *al);
+int lzma_index_cat(lzma_index *dest, lzma_index *src, lzma_allocator *al);
+int lzma_index_buffer_decode(lzma_index **i, uint64_t *memlimit,
+    lzma_allocator *allocator, const uint8_t *in, size_t *in_pos,
+    size_t in_size);
+lzma_vli lzma_index_block_count(const lzma_index *i);
+lzma_vli lzma_index_stream_size(const lzma_index *i);
+lzma_vli lzma_index_uncompressed_size(const lzma_index *i);
+lzma_vli lzma_index_size(const lzma_index *i);
+lzma_vli lzma_index_total_size(const lzma_index *i);
+
+// Blocks
+int lzma_block_header_decode(lzma_block *block, lzma_allocator *al,
+    const uint8_t *in);
+int lzma_block_compressed_size(lzma_block *block, lzma_vli unpadded_size);
+
+typedef struct {
+    // cffi doesn't support partial anonymous structs
+    // so we write the definition in full
+    struct {
+        const lzma_stream_flags *flags;
+        const void *reserved_ptr1;
+        const void *reserved_ptr2;
+        const void *reserved_ptr3;
+        lzma_vli number;
+        lzma_vli block_count;
+        lzma_vli compressed_offset;
+        lzma_vli uncompressed_offset;
+        lzma_vli compressed_size;
+        lzma_vli uncompressed_size;
+        lzma_vli padding;
+        lzma_vli reserved_vli1;
+        lzma_vli reserved_vli2;
+        lzma_vli reserved_vli3;
+        lzma_vli reserved_vli4;
+    } stream;
+    struct {
+        lzma_vli number_in_file;
+        lzma_vli compressed_file_offset;
+        lzma_vli uncompressed_file_offset;
+        lzma_vli number_in_stream;
+        lzma_vli compressed_stream_offset;
+        lzma_vli uncompressed_stream_offset;
+        lzma_vli uncompressed_size;
+        lzma_vli unpadded_size;
+        lzma_vli total_size;
+        lzma_vli reserved_vli1;
+        lzma_vli reserved_vli2;
+        lzma_vli reserved_vli3;
+        lzma_vli reserved_vli4;
+        const void *reserved_ptr1;
+        const void *reserved_ptr2;
+        const void *reserved_ptr3;
+        const void *reserved_ptr4;
+    } block;
+    ...;
+} lzma_index_iter;
+
+void lzma_index_iter_init(lzma_index_iter *iter, const lzma_index *i);
+int lzma_index_iter_next(lzma_index_iter *iter, int mode);
+int lzma_index_iter_locate(lzma_index_iter *iter, lzma_vli target);
+
+// Properties
+int lzma_properties_size(uint32_t *size, const lzma_filter *filter);
+int lzma_properties_encode(const lzma_filter *filter, uint8_t *props);
+int lzma_properties_decode(lzma_filter *filter, lzma_allocator *allocator,
+    const uint8_t *props, size_t props_size);
+int lzma_lzma_preset(lzma_options_lzma* options, uint32_t preset);
+
+// Special functions
+void _pylzma_stream_init(lzma_stream *strm);
+void _pylzma_block_header_size_decode(uint32_t b);
+
+void *malloc(size_t size);
+void free(void *ptr);
+void *realloc(void *ptr, size_t size);
+""")
+
+ffi.set_source('_lzma_cffi', """
+#include <lzma.h>
+#include <stdlib.h>
+void _pylzma_stream_init(lzma_stream *strm) {
+    lzma_stream tmp = LZMA_STREAM_INIT; // macro from lzma.h
+    *strm = tmp;
+}
+
+uint32_t _pylzma_block_header_size_decode(uint32_t b) {
+    return lzma_block_header_size_decode(b); // macro from lzma.h
+}
+""",
+    libraries=['lzma'])
+
+
+if __name__ == '__main__':
+    ffi.compile()
diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py
--- a/pypy/tool/release/package.py
+++ b/pypy/tool/release/package.py
@@ -61,7 +61,7 @@
     "syslog": "_syslog_build.py" if sys.platform != "win32" else None,
     "_gdbm": "_gdbm_build.py"  if sys.platform != "win32" else None,
     "pwdgrp": "_pwdgrp_build.py" if sys.platform != "win32" else None,
-    "lzma": None, # XXX change _lzma to use CFFI 1.0
+    "lzma": "_lzma_build.py",
     "_decimal": None, # XXX change _decimal to use CFFI 1.0
     "xx": None,    # for testing: 'None' should be completely ignored
     }

From noreply at buildbot.pypy.org  Sun Jun 21 12:15:02 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 21 Jun 2015 12:15:02 +0200 (CEST)
Subject: [pypy-commit] pypy default: fix this test
Message-ID: <20150621101502.5FC0C1C1DFC@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78225:6042b77b9402
Date: 2015-06-21 12:15 +0200
http://bitbucket.org/pypy/pypy/changeset/6042b77b9402/

Log:	fix this test

diff --git a/pypy/module/struct/test/test_struct.py b/pypy/module/struct/test/test_struct.py
--- a/pypy/module/struct/test/test_struct.py
+++ b/pypy/module/struct/test/test_struct.py
@@ -390,9 +390,9 @@
                                       self.struct.pack("ii", 17, 42) +
                                       '\x00' * (19-sz-2))
         exc = raises(TypeError, self.struct.pack_into, "ii", buffer(b), 0, 17, 42)
-        assert str(exc.value) == "argument must be read-write buffer, not buffer"
+        assert str(exc.value) == "must be read-write buffer, not buffer"
         exc = raises(TypeError, self.struct.pack_into, "ii", 'test', 0, 17, 42)
-        assert str(exc.value) == "argument must be read-write buffer, not str"
+        assert str(exc.value) == "must be read-write buffer, not str"
         exc = raises(self.struct.error, self.struct.pack_into, "ii", b[0:1], 0, 17, 42)
         assert str(exc.value) == "pack_into requires a buffer of at least 8 bytes"
 

From noreply at buildbot.pypy.org  Thu Jun 18 15:29:15 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 18 Jun 2015 15:29:15 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c8: translation fix
Message-ID: <20150618132915.63E6F1C1FCD@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c8
Changeset: r78180:bd18b5f3edec
Date: 2015-06-18 14:30 +0100
http://bitbucket.org/pypy/pypy/changeset/bd18b5f3edec/

Log:	translation fix

diff --git a/pypy/module/pypystm/queue.py b/pypy/module/pypystm/queue.py
--- a/pypy/module/pypystm/queue.py
+++ b/pypy/module/pypystm/queue.py
@@ -75,8 +75,9 @@
         """
         res = self.q.join()
         if res != 0:
-            raise oefmt('task_done() called too many times (%d more than '
-                        'there were items placed in the queue)', -res)
+            raise oefmt(space.w_ValueError,
+                'task_done() called too many times '
+                '(%d more than there were items placed in the queue)', -res)
 
 
 def W_Queue___new__(space, w_subtype):

From noreply at buildbot.pypy.org  Thu Jun 18 11:57:23 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 18 Jun 2015 11:57:23 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c8: can't use queue.get(block=True) in an
	atomic context
Message-ID: <20150618095723.68A061C1FBE@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c8
Changeset: r78173:49f32dc15a94
Date: 2015-06-18 11:57 +0200
http://bitbucket.org/pypy/pypy/changeset/49f32dc15a94/

Log:	can't use queue.get(block=True) in an atomic context

diff --git a/pypy/module/pypystm/queue.py b/pypy/module/pypystm/queue.py
--- a/pypy/module/pypystm/queue.py
+++ b/pypy/module/pypystm/queue.py
@@ -5,7 +5,8 @@
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.typedef import TypeDef
 from pypy.interpreter.gateway import interp2app, unwrap_spec
-from pypy.interpreter.error import OperationError
+from pypy.interpreter.error import OperationError, oefmt
+from pypy.module.thread.error import wrap_thread_error
 
 from rpython.rlib import rstm
 from rpython.rtyper.annlowlevel import cast_gcref_to_instance
@@ -46,6 +47,14 @@
             timeout = -1.0     # no timeout
         else:
             timeout = space.float_w(w_timeout)
+            if timeout < 0.0:
+                raise oefmt(space.w_ValueError,
+                            "'timeout' must be a non-negative number")
+
+        if space.config.translation.stm:    # for tests
+            if rstm.is_atomic() and timeout != 0.0:
+                raise wrap_thread_error(space,
+                    "can't use queue.get(block=True) in an atomic context")
 
         gcref = self.q.get(timeout)
         if not gcref:

From noreply at buildbot.pypy.org  Mon Jun 22 09:18:04 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 22 Jun 2015 09:18:04 +0200 (CEST)
Subject: [pypy-commit] pypy default: Never write a date in the docs without
 specifying the year
Message-ID: <20150622071804.6B9E51C200C@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78239:c7ccc6a7f9d7
Date: 2015-06-22 09:18 +0200
http://bitbucket.org/pypy/pypy/changeset/c7ccc6a7f9d7/

Log:	Never write a date in the docs without specifying the year

diff --git a/pypy/doc/embedding.rst b/pypy/doc/embedding.rst
--- a/pypy/doc/embedding.rst
+++ b/pypy/doc/embedding.rst
@@ -6,7 +6,7 @@
 C. It was developed in collaboration with Roberto De Ioris from the `uwsgi`_
 project. The `PyPy uwsgi plugin`_ is a good example of using the embedding API.
 
-**NOTE**: As of 1st of December, PyPy comes with ``--shared`` by default
+**NOTE**: As of 1st of December 2014, PyPy comes with ``--shared`` by default
 on linux, linux64 and windows. We will make it the default on all platforms
 by the time of the next release.
 

From noreply at buildbot.pypy.org  Thu Jun 18 18:56:16 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 18 Jun 2015 18:56:16 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c8: in-porgress: massively kill stuff in
 transaction.py thanks to the new queue
Message-ID: <20150618165616.CF0781C1FE2@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c8
Changeset: r78189:ad1b81c56740
Date: 2015-06-18 17:57 +0100
http://bitbucket.org/pypy/pypy/changeset/ad1b81c56740/

Log:	in-porgress: massively kill stuff in transaction.py thanks to the
	new queue

diff --git a/lib_pypy/pypy_test/test_transaction.py b/lib_pypy/pypy_test/test_transaction.py
--- a/lib_pypy/pypy_test/test_transaction.py
+++ b/lib_pypy/pypy_test/test_transaction.py
@@ -66,12 +66,14 @@
     for x in range(N):
         lsts = ([], [], [], [], [], [], [], [], [], [])
         def do_stuff(i, j):
+            print 'do_stuff', i, j
             lsts[i].append(j)
             j += 1
             if j < 5:
                 tq.add(do_stuff, i, j)
             else:
                 lsts[i].append('foo')
+                print 'raising FooError!'
                 raise FooError
         tq = transaction.TransactionQueue()
         for i in range(10):
@@ -94,7 +96,8 @@
         assert num_foos == 1, lsts
 
 
-def test_number_of_transactions_reported():
+# XXX reimplement or kill:
+def DONT_test_number_of_transactions_reported():
     tq = transaction.TransactionQueue()
     tq.add(lambda: None)
     tq.add(lambda: None)
diff --git a/lib_pypy/transaction.py b/lib_pypy/transaction.py
--- a/lib_pypy/transaction.py
+++ b/lib_pypy/transaction.py
@@ -51,6 +51,12 @@
     stmdict = dict
     from time import time, clock
 
+try:
+    from pypystm import queue, Empty
+except ImportError:
+    from Queue import Queue as queue
+    from Queue import Empty
+
 class stmidset(object):
     def __init__(self):
         self._hashtable = hashtable()
@@ -114,19 +120,12 @@
     """
 
     def __init__(self):
-        self._deque = collections.deque()
-        self._pending = self._deque
-        self._number_transactions_exec = 0
+        self._queue = queue()
 
     def add(self, f, *args, **kwds):
         """Register a new transaction to be done by 'f(*args, **kwds)'.
         """
-        # note: 'self._pending.append' can be two things here:
-        # * if we are outside run(), it is the regular deque.append method;
-        # * if we are inside run(), self._pending is a thread._local()
-        #   and then its append attribute is the append method of a
-        #   thread-local list.
-        self._pending.append((f, args, kwds))
+        self._queue.put((f, args, kwds))
 
     def add_generator(self, generator_iterator):
         """Register N new transactions to be done by a generator-iterator
@@ -144,141 +143,50 @@
     def run(self, nb_segments=0):
         """Run all transactions, and all transactions started by these
         ones, recursively, until the queue is empty.  If one transaction
-        raises, run() re-raises the exception and the unexecuted transaction
-        are left in the queue.
+        raises, run() re-raises the exception.
         """
         if is_atomic():
             raise TransactionError(
                 "TransactionQueue.run() cannot be called in an atomic context")
-        if not self._pending:
-            return
         if nb_segments <= 0:
             nb_segments = getsegmentlimit()
 
-        assert self._pending is self._deque, "broken state"
-        try:
-            self._pending = thread._local()
-            lock_done_running = thread.allocate_lock()
-            lock_done_running.acquire()
-            lock_deque = thread.allocate_lock()
-            locks = []
-            exception = []
-            args = (locks, lock_done_running, lock_deque,
-                    exception, nb_segments)
-            #
-            for i in range(nb_segments):
-                thread.start_new_thread(self._thread_runner, args)
-            #
-            # The threads run here, and they will release this lock when
-            # they are all finished.
-            lock_done_running.acquire()
-            #
-            assert len(locks) == nb_segments
-            for lock in locks:
-                lock.release()
-            #
-        finally:
-            self._pending = self._deque
+        self._exception = []
+        for i in range(nb_segments):
+            thread.start_new_thread(self._thread_runner, ())
         #
-        if exception:
-            exc_type, exc_value, exc_traceback = exception
+        # The threads run here until queue.join() returns, i.e. until
+        # all add()ed transactions are executed.
+        self._queue.join()
+        #
+        for i in range(nb_segments):
+            self._queue.put((None, None, None))
+        #
+        if self._exception:
+            exc_type, exc_value, exc_traceback = self._exception
+            self._exception = None
             raise exc_type, exc_value, exc_traceback
 
-    def number_of_transactions_executed(self):
-        if self._pending is self._deque:
-            return self._number_transactions_exec
-        raise TransactionError("TransactionQueue.run() is currently running")
+    #def number_of_transactions_executed(self):
+    #    disabled for now
 
-    def _thread_runner(self, locks, lock_done_running, lock_deque,
-                       exception, nb_segments):
-        pending = []
-        self._pending.append = pending.append
-        deque = self._deque
-        lock = thread.allocate_lock()
-        lock.acquire()
-        next_transaction = None
-        count = [0]
-        #
-        def _pause_thread():
-            self._number_transactions_exec += count[0]
-            count[0] = 0
-            locks.append(lock)
-            if len(locks) == nb_segments:
-                lock_done_running.release()
-            lock_deque.release()
-            #
-            # Now wait until our lock is released.
-            lock.acquire()
-            return len(locks) == nb_segments
-        #
-        while not exception:
-            assert next_transaction is None
-            #
-            # Look at the deque and try to fetch the next item on the left.
-            # If empty, we add our lock to the 'locks' list.
-            lock_deque.acquire()
-            if deque:
-                next_transaction = deque.popleft()
-                lock_deque.release()
-            else:
-                if _pause_thread():
-                    return
-                continue
-            #
-            # Now we have a next_transaction.  Run it.
-            assert len(pending) == 0
-            while True:
-                f, args, kwds = next_transaction
-                # The next hint_commit_soon() is essential: without it, the
-                # current transaction is short, so far, but includes everything
-                # after some lock.acquire() done recently.  That means that
-                # anything we do in the atomic section will run with the lock
-                # still acquired.  This prevents any parallelization.
-                hint_commit_soon()
+    def _thread_runner(self):
+        queue = self._queue
+        exception = self._exception
+        while True:
+            f, args, kwds = queue.get()
+            try:
+                if args is None:
+                    break
                 with atomic:
-                    if exception:
-                        break
-                    next_transaction = None
-                    try:
-                        with signals_enabled:
-                            count[0] += 1
-                            f(*args, **kwds)
-                    except:
-                        exception.extend(sys.exc_info())
-                        break
-                #
-                # If no new 'pending' transactions have been added, exit
-                # this loop and go back to fetch more from the deque.
-                if len(pending) == 0:
-                    break
-                #
-                # If we have some new 'pending' transactions, add them
-                # to the right of the deque and pop the next one from
-                # the left.  As we do this atomically with the
-                # 'lock_deque', we are sure that the deque cannot be
-                # empty before the popleft().  (We do that even when
-                # 'len(pending) == 1' instead of simply assigning the
-                # single item to 'next_transaction', because it looks
-                # like a good idea to preserve some first-in-first-out
-                # approximation.)
-                with lock_deque:
-                    deque.extend(pending)
-                    next_transaction = deque.popleft()
-                    try:
-                        for i in range(1, len(pending)):
-                            locks.pop().release()
-                    except IndexError:     # pop from empty list
-                        pass
-                del pending[:]
-        #
-        # We exit here with an exception.  Re-add 'next_transaction'
-        # if it is not None.
-        lock_deque.acquire()
-        if next_transaction is not None:
-            deque.appendleft(next_transaction)
-            next_transaction = None
-        while not _pause_thread():
-            lock_deque.acquire()
+                    if not exception:
+                        try:
+                            with signals_enabled:
+                                f(*args, **kwds)
+                        except:
+                            exception.extend(sys.exc_info())
+            finally:
+                queue.task_done()
 
 
 # ____________________________________________________________

From noreply at buildbot.pypy.org  Mon Jun 22 09:21:43 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 22 Jun 2015 09:21:43 +0200 (CEST)
Subject: [pypy-commit] pypy default: Actually,
 kill these paragraphs and replace them with "--shared is the
Message-ID: <20150622072143.3E2961C201E@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78240:37226b99d814
Date: 2015-06-22 09:21 +0200
http://bitbucket.org/pypy/pypy/changeset/37226b99d814/

Log:	Actually, kill these paragraphs and replace them with "--shared is
	the default in recent releases"

diff --git a/pypy/doc/embedding.rst b/pypy/doc/embedding.rst
--- a/pypy/doc/embedding.rst
+++ b/pypy/doc/embedding.rst
@@ -6,15 +6,9 @@
 C. It was developed in collaboration with Roberto De Ioris from the `uwsgi`_
 project. The `PyPy uwsgi plugin`_ is a good example of using the embedding API.
 
-**NOTE**: As of 1st of December 2014, PyPy comes with ``--shared`` by default
-on linux, linux64 and windows. We will make it the default on all platforms
-by the time of the next release.
-
-The first thing that you need is to compile PyPy yourself with the option
-``--shared``. We plan to make ``--shared`` the default in the future. Consult
-the `how to compile PyPy`_ doc for details. This will result in ``libpypy.so``
-or ``pypy.dll`` file or something similar, depending on your platform. Consult
-your platform specification for details.
+**NOTE**: You need a PyPy compiled with the option ``--shared``, i.e.
+with a ``libpypy-c.so`` or ``pypy-c.dll`` file.  This is the default in
+recent versions of PyPy.
 
 The resulting shared library exports very few functions, however they are
 enough to accomplish everything you need, provided you follow a few principles.

From noreply at buildbot.pypy.org  Thu Jun 18 09:42:26 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 18 Jun 2015 09:42:26 +0200 (CEST)
Subject: [pypy-commit] stmgc queue: Pushing roots around stm_queue_put was
 needed only when was had
Message-ID: <20150618074226.119101C118C@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: queue
Changeset: r1856:3ca830828468
Date: 2015-06-18 09:42 +0200
http://bitbucket.org/pypy/stmgc/changeset/3ca830828468/

Log:	Pushing roots around stm_queue_put was needed only when was had
	'stm_queue_entry' GC objects

diff --git a/c8/stmgc.h b/c8/stmgc.h
--- a/c8/stmgc.h
+++ b/c8/stmgc.h
@@ -739,7 +739,7 @@
 
 stm_queue_t *stm_queue_create(void);
 void stm_queue_free(stm_queue_t *);
-/* put() does not cause delays or transaction breaks (but push roots!) */
+/* put() does not cause delays or transaction breaks */
 void stm_queue_put(object_t *qobj, stm_queue_t *queue, object_t *newitem);
 /* get() can commit and wait outside a transaction (so push roots).
    Unsuitable if the current transaction is atomic!  With timeout < 0.0,

From noreply at buildbot.pypy.org  Thu Jun 18 13:10:44 2015
From: noreply at buildbot.pypy.org (cfbolz)
Date: Thu, 18 Jun 2015 13:10:44 +0200 (CEST)
Subject: [pypy-commit] pypy default: in the heapcache don't do the same dict
 lookups twice for every getfield and
Message-ID: <20150618111044.5DE9D1C1F86@cobra.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r78175:294f1462a787
Date: 2015-06-18 13:10 +0200
http://bitbucket.org/pypy/pypy/changeset/294f1462a787/

Log:	in the heapcache don't do the same dict lookups twice for every
	getfield and setfield that is being traced

diff --git a/rpython/jit/metainterp/heapcache.py b/rpython/jit/metainterp/heapcache.py
--- a/rpython/jit/metainterp/heapcache.py
+++ b/rpython/jit/metainterp/heapcache.py
@@ -60,6 +60,26 @@
             if not value.is_unescaped:
                 del d[value]
 
+
+class FieldUpdater(object):
+    def __init__(self, heapcache, value, cache, fieldvalue):
+        self.heapcache = heapcache
+        self.value = value
+        self.cache = cache
+        if fieldvalue is not None:
+            self.currfieldbox = fieldvalue.box
+        else:
+            self.currfieldbox = None
+
+    def getfield_now_known(self, fieldbox):
+        fieldvalue = self.heapcache.getvalue(fieldbox)
+        self.cache.read_now_known(self.value, fieldvalue)
+
+    def setfield(self, fieldbox):
+        fieldvalue = self.heapcache.getvalue(fieldbox)
+        self.cache.do_write_with_aliasing(self.value, fieldvalue)
+
+
 class HeapCache(object):
     def __init__(self):
         self.reset()
@@ -311,21 +331,23 @@
                     return tovalue.box
         return None
 
-    def getfield_now_known(self, box, descr, fieldbox):
+    def get_field_updater(self, box, descr):
         value = self.getvalue(box)
-        fieldvalue = self.getvalue(fieldbox)
         cache = self.heap_cache.get(descr, None)
         if cache is None:
             cache = self.heap_cache[descr] = CacheEntry()
-        cache.read_now_known(value, fieldvalue)
+            fieldvalue = None
+        else:
+            fieldvalue = cache.read(value)
+        return FieldUpdater(self, value, cache, fieldvalue)
+
+    def getfield_now_known(self, box, descr, fieldbox):
+        upd = self.get_field_updater(box, descr)
+        upd.getfield_now_known(fieldbox)
 
     def setfield(self, box, fieldbox, descr):
-        cache = self.heap_cache.get(descr, None)
-        if cache is None:
-            cache = self.heap_cache[descr] = CacheEntry()
-        value = self.getvalue(box)
-        fieldvalue = self.getvalue(fieldbox)
-        cache.do_write_with_aliasing(value, fieldvalue)
+        upd = self.get_field_updater(box, descr)
+        upd.setfield(fieldbox)
 
     def getarrayitem(self, box, indexbox, descr):
         if not isinstance(indexbox, ConstInt):
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -649,16 +649,16 @@
 
     @specialize.arg(1)
     def _opimpl_getfield_gc_any_pureornot(self, opnum, box, fielddescr):
-        tobox = self.metainterp.heapcache.getfield(box, fielddescr)
-        if tobox is not None:
+        upd = self.metainterp.heapcache.get_field_updater(box, fielddescr)
+        if upd.currfieldbox is not None:
             # sanity check: see whether the current struct value
             # corresponds to what the cache thinks the value is
             resbox = executor.execute(self.metainterp.cpu, self.metainterp,
                                       rop.GETFIELD_GC, fielddescr, box)
-            assert resbox.constbox().same_constant(tobox.constbox())
-            return tobox
+            assert resbox.constbox().same_constant(upd.currfieldbox.constbox())
+            return upd.currfieldbox
         resbox = self.execute_with_descr(opnum, fielddescr, box)
-        self.metainterp.heapcache.getfield_now_known(box, fielddescr, resbox)
+        upd.getfield_now_known(resbox)
         return resbox
 
     @arguments("box", "descr", "orgpc")
@@ -679,10 +679,11 @@
 
     @arguments("box", "box", "descr")
     def _opimpl_setfield_gc_any(self, box, valuebox, fielddescr):
-        tobox = self.metainterp.heapcache.getfield(box, fielddescr)
-        if tobox is valuebox:
+        upd = self.metainterp.heapcache.get_field_updater(box, fielddescr)
+        if upd.currfieldbox is valuebox:
             return
-        self.metainterp.execute_setfield_gc(fielddescr, box, valuebox)
+        self.metainterp.execute_and_record(rop.SETFIELD_GC, fielddescr, box, valuebox)
+        upd.setfield(valuebox)
         # The following logic is disabled because buggy.  It is supposed
         # to be: not(we're writing null into a freshly allocated object)
         # but the bug is that is_unescaped() can be True even after the

From noreply at buildbot.pypy.org  Sun Jun 21 21:43:23 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Sun, 21 Jun 2015 21:43:23 +0200 (CEST)
Subject: [pypy-commit] pypy run-create_cffi_imports: add try/except (arigato)
Message-ID: <20150621194323.3A3541C1F38@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: run-create_cffi_imports
Changeset: r78236:e281196f520a
Date: 2015-06-21 22:43 +0300
http://bitbucket.org/pypy/pypy/changeset/e281196f520a/

Log:	add try/except (arigato)

diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -315,7 +315,7 @@
 
         @taskdef(['compile_c'], "Create cffi bindings for modules")
         def task_build_cffi_imports(self):
-            from pypy.tool.build_cffi_imports import create_cffi_import_libraries
+            from pypy.tool.build_cffi_imports import create_cffi_import_libraries, MissingDependenciesError
             ''' Use cffi to compile cffi interfaces to modules'''
             exename = mkexename(driver.compute_exe_name())
             basedir = exename
@@ -328,7 +328,10 @@
             modules = self.config.objspace.usemodules.getpaths()
             options = Options()
             # XXX possibly adapt options using modules
-            create_cffi_import_libraries(exename, options, basedir)      
+            try:
+                create_cffi_import_libraries(exename, options, basedir)
+            except MissingDependenciesError:
+                pass
         driver.task_build_cffi_imports = types.MethodType(task_build_cffi_imports, driver)
         driver.tasks['build_cffi_imports'] = driver.task_build_cffi_imports, ['compile_c']
         driver.default_goal = 'build_cffi_imports'

From noreply at buildbot.pypy.org  Thu Jun 18 23:40:49 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 18 Jun 2015 23:40:49 +0200 (CEST)
Subject: [pypy-commit] stmgc queue: tweaks
Message-ID: <20150618214049.CB9601C1C7D@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: queue
Changeset: r1872:277dd2ad5226
Date: 2015-06-18 23:41 +0200
http://bitbucket.org/pypy/stmgc/changeset/277dd2ad5226/

Log:	tweaks

diff --git a/c8/stm/queue.c b/c8/stm/queue.c
--- a/c8/stm/queue.c
+++ b/c8/stm/queue.c
@@ -84,8 +84,6 @@
             bool ok = tree_delete_item(pseg->active_queues, (uintptr_t)queue);
             assert(ok);
             (void)ok;
-            queue_free_entries(seg->added_in_this_transaction);
-            queue_free_entries(seg->old_objects_popped);
         }
         else {
             assert(!seg->added_in_this_transaction);
@@ -94,6 +92,9 @@
         }
 
         spinlock_release(pseg->active_queues_lock);
+
+        queue_free_entries(seg->added_in_this_transaction);
+        queue_free_entries(seg->old_objects_popped);
     }
     free(queue);
 }
@@ -164,8 +165,11 @@
         if (head != NULL) {
             queue_entry_t *old;
             queue_entry_t *tail = head;
-            while (tail->next != NULL)
+            assert(!_is_in_nursery(head->object));
+            while (tail->next != NULL) {
                 tail = tail->next;
+                assert(!_is_in_nursery(tail->object));
+            }
             dprintf(("items move to old_entries in queue %p\n", queue));
 
             spinlock_acquire(queue->old_entries_lock);
@@ -271,6 +275,8 @@
         queue_activate(queue, seg);
 
         queue_check_entry(entry);
+        assert(!_is_in_nursery(entry->object));
+
         entry->next = seg->old_objects_popped;
         seg->old_objects_popped = entry;
         return entry->object;

From noreply at buildbot.pypy.org  Thu Jun 18 10:56:46 2015
From: noreply at buildbot.pypy.org (cfbolz)
Date: Thu, 18 Jun 2015 10:56:46 +0200 (CEST)
Subject: [pypy-commit] pypy default: don't create HeapCacheValue entries
	just to escape them
Message-ID: <20150618085646.803521C148B@cobra.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r78166:11818af6e56c
Date: 2015-06-18 10:44 +0200
http://bitbucket.org/pypy/pypy/changeset/11818af6e56c/

Log:	don't create HeapCacheValue entries just to escape them

diff --git a/rpython/jit/metainterp/heapcache.py b/rpython/jit/metainterp/heapcache.py
--- a/rpython/jit/metainterp/heapcache.py
+++ b/rpython/jit/metainterp/heapcache.py
@@ -98,9 +98,9 @@
         self.heap_cache = {}
         self.heap_array_cache = {}
 
-    def getvalue(self, box):
+    def getvalue(self, box, create=True):
         value = self.values.get(box, None)
-        if not value:
+        if not value and create:
             value = self.values[box] = HeapCacheValue(box)
         return value
 
@@ -111,25 +111,26 @@
         self.mark_escaped(opnum, descr, argboxes)
         self.clear_caches(opnum, descr, argboxes)
 
+    def _escape_from_write(self, box, fieldbox):
+        value = self.getvalue(box, create=False)
+        fieldvalue = self.getvalue(fieldbox, create=False)
+        if (value is not None and value.is_unescaped and
+                fieldvalue is not None and fieldvalue.is_unescaped):
+            if value.dependencies is None:
+                value.dependencies = []
+            value.dependencies.append(fieldvalue)
+        elif fieldvalue is not None:
+            self._escape(fieldvalue)
+
     def mark_escaped(self, opnum, descr, argboxes):
         if opnum == rop.SETFIELD_GC:
             assert len(argboxes) == 2
-            value, fieldvalue = self.getvalues(argboxes)
-            if value.is_unescaped and fieldvalue.is_unescaped:
-                if value.dependencies is None:
-                    value.dependencies = []
-                value.dependencies.append(fieldvalue)
-            else:
-                self._escape(fieldvalue)
+            box, fieldbox = argboxes
+            self._escape_from_write(box, fieldbox)
         elif opnum == rop.SETARRAYITEM_GC:
             assert len(argboxes) == 3
-            value, indexvalue, fieldvalue = self.getvalues(argboxes)
-            if value.is_unescaped and fieldvalue.is_unescaped:
-                if value.dependencies is None:
-                    value.dependencies = []
-                value.dependencies.append(fieldvalue)
-            else:
-                self._escape(fieldvalue)
+            box, indexbox, fieldbox = argboxes
+            self._escape_from_write(box, fieldbox)
         elif (opnum == rop.CALL and
               descr.get_extra_info().oopspecindex == descr.get_extra_info().OS_ARRAYCOPY and
               isinstance(argboxes[3], ConstInt) and
@@ -153,7 +154,7 @@
                 self._escape_box(box)
 
     def _escape_box(self, box):
-        value = self.values.get(box, None)
+        value = self.getvalue(box, create=False)
         if not value:
             return
         self._escape(value)
@@ -261,7 +262,7 @@
         self.reset_keep_likely_virtuals()
 
     def is_class_known(self, box):
-        value = self.values.get(box, None)
+        value = self.getvalue(box, create=False)
         if value:
             return value.known_class
         return False
@@ -270,7 +271,7 @@
         self.getvalue(box).known_class = True
 
     def is_nonstandard_virtualizable(self, box):
-        value = self.values.get(box, None)
+        value = self.getvalue(box, create=False)
         if value:
             return value.nonstandard_virtualizable
         return False
@@ -279,13 +280,13 @@
         self.getvalue(box).nonstandard_virtualizable = True
 
     def is_unescaped(self, box):
-        value = self.values.get(box, None)
+        value = self.getvalue(box, create=False)
         if value:
             return value.is_unescaped
         return False
 
     def is_likely_virtual(self, box):
-        value = self.values.get(box, None)
+        value = self.getvalue(box, create=False)
         if value:
             return value.likely_virtual
         return False
@@ -301,7 +302,7 @@
         self.arraylen_now_known(box, lengthbox)
 
     def getfield(self, box, descr):
-        value = self.values.get(box, None)
+        value = self.getvalue(box, create=False)
         if value:
             cache = self.heap_cache.get(descr, None)
             if cache:
@@ -329,7 +330,7 @@
     def getarrayitem(self, box, indexbox, descr):
         if not isinstance(indexbox, ConstInt):
             return None
-        value = self.values.get(box, None)
+        value = self.getvalue(box, create=False)
         if value is None:
             return None
         index = indexbox.getint()
@@ -373,7 +374,7 @@
             indexcache.do_write_with_aliasing(value, fieldvalue)
 
     def arraylen(self, box):
-        value = self.values.get(box, None)
+        value = self.getvalue(box, create=False)
         if value and value.length:
             return value.length.box
         return None
@@ -383,7 +384,7 @@
         value.length = self.getvalue(lengthbox)
 
     def replace_box(self, oldbox, newbox):
-        value = self.values.get(oldbox, None)
+        value = self.getvalue(oldbox, create=False)
         if value is None:
             return
         value.box = newbox

From noreply at buildbot.pypy.org  Thu Jun 18 09:48:08 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 18 Jun 2015 09:48:08 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c8: import stmgc branch 'queue',
 and add rlib support for stm queues
Message-ID: <20150618074808.B1D8E1C1C96@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c8
Changeset: r78163:91a280e13a9d
Date: 2015-06-18 09:48 +0200
http://bitbucket.org/pypy/pypy/changeset/91a280e13a9d/

Log:	import stmgc branch 'queue', and add rlib support for stm queues

diff --git a/rpython/memory/gctransform/stmframework.py b/rpython/memory/gctransform/stmframework.py
--- a/rpython/memory/gctransform/stmframework.py
+++ b/rpython/memory/gctransform/stmframework.py
@@ -224,6 +224,7 @@
     gct_stm_stop_all_other_threads                  = _gct_with_roots_pushed
     gct_stm_transaction_break                       = _gct_with_roots_pushed
     gct_stm_collect                                 = _gct_with_roots_pushed
+    gct_stm_queue_get                               = _gct_with_roots_pushed
 
 
 class StmRootWalker(BaseRootWalker):
diff --git a/rpython/rlib/rstm.py b/rpython/rlib/rstm.py
--- a/rpython/rlib/rstm.py
+++ b/rpython/rlib/rstm.py
@@ -356,3 +356,70 @@
                         " use h.writeobj() instead")
 
     object = property(_getobj, _setobj)
+
+# ____________________________________________________________
+
+_STM_QUEUE_P = rffi.COpaquePtr('stm_queue_t')
+
+ at dont_look_inside
+def _ll_queue_get(q, timeout=-1.0):
+    # Returns a GCREF.
+    return llop.stm_queue_get(llmemory.GCREF, q, q.ll_raw_queue, timeout)
+
+ at dont_look_inside
+def _ll_queue_put(q, newitem):
+    llop.stm_queue_put(lltype.Void, q, q.ll_raw_queue, newitem)
+
+_QUEUE_OBJ = lltype.GcStruct('QUEUE_OBJ',
+                             ('ll_raw_queue', _STM_QUEUE_P),
+                             hints={'immutable': True},
+                             rtti=True,
+                             adtmeths={'get': _ll_queue_get,
+                                       'put': _ll_queue_put})
+NULL_QUEUE = lltype.nullptr(_QUEUE_OBJ)
+
+def _ll_queue_trace(gc, obj, callback, arg):
+    from rpython.memory.gctransform.stmframework import get_visit_function
+    visit_fn = get_visit_function(callback, arg)
+    addr = obj + llmemory.offsetof(_QUEUE_OBJ, 'll_raw_queue')
+    llop.stm_queue_tracefn(lltype.Void, addr.address[0], visit_fn)
+lambda_queue_trace = lambda: _ll_queue_trace
+
+def _ll_queue_finalizer(q):
+    if q.ll_raw_queue:
+        llop.stm_queue_free(lltype.Void, q.ll_raw_queue)
+lambda_queue_finlz = lambda: _ll_queue_finalizer
+
+ at dont_look_inside
+def create_queue():
+    if not we_are_translated():
+        return QueueForTest()      # for tests
+    rgc.register_custom_light_finalizer(_QUEUE_OBJ, lambda_queue_finlz)
+    rgc.register_custom_trace_hook(_QUEUE_OBJ, lambda_queue_trace)
+    q = lltype.malloc(_QUEUE_OBJ, zero=True)
+    q.ll_raw_queue = llop.stm_queue_create(_STM_QUEUE_P)
+    return q
+
+class QueueForTest(object):
+    def __init__(self):
+        import Queue
+        self._content = Queue.Queue()
+        self._Empty = Queue.Empty
+
+    def _cleanup_(self):
+        raise Exception("cannot translate a prebuilt rstm.Queue object")
+
+    def get(self, timeout=-1.0):
+        if timeout < 0.0:
+            return self._content.get()
+        try:
+            if timeout == 0.0:
+                return self._content.get(block=False)
+            else:
+                return self._content.get(timeout=timeout)
+        except self._Empty:
+            return NULL_GCREF
+
+    def put(self, newitem):
+        assert lltype.typeOf(newitem) == llmemory.GCREF
+        self._content.put(newitem)
diff --git a/rpython/rtyper/llinterp.py b/rpython/rtyper/llinterp.py
--- a/rpython/rtyper/llinterp.py
+++ b/rpython/rtyper/llinterp.py
@@ -1002,6 +1002,11 @@
     op_stm_hashtable_length_upper_bound = _stm_not_implemented
     op_stm_hashtable_list = _stm_not_implemented
     op_stm_hashtable_free = _stm_not_implemented
+    op_stm_queue_create = _stm_not_implemented
+    op_stm_queue_free = _stm_not_implemented
+    op_stm_queue_get = _stm_not_implemented
+    op_stm_queue_put = _stm_not_implemented
+    op_stm_queue_tracefn = _stm_not_implemented
     op_stm_register_thread_local = _stm_not_implemented
     op_stm_unregister_thread_local = _stm_not_implemented
     op_stm_really_force_cast_ptr = _stm_not_implemented
diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -479,6 +479,12 @@
     'stm_hashtable_list'  :   LLOp(),
     'stm_hashtable_tracefn':  LLOp(),
 
+    'stm_queue_create':       LLOp(),
+    'stm_queue_free':         LLOp(),
+    'stm_queue_get':          LLOp(canmallocgc=True),   # push roots!
+    'stm_queue_put':          LLOp(),
+    'stm_queue_tracefn':      LLOp(),
+
     # __________ address operations __________
 
     'boehm_malloc':         LLOp(),
diff --git a/rpython/translator/backendopt/finalizer.py b/rpython/translator/backendopt/finalizer.py
--- a/rpython/translator/backendopt/finalizer.py
+++ b/rpython/translator/backendopt/finalizer.py
@@ -18,7 +18,8 @@
     """
     ok_operations = ['ptr_nonzero', 'ptr_eq', 'ptr_ne', 'free', 'same_as',
                      'direct_ptradd', 'force_cast', 'track_alloc_stop',
-                     'raw_free', 'debug_print', 'stm_hashtable_free']
+                     'raw_free', 'debug_print', 'stm_hashtable_free',
+                     'stm_queue_free']
 
     def analyze_light_finalizer(self, graph):
         result = self.analyze_direct_call(graph)
diff --git a/rpython/translator/c/src/mem.c b/rpython/translator/c/src/mem.c
--- a/rpython/translator/c/src/mem.c
+++ b/rpython/translator/c/src/mem.c
@@ -62,7 +62,7 @@
 
 #ifdef RPY_STM
 // spinlock_acquire/spinlock_release defined in ../../stm/src_stm/stmgcintf.h
-static Signed pypy_debug_alloc_lock = 0;
+static uint8_t pypy_debug_alloc_lock = 0;
 #else
 # define spinlock_acquire(lock)               /* nothing */
 # define spinlock_release(lock)               /* nothing */
diff --git a/rpython/translator/stm/breakfinder.py b/rpython/translator/stm/breakfinder.py
--- a/rpython/translator/stm/breakfinder.py
+++ b/rpython/translator/stm/breakfinder.py
@@ -9,6 +9,7 @@
     'stm_enter_callback_call',
     'stm_leave_callback_call',
     'stm_transaction_break',
+    'stm_queue_get',
     ])
 
 for tb in TRANSACTION_BREAK:
diff --git a/rpython/translator/stm/funcgen.py b/rpython/translator/stm/funcgen.py
--- a/rpython/translator/stm/funcgen.py
+++ b/rpython/translator/stm/funcgen.py
@@ -368,3 +368,32 @@
     arg2 = funcgen.expr(op.args[2])
     return ('stm_hashtable_tracefn(%s, (stm_hashtable_t *)%s, '
             ' (void(*)(object_t**))%s);' % (arg0, arg1, arg2))
+
+def stm_queue_create(funcgen, op):
+    result = funcgen.expr(op.result)
+    return '%s = stm_queue_create();' % (result,)
+
+def stm_queue_free(funcgen, op):
+    arg = funcgen.expr(op.args[0])
+    return 'stm_queue_free(%s);' % (arg,)
+
+def stm_queue_get(funcgen, op):
+    arg0 = funcgen.expr(op.args[0])
+    arg1 = funcgen.expr(op.args[1])
+    arg2 = funcgen.expr(op.args[2])
+    result = funcgen.expr(op.result)
+    return ('%s = (rpygcchar_t *)stm_queue_get((object_t *)%s, %s, %s, '
+            '&stm_thread_local);' % (result, arg0, arg1, arg2))
+
+def stm_queue_put(funcgen, op):
+    arg0 = funcgen.expr(op.args[0])
+    arg1 = funcgen.expr(op.args[1])
+    arg2 = funcgen.expr(op.args[2])
+    return 'stm_queue_put((object_t *)%s, %s, (object_t *)%s);' % (
+        arg0, arg1, arg2)
+
+def stm_queue_tracefn(funcgen, op):
+    arg0 = funcgen.expr(op.args[0])
+    arg1 = funcgen.expr(op.args[1])
+    return ('stm_queue_tracefn((stm_queue_t *)%s, '
+            ' (void(*)(object_t**))%s);' % (arg0, arg1))
diff --git a/rpython/translator/stm/src_stm/revision b/rpython/translator/stm/src_stm/revision
--- a/rpython/translator/stm/src_stm/revision
+++ b/rpython/translator/stm/src_stm/revision
@@ -1,1 +1,1 @@
-14536c2a2af4
+3ca830828468
diff --git a/rpython/translator/stm/src_stm/stm/atomic.h b/rpython/translator/stm/src_stm/stm/atomic.h
--- a/rpython/translator/stm/src_stm/stm/atomic.h
+++ b/rpython/translator/stm/src_stm/stm/atomic.h
@@ -42,12 +42,19 @@
 #endif
 
 
-#define spinlock_acquire(lock)                                          \
-    do { if (LIKELY(__sync_lock_test_and_set(&(lock), 1) == 0)) break;  \
-         spin_loop(); } while (1)
-#define spinlock_release(lock)                                          \
-    do { assert((lock) == 1);                                           \
-         __sync_lock_release(&(lock)); } while (0)
+static inline void _spinlock_acquire(uint8_t *plock) {
+ retry:
+    if (__builtin_expect(__sync_lock_test_and_set(plock, 1) != 0, 0)) {
+        spin_loop();
+        goto retry;
+    }
+}
+static inline void _spinlock_release(uint8_t *plock) {
+    assert(*plock == 1);
+    __sync_lock_release(plock);
+}
+#define spinlock_acquire(lock) _spinlock_acquire(&(lock))
+#define spinlock_release(lock) _spinlock_release(&(lock))
 
 
 #endif  /* _STM_ATOMIC_H */
diff --git a/rpython/translator/stm/src_stm/stm/core.c b/rpython/translator/stm/src_stm/stm/core.c
--- a/rpython/translator/stm/src_stm/stm/core.c
+++ b/rpython/translator/stm/src_stm/stm/core.c
@@ -1146,6 +1146,7 @@
     assert(tree_is_cleared(STM_PSEGMENT->callbacks_on_commit_and_abort[1]));
     assert(list_is_empty(STM_PSEGMENT->young_objects_with_light_finalizers));
     assert(STM_PSEGMENT->finalizers == NULL);
+    assert(STM_PSEGMENT->active_queues == NULL);
 #ifndef NDEBUG
     /* this should not be used when objects_pointing_to_nursery == NULL */
     STM_PSEGMENT->position_markers_len_old = 99999999999999999L;
@@ -1351,6 +1352,9 @@
         STM_PSEGMENT->overflow_number_has_been_used = false;
     }
 
+    if (STM_PSEGMENT->active_queues)
+        queues_deactivate_all(/*at_commit=*/true);
+
     invoke_and_clear_user_callbacks(0);   /* for commit */
 
     /* done */
@@ -1505,6 +1509,9 @@
     if (tl->mem_clear_on_abort)
         memset(tl->mem_clear_on_abort, 0, tl->mem_bytes_to_clear_on_abort);
 
+    if (STM_PSEGMENT->active_queues)
+        queues_deactivate_all(/*at_commit=*/false);
+
     invoke_and_clear_user_callbacks(1);   /* for abort */
 
     if (is_abort(STM_SEGMENT->nursery_end)) {
diff --git a/rpython/translator/stm/src_stm/stm/core.h b/rpython/translator/stm/src_stm/stm/core.h
--- a/rpython/translator/stm/src_stm/stm/core.h
+++ b/rpython/translator/stm/src_stm/stm/core.h
@@ -117,13 +117,15 @@
     bool minor_collect_will_commit_now;
 
     struct tree_s *callbacks_on_commit_and_abort[2];
+    struct tree_s *active_queues;
+    uint8_t active_queues_lock;
 
     /* This is the number stored in the overflowed objects (a multiple of
        GCFLAG_OVERFLOW_NUMBER_bit0).  It is incremented when the
        transaction is done, but only if we actually overflowed any
        object; otherwise, no object has got this number. */
+    bool overflow_number_has_been_used;
     uint32_t overflow_number;
-    bool overflow_number_has_been_used;
 
 
     struct stm_commit_log_entry_s *last_commit_log_entry;
diff --git a/rpython/translator/stm/src_stm/stm/gcpage.c b/rpython/translator/stm/src_stm/stm/gcpage.c
--- a/rpython/translator/stm/src_stm/stm/gcpage.c
+++ b/rpython/translator/stm/src_stm/stm/gcpage.c
@@ -48,7 +48,7 @@
 }
 
 
-static int lock_growth_large = 0;
+static uint8_t lock_growth_large = 0;
 
 static stm_char *allocate_outside_nursery_large(uint64_t size)
 {
diff --git a/rpython/translator/stm/src_stm/stm/largemalloc.c b/rpython/translator/stm/src_stm/stm/largemalloc.c
--- a/rpython/translator/stm/src_stm/stm/largemalloc.c
+++ b/rpython/translator/stm/src_stm/stm/largemalloc.c
@@ -108,7 +108,7 @@
 
 
 static struct {
-    int lock;
+    uint8_t lock;
     mchunk_t *first_chunk, *last_chunk;
     dlist_t largebins[N_BINS];
 } lm __attribute__((aligned(64)));
diff --git a/rpython/translator/stm/src_stm/stm/queue.c b/rpython/translator/stm/src_stm/stm/queue.c
new file mode 100644
--- /dev/null
+++ b/rpython/translator/stm/src_stm/stm/queue.c
@@ -0,0 +1,319 @@
+/* Imported by rpython/translator/stm/import_stmgc.py */
+typedef struct queue_entry_s {
+    object_t *object;
+    struct queue_entry_s *next;
+} queue_entry_t;
+
+typedef union stm_queue_segment_u {
+    struct {
+        /* a chained list of fresh entries that have been allocated and
+           added to this queue during the current transaction.  If the
+           transaction commits, these are moved to 'old_entries'. */
+        queue_entry_t *added_in_this_transaction;
+
+        /* a point inside the chained list above such that all items from
+           this point are known to contain non-young objects, for GC */
+        queue_entry_t *added_young_limit;
+
+        /* a chained list of old entries that the current transaction
+           popped.  only used if the transaction is not inevitable:
+           if it aborts, these entries are added back to 'old_entries'. */
+        queue_entry_t *old_objects_popped;
+
+        /* a queue is active when either of the two chained lists
+           above is not empty, until the transaction commits.  (this
+           notion is per segment.)  this flag says that the queue is
+           already in the tree STM_PSEGMENT->active_queues. */
+        bool active;
+    };
+    char pad[64];
+} stm_queue_segment_t;
+
+
+struct stm_queue_s {
+    /* this structure is always allocated on a multiple of 64 bytes,
+       and the 'segs' is an array of items 64 bytes each */
+    stm_queue_segment_t segs[STM_NB_SEGMENTS];
+
+    /* a chained list of old entries in the queue */
+    queue_entry_t *volatile old_entries;
+};
+
+
+stm_queue_t *stm_queue_create(void)
+{
+    void *mem;
+    int result = posix_memalign(&mem, 64, sizeof(stm_queue_t));
+    assert(result == 0);
+    (void)result;
+    memset(mem, 0, sizeof(stm_queue_t));
+    return (stm_queue_t *)mem;
+}
+
+static void queue_free_entries(queue_entry_t *lst)
+{
+    while (lst != NULL) {
+        queue_entry_t *next = lst->next;
+        free(lst);
+        lst = next;
+    }
+}
+
+void stm_queue_free(stm_queue_t *queue)
+{
+    long i;
+    dprintf(("free queue %p\n", queue));
+    for (i = 0; i < STM_NB_SEGMENTS; i++) {
+        stm_queue_segment_t *seg = &queue->segs[i];
+
+        /* it is possible that queues_deactivate_all() runs in parallel,
+           but it should not be possible at this point for another thread
+           to change 'active' from false to true.  if it is false, then
+           that's it */
+        if (!seg->active) {
+            assert(!seg->added_in_this_transaction);
+            assert(!seg->added_young_limit);
+            assert(!seg->old_objects_popped);
+            continue;
+        }
+
+        struct stm_priv_segment_info_s *pseg = get_priv_segment(i + 1);
+        spinlock_acquire(pseg->active_queues_lock);
+
+        if (seg->active) {
+            assert(pseg->active_queues != NULL);
+            bool ok = tree_delete_item(pseg->active_queues, (uintptr_t)queue);
+            assert(ok);
+            (void)ok;
+        }
+        queue_free_entries(seg->added_in_this_transaction);
+        queue_free_entries(seg->old_objects_popped);
+
+        spinlock_release(pseg->active_queues_lock);
+    }
+    free(queue);
+}
+
+static inline void queue_lock_acquire(void)
+{
+    int num = STM_SEGMENT->segment_num;
+    spinlock_acquire(get_priv_segment(num)->active_queues_lock);
+}
+static inline void queue_lock_release(void)
+{
+    int num = STM_SEGMENT->segment_num;
+    spinlock_release(get_priv_segment(num)->active_queues_lock);
+}
+
+static void queue_activate(stm_queue_t *queue)
+{
+    stm_queue_segment_t *seg = &queue->segs[STM_SEGMENT->segment_num - 1];
+
+    if (!seg->active) {
+        queue_lock_acquire();
+        if (STM_PSEGMENT->active_queues == NULL)
+            STM_PSEGMENT->active_queues = tree_create();
+        tree_insert(STM_PSEGMENT->active_queues, (uintptr_t)queue, 0);
+        assert(!seg->active);
+        seg->active = true;
+        dprintf(("activated queue %p\n", queue));
+        queue_lock_release();
+    }
+}
+
+static void queues_deactivate_all(bool at_commit)
+{
+    queue_lock_acquire();
+
+    bool added_any_old_entries = false;
+    wlog_t *item;
+    TREE_LOOP_FORWARD(STM_PSEGMENT->active_queues, item) {
+        stm_queue_t *queue = (stm_queue_t *)item->addr;
+        stm_queue_segment_t *seg = &queue->segs[STM_SEGMENT->segment_num - 1];
+        queue_entry_t *head, *freehead;
+
+        if (at_commit) {
+            head = seg->added_in_this_transaction;
+            freehead = seg->old_objects_popped;
+        }
+        else {
+            head = seg->old_objects_popped;
+            freehead = seg->added_in_this_transaction;
+        }
+
+        /* forget the two lists of entries */
+        seg->added_in_this_transaction = NULL;
+        seg->added_young_limit = NULL;
+        seg->old_objects_popped = NULL;
+
+        /* free the list of entries that must disappear */
+        queue_free_entries(freehead);
+
+        /* move the list of entries that must survive to 'old_entries' */
+        if (head != NULL) {
+            queue_entry_t *old;
+            queue_entry_t *tail = head;
+            while (tail->next != NULL)
+                tail = tail->next;
+            dprintf(("items move to old_entries in queue %p\n", queue));
+         retry:
+            old = queue->old_entries;
+            tail->next = old;
+            if (!__sync_bool_compare_and_swap(&queue->old_entries, old, head))
+                goto retry;
+            added_any_old_entries = true;
+        }
+
+        /* deactivate this queue */
+        assert(seg->active);
+        seg->active = false;
+        dprintf(("deactivated queue %p\n", queue));
+
+    } TREE_LOOP_END;
+
+    tree_free(STM_PSEGMENT->active_queues);
+    STM_PSEGMENT->active_queues = NULL;
+
+    queue_lock_release();
+
+    if (added_any_old_entries) {
+        assert(_has_mutex());
+        cond_broadcast(C_QUEUE_OLD_ENTRIES);
+    }
+}
+
+void stm_queue_put(object_t *qobj, stm_queue_t *queue, object_t *newitem)
+{
+    /* must be run in a transaction, but doesn't cause conflicts or
+       delays or transaction breaks.  you need to push roots!
+    */
+    stm_queue_segment_t *seg = &queue->segs[STM_SEGMENT->segment_num - 1];
+    queue_entry_t *entry = malloc(sizeof(queue_entry_t));
+    assert(entry);
+    entry->object = newitem;
+    entry->next = seg->added_in_this_transaction;
+    seg->added_in_this_transaction = entry;
+
+    queue_activate(queue);
+
+    /* add qobj to 'objects_pointing_to_nursery' if it has the
+       WRITE_BARRIER flag */
+    if (qobj->stm_flags & GCFLAG_WRITE_BARRIER) {
+        qobj->stm_flags &= ~GCFLAG_WRITE_BARRIER;
+        LIST_APPEND(STM_PSEGMENT->objects_pointing_to_nursery, qobj);
+    }
+}
+
+object_t *stm_queue_get(object_t *qobj, stm_queue_t *queue, double timeout,
+                        stm_thread_local_t *tl)
+{
+    /* if the queue is empty, this commits and waits outside a transaction.
+       must not be called if the transaction is atomic!  never causes
+       conflicts.  you need to push roots!
+    */
+    struct timespec t;
+    bool t_ready = false;
+    queue_entry_t *entry;
+    object_t *result;
+    stm_queue_segment_t *seg = &queue->segs[STM_SEGMENT->segment_num - 1];
+
+    if (seg->added_in_this_transaction) {
+        entry = seg->added_in_this_transaction;
+        seg->added_in_this_transaction = entry->next;
+        if (entry == seg->added_young_limit)
+            seg->added_young_limit = entry->next;
+        result = entry->object;
+        assert(result != NULL);
+        free(entry);
+        return result;
+    }
+
+ retry:
+    entry = queue->old_entries;
+    if (entry != NULL) {
+        if (!__sync_bool_compare_and_swap(&queue->old_entries,
+                                          entry, entry->next))
+            goto retry;
+
+        /* successfully popped the old 'entry'.  It remains in the
+           'old_objects_popped' list for now. */
+        entry->next = seg->old_objects_popped;
+        seg->old_objects_popped = entry;
+
+        queue_activate(queue);
+        assert(entry->object != NULL);
+        return entry->object;
+    }
+    else {
+        /* no pending entry, wait */
+#if STM_TESTS
+        assert(timeout == 0.0);   /* can't wait in the basic tests */
+#endif
+        if (timeout == 0.0) {
+            if (!stm_is_inevitable(tl)) {
+                stm_become_inevitable(tl, "stm_queue_get");
+                goto retry;
+            }
+            else
+                return NULL;
+        }
+
+        STM_PUSH_ROOT(*tl, qobj);
+        _stm_commit_transaction();
+
+        s_mutex_lock();
+        while (queue->old_entries == NULL) {
+            if (timeout < 0.0) {      /* no timeout */
+                cond_wait(C_QUEUE_OLD_ENTRIES);
+            }
+            else {
+                if (!t_ready) {
+                    timespec_delay(&t, timeout);
+                    t_ready = true;
+                }
+                if (!cond_wait_timespec(C_QUEUE_OLD_ENTRIES, &t)) {
+                    timeout = 0.0;   /* timed out! */
+                    break;
+                }
+            }
+        }
+        s_mutex_unlock();
+
+        _stm_start_transaction(tl);
+        STM_POP_ROOT(*tl, qobj);   /* 'queue' should stay alive until here */
+        goto retry;
+    }
+}
+
+static void queue_trace_list(queue_entry_t *entry, void trace(object_t **),
+                             queue_entry_t *stop_at)
+{
+    while (entry != stop_at) {
+        trace(&entry->object);
+        entry = entry->next;
+    }
+}
+
+void stm_queue_tracefn(stm_queue_t *queue, void trace(object_t **))
+{
+    if (trace == TRACE_FOR_MAJOR_COLLECTION) {
+        long i;
+        for (i = 0; i < STM_NB_SEGMENTS; i++) {
+            stm_queue_segment_t *seg = &queue->segs[i];
+            seg->added_young_limit = seg->added_in_this_transaction;
+            queue_trace_list(seg->added_in_this_transaction, trace, NULL);
+            queue_trace_list(seg->old_objects_popped, trace, NULL);
+        }
+        queue_trace_list(queue->old_entries, trace, NULL);
+    }
+    else {
+        /* for minor collections: it is enough to trace the objects
+           added in the current transaction.  All other objects are
+           old (or, worse, belong to a parallel thread and must not
+           be traced). */
+        stm_queue_segment_t *seg = &queue->segs[STM_SEGMENT->segment_num - 1];
+        queue_trace_list(seg->added_in_this_transaction, trace,
+                         seg->added_young_limit);
+        seg->added_young_limit = seg->added_in_this_transaction;
+    }
+}
diff --git a/rpython/translator/stm/src_stm/stm/queue.h b/rpython/translator/stm/src_stm/stm/queue.h
new file mode 100644
--- /dev/null
+++ b/rpython/translator/stm/src_stm/stm/queue.h
@@ -0,0 +1,2 @@
+/* Imported by rpython/translator/stm/import_stmgc.py */
+static void queues_deactivate_all(bool at_commit);
diff --git a/rpython/translator/stm/src_stm/stm/setup.c b/rpython/translator/stm/src_stm/stm/setup.c
--- a/rpython/translator/stm/src_stm/stm/setup.c
+++ b/rpython/translator/stm/src_stm/stm/setup.c
@@ -167,6 +167,7 @@
         tree_free(pr->callbacks_on_commit_and_abort[1]);
         list_free(pr->young_objects_with_light_finalizers);
         list_free(pr->old_objects_with_light_finalizers);
+        if (pr->active_queues) tree_free(pr->active_queues);
     }
 
     munmap(stm_object_pages, TOTAL_MEMORY);
diff --git a/rpython/translator/stm/src_stm/stm/smallmalloc.c b/rpython/translator/stm/src_stm/stm/smallmalloc.c
--- a/rpython/translator/stm/src_stm/stm/smallmalloc.c
+++ b/rpython/translator/stm/src_stm/stm/smallmalloc.c
@@ -44,7 +44,7 @@
     memset(full_pages_object_size, 0, sizeof(full_pages_object_size));
 }
 
-static int gmfp_lock = 0;
+static uint8_t gmfp_lock = 0;
 
 static void grab_more_free_pages_for_small_allocations(void)
 {
diff --git a/rpython/translator/stm/src_stm/stm/sync.c b/rpython/translator/stm/src_stm/stm/sync.c
--- a/rpython/translator/stm/src_stm/stm/sync.c
+++ b/rpython/translator/stm/src_stm/stm/sync.c
@@ -125,7 +125,7 @@
     t->tv_nsec = nsec;
 }
 
-static inline bool cond_wait_timeout(enum cond_type_e ctype, double delay)
+static bool cond_wait_timespec(enum cond_type_e ctype, struct timespec *pt)
 {
 #ifdef STM_NO_COND_WAIT
     stm_fatalerror("*** cond_wait/%d called!", (int)ctype);
@@ -133,11 +133,8 @@
 
     assert(_has_mutex_here);
 
-    struct timespec t;
-    timespec_delay(&t, delay);
-
     int err = pthread_cond_timedwait(&sync_ctl.cond[ctype],
-                                     &sync_ctl.global_mutex, &t);
+                                     &sync_ctl.global_mutex, pt);
     if (err == 0)
         return true;     /* success */
     if (LIKELY(err == ETIMEDOUT))
@@ -145,6 +142,13 @@
     stm_fatalerror("pthread_cond_timedwait/%d: %d", (int)ctype, err);
 }
 
+static bool cond_wait_timeout(enum cond_type_e ctype, double delay)
+{
+    struct timespec t;
+    timespec_delay(&t, delay);
+    return cond_wait_timespec(ctype, &t);
+}
+
 static inline void cond_signal(enum cond_type_e ctype)
 {
     int err = pthread_cond_signal(&sync_ctl.cond[ctype]);
diff --git a/rpython/translator/stm/src_stm/stm/sync.h b/rpython/translator/stm/src_stm/stm/sync.h
--- a/rpython/translator/stm/src_stm/stm/sync.h
+++ b/rpython/translator/stm/src_stm/stm/sync.h
@@ -6,6 +6,7 @@
     C_REQUEST_REMOVED,
     C_SEGMENT_FREE,
     C_SEGMENT_FREE_OR_SAFE_POINT,
+    C_QUEUE_OLD_ENTRIES,
     _C_TOTAL
 };
 
diff --git a/rpython/translator/stm/src_stm/stmgc.c b/rpython/translator/stm/src_stm/stmgc.c
--- a/rpython/translator/stm/src_stm/stmgc.c
+++ b/rpython/translator/stm/src_stm/stmgc.c
@@ -20,6 +20,7 @@
 #include "stm/finalizer.h"
 #include "stm/locks.h"
 #include "stm/detach.h"
+#include "stm/queue.h"
 #include "stm/misc.c"
 #include "stm/list.c"
 #include "stm/smallmalloc.c"
@@ -42,4 +43,5 @@
 #include "stm/rewind_setjmp.c"
 #include "stm/finalizer.c"
 #include "stm/hashtable.c"
+#include "stm/queue.c"
 #include "stm/detach.c"
diff --git a/rpython/translator/stm/src_stm/stmgc.h b/rpython/translator/stm/src_stm/stmgc.h
--- a/rpython/translator/stm/src_stm/stmgc.h
+++ b/rpython/translator/stm/src_stm/stmgc.h
@@ -729,6 +729,27 @@
     object_t *object;
 };
 
+
+/* Queues.  The items you put() and get() back are in random order.
+   Like hashtables, the type 'stm_queue_t' is not an object type at
+   all; you need to allocate and free it explicitly.  If you want to
+   embed the queue inside an 'object_t' you probably need a light
+   finalizer to do the freeing. */
+typedef struct stm_queue_s stm_queue_t;
+
+stm_queue_t *stm_queue_create(void);
+void stm_queue_free(stm_queue_t *);
+/* put() does not cause delays or transaction breaks */
+void stm_queue_put(object_t *qobj, stm_queue_t *queue, object_t *newitem);
+/* get() can commit and wait outside a transaction (so push roots).
+   Unsuitable if the current transaction is atomic!  With timeout < 0.0,
+   waits forever; with timeout >= 0.0, returns NULL in an *inevitable*
+   transaction (this is needed to ensure correctness). */
+object_t *stm_queue_get(object_t *qobj, stm_queue_t *queue, double timeout,
+                        stm_thread_local_t *tl);
+void stm_queue_tracefn(stm_queue_t *queue, void trace(object_t **));
+
+
 /* ==================== END ==================== */
 
 static void (*stmcb_expand_marker)(char *segment_base, uintptr_t odd_number,
diff --git a/rpython/translator/stm/test/test_ztranslated.py b/rpython/translator/stm/test/test_ztranslated.py
--- a/rpython/translator/stm/test/test_ztranslated.py
+++ b/rpython/translator/stm/test/test_ztranslated.py
@@ -608,6 +608,46 @@
         data = cbuilder.cmdexec('')
         assert 'ok!\n' in data
 
+    def test_queue(self):
+        class X(object):
+            pass
+
+        def main(argv):
+            q = rstm.create_queue()
+            p = q.get(0.0)
+            assert p == lltype.nullptr(llmemory.GCREF.TO)
+            p = q.get(0.001)
+            assert p == lltype.nullptr(llmemory.GCREF.TO)
+            #
+            x1 = X()
+            p1 = cast_instance_to_gcref(x1)
+            q.put(p1)
+            #
+            p2 = q.get()
+            x2 = cast_gcref_to_instance(X, p2)
+            assert x2 is x1
+            #
+            q.put(p1)
+            rgc.collect()
+            #
+            p2 = q.get()
+            x2 = cast_gcref_to_instance(X, p2)
+            assert x2 is x1
+            #
+            print "ok!"
+            return 0
+
+        res = main([])      # direct run
+        assert res == 0
+
+        t, cbuilder = self.compile(main)
+        data = cbuilder.cmdexec('')
+        assert 'ok!\n' in data
+
+        t, cbuilder = self.compile(main, backendopt=True)
+        data = cbuilder.cmdexec('')
+        assert 'ok!\n' in data
+
     def test_allocate_preexisting(self):
         py.test.skip("kill me or re-add me")
         S = lltype.GcStruct('S', ('n', lltype.Signed))

From noreply at buildbot.pypy.org  Mon Jun 22 11:48:15 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 22 Jun 2015 11:48:15 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c8: Use 'stm_dont_track_raw_accesses'
 instead of 'stm_ignore' here, which is
Message-ID: <20150622094815.ABDDC1C1F36@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c8
Changeset: r78241:c883fa17281d
Date: 2015-06-22 10:46 +0200
http://bitbucket.org/pypy/pypy/changeset/c883fa17281d/

Log:	Use 'stm_dont_track_raw_accesses' instead of 'stm_ignore' here,
	which is better supported by the JIT

diff --git a/pypy/module/pypystm/unsafe_op.py b/pypy/module/pypystm/unsafe_op.py
--- a/pypy/module/pypystm/unsafe_op.py
+++ b/pypy/module/pypystm/unsafe_op.py
@@ -1,18 +1,15 @@
 from pypy.interpreter.gateway import unwrap_spec
 from pypy.module._cffi_backend import cdataobj
-from rpython.rlib.rstm import stm_ignored
-from rpython.rlib.jit import dont_look_inside
-from rpython.rtyper.lltypesystem import rffi
+from rpython.rtyper.lltypesystem import lltype, rffi
 
 
- at dont_look_inside
-def unsafe_write(ptr, value):
-    with stm_ignored:
-        ptr[0] = value
+UNSAFE_INT = lltype.Struct('UNSAFE_INT', ('x', rffi.INT),
+                           hints = {'stm_dont_track_raw_accesses': True})
+UNSAFE_INT_P = lltype.Ptr(UNSAFE_INT)
+
 
 @unwrap_spec(w_cdata=cdataobj.W_CData, index=int, value='c_int')
 def unsafe_write_int32(space, w_cdata, index, value):
     with w_cdata as ptr:
-        ptr = rffi.cast(rffi.INTP, rffi.ptradd(ptr, index * 4))
-        value = rffi.cast(rffi.INT, value)
-        unsafe_write(ptr, value)
+        ptr = rffi.cast(UNSAFE_INT_P, rffi.ptradd(ptr, index * 4))
+        ptr.x = rffi.cast(rffi.INT, value)

From noreply at buildbot.pypy.org  Thu Jun 18 10:56:47 2015
From: noreply at buildbot.pypy.org (cfbolz)
Date: Thu, 18 Jun 2015 10:56:47 +0200 (CEST)
Subject: [pypy-commit] pypy default: move the check whether something is an
 ovf operation from
Message-ID: <20150618085647.A28CB1C1DB8@cobra.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r78167:7b929b68078d
Date: 2015-06-18 10:46 +0200
http://bitbucket.org/pypy/pypy/changeset/7b929b68078d/

Log:	move the check whether something is an ovf operation from
	_record_helper_nonpure_varargs (where it was run for every single
	operation emitted) to execute_and_record (where it is constant-
	folded because that is specialized to the opnum). This also means we
	don't create the list of argboxes in the case where the ovf
	operation is folded away.

diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -1922,9 +1922,10 @@
         resbox = executor.execute(self.cpu, self, opnum, descr, *argboxes)
         if rop._ALWAYS_PURE_FIRST <= opnum <= rop._ALWAYS_PURE_LAST:
             return self._record_helper_pure(opnum, resbox, descr, *argboxes)
-        else:
-            return self._record_helper_nonpure_varargs(opnum, resbox, descr,
-                                                       list(argboxes))
+        if rop._OVF_FIRST <= opnum <= rop._OVF_LAST:
+            return self._record_helper_ovf(opnum, resbox, descr, *argboxes)
+        return self._record_helper_nonpure_varargs(opnum, resbox, descr,
+                                                   list(argboxes))
 
     @specialize.arg(1)
     def execute_and_record_varargs(self, opnum, argboxes, descr=None):
@@ -1951,6 +1952,12 @@
             resbox = resbox.nonconstbox()    # ensure it is a Box
             return self._record_helper_nonpure_varargs(opnum, resbox, descr, list(argboxes))
 
+    def _record_helper_ovf(self, opnum, resbox, descr, *argboxes):
+        if (self.last_exc_value_box is None and
+                self._all_constants(*argboxes)):
+            return resbox.constbox()
+        return self._record_helper_nonpure_varargs(opnum, resbox, descr, list(argboxes))
+
     def _record_helper_pure_varargs(self, opnum, resbox, descr, argboxes):
         canfold = self._all_constants_varargs(argboxes)
         if canfold:
@@ -1962,10 +1969,6 @@
 
     def _record_helper_nonpure_varargs(self, opnum, resbox, descr, argboxes):
         assert resbox is None or isinstance(resbox, Box)
-        if (rop._OVF_FIRST <= opnum <= rop._OVF_LAST and
-            self.last_exc_value_box is None and
-            self._all_constants_varargs(argboxes)):
-            return resbox.constbox()
         # record the operation
         profiler = self.staticdata.profiler
         profiler.count_ops(opnum, Counters.RECORDED_OPS)

From noreply at buildbot.pypy.org  Sun Jun 21 12:26:28 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 21 Jun 2015 12:26:28 +0200 (CEST)
Subject: [pypy-commit] pypy default: Randomly try to fix test_io
Message-ID: <20150621102628.C8ABF1C1E92@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78226:9a1683dd96e2
Date: 2015-06-21 12:26 +0200
http://bitbucket.org/pypy/pypy/changeset/9a1683dd96e2/

Log:	Randomly try to fix test_io

diff --git a/pypy/module/_io/interp_textio.py b/pypy/module/_io/interp_textio.py
--- a/pypy/module/_io/interp_textio.py
+++ b/pypy/module/_io/interp_textio.py
@@ -548,6 +548,10 @@
         remain buffered in the decoder, yet to be converted."""
 
         if not self.w_decoder:
+            # very unsure about the following check, but some tests seem
+            # to expect a ValueError instead of an IOError in case the
+            # file was already closed.
+            self._check_closed(space)
             raise OperationError(space.w_IOError, space.wrap("not readable"))
 
         if self.telling:

From noreply at buildbot.pypy.org  Thu Jun 18 14:34:03 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 18 Jun 2015 14:34:03 +0200 (CEST)
Subject: [pypy-commit] stmgc queue: task_done() and join() interface of
	Queue.Queue
Message-ID: <20150618123403.3E68F1C1F89@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: queue
Changeset: r1859:e1de0881bd13
Date: 2015-06-18 14:34 +0200
http://bitbucket.org/pypy/stmgc/changeset/e1de0881bd13/

Log:	task_done() and join() interface of Queue.Queue

diff --git a/c8/stm/queue.c b/c8/stm/queue.c
--- a/c8/stm/queue.c
+++ b/c8/stm/queue.c
@@ -25,6 +25,10 @@
            notion is per segment.)  this flag says that the queue is
            already in the tree STM_PSEGMENT->active_queues. */
         bool active;
+
+        /* counts the number of put's done in this transaction, minus
+           the number of task_done's */
+        int64_t unfinished_tasks_in_this_transaction;
     };
     char pad[64];
 } stm_queue_segment_t;
@@ -37,6 +41,10 @@
 
     /* a chained list of old entries in the queue */
     queue_entry_t *volatile old_entries;
+
+    /* total of 'unfinished_tasks_in_this_transaction' for all
+       committed transactions */
+    volatile int64_t unfinished_tasks;
 };
 
 
@@ -126,6 +134,7 @@
     queue_lock_acquire();
 
     bool added_any_old_entries = false;
+    bool finished_more_tasks = false;
     wlog_t *item;
     TREE_LOOP_FORWARD(STM_PSEGMENT->active_queues, item) {
         stm_queue_t *queue = (stm_queue_t *)item->addr;
@@ -133,6 +142,11 @@
         queue_entry_t *head, *freehead;
 
         if (at_commit) {
+            int64_t d = seg->unfinished_tasks_in_this_transaction;
+            if (d != 0) {
+                finished_more_tasks |= (d < 0);
+                __sync_add_and_fetch(&queue->unfinished_tasks, d);
+            }
             head = seg->added_in_this_transaction;
             freehead = seg->old_objects_popped;
         }
@@ -145,6 +159,7 @@
         seg->added_in_this_transaction = NULL;
         seg->added_young_limit = NULL;
         seg->old_objects_popped = NULL;
+        seg->unfinished_tasks_in_this_transaction = 0;
 
         /* free the list of entries that must disappear */
         queue_free_entries(freehead);
@@ -176,10 +191,11 @@
 
     queue_lock_release();
 
-    if (added_any_old_entries) {
-        assert(_has_mutex());
+    assert(_has_mutex());
+    if (added_any_old_entries)
         cond_broadcast(C_QUEUE_OLD_ENTRIES);
-    }
+    if (finished_more_tasks)
+        cond_broadcast(C_QUEUE_FINISHED_MORE_TASKS);
 }
 
 void stm_queue_put(object_t *qobj, stm_queue_t *queue, object_t *newitem)
@@ -195,6 +211,7 @@
     seg->added_in_this_transaction = entry;
 
     queue_activate(queue);
+    seg->unfinished_tasks_in_this_transaction++;
 
     /* add qobj to 'objects_pointing_to_nursery' if it has the
        WRITE_BARRIER flag */
@@ -285,6 +302,41 @@
     }
 }
 
+void stm_queue_task_done(stm_queue_t *queue)
+{
+    queue_activate(queue);
+    stm_queue_segment_t *seg = &queue->segs[STM_SEGMENT->segment_num - 1];
+    seg->unfinished_tasks_in_this_transaction--;
+}
+
+int stm_queue_join(object_t *qobj, stm_queue_t *queue, stm_thread_local_t *tl)
+{
+    int64_t result;
+
+#if STM_TESTS
+    result = queue->unfinished_tasks;   /* can't wait in tests */
+    result += (queue->segs[STM_SEGMENT->segment_num - 1]
+               .unfinished_tasks_in_this_transaction);
+    if (result > 0)
+        return 42;
+#else
+    STM_PUSH_ROOT(*tl, qobj);
+    _stm_commit_transaction();
+
+    s_mutex_lock();
+    while ((result = queue->unfinished_tasks) > 0) {
+        cond_wait(C_QUEUE_FINISHED_MORE_TASKS);
+    }
+    s_mutex_unlock();
+
+    _stm_start_transaction(tl);
+    STM_POP_ROOT(*tl, qobj);   /* 'queue' should stay alive until here */
+#endif
+
+    /* returns 1 for 'ok', or 0 for error: negative 'unfinished_tasks' */
+    return (result == 0);
+}
+
 static void queue_trace_list(queue_entry_t *entry, void trace(object_t **),
                              queue_entry_t *stop_at)
 {
diff --git a/c8/stm/sync.h b/c8/stm/sync.h
--- a/c8/stm/sync.h
+++ b/c8/stm/sync.h
@@ -7,6 +7,7 @@
     C_SEGMENT_FREE,
     C_SEGMENT_FREE_OR_SAFE_POINT,
     C_QUEUE_OLD_ENTRIES,
+    C_QUEUE_FINISHED_MORE_TASKS,
     _C_TOTAL
 };
 
diff --git a/c8/stmgc.h b/c8/stmgc.h
--- a/c8/stmgc.h
+++ b/c8/stmgc.h
@@ -747,6 +747,11 @@
    transaction (this is needed to ensure correctness). */
 object_t *stm_queue_get(object_t *qobj, stm_queue_t *queue, double timeout,
                         stm_thread_local_t *tl);
+/* task_done() and join(): see https://docs.python.org/2/library/queue.html */
+void stm_queue_task_done(stm_queue_t *queue);
+/* join() commits and waits outside a transaction (so push roots).
+   Unsuitable if the current transaction is atomic! */
+int stm_queue_join(object_t *qobj, stm_queue_t *queue, stm_thread_local_t *tl);
 void stm_queue_tracefn(stm_queue_t *queue, void trace(object_t **));
 
 
diff --git a/c8/test/support.py b/c8/test/support.py
--- a/c8/test/support.py
+++ b/c8/test/support.py
@@ -225,6 +225,8 @@
 void stm_queue_put(object_t *qobj, stm_queue_t *queue, object_t *newitem);
 object_t *stm_queue_get(object_t *qobj, stm_queue_t *queue, double timeout,
                         stm_thread_local_t *tl);
+void stm_queue_task_done(stm_queue_t *queue);
+int stm_queue_join(object_t *qobj, stm_queue_t *queue, stm_thread_local_t *tl);
 void stm_queue_tracefn(stm_queue_t *queue, void trace(object_t **));
 
 void _set_queue(object_t *obj, stm_queue_t *q);
@@ -658,7 +660,9 @@
 
 def get_hashtable(o):
     assert lib._get_type_id(o) == 421419
-    return lib._get_hashtable(o)
+    h = lib._get_hashtable(o)
+    assert h
+    return h
 
 def stm_allocate_queue():
     o = lib.stm_allocate(16)
@@ -670,7 +674,9 @@
 
 def get_queue(o):
     assert lib._get_type_id(o) == 421417
-    return lib._get_queue(o)
+    q = lib._get_queue(o)
+    assert q
+    return q
 
 def stm_get_weakref(o):
     return lib._get_weakref(o)
diff --git a/c8/test/test_queue.py b/c8/test/test_queue.py
--- a/c8/test/test_queue.py
+++ b/c8/test/test_queue.py
@@ -18,7 +18,7 @@
             try:
                 assert lib._get_type_id(obj) == 421417
                 self.seen_queues -= 1
-                q = lib._get_queue(obj)
+                q = get_queue(obj)
                 lib.stm_queue_free(q)
             except:
                 self.errors.append(sys.exc_info()[2])
@@ -42,16 +42,30 @@
         return q
 
     def get(self, obj):
-        q = lib._get_queue(obj)
+        q = get_queue(obj)
         res = lib.stm_queue_get(obj, q, 0.0, self.tls[self.current_thread])
         if res == ffi.NULL:
             raise Empty
         return res
 
     def put(self, obj, newitem):
-        q = lib._get_queue(obj)
+        q = get_queue(obj)
         lib.stm_queue_put(obj, q, newitem)
 
+    def task_done(self, obj):
+        q = get_queue(obj)
+        lib.stm_queue_task_done(q)
+
+    def join(self, obj):
+        q = get_queue(obj)
+        res = lib.stm_queue_join(obj, q, self.tls[self.current_thread]);
+        if res == 1:
+            return
+        elif res == 42:
+            raise Conflict("join() cannot wait in tests")
+        else:
+            raise AssertionError("stm_queue_join error")
+
 
 class TestQueue(BaseTestQueue):
 
@@ -299,3 +313,51 @@
             self.push_root(qobj)
             stm_minor_collect()
             qobj = self.pop_root()
+
+    def test_task_done_1(self):
+        self.start_transaction()
+        qobj = self.allocate_queue()
+        self.push_root(qobj)
+        stm_minor_collect()
+        qobj = self.pop_root()
+        self.join(qobj)
+        obj1 = stm_allocate(32)
+        self.put(qobj, obj1)
+        py.test.raises(Conflict, self.join, qobj)
+        self.get(qobj)
+        py.test.raises(Conflict, self.join, qobj)
+        self.task_done(qobj)
+        self.join(qobj)
+
+    def test_task_done_2(self):
+        self.start_transaction()
+        qobj = self.allocate_queue()
+        self.push_root(qobj)
+        self.put(qobj, stm_allocate(32))
+        self.put(qobj, stm_allocate(32))
+        self.get(qobj)
+        self.get(qobj)
+        self.commit_transaction()
+        qobj = self.pop_root()
+        #
+        self.start_transaction()
+        py.test.raises(Conflict, self.join, qobj)
+        #
+        self.switch(1)
+        self.start_transaction()
+        py.test.raises(Conflict, self.join, qobj)
+        self.task_done(qobj)
+        py.test.raises(Conflict, self.join, qobj)
+        self.task_done(qobj)
+        self.join(qobj)
+        #
+        self.switch(0)
+        py.test.raises(Conflict, self.join, qobj)
+        #
+        self.switch(1)
+        self.commit_transaction()
+        #
+        self.switch(0)
+        self.join(qobj)
+        #
+        stm_major_collect()       # to get rid of the queue object

From noreply at buildbot.pypy.org  Thu Jun 18 11:00:03 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Thu, 18 Jun 2015 11:00:03 +0200 (CEST)
Subject: [pypy-commit] pypy regalloc: exchanged the heuristic for the
 register allocator. it now takes the live range whose next use is furthest
 in the future
Message-ID: <20150618090003.CB04B1C1F6D@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: regalloc
Changeset: r78168:0dcfffa6613e
Date: 2015-06-18 11:00 +0200
http://bitbucket.org/pypy/pypy/changeset/0dcfffa6613e/

Log:	exchanged the heuristic for the register allocator. it now takes the
	live range whose next use is furthest in the future added 2 tests to
	check this

diff --git a/rpython/jit/backend/llsupport/regalloc.py b/rpython/jit/backend/llsupport/regalloc.py
--- a/rpython/jit/backend/llsupport/regalloc.py
+++ b/rpython/jit/backend/llsupport/regalloc.py
@@ -1,6 +1,7 @@
 import os
 from rpython.jit.metainterp.history import Const, Box, REF, JitCellToken
 from rpython.rlib.objectmodel import we_are_translated, specialize
+from rpython.rlib.rbisect import bisect_left
 from rpython.jit.metainterp.resoperation import rop
 from rpython.rtyper.lltypesystem import lltype
 from rpython.rtyper.lltypesystem.lloperation import llop
@@ -409,9 +410,12 @@
 
     def _pick_variable_to_spill(self, v, forbidden_vars, selected_reg=None,
                                 need_lower_byte=False):
-        """ Slightly less silly algorithm.
+        """ Heuristic: take the variable which has it's next use
+            the furthest in the future.
+            The original heuristic proposed by Poletto & Sarkar takes
+            the live range that ends the last.
         """
-        cur_max_age = -1
+        furthest_pos = -1
         candidate = None
         for next in self.reg_bindings:
             reg = self.reg_bindings[next]
@@ -424,9 +428,9 @@
                     continue
             if need_lower_byte and reg in self.no_lower_byte_regs:
                 continue
-            max_age = self.longevity[next][1]
-            if cur_max_age < max_age:
-                cur_max_age = max_age
+            pos = next_var_usage(self.longevity[next], self.position)
+            if furthest_pos < pos:
+                furthest_pos = pos
                 candidate = next
         if candidate is None:
             raise NoVariableToSpill
@@ -673,6 +677,19 @@
         else:
             return [self.loc(op.getarg(0))]
 
+def next_var_usage(longevity, pos):
+    start, end, uses = longevity
+    if pos > end:
+        # there is no next usage, has already been passed
+        return -1
+    if pos < start:
+        return start
+    if uses is None:
+        # a live range with just a definition and one use
+        return end
+    i = bisect_left(uses, pos, len(uses))
+    return uses[i]
+
 def compute_vars_longevity(inputargs, operations):
     # compute a dictionary that maps variables to index in
     # operations that is a "last-time-seen"
diff --git a/rpython/jit/backend/llsupport/test/test_regalloc.py b/rpython/jit/backend/llsupport/test/test_regalloc.py
--- a/rpython/jit/backend/llsupport/test/test_regalloc.py
+++ b/rpython/jit/backend/llsupport/test/test_regalloc.py
@@ -3,7 +3,7 @@
     INT, FLOAT, BoxPtr)
 from rpython.jit.metainterp.resoperation import rop, ResOperation
 from rpython.jit.backend.llsupport.regalloc import (FrameManager,
-        LinkedList, compute_vars_longevity)
+        LinkedList, compute_vars_longevity, next_var_usage)
 from rpython.jit.backend.llsupport.regalloc import RegisterManager as BaseRegMan
 
 def newboxes(*values):
@@ -32,6 +32,13 @@
 
 class RegisterManager(BaseRegMan):
     all_regs = regs
+
+    def __init__(self, longevity, frame_manager=None, assembler=None):
+        for k,v in longevity.items():
+            if len(v) == 2:
+                longevity[k] = (v[0], v[1], None)
+        BaseRegMan.__init__(self, longevity, frame_manager, assembler)
+
     def convert_to_imm(self, v):
         return v
 
@@ -598,3 +605,40 @@
         assert longevity[d] == (4,5, None)
         assert longevity[e] == (0,2, None)
         assert longevity[f] == (2,5, None)
+
+
+    def test_next_var_usage(self):
+        assert next_var_usage( (0,9,None), 10 ) == -1
+        assert next_var_usage( (4,9,None), 2 ) == 4
+        assert next_var_usage( (0,9,None), -1 ) == 0
+        assert next_var_usage( (0,5,[0,1,2,3,4,5]), 3 ) == 3
+        assert next_var_usage( (1,10,[1,5,8,10]), 4 ) == 5
+        assert next_var_usage( (1,10,[1,5,8,10]), 1 ) == 1
+        assert next_var_usage( (1,10,[1,5,8,10]), 2 ) == 5
+        assert next_var_usage( (1,10,[1,5,8,10]), 9 ) == 10
+
+    def test_pick_to_spill(self):
+        b = [BoxInt() for i in range(0,10)]
+        longevity = {
+            b[0] : (1, 10, [1,2,8,10]),
+            b[1] : (3,  4, None),
+            b[2] : (0,  7, None),
+        }
+        rm = RegisterManager(longevity)
+
+        rm.reg_bindings[b[2]] = None
+        rm.position = 0
+        assert rm._pick_variable_to_spill(None, []) is b[2]
+
+        rm.reg_bindings[b[0]] = None
+        rm.position = 1
+        assert rm._pick_variable_to_spill(None, []) in (b[2],)
+
+        rm.reg_bindings[b[1]] = None
+        rm.position = 3
+        assert rm._pick_variable_to_spill(None, []) in (b[0],)
+
+        del rm.reg_bindings[b[1]]
+        rm.position = 5
+        assert rm._pick_variable_to_spill(None, []) in (b[0],)
+

From noreply at buildbot.pypy.org  Thu Jun 18 18:02:35 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 18 Jun 2015 18:02:35 +0200 (CEST)
Subject: [pypy-commit] pypy default: Don't give "libpypy-c.so" in the
 example programs, as this seems to
Message-ID: <20150618160235.F16961C1FD4@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78183:c864bf5c98a1
Date: 2015-06-18 18:02 +0200
http://bitbucket.org/pypy/pypy/changeset/c864bf5c98a1/

Log:	Don't give "libpypy-c.so" in the example programs, as this seems to
	confuse people into thinking they only need to have the
	"libpypy-c.so" somewhere.

	Fix pypy_setup_home("xx") to also accept as "xx" the path of the
	root directory of pypy directly; currently it has to be anything
	strictly inside it.

diff --git a/pypy/doc/embedding.rst b/pypy/doc/embedding.rst
--- a/pypy/doc/embedding.rst
+++ b/pypy/doc/embedding.rst
@@ -75,10 +75,12 @@
 Note that this API is a lot more minimal than say CPython C API, so at first
 it's obvious to think that you can't do much. However, the trick is to do
 all the logic in Python and expose it via `cffi`_ callbacks. Let's assume
-we're on linux and pypy is installed in ``/opt/pypy`` with the
+we're on linux and pypy is installed in ``/opt/pypy`` (with
+subdirectories like ``lib-python`` and ``lib_pypy``), and with the
 library in ``/opt/pypy/bin/libpypy-c.so``.  (It doesn't need to be
-installed; you can also replace this path with your local checkout.)
-We write a little C program:
+installed; you can also replace these paths with a local extract of the
+installation tarballs, or with your local checkout of pypy.) We write a
+little C program:
 
 .. code-block:: c
 
@@ -92,7 +94,9 @@
         int res;
 
         rpython_startup_code();
-        res = pypy_setup_home("/opt/pypy/bin/libpypy-c.so", 1);
+        /* note: in the path /opt/pypy/x, the final x is ignored and
+           replaced with lib-python and lib_pypy. */
+        res = pypy_setup_home("/opt/pypy/x", 1);
         if (res) {
             printf("Error setting pypy home!\n");
             return 1;
@@ -179,7 +183,7 @@
         int res;
 
         rpython_startup_code();
-        res = pypy_setup_home("/opt/pypy/bin/libpypy-c.so", 1);
+        res = pypy_setup_home("/opt/pypy/x", 1);
         if (res) {
             fprintf(stderr, "Error setting pypy home!\n");
             return -1;
@@ -220,9 +224,15 @@
 Finding pypy_home
 -----------------
 
-Function pypy_setup_home takes one parameter - the path to libpypy. There's 
-currently no "clean" way (pkg-config comes to mind) how to find this path. You 
-can try the following (GNU-specific) hack (don't forget to link against *dl*):
+The function pypy_setup_home() takes as first parameter the path to a
+file from which it can deduce the location of the standard library.
+More precisely, it tries to remove final components until it finds
+``lib-python`` and ``lib_pypy``.  There is currently no "clean" way
+(pkg-config comes to mind) to find this path.  You can try the following
+(GNU-specific) hack (don't forget to link against *dl*), which assumes
+that the ``libpypy-c.so`` is inside the standard library directory.
+(This must more-or-less be the case anyway, otherwise the ``pypy``
+program itself would not run.)
 
 .. code-block:: c
 
@@ -236,7 +246,7 @@
 
     // caller should free returned pointer to avoid memleaks
     // returns NULL on error
-    char* guess_pypyhome() {
+    char* guess_pypyhome(void) {
         // glibc-only (dladdr is why we #define _GNU_SOURCE)
         Dl_info info;
         void *_rpython_startup_code = dlsym(0,"rpython_startup_code");
diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -97,13 +97,16 @@
         from pypy.module.sys.initpath import pypy_find_stdlib
         verbose = rffi.cast(lltype.Signed, verbose)
         if ll_home:
-            home = rffi.charp2str(ll_home)
+            home1 = rffi.charp2str(ll_home)
+            home = os.path.join(home1, 'x') # <- so that 'll_home' can be
+                                            # directly the root directory
         else:
-            home = pypydir
+            home = home1 = pypydir
         w_path = pypy_find_stdlib(space, home)
         if space.is_none(w_path):
             if verbose:
-                debug("Failed to find library based on pypy_find_stdlib")
+                debug("pypy_setup_home: directories 'lib-python' and 'lib_pypy'"
+                      " not found in '%s' or in any parent directory" % home1)
             return rffi.cast(rffi.INT, 1)
         space.startup()
         space.call_function(w_pathsetter, w_path)

From noreply at buildbot.pypy.org  Sun Jun 21 12:38:04 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 21 Jun 2015 12:38:04 +0200 (CEST)
Subject: [pypy-commit] pypy default: import the cffi tests
Message-ID: <20150621103804.42D1D1C201F@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78227:67324d866b32
Date: 2015-06-21 12:38 +0200
http://bitbucket.org/pypy/pypy/changeset/67324d866b32/

Log:	import the cffi tests

diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
@@ -266,6 +266,15 @@
     """)
     lib.aa = 5
     assert dir(lib) == ['aa', 'ff', 'my_constant']
+    #
+    aaobj = lib.__dict__['aa']
+    assert not isinstance(aaobj, int)    # some internal object instead
+    assert lib.__dict__ == {
+        'ff': lib.ff,
+        'aa': aaobj,
+        'my_constant': -45}
+    lib.__dict__['ff'] = "??"
+    assert lib.ff(10) == 15
 
 def test_verify_opaque_struct():
     ffi = FFI()
@@ -1053,5 +1062,5 @@
     assert sys.modules['_CFFI_test_import_from_lib.lib'] is lib
     from _CFFI_test_import_from_lib.lib import MYFOO
     assert MYFOO == 42
-    assert not hasattr(lib, '__dict__')
+    assert hasattr(lib, '__dict__')
     assert lib.__all__ == ['MYFOO', 'mybar']   # but not 'myvar'

From noreply at buildbot.pypy.org  Thu Jun 18 09:58:31 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Thu, 18 Jun 2015 09:58:31 +0200 (CEST)
Subject: [pypy-commit] pypy regalloc: storing the variable use/def
 information in the tuple of longevity as third parameter
Message-ID: <20150618075831.61ACF1C1F7B@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: regalloc
Changeset: r78164:73506cd059de
Date: 2015-06-18 09:58 +0200
http://bitbucket.org/pypy/pypy/changeset/73506cd059de/

Log:	storing the variable use/def information in the tuple of longevity
	as third parameter

diff --git a/rpython/jit/backend/llsupport/regalloc.py b/rpython/jit/backend/llsupport/regalloc.py
--- a/rpython/jit/backend/llsupport/regalloc.py
+++ b/rpython/jit/backend/llsupport/regalloc.py
@@ -673,7 +673,6 @@
         else:
             return [self.loc(op.getarg(0))]
 
-
 def compute_vars_longevity(inputargs, operations):
     # compute a dictionary that maps variables to index in
     # operations that is a "last-time-seen"
@@ -685,6 +684,7 @@
     produced = {}
     last_used = {}
     last_real_usage = {}
+    usage_positions = {}
     for i in range(len(operations)-1, -1, -1):
         op = operations[i]
         if op.result:
@@ -692,11 +692,13 @@
                 continue
             assert op.result not in produced
             produced[op.result] = i
+            usage_positions.setdefault(op.result, []).insert(0, i)
         opnum = op.getopnum()
         for j in range(op.numargs()):
             arg = op.getarg(j)
             if not isinstance(arg, Box):
                 continue
+            usage_positions.setdefault(arg, []).insert(0, i)
             if arg not in last_used:
                 last_used[arg] = i
             if opnum != rop.JUMP and opnum != rop.LABEL:
@@ -707,6 +709,7 @@
                 if arg is None: # hole
                     continue
                 assert isinstance(arg, Box)
+                usage_positions.setdefault(arg, []).insert(0, i)
                 if arg not in last_used:
                     last_used[arg] = i
     #
@@ -715,14 +718,20 @@
         if arg in last_used:
             assert isinstance(arg, Box)
             assert produced[arg] < last_used[arg]
-            longevity[arg] = (produced[arg], last_used[arg])
+            upos = usage_positions[arg]
+            if len(upos) == 2:
+                upos = None
+            longevity[arg] = (produced[arg], last_used[arg], upos)
             del last_used[arg]
     for arg in inputargs:
         assert isinstance(arg, Box)
         if arg not in last_used:
-            longevity[arg] = (-1, -1)
+            longevity[arg] = (-1, -1, None)
         else:
-            longevity[arg] = (0, last_used[arg])
+            upos = usage_positions[arg]
+            if len(upos) == 2:
+                upos = None
+            longevity[arg] = (0, last_used[arg], upos)
             del last_used[arg]
     assert len(last_used) == 0
     return longevity, last_real_usage
diff --git a/rpython/jit/backend/llsupport/test/test_regalloc.py b/rpython/jit/backend/llsupport/test/test_regalloc.py
--- a/rpython/jit/backend/llsupport/test/test_regalloc.py
+++ b/rpython/jit/backend/llsupport/test/test_regalloc.py
@@ -574,16 +574,27 @@
         a = BoxInt()
         b = BoxInt()
         c = BoxInt()
-        inputargs = [a, b]
+        d = BoxInt()
+        e = BoxInt()
+        f = BoxInt()
+        a0 = BoxInt()
+        a1 = BoxInt()
+        a2 = BoxInt()
+        inputargs = [a, b,e,a0]
 
         operations = [
-            ResOperation(rop.LABEL, [a,b], None),
+            ResOperation(rop.LABEL, [a,b,e,a0], None),
             ResOperation(rop.INT_ADD, [a,b], c),
-            ResOperation(rop.INT_ADD, [c,a], BoxInt()),
-            ResOperation(rop.INT_ADD, [c,b], BoxInt()),
-            ResOperation(rop.JUMP, [a,c], None),
+            ResOperation(rop.INT_ADD, [e,a], f),
+            ResOperation(rop.INT_ADD, [a0,a], a1),
+            ResOperation(rop.INT_ADD, [c,b], d),
+            ResOperation(rop.JUMP, [c,d,f,a1], None),
         ]
         longevity, lru = compute_vars_longevity(inputargs, operations)
-        assert lru[c] == 3
-        assert longevity[c][0] == 0
-        assert longevity[a][1] == 1
+        assert lru[c] == 4
+        assert longevity[a] == (0,3, [0,1,2,3])
+        assert longevity[b] == (0,4, [0,1,4])
+        assert longevity[c] == (1,5, [1,4,5])
+        assert longevity[d] == (4,5, None)
+        assert longevity[e] == (0,2, None)
+        assert longevity[f] == (2,5, None)
diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -139,14 +139,15 @@
         operations = cpu.gc_ll_descr.rewrite_assembler(cpu, operations,
                                                        allgcrefs)
         # compute longevity of variables
-        longevity, last_real_usage = compute_vars_longevity(
-                                                    inputargs, operations)
+        longevity, last_real_usage = \
+            compute_vars_longevity(inputargs, operations)
         self.longevity = longevity
         self.last_real_usage = last_real_usage
         self.rm = gpr_reg_mgr_cls(self.longevity,
                                   frame_manager = self.fm,
                                   assembler = self.assembler)
-        self.xrm = xmm_reg_mgr_cls(self.longevity, frame_manager = self.fm,
+        self.xrm = xmm_reg_mgr_cls(self.longevity,
+                                   frame_manager = self.fm,
                                    assembler = self.assembler)
         return operations
 

From noreply at buildbot.pypy.org  Mon Jun 22 21:14:45 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Mon, 22 Jun 2015 21:14:45 +0200 (CEST)
Subject: [pypy-commit] pypy run-create_cffi_imports:
 create_cffi_import_libraries does not raise rather indicates which imports
 failed
Message-ID: <20150622191445.90AD41C1FA0@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: run-create_cffi_imports
Changeset: r78246:c537451c6156
Date: 2015-06-22 22:14 +0300
http://bitbucket.org/pypy/pypy/changeset/c537451c6156/

Log:	create_cffi_import_libraries does not raise rather indicates which
	imports failed

diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -315,7 +315,7 @@
 
         @taskdef(['compile_c'], "Create cffi bindings for modules")
         def task_build_cffi_imports(self):
-            from pypy.tool.build_cffi_imports import create_cffi_import_libraries, MissingDependenciesError
+            from pypy.tool.build_cffi_imports import create_cffi_import_libraries
             ''' Use cffi to compile cffi interfaces to modules'''
             exename = mkexename(driver.compute_exe_name())
             basedir = exename
@@ -328,10 +328,8 @@
             modules = self.config.objspace.usemodules.getpaths()
             options = Options()
             # XXX possibly adapt options using modules
-            try:
-                create_cffi_import_libraries(exename, options, basedir)
-            except MissingDependenciesError:
-                pass
+            failures = create_cffi_import_libraries(exename, options, basedir)
+            # if failures, they were already printed
         driver.task_build_cffi_imports = types.MethodType(task_build_cffi_imports, driver)
         driver.tasks['build_cffi_imports'] = driver.task_build_cffi_imports, ['compile_c']
         driver.default_goal = 'build_cffi_imports'
diff --git a/pypy/tool/build_cffi_imports.py b/pypy/tool/build_cffi_imports.py
--- a/pypy/tool/build_cffi_imports.py
+++ b/pypy/tool/build_cffi_imports.py
@@ -19,6 +19,7 @@
 def create_cffi_import_libraries(pypy_c, options, basedir):
     shutil.rmtree(str(basedir.join('lib_pypy', '__pycache__')),
                   ignore_errors=True)
+    failures = []
     for key, module in sorted(cffi_build_scripts.items()):
         if module is None or getattr(options, 'no_' + key, False):
             continue
@@ -30,17 +31,17 @@
             cwd = None
         print >> sys.stderr, '*', ' '.join(args)
         try:
-            results = run_subprocess(str(pypy_c), args, cwd=cwd)
+            status, stdout, stderr = run_subprocess(str(pypy_c), args, cwd=cwd)
+            if status != 0:
+                print >> sys.stderr, stdout, stderr
+                failures.append((key, module))
         except:
             import traceback;traceback.print_exc()
-            print >>sys.stderr, """!!!!!!!!!!\nBuilding {0} bindings failed.
-You can either install development headers package,
-add the --without-{0} option to skip packaging this
-binary CFFI extension, or say --without-cffi.""".format(key)
-            raise MissingDependenciesError(module)
+            failures.append((key, module))
+    return failures
 
 if __name__ == '__main__':
-    import py
+    import py, os
     if '__pypy__' not in sys.builtin_module_names:
         print 'Call with a pypy interpreter'
         sys.exit(-1)
@@ -57,4 +58,18 @@
                                  str(exename))
         basedir = _basedir
     options = Options()
-    create_cffi_import_libraries(exename, options, basedir) 
+    print >> sys.stderr, "There should be no failures here"
+    failures = create_cffi_import_libraries(exename, options, basedir)
+    if len(failures) > 0:
+        print 'failed to build', [f[1] for f in failures]
+        assert False
+
+    # monkey patch a failure, just to test
+    print >> sys.stderr, 'This line should be followed by a traceback'
+    for k in cffi_build_scripts:
+        setattr(options, 'no_' + k, True)
+    must_fail = '_missing_build_script.py'
+    assert not os.path.exists(str(basedir.join('lib_pypy').join(must_fail)))
+    cffi_build_scripts['should_fail'] = must_fail
+    failures = create_cffi_import_libraries(exename, options, basedir)
+    assert len(failures) == 1
diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py
--- a/pypy/tool/release/package.py
+++ b/pypy/tool/release/package.py
@@ -81,9 +81,13 @@
     if not _fake and not pypy_runs(pypy_c):
         raise OSError("Running %r failed!" % (str(pypy_c),))
     if not options.no_cffi:
-        try:
-            create_cffi_import_libraries(pypy_c, options, basedir)
-        except MissingDependenciesError:
+        failures = create_cffi_import_libraries(pypy_c, options, basedir)
+        for key, module in failures:
+            print >>sys.stderr, """!!!!!!!!!!\nBuilding {0} bindings failed.
+                You can either install development headers package,
+                add the --without-{0} option to skip packaging this
+                binary CFFI extension, or say --without-cffi.""".format(key)
+        if len(failures) > 0:
             return 1, None
 
     if sys.platform == 'win32' and not rename_pypy_c.lower().endswith('.exe'):

From noreply at buildbot.pypy.org  Fri Jun 19 00:02:38 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 19 Jun 2015 00:02:38 +0200 (CEST)
Subject: [pypy-commit] pypy cffi-callback-onerror: A branch to try out the
	cffi issue #152
Message-ID: <20150618220238.39F0C1C1FED@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: cffi-callback-onerror
Changeset: r78194:5ceea5c3249b
Date: 2015-06-18 23:44 +0200
http://bitbucket.org/pypy/pypy/changeset/5ceea5c3249b/

Log:	A branch to try out the cffi issue #152


From noreply at buildbot.pypy.org  Thu Jun 18 20:39:43 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 18 Jun 2015 20:39:43 +0200 (CEST)
Subject: [pypy-commit] stmgc queue: Use a regular lock instead of
 compare-and-swap for 'old_entries'.
Message-ID: <20150618183943.CAE231C1FBF@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: queue
Changeset: r1867:2cedc7732a5e
Date: 2015-06-18 20:40 +0200
http://bitbucket.org/pypy/stmgc/changeset/2cedc7732a5e/

Log:	Use a regular lock instead of compare-and-swap for 'old_entries'.
	See comment for why compare-and-swap has subtle issues with linked
	list in the presence of free()

diff --git a/c8/stm/queue.c b/c8/stm/queue.c
--- a/c8/stm/queue.c
+++ b/c8/stm/queue.c
@@ -39,8 +39,10 @@
        and the 'segs' is an array of items 64 bytes each */
     stm_queue_segment_t segs[STM_NB_SEGMENTS];
 
-    /* a chained list of old entries in the queue */
-    queue_entry_t *volatile old_entries;
+    /* a chained list of old entries in the queue; modified only
+       with the lock */
+    queue_entry_t *old_entries;
+    uint8_t old_entries_lock;
 
     /* total of 'unfinished_tasks_in_this_transaction' for all
        committed transactions */
@@ -74,17 +76,6 @@
     for (i = 0; i < STM_NB_SEGMENTS; i++) {
         stm_queue_segment_t *seg = &queue->segs[i];
 
-        /* it is possible that queues_deactivate_all() runs in parallel,
-           but it should not be possible at this point for another thread
-           to change 'active' from false to true.  if it is false, then
-           that's it */
-        if (!seg->active) {
-            assert(!seg->added_in_this_transaction);
-            assert(!seg->added_young_limit);
-            assert(!seg->old_objects_popped);
-            continue;
-        }
-
         struct stm_priv_segment_info_s *pseg = get_priv_segment(i + 1);
         spinlock_acquire(pseg->active_queues_lock);
 
@@ -93,9 +84,14 @@
             bool ok = tree_delete_item(pseg->active_queues, (uintptr_t)queue);
             assert(ok);
             (void)ok;
+            queue_free_entries(seg->added_in_this_transaction);
+            queue_free_entries(seg->old_objects_popped);
         }
-        queue_free_entries(seg->added_in_this_transaction);
-        queue_free_entries(seg->old_objects_popped);
+        else {
+            assert(!seg->added_in_this_transaction);
+            assert(!seg->added_young_limit);
+            assert(!seg->old_objects_popped);
+        }
 
         spinlock_release(pseg->active_queues_lock);
     }
@@ -171,11 +167,13 @@
             while (tail->next != NULL)
                 tail = tail->next;
             dprintf(("items move to old_entries in queue %p\n", queue));
-         retry:
+
+            spinlock_acquire(queue->old_entries_lock);
             old = queue->old_entries;
             tail->next = old;
-            if (!__sync_bool_compare_and_swap(&queue->old_entries, old, head))
-                goto retry;
+            queue->old_entries = head;
+            spinlock_release(queue->old_entries_lock);
+
             added_any_old_entries = true;
         }
 
@@ -221,6 +219,12 @@
     }
 }
 
+static void queue_check_entry(queue_entry_t *entry)
+{
+    assert(entry->object != NULL);
+    assert(((TLPREFIX int *)entry->object)[1] != 0);   /* userdata != 0 */
+}
+
 object_t *stm_queue_get(object_t *qobj, stm_queue_t *queue, double timeout,
                         stm_thread_local_t *tl)
 {
@@ -239,26 +243,38 @@
         seg->added_in_this_transaction = entry->next;
         if (entry == seg->added_young_limit)
             seg->added_young_limit = entry->next;
+        queue_check_entry(entry);
         result = entry->object;
-        assert(result != NULL);
         free(entry);
         return result;
     }
 
  retry:
+    /* can't easily use compare_and_swap here.  The issue is that
+       if we do "compare_and_swap(&old_entry, entry, entry->next)",
+       then we need to read entry->next, but a parallel thread
+       could have grabbed the same entry and already freed it.
+       More subtly, there is also an ABA problem: even if we
+       read the correct entry->next, maybe a parallel thread
+       can free and reuse this entry.  Then the compare_and_swap
+       succeeds, but the value written is outdated nonsense.
+    */
+    spinlock_acquire(queue->old_entries_lock);
     entry = queue->old_entries;
+    if (entry != NULL)
+        queue->old_entries = entry->next;
+    spinlock_release(queue->old_entries_lock);
+
     if (entry != NULL) {
-        if (!__sync_bool_compare_and_swap(&queue->old_entries,
-                                          entry, entry->next))
-            goto retry;
-
         /* successfully popped the old 'entry'.  It remains in the
-           'old_objects_popped' list for now. */
+           'old_objects_popped' list for now.  From now on, this entry
+           "belongs" to this segment and should never be read by
+           another segment. */
+        queue_check_entry(entry);
         entry->next = seg->old_objects_popped;
         seg->old_objects_popped = entry;
 
         queue_activate(queue);
-        assert(entry->object != NULL);
         return entry->object;
     }
     else {

From noreply at buildbot.pypy.org  Fri Jun 19 00:02:39 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 19 Jun 2015 00:02:39 +0200 (CEST)
Subject: [pypy-commit] pypy cffi-callback-onerror: add the "onerror"
 argument to ffi.callback(), at least the out-of-line one
Message-ID: <20150618220239.7B5891C1FEE@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: cffi-callback-onerror
Changeset: r78195:bc928ced3d92
Date: 2015-06-19 00:02 +0200
http://bitbucket.org/pypy/pypy/changeset/bc928ced3d92/

Log:	add the "onerror" argument to ffi.callback(), at least the out-of-
	line one

diff --git a/pypy/module/_cffi_backend/ccallback.py b/pypy/module/_cffi_backend/ccallback.py
--- a/pypy/module/_cffi_backend/ccallback.py
+++ b/pypy/module/_cffi_backend/ccallback.py
@@ -22,8 +22,9 @@
 class W_CDataCallback(W_CData):
     #_immutable_fields_ = ...
     ll_error = lltype.nullptr(rffi.CCHARP.TO)
+    w_onerror = None
 
-    def __init__(self, space, ctype, w_callable, w_error):
+    def __init__(self, space, ctype, w_callable, w_error, w_onerror):
         raw_closure = rffi.cast(rffi.CCHARP, clibffi.closureHeap.alloc())
         W_CData.__init__(self, space, raw_closure, ctype)
         #
@@ -31,6 +32,12 @@
             raise oefmt(space.w_TypeError,
                         "expected a callable object, not %T", w_callable)
         self.w_callable = w_callable
+        if not space.is_none(w_onerror):
+            if not space.is_true(space.callable(w_onerror)):
+                raise oefmt(space.w_TypeError,
+                            "expected a callable object for 'onerror', not %T",
+                            w_onerror)
+            self.w_onerror = w_onerror
         #
         fresult = self.getfunctype().ctitem
         size = fresult.size
@@ -196,6 +203,15 @@
             callback.convert_result(ll_res, w_res)
         except OperationError, e:
             # got an app-level exception
+            if callback.w_onerror is not None:
+                try:
+                    e.normalize_exception(space)
+                    w_t = e.w_type
+                    w_v = e.get_w_value(space)
+                    w_tb = space.wrap(e.get_traceback())
+                    space.call_function(callback.w_onerror, w_t, w_v, w_tb)
+                except OperationError, e2:
+                    e = e2
             callback.print_error(e, extra_line)
             callback.write_error_return_value(ll_res)
         #
diff --git a/pypy/module/_cffi_backend/ffi_obj.py b/pypy/module/_cffi_backend/ffi_obj.py
--- a/pypy/module/_cffi_backend/ffi_obj.py
+++ b/pypy/module/_cffi_backend/ffi_obj.py
@@ -276,8 +276,9 @@
 
 
     @unwrap_spec(w_python_callable=WrappedDefault(None),
-                 w_error=WrappedDefault(None))
-    def descr_callback(self, w_cdecl, w_python_callable, w_error):
+                 w_error=WrappedDefault(None),
+                 w_onerror=WrappedDefault(None))
+    def descr_callback(self, w_cdecl, w_python_callable, w_error, w_onerror):
         """\
 Return a callback object or a decorator making such a callback object.
 'cdecl' must name a C function pointer type.  The callback invokes the
@@ -290,14 +291,16 @@
         space = self.space
         if not space.is_none(w_python_callable):
             return ccallback.W_CDataCallback(space, w_ctype,
-                                             w_python_callable, w_error)
+                                             w_python_callable, w_error,
+                                             w_onerror)
         else:
             # decorator mode: returns a single-argument function
-            return space.appexec([w_ctype, w_error],
-            """(ctype, error):
+            return space.appexec([w_ctype, w_error, w_onerror],
+            """(ctype, error, onerror):
                 import _cffi_backend
                 return lambda python_callable: (
-                    _cffi_backend.callback(ctype, python_callable, error))""")
+                    _cffi_backend.callback(ctype, python_callable,
+                                           error, onerror))""")
 
 
     def descr_cast(self, w_arg, w_ob):
diff --git a/pypy/module/_cffi_backend/func.py b/pypy/module/_cffi_backend/func.py
--- a/pypy/module/_cffi_backend/func.py
+++ b/pypy/module/_cffi_backend/func.py
@@ -18,9 +18,9 @@
 # ____________________________________________________________
 
 @unwrap_spec(w_ctype=ctypeobj.W_CType)
-def callback(space, w_ctype, w_callable, w_error=None):
+def callback(space, w_ctype, w_callable, w_error=None, w_onerror=None):
     from pypy.module._cffi_backend.ccallback import W_CDataCallback
-    return W_CDataCallback(space, w_ctype, w_callable, w_error)
+    return W_CDataCallback(space, w_ctype, w_callable, w_error, w_onerror)
 
 # ____________________________________________________________
 
diff --git a/pypy/module/_cffi_backend/test/test_ffi_obj.py b/pypy/module/_cffi_backend/test/test_ffi_obj.py
--- a/pypy/module/_cffi_backend/test/test_ffi_obj.py
+++ b/pypy/module/_cffi_backend/test/test_ffi_obj.py
@@ -114,6 +114,18 @@
         assert ffi.callback("int(int)", lambda x: x + "", -66)(10) == -66
         assert ffi.callback("int(int)", lambda x: x + "", error=-66)(10) == -66
 
+    def test_ffi_callback_onerror(self):
+        import _cffi_backend as _cffi1_backend
+        ffi = _cffi1_backend.FFI()
+        seen = []
+        def myerror(exc, val, tb):
+            seen.append(exc)
+        cb = ffi.callback("int(int)", lambda x: x + "", onerror=myerror)
+        assert cb(10) == 0
+        cb = ffi.callback("int(int)", lambda x:int(1E100), -66, onerror=myerror)
+        assert cb(10) == -66
+        assert seen == [TypeError, OverflowError]
+
     def test_ffi_callback_decorator(self):
         import _cffi_backend as _cffi1_backend
         ffi = _cffi1_backend.FFI()

From noreply at buildbot.pypy.org  Thu Jun 18 14:47:56 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 18 Jun 2015 14:47:56 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c8: add task_done() and join() to rstm's
	queues
Message-ID: <20150618124756.B437C1C1FC5@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c8
Changeset: r78177:f597af41b44b
Date: 2015-06-18 14:48 +0200
http://bitbucket.org/pypy/pypy/changeset/f597af41b44b/

Log:	add task_done() and join() to rstm's queues

diff --git a/rpython/memory/gctransform/stmframework.py b/rpython/memory/gctransform/stmframework.py
--- a/rpython/memory/gctransform/stmframework.py
+++ b/rpython/memory/gctransform/stmframework.py
@@ -225,6 +225,7 @@
     gct_stm_transaction_break                       = _gct_with_roots_pushed
     gct_stm_collect                                 = _gct_with_roots_pushed
     gct_stm_queue_get                               = _gct_with_roots_pushed
+    gct_stm_queue_join                              = _gct_with_roots_pushed
 
 
 class StmRootWalker(BaseRootWalker):
diff --git a/rpython/rlib/rstm.py b/rpython/rlib/rstm.py
--- a/rpython/rlib/rstm.py
+++ b/rpython/rlib/rstm.py
@@ -370,12 +370,22 @@
 def _ll_queue_put(q, newitem):
     llop.stm_queue_put(lltype.Void, q, q.ll_raw_queue, newitem)
 
+ at dont_look_inside
+def _ll_queue_task_done(q):
+    llop.stm_queue_task_done(lltype.Void, q.ll_raw_queue)
+
+ at dont_look_inside
+def _ll_queue_join(q):
+    return llop.stm_queue_join(lltype.Signed, q, q.ll_raw_queue)
+
 _QUEUE_OBJ = lltype.GcStruct('QUEUE_OBJ',
                              ('ll_raw_queue', _STM_QUEUE_P),
                              hints={'immutable': True},
                              rtti=True,
                              adtmeths={'get': _ll_queue_get,
-                                       'put': _ll_queue_put})
+                                       'put': _ll_queue_put,
+                                       'task_done': _ll_queue_task_done,
+                                       'join': _ll_queue_join})
 NULL_QUEUE = lltype.nullptr(_QUEUE_OBJ)
 
 def _ll_queue_trace(gc, obj, callback, arg):
@@ -423,3 +433,10 @@
     def put(self, newitem):
         assert lltype.typeOf(newitem) == llmemory.GCREF
         self._content.put(newitem)
+
+    def task_done(self):
+        self._content.task_done()
+
+    def join(self):
+        self._content.join()
+        return 0
diff --git a/rpython/rtyper/llinterp.py b/rpython/rtyper/llinterp.py
--- a/rpython/rtyper/llinterp.py
+++ b/rpython/rtyper/llinterp.py
@@ -1006,6 +1006,8 @@
     op_stm_queue_free = _stm_not_implemented
     op_stm_queue_get = _stm_not_implemented
     op_stm_queue_put = _stm_not_implemented
+    op_stm_queue_task_done = _stm_not_implemented
+    op_stm_queue_join = _stm_not_implemented
     op_stm_queue_tracefn = _stm_not_implemented
     op_stm_register_thread_local = _stm_not_implemented
     op_stm_unregister_thread_local = _stm_not_implemented
diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -483,6 +483,8 @@
     'stm_queue_free':         LLOp(),
     'stm_queue_get':          LLOp(canmallocgc=True),   # push roots!
     'stm_queue_put':          LLOp(),
+    'stm_queue_task_done':    LLOp(),
+    'stm_queue_join':         LLOp(canmallocgc=True),   # push roots!
     'stm_queue_tracefn':      LLOp(),
 
     # __________ address operations __________
diff --git a/rpython/translator/stm/breakfinder.py b/rpython/translator/stm/breakfinder.py
--- a/rpython/translator/stm/breakfinder.py
+++ b/rpython/translator/stm/breakfinder.py
@@ -10,6 +10,7 @@
     'stm_leave_callback_call',
     'stm_transaction_break',
     'stm_queue_get',
+    'stm_queue_join',
     ])
 
 for tb in TRANSACTION_BREAK:
diff --git a/rpython/translator/stm/funcgen.py b/rpython/translator/stm/funcgen.py
--- a/rpython/translator/stm/funcgen.py
+++ b/rpython/translator/stm/funcgen.py
@@ -392,6 +392,17 @@
     return 'stm_queue_put((object_t *)%s, %s, (object_t *)%s);' % (
         arg0, arg1, arg2)
 
+def stm_queue_task_done(funcgen, op):
+    arg0 = funcgen.expr(op.args[0])
+    return 'stm_queue_task_done(%s);' % (arg0,)
+
+def stm_queue_join(funcgen, op):
+    arg0 = funcgen.expr(op.args[0])
+    arg1 = funcgen.expr(op.args[1])
+    result = funcgen.expr(op.result)
+    return ('%s = stm_queue_join((object_t *)%s, %s, '
+            '&stm_thread_local);' % (result, arg0, arg1,))
+
 def stm_queue_tracefn(funcgen, op):
     arg0 = funcgen.expr(op.args[0])
     arg1 = funcgen.expr(op.args[1])
diff --git a/rpython/translator/stm/src_stm/revision b/rpython/translator/stm/src_stm/revision
--- a/rpython/translator/stm/src_stm/revision
+++ b/rpython/translator/stm/src_stm/revision
@@ -1,1 +1,1 @@
-7592a0f11ac2
+d083e426a17d
diff --git a/rpython/translator/stm/src_stm/stm/queue.c b/rpython/translator/stm/src_stm/stm/queue.c
--- a/rpython/translator/stm/src_stm/stm/queue.c
+++ b/rpython/translator/stm/src_stm/stm/queue.c
@@ -25,6 +25,10 @@
            notion is per segment.)  this flag says that the queue is
            already in the tree STM_PSEGMENT->active_queues. */
         bool active;
+
+        /* counts the number of put's done in this transaction, minus
+           the number of task_done's */
+        int64_t unfinished_tasks_in_this_transaction;
     };
     char pad[64];
 } stm_queue_segment_t;
@@ -37,6 +41,10 @@
 
     /* a chained list of old entries in the queue */
     queue_entry_t *volatile old_entries;
+
+    /* total of 'unfinished_tasks_in_this_transaction' for all
+       committed transactions */
+    volatile int64_t unfinished_tasks;
 };
 
 
@@ -126,6 +134,7 @@
     queue_lock_acquire();
 
     bool added_any_old_entries = false;
+    bool finished_more_tasks = false;
     wlog_t *item;
     TREE_LOOP_FORWARD(STM_PSEGMENT->active_queues, item) {
         stm_queue_t *queue = (stm_queue_t *)item->addr;
@@ -133,6 +142,11 @@
         queue_entry_t *head, *freehead;
 
         if (at_commit) {
+            int64_t d = seg->unfinished_tasks_in_this_transaction;
+            if (d != 0) {
+                finished_more_tasks |= (d < 0);
+                __sync_add_and_fetch(&queue->unfinished_tasks, d);
+            }
             head = seg->added_in_this_transaction;
             freehead = seg->old_objects_popped;
         }
@@ -145,6 +159,7 @@
         seg->added_in_this_transaction = NULL;
         seg->added_young_limit = NULL;
         seg->old_objects_popped = NULL;
+        seg->unfinished_tasks_in_this_transaction = 0;
 
         /* free the list of entries that must disappear */
         queue_free_entries(freehead);
@@ -176,10 +191,11 @@
 
     queue_lock_release();
 
-    if (added_any_old_entries) {
-        assert(_has_mutex());
+    assert(_has_mutex());
+    if (added_any_old_entries)
         cond_broadcast(C_QUEUE_OLD_ENTRIES);
-    }
+    if (finished_more_tasks)
+        cond_broadcast(C_QUEUE_FINISHED_MORE_TASKS);
 }
 
 void stm_queue_put(object_t *qobj, stm_queue_t *queue, object_t *newitem)
@@ -195,6 +211,7 @@
     seg->added_in_this_transaction = entry;
 
     queue_activate(queue);
+    seg->unfinished_tasks_in_this_transaction++;
 
     /* add qobj to 'objects_pointing_to_nursery' if it has the
        WRITE_BARRIER flag */
@@ -285,6 +302,41 @@
     }
 }
 
+void stm_queue_task_done(stm_queue_t *queue)
+{
+    queue_activate(queue);
+    stm_queue_segment_t *seg = &queue->segs[STM_SEGMENT->segment_num - 1];
+    seg->unfinished_tasks_in_this_transaction--;
+}
+
+long stm_queue_join(object_t *qobj, stm_queue_t *queue, stm_thread_local_t *tl)
+{
+    int64_t result;
+
+#if STM_TESTS
+    result = queue->unfinished_tasks;   /* can't wait in tests */
+    result += (queue->segs[STM_SEGMENT->segment_num - 1]
+               .unfinished_tasks_in_this_transaction);
+    return result;
+#else
+    STM_PUSH_ROOT(*tl, qobj);
+    _stm_commit_transaction();
+
+    s_mutex_lock();
+    while ((result = queue->unfinished_tasks) > 0) {
+        cond_wait(C_QUEUE_FINISHED_MORE_TASKS);
+    }
+    s_mutex_unlock();
+
+    _stm_start_transaction(tl);
+    STM_POP_ROOT(*tl, qobj);   /* 'queue' should stay alive until here */
+#endif
+
+    /* returns 0 for 'ok', or negative if there was more task_done()
+       than put() so far */
+    return result;
+}
+
 static void queue_trace_list(queue_entry_t *entry, void trace(object_t **),
                              queue_entry_t *stop_at)
 {
diff --git a/rpython/translator/stm/src_stm/stm/sync.h b/rpython/translator/stm/src_stm/stm/sync.h
--- a/rpython/translator/stm/src_stm/stm/sync.h
+++ b/rpython/translator/stm/src_stm/stm/sync.h
@@ -7,6 +7,7 @@
     C_SEGMENT_FREE,
     C_SEGMENT_FREE_OR_SAFE_POINT,
     C_QUEUE_OLD_ENTRIES,
+    C_QUEUE_FINISHED_MORE_TASKS,
     _C_TOTAL
 };
 
diff --git a/rpython/translator/stm/src_stm/stmgc.h b/rpython/translator/stm/src_stm/stmgc.h
--- a/rpython/translator/stm/src_stm/stmgc.h
+++ b/rpython/translator/stm/src_stm/stmgc.h
@@ -747,6 +747,11 @@
    transaction (this is needed to ensure correctness). */
 object_t *stm_queue_get(object_t *qobj, stm_queue_t *queue, double timeout,
                         stm_thread_local_t *tl);
+/* task_done() and join(): see https://docs.python.org/2/library/queue.html */
+void stm_queue_task_done(stm_queue_t *queue);
+/* join() commits and waits outside a transaction (so push roots).
+   Unsuitable if the current transaction is atomic! */
+long stm_queue_join(object_t *qobj, stm_queue_t *queue, stm_thread_local_t *tl);
 void stm_queue_tracefn(stm_queue_t *queue, void trace(object_t **));
 
 
diff --git a/rpython/translator/stm/test/test_ztranslated.py b/rpython/translator/stm/test/test_ztranslated.py
--- a/rpython/translator/stm/test/test_ztranslated.py
+++ b/rpython/translator/stm/test/test_ztranslated.py
@@ -634,6 +634,18 @@
             x2 = cast_gcref_to_instance(X, p2)
             assert x2 is x1
             #
+            q.task_done()
+            q.task_done()
+            res = q.join()
+            assert res == 0
+            res = q.join()
+            assert res == 0
+            if objectmodel.we_are_translated():
+                q.task_done()
+                q.task_done()
+                res = q.join()
+                assert res == -2
+            #
             print "ok!"
             return 0
 

From noreply at buildbot.pypy.org  Sun Jun 21 11:38:55 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 21 Jun 2015 11:38:55 +0200 (CEST)
Subject: [pypy-commit] pypy default: issue #2064: trying to call
 socket.close() on all sockets at exit, on Windows
Message-ID: <20150621093855.E73CF1C1E3A@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78222:ddbc9f97d312
Date: 2015-06-21 11:39 +0200
http://bitbucket.org/pypy/pypy/changeset/ddbc9f97d312/

Log:	issue #2064: trying to call socket.close() on all sockets at exit,
	on Windows

diff --git a/pypy/module/_socket/__init__.py b/pypy/module/_socket/__init__.py
--- a/pypy/module/_socket/__init__.py
+++ b/pypy/module/_socket/__init__.py
@@ -18,6 +18,10 @@
         from rpython.rlib.rsocket import rsocket_startup
         rsocket_startup()
 
+    def shutdown(self, space):
+        from pypy.module._socket.interp_socket import close_all_sockets
+        close_all_sockets(space)
+
     def buildloaders(cls):
         from rpython.rlib import rsocket
         for name in """
diff --git a/pypy/module/_socket/interp_func.py b/pypy/module/_socket/interp_func.py
--- a/pypy/module/_socket/interp_func.py
+++ b/pypy/module/_socket/interp_func.py
@@ -142,7 +142,7 @@
         sock = rsocket.fromfd(fd, family, type, proto)
     except SocketError, e:
         raise converted_error(space, e)
-    return space.wrap(W_Socket(sock))
+    return space.wrap(W_Socket(space, sock))
 
 @unwrap_spec(family=int, type=int, proto=int)
 def socketpair(space, family=rsocket.socketpair_default_family,
@@ -160,8 +160,8 @@
     except SocketError, e:
         raise converted_error(space, e)
     return space.newtuple([
-        space.wrap(W_Socket(sock1)),
-        space.wrap(W_Socket(sock2))
+        space.wrap(W_Socket(space, sock1)),
+        space.wrap(W_Socket(space, sock2))
     ])
 
 # The following 4 functions refuse all negative numbers, like CPython 2.6.
diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py
--- a/pypy/module/_socket/interp_socket.py
+++ b/pypy/module/_socket/interp_socket.py
@@ -1,4 +1,5 @@
-from rpython.rlib import rsocket
+import sys
+from rpython.rlib import rsocket, rweaklist
 from rpython.rlib.rarithmetic import intmask
 from rpython.rlib.rsocket import (
     RSocket, AF_INET, SOCK_STREAM, SocketError, SocketErrorWithErrno,
@@ -153,8 +154,9 @@
 
 
 class W_Socket(W_Root):
-    def __init__(self, sock):
+    def __init__(self, space, sock):
         self.sock = sock
+        register_socket(space, sock)
 
     def get_type_w(self, space):
         return space.wrap(self.sock.type)
@@ -183,7 +185,7 @@
             fd, addr = self.sock.accept()
             sock = rsocket.make_socket(
                 fd, self.sock.family, self.sock.type, self.sock.proto)
-            return space.newtuple([space.wrap(W_Socket(sock)),
+            return space.newtuple([space.wrap(W_Socket(space, sock)),
                                    addr_as_object(addr, sock.fd, space)])
         except SocketError as e:
             raise converted_error(space, e)
@@ -248,7 +250,7 @@
     def dup_w(self, space):
         try:
             sock = self.sock.dup()
-            return W_Socket(sock)
+            return W_Socket(space, sock)
         except SocketError as e:
             raise converted_error(space, e)
 
@@ -592,10 +594,50 @@
         sock = RSocket(family, type, proto)
     except SocketError as e:
         raise converted_error(space, e)
-    W_Socket.__init__(self, sock)
+    W_Socket.__init__(self, space, sock)
     return space.wrap(self)
 descr_socket_new = interp2app(newsocket)
 
+
+# ____________________________________________________________
+# Automatic shutdown()/close()
+
+# On some systems, the C library does not guarantee that when the program
+# finishes, all data sent so far is really sent even if the socket is not
+# explicitly closed.  This behavior has been observed on Windows but not
+# on Linux, so far.
+NEED_EXPLICIT_CLOSE = (sys.platform == 'win32')
+
+class OpenRSockets(rweaklist.RWeakListMixin):
+    pass
+class OpenRSocketsState:
+    def __init__(self, space):
+        self.openrsockets = OpenRSockets()
+        self.openrsockets.initialize()
+
+def getopenrsockets(space):
+    if NEED_EXPLICIT_CLOSE and space.config.translation.rweakref:
+        return space.fromcache(OpenRSocketsState).openrsockets
+    else:
+        return None
+
+def register_socket(space, socket):
+    openrsockets = getopenrsockets(space)
+    if openrsockets is not None:
+        openrsockets.add_handle(socket)
+
+def close_all_sockets(space):
+    openrsockets = getopenrsockets(space)
+    if openrsockets is not None:
+        for sock_wref in openrsockets.get_all_handles():
+            sock = sock_wref()
+            if sock is not None:
+                try:
+                    sock.close()
+                except SocketError:
+                    pass
+
+
 # ____________________________________________________________
 # Error handling
 
diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py
--- a/pypy/module/_socket/test/test_sock_app.py
+++ b/pypy/module/_socket/test/test_sock_app.py
@@ -309,10 +309,15 @@
 
 
 class AppTestSocket:
+    spaceconfig = dict(usemodules=['_socket', '_weakref', 'struct'])
+
     def setup_class(cls):
         cls.space = space
         cls.w_udir = space.wrap(str(udir))
 
+    def teardown_class(cls):
+        cls.space.sys.getmodule('_socket').shutdown(cls.space)
+
     def test_module(self):
         import _socket
         assert _socket.socket.__name__ == 'socket'
@@ -614,6 +619,12 @@
         finally:
             os.chdir(oldcwd)
 
+    def test_automatic_shutdown(self):
+        # doesn't really test anything, but at least should not explode
+        # in close_all_sockets()
+        import _socket
+        self.foo = _socket.socket()
+
 
 class AppTestPacket:
     def setup_class(cls):

From noreply at buildbot.pypy.org  Mon Jun 22 12:08:44 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Mon, 22 Jun 2015 12:08:44 +0200 (CEST)
Subject: [pypy-commit] pypy regalloc: added more information to the output
	of regalloc.py
Message-ID: <20150622100844.B0C0A1C1F7E@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: regalloc
Changeset: r78242:ce341006d844
Date: 2015-06-22 12:08 +0200
http://bitbucket.org/pypy/pypy/changeset/ce341006d844/

Log:	added more information to the output of regalloc.py

diff --git a/rpython/jit/backend/tool/regalloc.py b/rpython/jit/backend/tool/regalloc.py
--- a/rpython/jit/backend/tool/regalloc.py
+++ b/rpython/jit/backend/tool/regalloc.py
@@ -1,6 +1,9 @@
 #! /usr/bin/env python
 """
-    ./regalloc.py log
+    Prints information about the live range of a trace recorded in the specified logfiles.
+    It aggregates the information over all traces in the logfiles.
+
+    ./regalloc.py [--histogram] logfile1 [logfile2 ...]
 """
 
 import new
@@ -93,7 +96,7 @@
         if not isinstance(typedescr,str):
             typename = typedescr[0]
         lrt = self.entries.setdefault(typename,[])
-        lrt.append((arg, lr))
+        lrt.append((self.operations, lr))
 
     def active_live_ranges(self, position):
         active = []
@@ -120,8 +123,6 @@
             if start == loop_start and end == loop_end:
                 self.found_live_range(LR.WHOLE, arg, lr)
                 self.find_fail_type(LR.WHOLE, arg, lr)
-                if len(uses) == 2:
-                    self.found_live_range(LR.DUMMY, arg, lr)
             elif start == loop_start and end != loop_end:
                 self.found_live_range(LR.ENTER, arg, lr)
                 self.find_fail_type(LR.ENTER, arg, lr)
@@ -150,6 +151,8 @@
             op = self.operations[uses[-1]]
             if op.getfailargs() is not None and arg in op.getfailargs():
                 self.found_live_range('-'.join((typename, LR.FAIL[0], LR.END_FAIL[0])), arg, lr)
+            if len(uses) - 1 == used_in_guard:
+                self.found_live_range('-'.join((typename, LR.FAIL[0], LR.ONLY_FAIL[0])), arg, lr)
 
     def count(self, typedescr):
         return len(self.entries.get(typedescr[0],[]))
@@ -162,35 +165,34 @@
 class LR(object):
 
     WHOLE = ('whole', """
-    a live range that spans over the whole trace, use x times. (x > 0)
+    a live range that spans over the whole trace, used x times. (x > 0)
     """)
     ENTER = ('enter', """
-    a live range that spans from the label to an operation (not jump/label)
+    a live range that spans from the label to an operation (but not jump/label)
     """)
     EXIT = ('exit', """
-    a live range that starts at operation X (not at a label) and exits the trace in a jump or guard
+    a live range that starts at operation X (not at a label) and exits the trace in a jump
     """)
     VOLATILE = ('volatile', """
     a live range that starts at operation X (not a label) and ends at operation Y (not a jump/label)
     """)
     ALL_TYPES = [WHOLE, ENTER, EXIT, VOLATILE]
 
+    ONLY_FAIL = ('only failarg', """
+    a live range that is used only as fail arguments
+    """)
     FAIL = ('used as failargs', """
-    a live range that spans from the label to a guard exit may be used several times in between
+    a live range that is used in a guard exit as fail argument
     """)
     END_FAIL = ('end in failargs',"""
-    a live range that spans from the label to a guard exit (can be used only in guard fail args)
+    a live range that ends in a guard exit
     """)
     NO_FAIL = ('not in any failargs', """
-    same as enter, but is not used in guard exit
+    a live range that is not used as a fail argument
     """)
 
     FAIL_TYPES = [FAIL, NO_FAIL]
 
-    DUMMY = ('dummy', """
-    a live range that spans over the whole trace, but is never used
-    """)
-
     def __init__(self):
         self.loops = []
 
@@ -207,53 +209,107 @@
     def header(self, name):
         print name
 
-    def print_stats(self):
+    def print_stats(self, histogram=False):
         print
         self.header("STATS")
         normal = [l for l in self.loops if l.type == 'normal']
         loop_count = len(normal)
         peeled = [l for l in self.loops if l.type == 'peeled']
+        bridges = [l for l in self.loops if l.type == 'bridge']
         peeled_count = len(peeled)
         self.show("loop count", loop_count)
         self.show("peeled count", peeled_count)
+        self.show("bridge count", len(bridges))
+
+        self.header("")
+        self.header("BRIDGES")
+        self.print_for_loops(bridges, hist=histogram)
 
         self.header("")
         self.header("SHELL LOOPS (loop that are not unrolled or enter a peeled loop)")
-        self.print_for_loops(normal)
+        self.print_for_loops(normal, hist=histogram)
 
         self.header("")
         self.header("PEELED LOOPS")
-        self.print_for_loops(peeled)
+        self.print_for_loops(peeled, hist=histogram)
 
 
-    def print_for_loops(self, loops):
+    def show_help(self, help, descr, indent):
+        if help:
+            print " " * (indent * 2), "%s: %s" % (descr[0], descr[1].lstrip().rstrip())
+
+    def print_for_loops(self, loops, help=True, hist=True):
         lr_counts = []
         for loop in loops:
             lr_counts.append(len(loop.longevity))
-        self.show_cmv('lr count', lr_counts)
-        self.show_cmv('lr (overlap) max', map(lambda x: getattr(x, 'lr_active_max'), loops))
-        self.show_cmv('lr (overlap) min', map(lambda x: getattr(x, 'lr_active_min'), loops))
+        self.show_help(True, ('lr (overlap) max', 'the max number of live ranges that overlap in a trace'), 0)
+        self.show_cmv('lr (overlap) max', map(lambda x: getattr(x, 'lr_active_max'), loops), histogram=hist, integer=True)
+        self.show_help(True, ('lr (overlap) min', 'the min number of live ranges that overlap in a trace'), 0)
+        self.show_cmv('lr (overlap) min', map(lambda x: getattr(x, 'lr_active_min'), loops), histogram=False, integer=True)
+        self.show_help(True, ('lr count', 'the live range count'), 0)
+        self.show_cmv('lr count', lr_counts, histogram=hist, integer=True)
         for typedescr in LR.ALL_TYPES:
             typename = typedescr[0]
             lrs = self.all_entries(loops, typename)
-            counts = map(lambda e: len(e), lrs)
-            self.show_cmv(typename, counts, 0)
+            self.show_help(help, typedescr, 0)
+            self.show_cmv(typename, lrs, 0, histogram=hist)
             #
             for failtypedescr in LR.FAIL_TYPES:
                 failtypename = typename + '-' + failtypedescr[0]
                 lrs = self.all_entries(loops, failtypename)
-                counts = map(lambda e: len(e), lrs)
-                self.show_cmv(failtypename, counts, 1)
+                self.show_help(help, failtypedescr, 1)
+                self.show_cmv(failtypename, lrs, 1, histogram=hist)
 
                 if failtypedescr == LR.FAIL:
+                    self.show_help(help, LR.END_FAIL, 2)
                     failtypename = failtypename + '-' + LR.END_FAIL[0]
                     lrs = self.all_entries(loops, failtypename)
-                    counts = map(lambda e: len(e), lrs)
-                    self.show_cmv(failtypename, counts, 2)
+                    self.show_cmv(failtypename, lrs, 2, histogram=hist)
 
-    def show_cmv(self, name, counts, indent=0):
-        total = sum(counts)
-        self.show(name, "mean %.2f\tvar %.2f\tcount %d" % (self.mean(counts), self.var(counts), total), indent=indent)
+                    self.show_help(help, LR.ONLY_FAIL, 2)
+                    failtypename = failtypename + '-' + LR.ONLY_FAIL[0]
+                    lrs = self.all_entries(loops, failtypename)
+                    self.show_cmv(failtypename, lrs, 2, histogram=hist)
+
+    def show_cmv(self, name, loop_lrs, indent=0, histogram=True, integer=False):
+        indent = " " * (indent * 2)
+        if integer:
+            counts = loop_lrs
+        else:
+            counts = map(lambda e: len(e), loop_lrs)
+            use_count = []
+            use_guard_count = []
+            for lrs in loop_lrs:
+                for ops, lr in lrs:
+                    count = 0
+                    gcount = 0
+                    for use in lr[2][1:]:
+                        op = ops[use]
+                        if op.is_guard():
+                            gcount += 1
+                        count += 1
+                    use_count.append(count)
+                    use_guard_count.append(gcount)
+
+            if len(use_count) > 0:
+                print indent, " #use: mean %.2f std %.2f" % (self.mean(use_count), self.var(use_count))
+            if len(use_guard_count) > 0:
+                print indent, " guard #use: mean %.2f std %.2f" % (self.mean(use_guard_count), self.var(use_guard_count))
+
+        total = len(counts)
+        total_sum = sum(counts)
+        min_counts = min(counts)
+        max_counts = max(counts)
+        print indent," mean %.2f std %.2f" % (self.mean(counts),self.var(counts))
+        print indent," min %d max %d" % (min_counts,max_counts)
+        if histogram:
+            import numpy
+            hist, bins = numpy.histogram(counts,bins=5)
+            for i in range(5):
+                l = bins[i]
+                u = bins[i+1]
+                v = hist[i]
+                print indent, " [%.1f-%.1f): %d (%.1f%%)" % (l, u, int(v), (100*float(v)/len(counts)))
 
     def mean(self, values):
         if len(values) == 0:
@@ -270,74 +326,97 @@
             type = type[0]
         entries = []
         for loop in loops:
-            e = loop.entries.get(type, [])
-            entries.append(e)
+            lrs = loop.entries.get(type, [])
+            entries.append(lrs)
         return entries
 
 
-    def examine(self, inputargs, operations, peeled=False):
+    def examine(self, inputargs, operations, peeled=False, bridge=False):
         llr = LoopLiveRanges(inputargs, operations)
         llr.type = 'normal'
         if peeled:
             llr.type = 'peeled'
+        if bridge:
+            llr.type = 'bridge'
         self.loops.append(llr)
 
 # ____________________________________________________________
 
 if __name__ == '__main__':
     from rpython.tool import logparser
-    log1 = logparser.parse_log_file(sys.argv[1])
-    loops = logparser.extract_category(log1, catprefix='jit-log-compiling-loop')
+    histogram = False
+    if '--histogram' in sys.argv:
+        histogram = True
+        sys.argv.remove('--histogram')
     lr = LR()
-    ns = {}
-    loop_count = len(loops)
-    print
     skipped = []
     skipped_not_loop = []
-    for j,text in enumerate(loops):
-        parser = RegallocParser(text)
-        loop = parser.parse()
-        unrolled_label = -1
-        first_label = -1
-        for i,op in enumerate(loop.operations):
-            if op.getopnum() == rop.LABEL:
+    total_trace_count = 0
+    for logfile in sys.argv[1:]:
+        print "reading",logfile,"..."
+        log1 = logparser.parse_log_file(logfile)
+        loops = logparser.extract_category(log1, catprefix='jit-log-opt-loop')
+        ns = {}
+        loop_count = len(loops)
+        total_trace_count += loop_count
+        for j,text in enumerate(loops):
+            parser = RegallocParser(text)
+            loop = parser.parse()
+            unrolled_label = -1
+            first_label = -1
+            for i,op in enumerate(loop.operations):
+                if op.getopnum() == rop.LABEL:
+                    if first_label == -1:
+                        first_label = i
+                    else:
+                        unrolled_label = i
+
+            if loop.operations[-1].getopnum() != rop.JUMP:
+                assert loop.operations[-1].getopnum() == rop.FINISH
+                skipped_not_loop.append(loop)
+                continue
+
+            if first_label != 0:
+                if first_label == -1 and loop.operations[-1].getopnum() == rop.JUMP:
+                    assert unrolled_label == -1
+                    # add an artificial instruction to support the live range computation
+                    #loop.operations.insert(0, ResOperation(rop.LABEL, [inputargs], None, None))
+                else:
+                    first_label = 0
+                    #skipped.append(loop)
+                    #continue
+
+            if unrolled_label > 0:
+                ops = loop.operations[first_label:unrolled_label+1]
+                #    for op in ops:
+                #        print op
+                #    print '=' * 80
+                inputargs = loop.inputargs
+                lr.examine(inputargs, ops, peeled=False)
+                # peeled loop
+                ops = loop.operations[unrolled_label:]
+                #for op in ops:
+                #    print op
+                label = ops[0]
+                inputargs = label.getarglist()
+                lr.examine(inputargs, ops, peeled=True)
+                #print '-' * 80
+            else:
                 if first_label == -1:
-                    first_label = i
+                    ops = loop.operations
+                    bridge = True
                 else:
-                    unrolled_label = i
-
-        if loop.operations[-1].getopnum() != rop.JUMP:
-            skipped_not_loop.append(loop)
-            continue
-        
-        if first_label != 0:
-            skipped.append(loop)
-            continue
-
-        if unrolled_label > 0:
-            ops = loop.operations[first_label:unrolled_label+1]
-            #    for op in ops:
-            #        print op
-            #    print '=' * 80
-            inputargs = loop.inputargs
-            lr.examine(inputargs, ops, peeled=False)
-            # peeled loop
-            ops = loop.operations[unrolled_label:]
-            #for op in ops:
-            #    print op
-            label = ops[0]
-            inputargs = label.getarglist()
-            lr.examine(inputargs, ops, peeled=True)
-            #print '-' * 80
-        else:
-            ops = loop.operations[first_label:]
-            #for op in ops:
-            #    print op
-            #print '-' * 80
-            inputargs = loop.inputargs
-            lr.examine(inputargs, ops, peeled=False)
-        print "\rloop %d/%d (%d%%)" % (j, loop_count, int(100.0 * j / loop_count)),
-        sys.stdout.flush()
+                    ops = loop.operations[first_label:]
+                    bridge = False
+                #for op in ops:
+                #    print op
+                #print '-' * 80
+                inputargs = loop.inputargs
+                lr.examine(inputargs, ops, peeled=False, bridge=bridge)
+            print "\rloop %d/%d (%d%%)" % (j, loop_count, int(100.0 * j / loop_count)),
+            sys.stdout.flush()
+        print
+    print "total trace count:", total_trace_count
     
     if len(skipped) > 0:
         print
@@ -347,4 +426,4 @@
         print
         print "skipped %d traces (not loops but traces)" % len(skipped_not_loop)
 
-    lr.print_stats()
+    lr.print_stats(histogram)

From noreply at buildbot.pypy.org  Thu Jun 18 18:20:48 2015
From: noreply at buildbot.pypy.org (cfbolz)
Date: Thu, 18 Jun 2015 18:20:48 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: disable the unroll-related change for
 now, it breaks stuff
Message-ID: <20150618162048.2E5B11C1FD3@cobra.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: optresult
Changeset: r78185:c76bc6248751
Date: 2015-06-18 17:28 +0200
http://bitbucket.org/pypy/pypy/changeset/c76bc6248751/

Log:	disable the unroll-related change for now, it breaks stuff

diff --git a/rpython/jit/metainterp/optimizeopt/pure.py b/rpython/jit/metainterp/optimizeopt/pure.py
--- a/rpython/jit/metainterp/optimizeopt/pure.py
+++ b/rpython/jit/metainterp/optimizeopt/pure.py
@@ -144,8 +144,15 @@
         if self.optimizer.emitting_dissabled:
             self.extra_call_pure.append(op) # XXX
         else:
-            effectinfo = op.getdescr().get_extra_info()
-            if not effectinfo.check_can_raise(ignore_memoryerror=True):
+            # don't move call_pure_with_exception in the short preamble...
+            # issue #2015
+
+            # XXX default has this code:
+            # this does not work with how pure calls are done on this branch
+            # fix together with unroll
+            #effectinfo = op.getdescr().get_extra_info()
+            #if not effectinfo.check_can_raise(ignore_memoryerror=True):
+            if 1:
                 self.call_pure_positions.append(
                     len(self.optimizer._newoperations) - 1)
 

From noreply at buildbot.pypy.org  Mon Jun 22 18:01:27 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 22 Jun 2015 18:01:27 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c8: fixes for the merge
Message-ID: <20150622160127.8DFE51C1FAD@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c8
Changeset: r78245:329874ff059b
Date: 2015-06-22 16:36 +0200
http://bitbucket.org/pypy/pypy/changeset/329874ff059b/

Log:	fixes for the merge

diff --git a/rpython/annotator/bookkeeper.py b/rpython/annotator/bookkeeper.py
--- a/rpython/annotator/bookkeeper.py
+++ b/rpython/annotator/bookkeeper.py
@@ -146,7 +146,7 @@
             if self.emulated_pbc_calls:
                 for pbc, args_s in self.emulated_pbc_calls.itervalues():
                     args = simple_args(args_s)
-                    self.consider_call_site(args, s_ImpossibleValue, None)
+                    pbc.consider_call_site(args, s_ImpossibleValue, None)
                 self.emulated_pbc_calls.clear()
         finally:
             self.leave()
diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -139,7 +139,7 @@
         self._pop_all_regs_from_frame(mc, [eax], True, callee_only=True)
         mc.RET()
         self._stm_leave_noninevitable_tr_slowpath = mc.materialize(
-            self.cpu.asmmemmgr, [])
+            self.cpu, [])
         #
         # a second helper to call _stm_reattach_transaction(tl),
         # preserving only registers that might store the result of a call
@@ -155,7 +155,7 @@
         mc.MOV_rs(eax.value, 0)
         mc.ADD_ri(esp.value, 3 * WORD)
         mc.RET()
-        self._stm_reattach_tr_slowpath = mc.materialize(self.cpu.asmmemmgr, [])
+        self._stm_reattach_tr_slowpath = mc.materialize(self.cpu, [])
 
 
     def set_extra_stack_depth(self, mc, value):
diff --git a/rpython/jit/backend/x86/callbuilder.py b/rpython/jit/backend/x86/callbuilder.py
--- a/rpython/jit/backend/x86/callbuilder.py
+++ b/rpython/jit/backend/x86/callbuilder.py
@@ -107,7 +107,7 @@
             self.asm.set_extra_stack_depth(self.mc, -current_esp)
         noregs = self.asm.cpu.gc_ll_descr.is_shadow_stack()
         gcmap = self.asm._regalloc.get_gcmap([eax], noregs=noregs)
-        self.asm.update_stm_location(self.mc, -self.current_esp)
+        self.asm.update_stm_location(self.mc, -self.get_current_esp())
         self.asm.push_gcmap(self.mc, gcmap, store=True)
 
     def pop_gcmap(self):
diff --git a/rpython/jit/backend/x86/rx86.py b/rpython/jit/backend/x86/rx86.py
--- a/rpython/jit/backend/x86/rx86.py
+++ b/rpython/jit/backend/x86/rx86.py
@@ -644,7 +644,7 @@
         self.PO1_b(ofs)
         self.stack_frame_size_delta(-self.WORD)
 
-    def POP_m(self, arg)
+    def POP_m(self, arg):
         self.PO1_m(arg)
         self.stack_frame_size_delta(-self.WORD)
 

From noreply at buildbot.pypy.org  Thu Jun 18 18:20:49 2015
From: noreply at buildbot.pypy.org (cfbolz)
Date: Thu, 18 Jun 2015 18:20:49 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: typo
Message-ID: <20150618162049.50F071C1FD5@cobra.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: optresult
Changeset: r78186:f92232236af0
Date: 2015-06-18 17:32 +0200
http://bitbucket.org/pypy/pypy/changeset/f92232236af0/

Log:	typo

diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -2105,11 +2105,11 @@
             return self._record_helper_nonpure_varargs(opnum, resvalue, descr,
                                                        list(argboxes))
 
-    def _record_helper_ovf(self, opnum, resbox, descr, *argboxes):
+    def _record_helper_ovf(self, opnum, resvalue, descr, *argboxes):
         if (not self.last_exc_value and
                 self._all_constants(*argboxes)):
             return history.newconst(resvalue)
-        return self._record_helper_nonpure_varargs(opnum, resbox, descr, list(argboxes))
+        return self._record_helper_nonpure_varargs(opnum, resvalue, descr, list(argboxes))
 
     @specialize.argtype(2)
     def _record_helper_pure_varargs(self, opnum, resvalue, descr, argboxes):

From noreply at buildbot.pypy.org  Thu Jun 18 18:20:50 2015
From: noreply at buildbot.pypy.org (cfbolz)
Date: Thu, 18 Jun 2015 18:20:50 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: use proper new interface
Message-ID: <20150618162050.68F0E1C1FD8@cobra.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: optresult
Changeset: r78187:1ba1b0efab29
Date: 2015-06-18 18:15 +0200
http://bitbucket.org/pypy/pypy/changeset/1ba1b0efab29/

Log:	use proper new interface

diff --git a/rpython/jit/metainterp/optimizeopt/intbounds.py b/rpython/jit/metainterp/optimizeopt/intbounds.py
--- a/rpython/jit/metainterp/optimizeopt/intbounds.py
+++ b/rpython/jit/metainterp/optimizeopt/intbounds.py
@@ -150,7 +150,7 @@
                         sum = intmask(arg2.getint() + prod_arg2.getint())
                         arg1 = prod_arg1
                         arg2 = ConstInt(sum)
-                        op = op.copy_and_change(rop.INT_ADD, args=[arg1, arg2])
+                        op = self.replace_op_with(op, rop.INT_ADD, args=[arg1, arg2])
 
         self.emit_operation(op)
         if self.is_raw_ptr(op.getarg(0)) or self.is_raw_ptr(op.getarg(1)):

From noreply at buildbot.pypy.org  Thu Jun 18 18:20:47 2015
From: noreply at buildbot.pypy.org (cfbolz)
Date: Thu, 18 Jun 2015 18:20:47 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: merge default
Message-ID: <20150618162047.07EB51C1FD2@cobra.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: optresult
Changeset: r78184:f191fe52a258
Date: 2015-06-18 17:26 +0200
http://bitbucket.org/pypy/pypy/changeset/f191fe52a258/

Log:	merge default

diff too long, truncating to 2000 out of 73369 lines

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -3,11 +3,15 @@
 d8ac7d23d3ec5f9a0fa1264972f74a010dbfd07f release-1.6
 ff4af8f318821f7f5ca998613a60fca09aa137da release-1.7
 07e08e9c885ca67d89bcc304e45a32346daea2fa release-2.0-beta-1
-9b623bc48b5950cf07184462a0e48f2c4df0d720 pypy-2.1-beta1-arm
-9b623bc48b5950cf07184462a0e48f2c4df0d720 pypy-2.1-beta1-arm
 ab0dd631c22015ed88e583d9fdd4c43eebf0be21 pypy-2.1-beta1-arm
 20e51c4389ed4469b66bb9d6289ce0ecfc82c4b9 release-2.3.0
 394146e9bb673514c61f0150ab2013ccf78e8de7 release-2.3
 32f35069a16d819b58c1b6efb17c44e3e53397b2 release-2.2=3.1
 32f35069a16d819b58c1b6efb17c44e3e53397b2 release-2.3.1
 10f1b29a2bd21f837090286174a9ca030b8680b2 release-2.5.0
+9c4588d731b7fe0b08669bd732c2b676cb0a8233 release-2.5.1
+fcdb941565156385cbac04cfb891f8f4c7a92ef6 release-2.6.0
+fcdb941565156385cbac04cfb891f8f4c7a92ef6 release-2.6.0
+e03971291f3a0729ecd3ee7fae7ddb0bb82d476c release-2.6.0
+e03971291f3a0729ecd3ee7fae7ddb0bb82d476c release-2.6.0
+295ee98b69288471b0fcf2e0ede82ce5209eb90b release-2.6.0
diff --git a/.tddium.requirements.txt b/.tddium.requirements.txt
deleted file mode 100644
--- a/.tddium.requirements.txt
+++ /dev/null
@@ -1,1 +0,0 @@
-pytest
diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -38,8 +38,8 @@
   Armin Rigo
   Maciej Fijalkowski
   Carl Friedrich Bolz
+  Amaury Forgeot d'Arc
   Antonio Cuni
-  Amaury Forgeot d'Arc
   Samuele Pedroni
   Alex Gaynor
   Brian Kearns
@@ -50,9 +50,9 @@
   Holger Krekel
   Christian Tismer
   Hakan Ardo
-  Benjamin Peterson
   Manuel Jacob
   Ronan Lamy
+  Benjamin Peterson
   Anders Chrigstrom
   Eric van Riet Paap
   Wim Lavrijsen
@@ -63,8 +63,8 @@
   Sven Hager
   Anders Lehmann
   Aurelien Campeas
+  Remi Meier
   Niklaus Haldimann
-  Remi Meier
   Camillo Bruni
   Laura Creighton
   Toon Verwaest
@@ -76,10 +76,10 @@
   David Edelsohn
   Anders Hammarquist
   Jakub Gustak
+  Gregor Wegberg
   Guido Wesdorp
   Lawrence Oluyede
   Bartosz Skowron
-  Gregor Wegberg
   Daniel Roberts
   Niko Matsakis
   Adrien Di Mascio
@@ -87,10 +87,11 @@
   Ludovic Aubry
   Jacob Hallen
   Jason Creighton
+  Richard Plangger
   Alex Martelli
   Michal Bendowski
+  stian
   Jan de Mooij
-  stian
   Tyler Wade
   Michael Foord
   Stephan Diehl
@@ -133,15 +134,15 @@
   Georg Brandl
   Bert Freudenberg
   Stian Andreassen
+  Edd Barrett
   Wanja Saatkamp
   Gerald Klix
   Mike Blume
+  Tobias Pape
   Oscar Nierstrasz
   Stefan H. Muller
-  Edd Barrett
   Jeremy Thurgood
   Rami Chowdhury
-  Tobias Pape
   Eugene Oden
   Henry Mason
   Vasily Kuznetsov
@@ -167,11 +168,13 @@
   Michael Twomey
   Lucian Branescu Mihaila
   Yichao Yu
+  Anton Gulenko
   Gabriel Lavoie
   Olivier Dormond
   Jared Grubb
   Karl Bartel
   Wouter van Heyst
+  Sebastian Pawlu&#347;
   Brian Dorsey
   Victor Stinner
   Andrews Medina
@@ -188,6 +191,7 @@
   Neil Shepperd
   Stanislaw Halik
   Mikael Sch&#246;nenberg
+  Berkin Ilbeyi
   Elmo M?ntynen
   Jonathan David Riehl
   Anders Qvist
@@ -211,11 +215,11 @@
   Carl Meyer
   Karl Ramm
   Pieter Zieschang
-  Sebastian Pawlu&#347;
   Gabriel
   Lukas Vacek
   Andrew Dalke
   Sylvain Thenault
+  Jakub Stasiak
   Nathan Taylor
   Vladimir Kryachko
   Jacek Generowicz
@@ -242,6 +246,7 @@
   Tomo Cocoa
   Toni Mattis
   Lucas Stadler
+  Julian Berman
   roberto at goyle
   Yury V. Zaytsev
   Anna Katrina Dominguez
@@ -253,6 +258,8 @@
   Ben Darnell
   Roberto De Ioris
   Juan Francisco Cantero Hurtado
+  Ruochen Huang
+  Jeong YunWon
   Godefroid Chappelle
   Joshua Gilbert
   Dan Colish
@@ -271,6 +278,7 @@
   Christian Muirhead
   Berker Peksag
   James Lan
+  Volodymyr Vladymyrov
   shoma hosaka
   Daniel Neuh&#228;user
   Ben Mather
@@ -316,6 +324,7 @@
   yasirs
   Michael Chermside
   Anna Ravencroft
+  Andrey Churin
   Dan Crosta
   Julien Phalip
   Roman Podoliaka
@@ -420,3 +429,10 @@
 the terms of the GPL license version 2 or any later version.  Thus the
 gdbm module, provided in the file lib_pypy/gdbm.py, is redistributed
 under the terms of the GPL license as well.
+
+License for 'pypy/module/_vmprof/src'
+--------------------------------------
+
+The code is based on gperftools. You may see a copy of the License for it at
+
+    https://code.google.com/p/gperftools/source/browse/COPYING
diff --git a/lib-python/2.7/Cookie.py b/lib-python/2.7/Cookie.py
--- a/lib-python/2.7/Cookie.py
+++ b/lib-python/2.7/Cookie.py
@@ -528,12 +528,13 @@
 # result, the parsing rules here are less strict.
 #
 
-_LegalCharsPatt  = r"[\w\d!#%&'~_`><@,:/\$\*\+\-\.\^\|\)\(\?\}\{\=]"
+_LegalKeyChars  = r"\w\d!#%&'~_`><@,:/\$\*\+\-\.\^\|\)\(\?\}\{\="
+_LegalValueChars = _LegalKeyChars + r"\[\]"
 _CookiePattern = re.compile(
     r"(?x)"                       # This is a Verbose pattern
     r"\s*"                        # Optional whitespace at start of cookie
     r"(?P<key>"                   # Start of group 'key'
-    ""+ _LegalCharsPatt +"+?"     # Any word of at least one letter, nongreedy
+    "["+ _LegalKeyChars +"]+?"     # Any word of at least one letter, nongreedy
     r")"                          # End of group 'key'
     r"("                          # Optional group: there may not be a value.
     r"\s*=\s*"                    # Equal Sign
@@ -542,7 +543,7 @@
     r"|"                            # or
     r"\w{3},\s[\s\w\d-]{9,11}\s[\d:]{8}\sGMT" # Special case for "expires" attr
     r"|"                            # or
-    ""+ _LegalCharsPatt +"*"        # Any word or empty string
+    "["+ _LegalValueChars +"]*"        # Any word or empty string
     r")"                          # End of group 'val'
     r")?"                         # End of optional value group
     r"\s*"                        # Any number of spaces.
diff --git a/lib-python/2.7/SimpleHTTPServer.py b/lib-python/2.7/SimpleHTTPServer.py
--- a/lib-python/2.7/SimpleHTTPServer.py
+++ b/lib-python/2.7/SimpleHTTPServer.py
@@ -14,6 +14,7 @@
 import posixpath
 import BaseHTTPServer
 import urllib
+import urlparse
 import cgi
 import sys
 import shutil
@@ -68,10 +69,14 @@
         path = self.translate_path(self.path)
         f = None
         if os.path.isdir(path):
-            if not self.path.endswith('/'):
+            parts = urlparse.urlsplit(self.path)
+            if not parts.path.endswith('/'):
                 # redirect browser - doing basically what apache does
                 self.send_response(301)
-                self.send_header("Location", self.path + "/")
+                new_parts = (parts[0], parts[1], parts[2] + '/',
+                             parts[3], parts[4])
+                new_url = urlparse.urlunsplit(new_parts)
+                self.send_header("Location", new_url)
                 self.end_headers()
                 return None
             for index in "index.html", "index.htm":
diff --git a/lib-python/2.7/_LWPCookieJar.py b/lib-python/2.7/_LWPCookieJar.py
--- a/lib-python/2.7/_LWPCookieJar.py
+++ b/lib-python/2.7/_LWPCookieJar.py
@@ -18,7 +18,7 @@
                        iso2time, time2isoz)
 
 def lwp_cookie_str(cookie):
-    """Return string representation of Cookie in an the LWP cookie file format.
+    """Return string representation of Cookie in the LWP cookie file format.
 
     Actually, the format is extended a bit -- see module docstring.
 
diff --git a/lib-python/2.7/_abcoll.py b/lib-python/2.7/_abcoll.py
--- a/lib-python/2.7/_abcoll.py
+++ b/lib-python/2.7/_abcoll.py
@@ -548,23 +548,25 @@
             If E present and lacks .keys() method, does:     for (k, v) in E: D[k] = v
             In either case, this is followed by: for k, v in F.items(): D[k] = v
         '''
-        if len(args) > 2:
-            raise TypeError("update() takes at most 2 positional "
-                            "arguments ({} given)".format(len(args)))
-        elif not args:
-            raise TypeError("update() takes at least 1 argument (0 given)")
+        if not args:
+            raise TypeError("descriptor 'update' of 'MutableMapping' object "
+                            "needs an argument")
         self = args[0]
-        other = args[1] if len(args) >= 2 else ()
-
-        if isinstance(other, Mapping):
-            for key in other:
-                self[key] = other[key]
-        elif hasattr(other, "keys"):
-            for key in other.keys():
-                self[key] = other[key]
-        else:
-            for key, value in other:
-                self[key] = value
+        args = args[1:]
+        if len(args) > 1:
+            raise TypeError('update expected at most 1 arguments, got %d' %
+                            len(args))
+        if args:
+            other = args[0]
+            if isinstance(other, Mapping):
+                for key in other:
+                    self[key] = other[key]
+            elif hasattr(other, "keys"):
+                for key in other.keys():
+                    self[key] = other[key]
+            else:
+                for key, value in other:
+                    self[key] = value
         for key, value in kwds.items():
             self[key] = value
 
diff --git a/lib-python/2.7/_pyio.py b/lib-python/2.7/_pyio.py
--- a/lib-python/2.7/_pyio.py
+++ b/lib-python/2.7/_pyio.py
@@ -25,8 +25,8 @@
 DEFAULT_BUFFER_SIZE = 8 * 1024  # bytes
 
 # NOTE: Base classes defined here are registered with the "official" ABCs
-# defined in io.py. We don't use real inheritance though, because we don't
-# want to inherit the C implementations.
+# defined in io.py. We don't use real inheritance though, because we don't want
+# to inherit the C implementations.
 
 
 class BlockingIOError(IOError):
@@ -775,7 +775,7 @@
         clsname = self.__class__.__name__
         try:
             name = self.name
-        except AttributeError:
+        except Exception:
             return "<_pyio.{0}>".format(clsname)
         else:
             return "<_pyio.{0} name={1!r}>".format(clsname, name)
@@ -1216,8 +1216,10 @@
         return self.writer.flush()
 
     def close(self):
-        self.writer.close()
-        self.reader.close()
+        try:
+            self.writer.close()
+        finally:
+            self.reader.close()
 
     def isatty(self):
         return self.reader.isatty() or self.writer.isatty()
@@ -1538,7 +1540,7 @@
     def __repr__(self):
         try:
             name = self.name
-        except AttributeError:
+        except Exception:
             return "<_pyio.TextIOWrapper encoding='{0}'>".format(self.encoding)
         else:
             return "<_pyio.TextIOWrapper name={0!r} encoding='{1}'>".format(
diff --git a/lib-python/2.7/_strptime.py b/lib-python/2.7/_strptime.py
--- a/lib-python/2.7/_strptime.py
+++ b/lib-python/2.7/_strptime.py
@@ -335,9 +335,9 @@
     # though
     week_of_year = -1
     week_of_year_start = -1
-    # weekday and julian defaulted to -1 so as to signal need to calculate
+    # weekday and julian defaulted to None so as to signal need to calculate
     # values
-    weekday = julian = -1
+    weekday = julian = None
     found_dict = found.groupdict()
     for group_key in found_dict.iterkeys():
         # Directives not explicitly handled below:
@@ -434,14 +434,14 @@
         year = 1900
     # If we know the week of the year and what day of that week, we can figure
     # out the Julian day of the year.
-    if julian == -1 and week_of_year != -1 and weekday != -1:
+    if julian is None and week_of_year != -1 and weekday is not None:
         week_starts_Mon = True if week_of_year_start == 0 else False
         julian = _calc_julian_from_U_or_W(year, week_of_year, weekday,
                                             week_starts_Mon)
     # Cannot pre-calculate datetime_date() since can change in Julian
     # calculation and thus could have different value for the day of the week
     # calculation.
-    if julian == -1:
+    if julian is None:
         # Need to add 1 to result since first day of the year is 1, not 0.
         julian = datetime_date(year, month, day).toordinal() - \
                   datetime_date(year, 1, 1).toordinal() + 1
@@ -451,7 +451,7 @@
         year = datetime_result.year
         month = datetime_result.month
         day = datetime_result.day
-    if weekday == -1:
+    if weekday is None:
         weekday = datetime_date(year, month, day).weekday()
     if leap_year_fix:
         # the caller didn't supply a year but asked for Feb 29th. We couldn't
diff --git a/lib-python/2.7/aifc.py b/lib-python/2.7/aifc.py
--- a/lib-python/2.7/aifc.py
+++ b/lib-python/2.7/aifc.py
@@ -357,10 +357,13 @@
         self._soundpos = 0
 
     def close(self):
-        if self._decomp:
-            self._decomp.CloseDecompressor()
-            self._decomp = None
-        self._file.close()
+        decomp = self._decomp
+        try:
+            if decomp:
+                self._decomp = None
+                decomp.CloseDecompressor()
+        finally:
+            self._file.close()
 
     def tell(self):
         return self._soundpos
diff --git a/lib-python/2.7/binhex.py b/lib-python/2.7/binhex.py
--- a/lib-python/2.7/binhex.py
+++ b/lib-python/2.7/binhex.py
@@ -32,7 +32,8 @@
     pass
 
 # States (what have we written)
-[_DID_HEADER, _DID_DATA, _DID_RSRC] = range(3)
+_DID_HEADER = 0
+_DID_DATA = 1
 
 # Various constants
 REASONABLY_LARGE=32768  # Minimal amount we pass the rle-coder
@@ -235,17 +236,22 @@
         self._write(data)
 
     def close(self):
-        if self.state < _DID_DATA:
-            self.close_data()
-        if self.state != _DID_DATA:
-            raise Error, 'Close at the wrong time'
-        if self.rlen != 0:
-            raise Error, \
-                  "Incorrect resource-datasize, diff=%r" % (self.rlen,)
-        self._writecrc()
-        self.ofp.close()
-        self.state = None
-        del self.ofp
+        if self.state is None:
+            return
+        try:
+            if self.state < _DID_DATA:
+                self.close_data()
+            if self.state != _DID_DATA:
+                raise Error, 'Close at the wrong time'
+            if self.rlen != 0:
+                raise Error, \
+                    "Incorrect resource-datasize, diff=%r" % (self.rlen,)
+            self._writecrc()
+        finally:
+            self.state = None
+            ofp = self.ofp
+            del self.ofp
+            ofp.close()
 
 def binhex(inp, out):
     """(infilename, outfilename) - Create binhex-encoded copy of a file"""
@@ -463,11 +469,15 @@
         return self._read(n)
 
     def close(self):
-        if self.rlen:
-            dummy = self.read_rsrc(self.rlen)
-        self._checkcrc()
-        self.state = _DID_RSRC
-        self.ifp.close()
+        if self.state is None:
+            return
+        try:
+            if self.rlen:
+                dummy = self.read_rsrc(self.rlen)
+            self._checkcrc()
+        finally:
+            self.state = None
+            self.ifp.close()
 
 def hexbin(inp, out):
     """(infilename, outfilename) - Decode binhexed file"""
diff --git a/lib-python/2.7/bsddb/test/test_all.py b/lib-python/2.7/bsddb/test/test_all.py
--- a/lib-python/2.7/bsddb/test/test_all.py
+++ b/lib-python/2.7/bsddb/test/test_all.py
@@ -412,9 +412,6 @@
         def get_dbp(self) :
             return self._db
 
-    import string
-    string.letters=[chr(i) for i in xrange(65,91)]
-
     bsddb._db.DBEnv_orig = bsddb._db.DBEnv
     bsddb._db.DB_orig = bsddb._db.DB
     if bsddb.db.version() <= (4, 3) :
diff --git a/lib-python/2.7/bsddb/test/test_basics.py b/lib-python/2.7/bsddb/test/test_basics.py
--- a/lib-python/2.7/bsddb/test/test_basics.py
+++ b/lib-python/2.7/bsddb/test/test_basics.py
@@ -999,7 +999,7 @@
         for x in "The quick brown fox jumped over the lazy dog".split():
             d2.put(x, self.makeData(x))
 
-        for x in string.letters:
+        for x in string.ascii_letters:
             d3.put(x, x*70)
 
         d1.sync()
@@ -1047,7 +1047,7 @@
             if verbose:
                 print rec
             rec = c3.next()
-        self.assertEqual(count, len(string.letters))
+        self.assertEqual(count, len(string.ascii_letters))
 
 
         c1.close()
diff --git a/lib-python/2.7/bsddb/test/test_dbshelve.py b/lib-python/2.7/bsddb/test/test_dbshelve.py
--- a/lib-python/2.7/bsddb/test/test_dbshelve.py
+++ b/lib-python/2.7/bsddb/test/test_dbshelve.py
@@ -59,7 +59,7 @@
             return bytes(key, "iso8859-1")  # 8 bits
 
     def populateDB(self, d):
-        for x in string.letters:
+        for x in string.ascii_letters:
             d[self.mk('S' + x)] = 10 * x           # add a string
             d[self.mk('I' + x)] = ord(x)           # add an integer
             d[self.mk('L' + x)] = [x] * 10         # add a list
diff --git a/lib-python/2.7/bsddb/test/test_get_none.py b/lib-python/2.7/bsddb/test/test_get_none.py
--- a/lib-python/2.7/bsddb/test/test_get_none.py
+++ b/lib-python/2.7/bsddb/test/test_get_none.py
@@ -26,14 +26,14 @@
         d.open(self.filename, db.DB_BTREE, db.DB_CREATE)
         d.set_get_returns_none(1)
 
-        for x in string.letters:
+        for x in string.ascii_letters:
             d.put(x, x * 40)
 
         data = d.get('bad key')
         self.assertEqual(data, None)
 
-        data = d.get(string.letters[0])
-        self.assertEqual(data, string.letters[0]*40)
+        data = d.get(string.ascii_letters[0])
+        self.assertEqual(data, string.ascii_letters[0]*40)
 
         count = 0
         c = d.cursor()
@@ -43,7 +43,7 @@
             rec = c.next()
 
         self.assertEqual(rec, None)
-        self.assertEqual(count, len(string.letters))
+        self.assertEqual(count, len(string.ascii_letters))
 
         c.close()
         d.close()
@@ -54,14 +54,14 @@
         d.open(self.filename, db.DB_BTREE, db.DB_CREATE)
         d.set_get_returns_none(0)
 
-        for x in string.letters:
+        for x in string.ascii_letters:
             d.put(x, x * 40)
 
         self.assertRaises(db.DBNotFoundError, d.get, 'bad key')
         self.assertRaises(KeyError, d.get, 'bad key')
 
-        data = d.get(string.letters[0])
-        self.assertEqual(data, string.letters[0]*40)
+        data = d.get(string.ascii_letters[0])
+        self.assertEqual(data, string.ascii_letters[0]*40)
 
         count = 0
         exceptionHappened = 0
@@ -77,7 +77,7 @@
 
         self.assertNotEqual(rec, None)
         self.assertTrue(exceptionHappened)
-        self.assertEqual(count, len(string.letters))
+        self.assertEqual(count, len(string.ascii_letters))
 
         c.close()
         d.close()
diff --git a/lib-python/2.7/bsddb/test/test_queue.py b/lib-python/2.7/bsddb/test/test_queue.py
--- a/lib-python/2.7/bsddb/test/test_queue.py
+++ b/lib-python/2.7/bsddb/test/test_queue.py
@@ -10,7 +10,6 @@
 
 #----------------------------------------------------------------------
 
- at unittest.skip("fails on Windows; see issue 22943")
 class SimpleQueueTestCase(unittest.TestCase):
     def setUp(self):
         self.filename = get_new_database_path()
@@ -37,17 +36,17 @@
             print "before appends" + '-' * 30
             pprint(d.stat())
 
-        for x in string.letters:
+        for x in string.ascii_letters:
             d.append(x * 40)
 
-        self.assertEqual(len(d), len(string.letters))
+        self.assertEqual(len(d), len(string.ascii_letters))
 
         d.put(100, "some more data")
         d.put(101, "and some more ")
         d.put(75,  "out of order")
         d.put(1,   "replacement data")
 
-        self.assertEqual(len(d), len(string.letters)+3)
+        self.assertEqual(len(d), len(string.ascii_letters)+3)
 
         if verbose:
             print "before close" + '-' * 30
@@ -108,17 +107,17 @@
             print "before appends" + '-' * 30
             pprint(d.stat())
 
-        for x in string.letters:
+        for x in string.ascii_letters:
             d.append(x * 40)
 
-        self.assertEqual(len(d), len(string.letters))
+        self.assertEqual(len(d), len(string.ascii_letters))
 
         d.put(100, "some more data")
         d.put(101, "and some more ")
         d.put(75,  "out of order")
         d.put(1,   "replacement data")
 
-        self.assertEqual(len(d), len(string.letters)+3)
+        self.assertEqual(len(d), len(string.ascii_letters)+3)
 
         if verbose:
             print "before close" + '-' * 30
diff --git a/lib-python/2.7/bsddb/test/test_recno.py b/lib-python/2.7/bsddb/test/test_recno.py
--- a/lib-python/2.7/bsddb/test/test_recno.py
+++ b/lib-python/2.7/bsddb/test/test_recno.py
@@ -4,12 +4,11 @@
 import os, sys
 import errno
 from pprint import pprint
+import string
 import unittest
 
 from test_all import db, test_support, verbose, get_new_environment_path, get_new_database_path
 
-letters = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
-
 
 #----------------------------------------------------------------------
 
@@ -39,7 +38,7 @@
 
         d.open(self.filename, db.DB_RECNO, db.DB_CREATE)
 
-        for x in letters:
+        for x in string.ascii_letters:
             recno = d.append(x * 60)
             self.assertIsInstance(recno, int)
             self.assertGreaterEqual(recno, 1)
@@ -270,7 +269,7 @@
         d.set_re_pad(45)  # ...test both int and char
         d.open(self.filename, db.DB_RECNO, db.DB_CREATE)
 
-        for x in letters:
+        for x in string.ascii_letters:
             d.append(x * 35)    # These will be padded
 
         d.append('.' * 40)      # this one will be exact
diff --git a/lib-python/2.7/chunk.py b/lib-python/2.7/chunk.py
--- a/lib-python/2.7/chunk.py
+++ b/lib-python/2.7/chunk.py
@@ -85,8 +85,10 @@
 
     def close(self):
         if not self.closed:
-            self.skip()
-            self.closed = True
+            try:
+                self.skip()
+            finally:
+                self.closed = True
 
     def isatty(self):
         if self.closed:
diff --git a/lib-python/2.7/codecs.py b/lib-python/2.7/codecs.py
--- a/lib-python/2.7/codecs.py
+++ b/lib-python/2.7/codecs.py
@@ -20,8 +20,14 @@
            "BOM_LE", "BOM32_BE", "BOM32_LE", "BOM64_BE", "BOM64_LE",
            "BOM_UTF8", "BOM_UTF16", "BOM_UTF16_LE", "BOM_UTF16_BE",
            "BOM_UTF32", "BOM_UTF32_LE", "BOM_UTF32_BE",
+           "CodecInfo", "Codec", "IncrementalEncoder", "IncrementalDecoder",
+           "StreamReader", "StreamWriter",
+           "StreamReaderWriter", "StreamRecoder",
+           "getencoder", "getdecoder", "getincrementalencoder",
+           "getincrementaldecoder", "getreader", "getwriter",
+           "encode", "decode", "iterencode", "iterdecode",
            "strict_errors", "ignore_errors", "replace_errors",
-           "xmlcharrefreplace_errors",
+           "xmlcharrefreplace_errors", "backslashreplace_errors",
            "register_error", "lookup_error"]
 
 ### Constants
@@ -1051,7 +1057,7 @@
         during translation.
 
         One example where this happens is cp875.py which decodes
-        multiple character to \u001a.
+        multiple character to \\u001a.
 
     """
     m = {}
diff --git a/lib-python/2.7/collections.py b/lib-python/2.7/collections.py
--- a/lib-python/2.7/collections.py
+++ b/lib-python/2.7/collections.py
@@ -330,7 +330,7 @@
     #   http://code.activestate.com/recipes/259174/
     #   Knuth, TAOCP Vol. II section 4.6.3
 
-    def __init__(self, iterable=None, **kwds):
+    def __init__(*args, **kwds):
         '''Create a new, empty Counter object.  And if given, count elements
         from an input iterable.  Or, initialize the count from another mapping
         of elements to their counts.
@@ -341,8 +341,15 @@
         >>> c = Counter(a=4, b=2)                   # a new counter from keyword args
 
         '''
+        if not args:
+            raise TypeError("descriptor '__init__' of 'Counter' object "
+                            "needs an argument")
+        self = args[0]
+        args = args[1:]
+        if len(args) > 1:
+            raise TypeError('expected at most 1 arguments, got %d' % len(args))
         super(Counter, self).__init__()
-        self.update(iterable, **kwds)
+        self.update(*args, **kwds)
 
     def __missing__(self, key):
         'The count of elements not in the Counter is zero.'
@@ -393,7 +400,7 @@
         raise NotImplementedError(
             'Counter.fromkeys() is undefined.  Use Counter(iterable) instead.')
 
-    def update(self, iterable=None, **kwds):
+    def update(*args, **kwds):
         '''Like dict.update() but add counts instead of replacing them.
 
         Source can be an iterable, a dictionary, or another Counter instance.
@@ -413,6 +420,14 @@
         # contexts.  Instead, we implement straight-addition.  Both the inputs
         # and outputs are allowed to contain zero and negative counts.
 
+        if not args:
+            raise TypeError("descriptor 'update' of 'Counter' object "
+                            "needs an argument")
+        self = args[0]
+        args = args[1:]
+        if len(args) > 1:
+            raise TypeError('expected at most 1 arguments, got %d' % len(args))
+        iterable = args[0] if args else None
         if iterable is not None:
             if isinstance(iterable, Mapping):
                 if self:
@@ -428,7 +443,7 @@
         if kwds:
             self.update(kwds)
 
-    def subtract(self, iterable=None, **kwds):
+    def subtract(*args, **kwds):
         '''Like dict.update() but subtracts counts instead of replacing them.
         Counts can be reduced below zero.  Both the inputs and outputs are
         allowed to contain zero and negative counts.
@@ -444,6 +459,14 @@
         -1
 
         '''
+        if not args:
+            raise TypeError("descriptor 'subtract' of 'Counter' object "
+                            "needs an argument")
+        self = args[0]
+        args = args[1:]
+        if len(args) > 1:
+            raise TypeError('expected at most 1 arguments, got %d' % len(args))
+        iterable = args[0] if args else None
         if iterable is not None:
             self_get = self.get
             if isinstance(iterable, Mapping):
diff --git a/lib-python/2.7/cookielib.py b/lib-python/2.7/cookielib.py
--- a/lib-python/2.7/cookielib.py
+++ b/lib-python/2.7/cookielib.py
@@ -464,26 +464,42 @@
     for ns_header in ns_headers:
         pairs = []
         version_set = False
-        for ii, param in enumerate(re.split(r";\s*", ns_header)):
-            param = param.rstrip()
-            if param == "": continue
-            if "=" not in param:
-                k, v = param, None
-            else:
-                k, v = re.split(r"\s*=\s*", param, 1)
-                k = k.lstrip()
+
+        # XXX: The following does not strictly adhere to RFCs in that empty
+        # names and values are legal (the former will only appear once and will
+        # be overwritten if multiple occurrences are present). This is
+        # mostly to deal with backwards compatibility.
+        for ii, param in enumerate(ns_header.split(';')):
+            param = param.strip()
+
+            key, sep, val = param.partition('=')
+            key = key.strip()
+
+            if not key:
+                if ii == 0:
+                    break
+                else:
+                    continue
+
+            # allow for a distinction between present and empty and missing
+            # altogether
+            val = val.strip() if sep else None
+
             if ii != 0:
-                lc = k.lower()
+                lc = key.lower()
                 if lc in known_attrs:
-                    k = lc
-                if k == "version":
+                    key = lc
+
+                if key == "version":
                     # This is an RFC 2109 cookie.
-                    v = _strip_quotes(v)
+                    if val is not None:
+                        val = _strip_quotes(val)
                     version_set = True
-                if k == "expires":
+                elif key == "expires":
                     # convert expires date to seconds since epoch
-                    v = http2time(_strip_quotes(v))  # None if invalid
-            pairs.append((k, v))
+                    if val is not None:
+                        val = http2time(_strip_quotes(val))  # None if invalid
+            pairs.append((key, val))
 
         if pairs:
             if not version_set:
diff --git a/lib-python/2.7/ctypes/macholib/fetch_macholib.bat b/lib-python/2.7/ctypes/macholib/fetch_macholib.bat
--- a/lib-python/2.7/ctypes/macholib/fetch_macholib.bat
+++ b/lib-python/2.7/ctypes/macholib/fetch_macholib.bat
@@ -1,1 +1,1 @@
-svn export --force http://svn.red-bean.com/bob/macholib/trunk/macholib/ .
+svn export --force http://svn.red-bean.com/bob/macholib/trunk/macholib/ .
diff --git a/lib-python/2.7/ctypes/test/test_find.py b/lib-python/2.7/ctypes/test/test_find.py
--- a/lib-python/2.7/ctypes/test/test_find.py
+++ b/lib-python/2.7/ctypes/test/test_find.py
@@ -32,15 +32,24 @@
     def setUp(self):
         self.gl = self.glu = self.gle = None
         if lib_gl:
-            self.gl = CDLL(lib_gl, mode=RTLD_GLOBAL)
+            try:
+                self.gl = CDLL(lib_gl, mode=RTLD_GLOBAL)
+            except OSError:
+                pass
         if lib_glu:
-            self.glu = CDLL(lib_glu, RTLD_GLOBAL)
+            try:
+                self.glu = CDLL(lib_glu, RTLD_GLOBAL)
+            except OSError:
+                pass
         if lib_gle:
             try:
                 self.gle = CDLL(lib_gle)
             except OSError:
                 pass
 
+    def tearDown(self):
+        self.gl = self.glu = self.gle = None
+
     @unittest.skipUnless(lib_gl, 'lib_gl not available')
     def test_gl(self):
         if self.gl:
diff --git a/lib-python/2.7/ctypes/test/test_pickling.py b/lib-python/2.7/ctypes/test/test_pickling.py
--- a/lib-python/2.7/ctypes/test/test_pickling.py
+++ b/lib-python/2.7/ctypes/test/test_pickling.py
@@ -15,9 +15,9 @@
 class Y(X):
     _fields_ = [("str", c_char_p)]
 
-class PickleTest(unittest.TestCase):
+class PickleTest:
     def dumps(self, item):
-        return pickle.dumps(item)
+        return pickle.dumps(item, self.proto)
 
     def loads(self, item):
         return pickle.loads(item)
@@ -72,17 +72,15 @@
 
     @xfail
     def test_wchar(self):
-        pickle.dumps(c_char("x"))
+        self.dumps(c_char(b"x"))
         # Issue 5049
-        pickle.dumps(c_wchar(u"x"))
+        self.dumps(c_wchar(u"x"))
 
-class PickleTest_1(PickleTest):
-    def dumps(self, item):
-        return pickle.dumps(item, 1)
-
-class PickleTest_2(PickleTest):
-    def dumps(self, item):
-        return pickle.dumps(item, 2)
+for proto in range(pickle.HIGHEST_PROTOCOL + 1):
+    name = 'PickleTest_%s' % proto
+    globals()[name] = type(name,
+                           (PickleTest, unittest.TestCase),
+                           {'proto': proto})
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/lib-python/2.7/ctypes/test/test_pointers.py b/lib-python/2.7/ctypes/test/test_pointers.py
--- a/lib-python/2.7/ctypes/test/test_pointers.py
+++ b/lib-python/2.7/ctypes/test/test_pointers.py
@@ -7,8 +7,6 @@
                  c_long, c_ulong, c_longlong, c_ulonglong, c_double, c_float]
 python_types = [int, int, int, int, int, long,
                 int, long, long, long, float, float]
-LargeNamedType = type('T' * 2 ** 25, (Structure,), {})
-large_string = 'T' * 2 ** 25
 
 class PointersTestCase(unittest.TestCase):
 
@@ -191,9 +189,11 @@
             self.assertEqual(bool(mth), True)
 
     def test_pointer_type_name(self):
+        LargeNamedType = type('T' * 2 ** 25, (Structure,), {})
         self.assertTrue(POINTER(LargeNamedType))
 
     def test_pointer_type_str_name(self):
+        large_string = 'T' * 2 ** 25
         self.assertTrue(POINTER(large_string))
 
 if __name__ == '__main__':
diff --git a/lib-python/2.7/ctypes/util.py b/lib-python/2.7/ctypes/util.py
--- a/lib-python/2.7/ctypes/util.py
+++ b/lib-python/2.7/ctypes/util.py
@@ -178,7 +178,7 @@
             res = re.findall(expr, data)
             if not res:
                 return _get_soname(_findLib_gcc(name))
-            res.sort(cmp= lambda x,y: cmp(_num_version(x), _num_version(y)))
+            res.sort(key=_num_version)
             return res[-1]
 
     elif sys.platform == "sunos5":
diff --git a/lib-python/2.7/distutils/__init__.py b/lib-python/2.7/distutils/__init__.py
--- a/lib-python/2.7/distutils/__init__.py
+++ b/lib-python/2.7/distutils/__init__.py
@@ -15,5 +15,5 @@
 # Updated automatically by the Python release process.
 #
 #--start constants--
-__version__ = "2.7.9"
+__version__ = "2.7.10"
 #--end constants--
diff --git a/lib-python/2.7/distutils/command/check.py b/lib-python/2.7/distutils/command/check.py
--- a/lib-python/2.7/distutils/command/check.py
+++ b/lib-python/2.7/distutils/command/check.py
@@ -126,7 +126,7 @@
         """Returns warnings when the provided data doesn't compile."""
         source_path = StringIO()
         parser = Parser()
-        settings = frontend.OptionParser().get_default_values()
+        settings = frontend.OptionParser(components=(Parser,)).get_default_values()
         settings.tab_width = 4
         settings.pep_references = None
         settings.rfc_references = None
@@ -142,8 +142,8 @@
         document.note_source(source_path, -1)
         try:
             parser.parse(data, document)
-        except AttributeError:
-            reporter.messages.append((-1, 'Could not finish the parsing.',
-                                      '', {}))
+        except AttributeError as e:
+            reporter.messages.append(
+                (-1, 'Could not finish the parsing: %s.' % e, '', {}))
 
         return reporter.messages
diff --git a/lib-python/2.7/distutils/dir_util.py b/lib-python/2.7/distutils/dir_util.py
--- a/lib-python/2.7/distutils/dir_util.py
+++ b/lib-python/2.7/distutils/dir_util.py
@@ -83,7 +83,7 @@
     """Create all the empty directories under 'base_dir' needed to put 'files'
     there.
 
-    'base_dir' is just the a name of a directory which doesn't necessarily
+    'base_dir' is just the name of a directory which doesn't necessarily
     exist yet; 'files' is a list of filenames to be interpreted relative to
     'base_dir'.  'base_dir' + the directory portion of every file in 'files'
     will be created if it doesn't already exist.  'mode', 'verbose' and
diff --git a/lib-python/2.7/distutils/tests/test_check.py b/lib-python/2.7/distutils/tests/test_check.py
--- a/lib-python/2.7/distutils/tests/test_check.py
+++ b/lib-python/2.7/distutils/tests/test_check.py
@@ -1,5 +1,6 @@
 # -*- encoding: utf8 -*-
 """Tests for distutils.command.check."""
+import textwrap
 import unittest
 from test.test_support import run_unittest
 
@@ -93,6 +94,36 @@
         cmd = self._run(metadata, strict=1, restructuredtext=1)
         self.assertEqual(cmd._warnings, 0)
 
+    @unittest.skipUnless(HAS_DOCUTILS, "won't test without docutils")
+    def test_check_restructuredtext_with_syntax_highlight(self):
+        # Don't fail if there is a `code` or `code-block` directive
+
+        example_rst_docs = []
+        example_rst_docs.append(textwrap.dedent("""\
+            Here's some code:
+
+            .. code:: python
+
+                def foo():
+                    pass
+            """))
+        example_rst_docs.append(textwrap.dedent("""\
+            Here's some code:
+
+            .. code-block:: python
+
+                def foo():
+                    pass
+            """))
+
+        for rest_with_code in example_rst_docs:
+            pkg_info, dist = self.create_dist(long_description=rest_with_code)
+            cmd = check(dist)
+            cmd.check_restructuredtext()
+            self.assertEqual(cmd._warnings, 0)
+            msgs = cmd._check_rst_data(rest_with_code)
+            self.assertEqual(len(msgs), 0)
+
     def test_check_all(self):
 
         metadata = {'url': 'xxx', 'author': 'xxx'}
diff --git a/lib-python/2.7/distutils/text_file.py b/lib-python/2.7/distutils/text_file.py
--- a/lib-python/2.7/distutils/text_file.py
+++ b/lib-python/2.7/distutils/text_file.py
@@ -124,11 +124,11 @@
     def close (self):
         """Close the current file and forget everything we know about it
            (filename, current line number)."""
-
-        self.file.close ()
+        file = self.file
         self.file = None
         self.filename = None
         self.current_line = None
+        file.close()
 
 
     def gen_error (self, msg, line=None):
diff --git a/lib-python/2.7/dumbdbm.py b/lib-python/2.7/dumbdbm.py
--- a/lib-python/2.7/dumbdbm.py
+++ b/lib-python/2.7/dumbdbm.py
@@ -21,6 +21,7 @@
 
 """
 
+import ast as _ast
 import os as _os
 import __builtin__
 import UserDict
@@ -85,7 +86,7 @@
             with f:
                 for line in f:
                     line = line.rstrip()
-                    key, pos_and_siz_pair = eval(line)
+                    key, pos_and_siz_pair = _ast.literal_eval(line)
                     self._index[key] = pos_and_siz_pair
 
     # Write the index dict to the directory file.  The original directory
@@ -208,8 +209,10 @@
         return len(self._index)
 
     def close(self):
-        self._commit()
-        self._index = self._datfile = self._dirfile = self._bakfile = None
+        try:
+            self._commit()
+        finally:
+            self._index = self._datfile = self._dirfile = self._bakfile = None
 
     __del__ = close
 
diff --git a/lib-python/2.7/encodings/uu_codec.py b/lib-python/2.7/encodings/uu_codec.py
--- a/lib-python/2.7/encodings/uu_codec.py
+++ b/lib-python/2.7/encodings/uu_codec.py
@@ -84,7 +84,7 @@
             data = a2b_uu(s)
         except binascii.Error, v:
             # Workaround for broken uuencoders by /Fredrik Lundh
-            nbytes = (((ord(s[0])-32) & 63) * 4 + 5) / 3
+            nbytes = (((ord(s[0])-32) & 63) * 4 + 5) // 3
             data = a2b_uu(s[:nbytes])
             #sys.stderr.write("Warning: %s\n" % str(v))
         write(data)
diff --git a/lib-python/2.7/ensurepip/__init__.py b/lib-python/2.7/ensurepip/__init__.py
--- a/lib-python/2.7/ensurepip/__init__.py
+++ b/lib-python/2.7/ensurepip/__init__.py
@@ -12,9 +12,9 @@
 __all__ = ["version", "bootstrap"]
 
 
-_SETUPTOOLS_VERSION = "7.0"
+_SETUPTOOLS_VERSION = "15.2"
 
-_PIP_VERSION = "1.5.6"
+_PIP_VERSION = "6.1.1"
 
 # pip currently requires ssl support, so we try to provide a nicer
 # error message when that is missing (http://bugs.python.org/issue19744)
diff --git a/lib-python/2.7/ensurepip/_bundled/pip-1.5.6-py2.py3-none-any.whl b/lib-python/2.7/ensurepip/_bundled/pip-1.5.6-py2.py3-none-any.whl
deleted file mode 100644
Binary file lib-python/2.7/ensurepip/_bundled/pip-1.5.6-py2.py3-none-any.whl has changed
diff --git a/lib-python/2.7/ensurepip/_bundled/pip-6.1.1-py2.py3-none-any.whl b/lib-python/2.7/ensurepip/_bundled/pip-6.1.1-py2.py3-none-any.whl
new file mode 100644
index 0000000000000000000000000000000000000000..e59694a019051d58b9a378a1adfc9461b8cec9c3
GIT binary patch

[cut]

diff --git a/lib-python/2.7/ensurepip/_bundled/setuptools-15.2-py2.py3-none-any.whl b/lib-python/2.7/ensurepip/_bundled/setuptools-15.2-py2.py3-none-any.whl
new file mode 100644
index 0000000000000000000000000000000000000000..f153ed376684275e08fcfebdb2de8352fb074171
GIT binary patch

[cut]

diff --git a/lib-python/2.7/ensurepip/_bundled/setuptools-7.0-py2.py3-none-any.whl b/lib-python/2.7/ensurepip/_bundled/setuptools-7.0-py2.py3-none-any.whl
deleted file mode 100644
Binary file lib-python/2.7/ensurepip/_bundled/setuptools-7.0-py2.py3-none-any.whl has changed
diff --git a/lib-python/2.7/fileinput.py b/lib-python/2.7/fileinput.py
--- a/lib-python/2.7/fileinput.py
+++ b/lib-python/2.7/fileinput.py
@@ -233,8 +233,10 @@
         self.close()
 
     def close(self):
-        self.nextfile()
-        self._files = ()
+        try:
+            self.nextfile()
+        finally:
+            self._files = ()
 
     def __iter__(self):
         return self
@@ -270,23 +272,25 @@
 
         output = self._output
         self._output = 0
-        if output:
-            output.close()
+        try:
+            if output:
+                output.close()
+        finally:
+            file = self._file
+            self._file = 0
+            try:
+                if file and not self._isstdin:
+                    file.close()
+            finally:
+                backupfilename = self._backupfilename
+                self._backupfilename = 0
+                if backupfilename and not self._backup:
+                    try: os.unlink(backupfilename)
+                    except OSError: pass
 
-        file = self._file
-        self._file = 0
-        if file and not self._isstdin:
-            file.close()
-
-        backupfilename = self._backupfilename
-        self._backupfilename = 0
-        if backupfilename and not self._backup:
-            try: os.unlink(backupfilename)
-            except OSError: pass
-
-        self._isstdin = False
-        self._buffer = []
-        self._bufindex = 0
+                self._isstdin = False
+                self._buffer = []
+                self._bufindex = 0
 
     def readline(self):
         try:
diff --git a/lib-python/2.7/fnmatch.py b/lib-python/2.7/fnmatch.py
--- a/lib-python/2.7/fnmatch.py
+++ b/lib-python/2.7/fnmatch.py
@@ -47,12 +47,14 @@
     import os,posixpath
     result=[]
     pat=os.path.normcase(pat)
-    if not pat in _cache:
+    try:
+        re_pat = _cache[pat]
+    except KeyError:
         res = translate(pat)
         if len(_cache) >= _MAXCACHE:
             _cache.clear()
-        _cache[pat] = re.compile(res)
-    match=_cache[pat].match
+        _cache[pat] = re_pat = re.compile(res)
+    match = re_pat.match
     if os.path is posixpath:
         # normcase on posix is NOP. Optimize it away from the loop.
         for name in names:
@@ -71,12 +73,14 @@
     its arguments.
     """
 
-    if not pat in _cache:
+    try:
+        re_pat = _cache[pat]
+    except KeyError:
         res = translate(pat)
         if len(_cache) >= _MAXCACHE:
             _cache.clear()
-        _cache[pat] = re.compile(res)
-    return _cache[pat].match(name) is not None
+        _cache[pat] = re_pat = re.compile(res)
+    return re_pat.match(name) is not None
 
 def translate(pat):
     """Translate a shell PATTERN to a regular expression.
diff --git a/lib-python/2.7/ftplib.py b/lib-python/2.7/ftplib.py
--- a/lib-python/2.7/ftplib.py
+++ b/lib-python/2.7/ftplib.py
@@ -594,11 +594,16 @@
 
     def close(self):
         '''Close the connection without assuming anything about it.'''
-        if self.file is not None:
-            self.file.close()
-        if self.sock is not None:
-            self.sock.close()
-        self.file = self.sock = None
+        try:
+            file = self.file
+            self.file = None
+            if file is not None:
+                file.close()
+        finally:
+            sock = self.sock
+            self.sock = None
+            if sock is not None:
+                sock.close()
 
 try:
     import ssl
@@ -638,12 +643,24 @@
         '221 Goodbye.'
         >>>
         '''
-        ssl_version = ssl.PROTOCOL_TLSv1
+        ssl_version = ssl.PROTOCOL_SSLv23
 
         def __init__(self, host='', user='', passwd='', acct='', keyfile=None,
-                     certfile=None, timeout=_GLOBAL_DEFAULT_TIMEOUT):
+                     certfile=None, context=None,
+                     timeout=_GLOBAL_DEFAULT_TIMEOUT, source_address=None):
+            if context is not None and keyfile is not None:
+                raise ValueError("context and keyfile arguments are mutually "
+                                 "exclusive")
+            if context is not None and certfile is not None:
+                raise ValueError("context and certfile arguments are mutually "
+                                 "exclusive")
             self.keyfile = keyfile
             self.certfile = certfile
+            if context is None:
+                context = ssl._create_stdlib_context(self.ssl_version,
+                                                     certfile=certfile,
+                                                     keyfile=keyfile)
+            self.context = context
             self._prot_p = False
             FTP.__init__(self, host, user, passwd, acct, timeout)
 
@@ -656,12 +673,12 @@
             '''Set up secure control connection by using TLS/SSL.'''
             if isinstance(self.sock, ssl.SSLSocket):
                 raise ValueError("Already using TLS")
-            if self.ssl_version == ssl.PROTOCOL_TLSv1:
+            if self.ssl_version >= ssl.PROTOCOL_SSLv23:
                 resp = self.voidcmd('AUTH TLS')
             else:
                 resp = self.voidcmd('AUTH SSL')
-            self.sock = ssl.wrap_socket(self.sock, self.keyfile, self.certfile,
-                                        ssl_version=self.ssl_version)
+            self.sock = self.context.wrap_socket(self.sock,
+                                                 server_hostname=self.host)
             self.file = self.sock.makefile(mode='rb')
             return resp
 
@@ -692,8 +709,8 @@
         def ntransfercmd(self, cmd, rest=None):
             conn, size = FTP.ntransfercmd(self, cmd, rest)
             if self._prot_p:
-                conn = ssl.wrap_socket(conn, self.keyfile, self.certfile,
-                                       ssl_version=self.ssl_version)
+                conn = self.context.wrap_socket(conn,
+                                                server_hostname=self.host)
             return conn, size
 
         def retrbinary(self, cmd, callback, blocksize=8192, rest=None):
diff --git a/lib-python/2.7/genericpath.py b/lib-python/2.7/genericpath.py
--- a/lib-python/2.7/genericpath.py
+++ b/lib-python/2.7/genericpath.py
@@ -10,6 +10,14 @@
            'getsize', 'isdir', 'isfile']
 
 
+try:
+    _unicode = unicode
+except NameError:
+    # If Python is built without Unicode support, the unicode type
+    # will not exist. Fake one.
+    class _unicode(object):
+        pass
+
 # Does a path exist?
 # This is false for dangling symbolic links on systems that support them.
 def exists(path):
diff --git a/lib-python/2.7/gettext.py b/lib-python/2.7/gettext.py
--- a/lib-python/2.7/gettext.py
+++ b/lib-python/2.7/gettext.py
@@ -52,7 +52,9 @@
 
 __all__ = ['NullTranslations', 'GNUTranslations', 'Catalog',
            'find', 'translation', 'install', 'textdomain', 'bindtextdomain',
-           'dgettext', 'dngettext', 'gettext', 'ngettext',
+           'bind_textdomain_codeset',
+           'dgettext', 'dngettext', 'gettext', 'lgettext', 'ldgettext',
+           'ldngettext', 'lngettext', 'ngettext',
            ]
 
 _default_localedir = os.path.join(sys.prefix, 'share', 'locale')
@@ -294,11 +296,12 @@
             # See if we're looking at GNU .mo conventions for metadata
             if mlen == 0:
                 # Catalog description
-                lastk = k = None
+                lastk = None
                 for item in tmsg.splitlines():
                     item = item.strip()
                     if not item:
                         continue
+                    k = v = None
                     if ':' in item:
                         k, v = item.split(':', 1)
                         k = k.strip().lower()
diff --git a/lib-python/2.7/gzip.py b/lib-python/2.7/gzip.py
--- a/lib-python/2.7/gzip.py
+++ b/lib-python/2.7/gzip.py
@@ -238,9 +238,9 @@
             data = data.tobytes()
 
         if len(data) > 0:
-            self.size = self.size + len(data)
+            self.fileobj.write(self.compress.compress(data))
+            self.size += len(data)
             self.crc = zlib.crc32(data, self.crc) & 0xffffffffL
-            self.fileobj.write( self.compress.compress(data) )
             self.offset += len(data)
 
         return len(data)
@@ -369,19 +369,21 @@
         return self.fileobj is None
 
     def close(self):
-        if self.fileobj is None:
+        fileobj = self.fileobj
+        if fileobj is None:
             return
-        if self.mode == WRITE:
-            self.fileobj.write(self.compress.flush())
-            write32u(self.fileobj, self.crc)
-            # self.size may exceed 2GB, or even 4GB
-            write32u(self.fileobj, self.size & 0xffffffffL)
-            self.fileobj = None
-        elif self.mode == READ:
-            self.fileobj = None
-        if self.myfileobj:
-            self.myfileobj.close()
-            self.myfileobj = None
+        self.fileobj = None
+        try:
+            if self.mode == WRITE:
+                fileobj.write(self.compress.flush())
+                write32u(fileobj, self.crc)
+                # self.size may exceed 2GB, or even 4GB
+                write32u(fileobj, self.size & 0xffffffffL)
+        finally:
+            myfileobj = self.myfileobj
+            if myfileobj:
+                self.myfileobj = None
+                myfileobj.close()
 
     def flush(self,zlib_mode=zlib.Z_SYNC_FLUSH):
         self._check_closed()
diff --git a/lib-python/2.7/hashlib.py b/lib-python/2.7/hashlib.py
--- a/lib-python/2.7/hashlib.py
+++ b/lib-python/2.7/hashlib.py
@@ -187,7 +187,7 @@
 
         def prf(msg, inner=inner, outer=outer):
             # PBKDF2_HMAC uses the password as key. We can re-use the same
-            # digest objects and and just update copies to skip initialization.
+            # digest objects and just update copies to skip initialization.
             icpy = inner.copy()
             ocpy = outer.copy()
             icpy.update(msg)
diff --git a/lib-python/2.7/htmlentitydefs.py b/lib-python/2.7/htmlentitydefs.py
--- a/lib-python/2.7/htmlentitydefs.py
+++ b/lib-python/2.7/htmlentitydefs.py
@@ -1,6 +1,6 @@
 """HTML character entity references."""
 
-# maps the HTML entity name to the Unicode codepoint
+# maps the HTML entity name to the Unicode code point
 name2codepoint = {
     'AElig':    0x00c6, # latin capital letter AE = latin capital ligature AE, U+00C6 ISOlat1
     'Aacute':   0x00c1, # latin capital letter A with acute, U+00C1 ISOlat1
@@ -256,7 +256,7 @@
     'zwnj':     0x200c, # zero width non-joiner, U+200C NEW RFC 2070
 }
 
-# maps the Unicode codepoint to the HTML entity name
+# maps the Unicode code point to the HTML entity name
 codepoint2name = {}
 
 # maps the HTML entity name to the character
diff --git a/lib-python/2.7/httplib.py b/lib-python/2.7/httplib.py
--- a/lib-python/2.7/httplib.py
+++ b/lib-python/2.7/httplib.py
@@ -68,6 +68,7 @@
 
 from array import array
 import os
+import re
 import socket
 from sys import py3kwarning
 from urlparse import urlsplit
@@ -218,6 +219,38 @@
 # maximum amount of headers accepted
 _MAXHEADERS = 100
 
+# Header name/value ABNF (http://tools.ietf.org/html/rfc7230#section-3.2)
+#
+# VCHAR          = %x21-7E
+# obs-text       = %x80-FF
+# header-field   = field-name ":" OWS field-value OWS
+# field-name     = token
+# field-value    = *( field-content / obs-fold )
+# field-content  = field-vchar [ 1*( SP / HTAB ) field-vchar ]
+# field-vchar    = VCHAR / obs-text
+#
+# obs-fold       = CRLF 1*( SP / HTAB )
+#                ; obsolete line folding
+#                ; see Section 3.2.4
+
+# token          = 1*tchar
+#
+# tchar          = "!" / "#" / "$" / "%" / "&" / "'" / "*"
+#                / "+" / "-" / "." / "^" / "_" / "`" / "|" / "~"
+#                / DIGIT / ALPHA
+#                ; any VCHAR, except delimiters
+#
+# VCHAR defined in http://tools.ietf.org/html/rfc5234#appendix-B.1
+
+# the patterns for both name and value are more leniant than RFC
+# definitions to allow for backwards compatibility
+_is_legal_header_name = re.compile(r'\A[^:\s][^:\r\n]*\Z').match
+_is_illegal_header_value = re.compile(r'\n(?![ \t])|\r(?![ \t\n])').search
+
+# We always set the Content-Length header for these methods because some
+# servers will otherwise respond with a 411
+_METHODS_EXPECTING_BODY = {'PATCH', 'POST', 'PUT'}
+
 
 class HTTPMessage(mimetools.Message):
 
@@ -313,6 +346,11 @@
                 hlist.append(line)
                 self.addheader(headerseen, line[len(headerseen)+1:].strip())
                 continue
+            elif headerseen is not None:
+                # An empty header name. These aren't allowed in HTTP, but it's
+                # probably a benign mistake. Don't add the header, just keep
+                # going.
+                continue
             else:
                 # It's not a header line; throw it back and stop here.
                 if not self.dict:
@@ -522,9 +560,10 @@
         return True
 
     def close(self):
-        if self.fp:
-            self.fp.close()
+        fp = self.fp
+        if fp:
             self.fp = None
+            fp.close()
 
     def isclosed(self):
         # NOTE: it is possible that we will not ever call self.close(). This
@@ -723,7 +762,7 @@
         endpoint passed to set_tunnel. This is done by sending a HTTP CONNECT
         request to the proxy server when the connection is established.
 
-        This method must be called before the HTML connection has been
+        This method must be called before the HTTP connection has been
         established.
 
         The headers argument should be a mapping of extra HTTP headers
@@ -797,13 +836,17 @@
 
     def close(self):
         """Close the connection to the HTTP server."""
-        if self.sock:
-            self.sock.close()   # close it manually... there may be other refs
-            self.sock = None
-        if self.__response:
-            self.__response.close()
-            self.__response = None
         self.__state = _CS_IDLE
+        try:
+            sock = self.sock
+            if sock:
+                self.sock = None
+                sock.close()   # close it manually... there may be other refs
+        finally:
+            response = self.__response
+            if response:
+                self.__response = None
+                response.close()
 
     def send(self, data):
         """Send `data' to the server."""
@@ -978,7 +1021,16 @@
         if self.__state != _CS_REQ_STARTED:
             raise CannotSendHeader()
 
-        hdr = '%s: %s' % (header, '\r\n\t'.join([str(v) for v in values]))
+        header = '%s' % header
+        if not _is_legal_header_name(header):
+            raise ValueError('Invalid header name %r' % (header,))
+
+        values = [str(v) for v in values]
+        for one_value in values:
+            if _is_illegal_header_value(one_value):
+                raise ValueError('Invalid header value %r' % (one_value,))
+
+        hdr = '%s: %s' % (header, '\r\n\t'.join(values))
         self._output(hdr)
 
     def endheaders(self, message_body=None):
@@ -1000,19 +1052,25 @@
         """Send a complete request to the server."""
         self._send_request(method, url, body, headers)
 
-    def _set_content_length(self, body):
-        # Set the content-length based on the body.
+    def _set_content_length(self, body, method):
+        # Set the content-length based on the body. If the body is "empty", we
+        # set Content-Length: 0 for methods that expect a body (RFC 7230,
+        # Section 3.3.2). If the body is set for other methods, we set the
+        # header provided we can figure out what the length is.
         thelen = None
-        try:
-            thelen = str(len(body))
-        except TypeError, te:
-            # If this is a file-like object, try to
-            # fstat its file descriptor
+        if body is None and method.upper() in _METHODS_EXPECTING_BODY:
+            thelen = '0'
+        elif body is not None:
             try:
-                thelen = str(os.fstat(body.fileno()).st_size)
-            except (AttributeError, OSError):
-                # Don't send a length if this failed
-                if self.debuglevel > 0: print "Cannot stat!!"
+                thelen = str(len(body))
+            except TypeError:
+                # If this is a file-like object, try to
+                # fstat its file descriptor
+                try:
+                    thelen = str(os.fstat(body.fileno()).st_size)
+                except (AttributeError, OSError):
+                    # Don't send a length if this failed
+                    if self.debuglevel > 0: print "Cannot stat!!"
 
         if thelen is not None:
             self.putheader('Content-Length', thelen)
@@ -1028,8 +1086,8 @@
 
         self.putrequest(method, url, **skips)
 
-        if body is not None and 'content-length' not in header_names:
-            self._set_content_length(body)
+        if 'content-length' not in header_names:
+            self._set_content_length(body, method)
         for hdr, value in headers.iteritems():
             self.putheader(hdr, value)
         self.endheaders(body)
@@ -1072,20 +1130,20 @@
 
         try:
             response.begin()
+            assert response.will_close != _UNKNOWN
+            self.__state = _CS_IDLE
+
+            if response.will_close:
+                # this effectively passes the connection to the response
+                self.close()
+            else:
+                # remember this, so we can tell when it is complete
+                self.__response = response
+
+            return response
         except:
             response.close()
             raise
-        assert response.will_close != _UNKNOWN
-        self.__state = _CS_IDLE
-
-        if response.will_close:
-            # this effectively passes the connection to the response
-            self.close()
-        else:
-            # remember this, so we can tell when it is complete
-            self.__response = response
-
-        return response
 
 
 class HTTP:
@@ -1129,7 +1187,7 @@
         "Accept arguments to set the host/port, since the superclass doesn't."
 
         if host is not None:
-            self._conn._set_hostport(host, port)
+            (self._conn.host, self._conn.port) = self._conn._get_hostport(host, port)
         self._conn.connect()
 
     def getfile(self):
diff --git a/lib-python/2.7/idlelib/CodeContext.py b/lib-python/2.7/idlelib/CodeContext.py
--- a/lib-python/2.7/idlelib/CodeContext.py
+++ b/lib-python/2.7/idlelib/CodeContext.py
@@ -15,8 +15,8 @@
 from sys import maxint as INFINITY
 from idlelib.configHandler import idleConf
 
-BLOCKOPENERS = set(["class", "def", "elif", "else", "except", "finally", "for",
-                    "if", "try", "while", "with"])
+BLOCKOPENERS = {"class", "def", "elif", "else", "except", "finally", "for",
+                    "if", "try", "while", "with"}
 UPDATEINTERVAL = 100 # millisec
 FONTUPDATEINTERVAL = 1000 # millisec
 
diff --git a/lib-python/2.7/idlelib/EditorWindow.py b/lib-python/2.7/idlelib/EditorWindow.py
--- a/lib-python/2.7/idlelib/EditorWindow.py
+++ b/lib-python/2.7/idlelib/EditorWindow.py
@@ -469,13 +469,10 @@
         ("format", "F_ormat"),
         ("run", "_Run"),
         ("options", "_Options"),
-        ("windows", "_Windows"),
+        ("windows", "_Window"),
         ("help", "_Help"),
     ]
 
-    if sys.platform == "darwin":
-        menu_specs[-2] = ("windows", "_Window")
-
 
     def createmenubar(self):
         mbar = self.menubar
diff --git a/lib-python/2.7/idlelib/FormatParagraph.py b/lib-python/2.7/idlelib/FormatParagraph.py
--- a/lib-python/2.7/idlelib/FormatParagraph.py
+++ b/lib-python/2.7/idlelib/FormatParagraph.py
@@ -44,9 +44,11 @@
 
         The length limit parameter is for testing with a known value.
         """
-        if limit == None:
+        if limit is None:
+            # The default length limit is that defined by pep8
             limit = idleConf.GetOption(
-                    'main', 'FormatParagraph', 'paragraph', type='int')
+                'extensions', 'FormatParagraph', 'max-width',
+                type='int', default=72)
         text = self.editwin.text
         first, last = self.editwin.get_selection_indices()
         if first and last:
diff --git a/lib-python/2.7/idlelib/PyShell.py b/lib-python/2.7/idlelib/PyShell.py
--- a/lib-python/2.7/idlelib/PyShell.py
+++ b/lib-python/2.7/idlelib/PyShell.py
@@ -871,13 +871,10 @@
         ("edit", "_Edit"),
         ("debug", "_Debug"),
         ("options", "_Options"),
-        ("windows", "_Windows"),
+        ("windows", "_Window"),
         ("help", "_Help"),
     ]
 
-    if sys.platform == "darwin":
-        menu_specs[-2] = ("windows", "_Window")
-
 
     # New classes
     from idlelib.IdleHistory import History
@@ -1350,7 +1347,7 @@
         if type(s) not in (unicode, str, bytearray):
             # See issue #19481
             if isinstance(s, unicode):
-                s = unicode.__getslice__(s, None, None)
+                s = unicode.__getitem__(s, slice(None))
             elif isinstance(s, str):
                 s = str.__str__(s)
             elif isinstance(s, bytearray):
diff --git a/lib-python/2.7/idlelib/SearchEngine.py b/lib-python/2.7/idlelib/SearchEngine.py
--- a/lib-python/2.7/idlelib/SearchEngine.py
+++ b/lib-python/2.7/idlelib/SearchEngine.py
@@ -191,7 +191,7 @@
 
     This is done by searching forwards until there is no match.
     Prog: compiled re object with a search method returning a match.
-    Chars: line of text, without \n.
+    Chars: line of text, without \\n.
     Col: stop index for the search; the limit for match.end().
     '''
     m = prog.search(chars)
diff --git a/lib-python/2.7/idlelib/config-extensions.def b/lib-python/2.7/idlelib/config-extensions.def
--- a/lib-python/2.7/idlelib/config-extensions.def
+++ b/lib-python/2.7/idlelib/config-extensions.def
@@ -66,6 +66,7 @@
 
 [FormatParagraph]
 enable=True
+max-width=72
 [FormatParagraph_cfgBindings]
 format-paragraph=<Alt-Key-q>
 
diff --git a/lib-python/2.7/idlelib/config-main.def b/lib-python/2.7/idlelib/config-main.def
--- a/lib-python/2.7/idlelib/config-main.def
+++ b/lib-python/2.7/idlelib/config-main.def
@@ -58,9 +58,6 @@
 font-bold= 0
 encoding= none
 
-[FormatParagraph]
-paragraph=72
-
 [Indent]
 use-spaces= 1
 num-spaces= 4
diff --git a/lib-python/2.7/idlelib/configDialog.py b/lib-python/2.7/idlelib/configDialog.py
--- a/lib-python/2.7/idlelib/configDialog.py
+++ b/lib-python/2.7/idlelib/configDialog.py
@@ -371,7 +371,6 @@
         parent = self.parent
         self.winWidth = StringVar(parent)
         self.winHeight = StringVar(parent)
-        self.paraWidth = StringVar(parent)
         self.startupEdit = IntVar(parent)
         self.autoSave = IntVar(parent)
         self.encoding = StringVar(parent)
@@ -387,7 +386,6 @@
         frameSave = LabelFrame(frame, borderwidth=2, relief=GROOVE,
                                text=' Autosave Preferences ')
         frameWinSize = Frame(frame, borderwidth=2, relief=GROOVE)
-        frameParaSize = Frame(frame, borderwidth=2, relief=GROOVE)
         frameEncoding = Frame(frame, borderwidth=2, relief=GROOVE)
         frameHelp = LabelFrame(frame, borderwidth=2, relief=GROOVE,
                                text=' Additional Help Sources ')
@@ -416,11 +414,6 @@
         labelWinHeightTitle = Label(frameWinSize, text='Height')
         entryWinHeight = Entry(
                 frameWinSize, textvariable=self.winHeight, width=3)
-        #paragraphFormatWidth
-        labelParaWidthTitle = Label(
-                frameParaSize, text='Paragraph reformat width (in characters)')
-        entryParaWidth = Entry(
-                frameParaSize, textvariable=self.paraWidth, width=3)
         #frameEncoding
         labelEncodingTitle = Label(
                 frameEncoding, text="Default Source Encoding")
@@ -458,7 +451,6 @@
         frameRun.pack(side=TOP, padx=5, pady=5, fill=X)
         frameSave.pack(side=TOP, padx=5, pady=5, fill=X)
         frameWinSize.pack(side=TOP, padx=5, pady=5, fill=X)
-        frameParaSize.pack(side=TOP, padx=5, pady=5, fill=X)
         frameEncoding.pack(side=TOP, padx=5, pady=5, fill=X)
         frameHelp.pack(side=TOP, padx=5, pady=5, expand=TRUE, fill=BOTH)
         #frameRun
@@ -475,9 +467,6 @@
         labelWinHeightTitle.pack(side=RIGHT, anchor=E, pady=5)
         entryWinWidth.pack(side=RIGHT, anchor=E, padx=10, pady=5)
         labelWinWidthTitle.pack(side=RIGHT, anchor=E, pady=5)
-        #paragraphFormatWidth
-        labelParaWidthTitle.pack(side=LEFT, anchor=W, padx=5, pady=5)
-        entryParaWidth.pack(side=RIGHT, anchor=E, padx=10, pady=5)
         #frameEncoding
         labelEncodingTitle.pack(side=LEFT, anchor=W, padx=5, pady=5)
         radioEncNone.pack(side=RIGHT, anchor=E, pady=5)
@@ -509,7 +498,6 @@
         self.keysAreBuiltin.trace_variable('w', self.VarChanged_keysAreBuiltin)
         self.winWidth.trace_variable('w', self.VarChanged_winWidth)
         self.winHeight.trace_variable('w', self.VarChanged_winHeight)
-        self.paraWidth.trace_variable('w', self.VarChanged_paraWidth)
         self.startupEdit.trace_variable('w', self.VarChanged_startupEdit)
         self.autoSave.trace_variable('w', self.VarChanged_autoSave)
         self.encoding.trace_variable('w', self.VarChanged_encoding)
@@ -594,10 +582,6 @@
         value = self.winHeight.get()
         self.AddChangedItem('main', 'EditorWindow', 'height', value)
 
-    def VarChanged_paraWidth(self, *params):
-        value = self.paraWidth.get()
-        self.AddChangedItem('main', 'FormatParagraph', 'paragraph', value)
-
     def VarChanged_startupEdit(self, *params):
         value = self.startupEdit.get()
         self.AddChangedItem('main', 'General', 'editor-on-startup', value)
@@ -1094,9 +1078,6 @@
                 'main', 'EditorWindow', 'width', type='int'))
         self.winHeight.set(idleConf.GetOption(
                 'main', 'EditorWindow', 'height', type='int'))
-        #initial paragraph reformat size
-        self.paraWidth.set(idleConf.GetOption(
-                'main', 'FormatParagraph', 'paragraph', type='int'))
         # default source encoding
         self.encoding.set(idleConf.GetOption(
                 'main', 'EditorWindow', 'encoding', default='none'))
diff --git a/lib-python/2.7/idlelib/help.txt b/lib-python/2.7/idlelib/help.txt
--- a/lib-python/2.7/idlelib/help.txt
+++ b/lib-python/2.7/idlelib/help.txt
@@ -100,7 +100,7 @@
 			  which is scrolling off the top or the window.
 			  (Not present in Shell window.)
 
-Windows Menu:
+Window Menu:
 
 	Zoom Height -- toggles the window between configured size
 	and maximum height.
diff --git a/lib-python/2.7/idlelib/idle.bat b/lib-python/2.7/idlelib/idle.bat
--- a/lib-python/2.7/idlelib/idle.bat
+++ b/lib-python/2.7/idlelib/idle.bat
@@ -1,4 +1,4 @@
- at echo off
-rem Start IDLE using the appropriate Python interpreter
-set CURRDIR=%~dp0
-start "IDLE" "%CURRDIR%..\..\pythonw.exe" "%CURRDIR%idle.pyw" %1 %2 %3 %4 %5 %6 %7 %8 %9
+ at echo off
+rem Start IDLE using the appropriate Python interpreter
+set CURRDIR=%~dp0
+start "IDLE" "%CURRDIR%..\..\pythonw.exe" "%CURRDIR%idle.pyw" %1 %2 %3 %4 %5 %6 %7 %8 %9
diff --git a/lib-python/2.7/idlelib/idle_test/test_calltips.py b/lib-python/2.7/idlelib/idle_test/test_calltips.py
--- a/lib-python/2.7/idlelib/idle_test/test_calltips.py
+++ b/lib-python/2.7/idlelib/idle_test/test_calltips.py
@@ -55,7 +55,8 @@
         def gtest(obj, out):
             self.assertEqual(signature(obj), out)
 
-        gtest(List, '()\n' + List.__doc__)
+        if List.__doc__ is not None:
+            gtest(List, '()\n' + List.__doc__)
         gtest(list.__new__,
                'T.__new__(S, ...) -> a new object with type S, a subtype of T')
         gtest(list.__init__,
@@ -70,7 +71,8 @@
 
     def test_signature_wrap(self):
         # This is also a test of an old-style class
-        self.assertEqual(signature(textwrap.TextWrapper), '''\
+        if textwrap.TextWrapper.__doc__ is not None:
+            self.assertEqual(signature(textwrap.TextWrapper), '''\
 (width=70, initial_indent='', subsequent_indent='', expand_tabs=True,
     replace_whitespace=True, fix_sentence_endings=False, break_long_words=True,
     drop_whitespace=True, break_on_hyphens=True)''')
@@ -106,20 +108,23 @@
         def t5(a, b=None, *args, **kwds): 'doc'
         t5.tip = "(a, b=None, *args, **kwargs)"
 
+        doc = '\ndoc' if t1.__doc__ is not None else ''
         for func in (t1, t2, t3, t4, t5, TC):
-            self.assertEqual(signature(func), func.tip + '\ndoc')
+            self.assertEqual(signature(func), func.tip + doc)
 
     def test_methods(self):
+        doc = '\ndoc' if TC.__doc__ is not None else ''
         for meth in (TC.t1, TC.t2, TC.t3, TC.t4, TC.t5, TC.t6, TC.__call__):
-            self.assertEqual(signature(meth), meth.tip + "\ndoc")
-        self.assertEqual(signature(TC.cm), "(a)\ndoc")
-        self.assertEqual(signature(TC.sm), "(b)\ndoc")
+            self.assertEqual(signature(meth), meth.tip + doc)
+        self.assertEqual(signature(TC.cm), "(a)" + doc)
+        self.assertEqual(signature(TC.sm), "(b)" + doc)
 
     def test_bound_methods(self):
         # test that first parameter is correctly removed from argspec
+        doc = '\ndoc' if TC.__doc__ is not None else ''
         for meth, mtip  in ((tc.t1, "()"), (tc.t4, "(*args)"), (tc.t6, "(self)"),
                             (tc.__call__, '(ci)'), (tc, '(ci)'), (TC.cm, "(a)"),):
-            self.assertEqual(signature(meth), mtip + "\ndoc")
+            self.assertEqual(signature(meth), mtip + doc)
 
     def test_starred_parameter(self):
         # test that starred first parameter is *not* removed from argspec
diff --git a/lib-python/2.7/idlelib/idle_test/test_io.py b/lib-python/2.7/idlelib/idle_test/test_io.py
new file mode 100644
--- /dev/null
+++ b/lib-python/2.7/idlelib/idle_test/test_io.py
@@ -0,0 +1,267 @@
+import unittest
+import io
+from idlelib.PyShell import PseudoInputFile, PseudoOutputFile
+from test import test_support as support
+
+
+class Base(object):
+    def __str__(self):
+        return '%s:str' % type(self).__name__
+    def __unicode__(self):
+        return '%s:unicode' % type(self).__name__
+    def __len__(self):
+        return 3
+    def __iter__(self):
+        return iter('abc')
+    def __getitem__(self, *args):
+        return '%s:item' % type(self).__name__
+    def __getslice__(self, *args):
+        return '%s:slice' % type(self).__name__
+
+class S(Base, str):
+    pass
+
+class U(Base, unicode):
+    pass
+
+class BA(Base, bytearray):
+    pass
+
+class MockShell:
+    def __init__(self):
+        self.reset()
+
+    def write(self, *args):
+        self.written.append(args)
+
+    def readline(self):
+        return self.lines.pop()
+
+    def close(self):
+        pass
+
+    def reset(self):
+        self.written = []
+
+    def push(self, lines):
+        self.lines = list(lines)[::-1]
+
+
+class PseudeOutputFilesTest(unittest.TestCase):
+    def test_misc(self):
+        shell = MockShell()
+        f = PseudoOutputFile(shell, 'stdout', 'utf-8')
+        self.assertIsInstance(f, io.TextIOBase)
+        self.assertEqual(f.encoding, 'utf-8')
+        self.assertIsNone(f.errors)
+        self.assertIsNone(f.newlines)
+        self.assertEqual(f.name, '<stdout>')
+        self.assertFalse(f.closed)
+        self.assertTrue(f.isatty())
+        self.assertFalse(f.readable())
+        self.assertTrue(f.writable())
+        self.assertFalse(f.seekable())
+
+    def test_unsupported(self):
+        shell = MockShell()
+        f = PseudoOutputFile(shell, 'stdout', 'utf-8')

From noreply at buildbot.pypy.org  Mon Jun 22 18:01:26 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 22 Jun 2015 18:01:26 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c8: hg merge release-2.6.x
Message-ID: <20150622160126.5BA2F1C1FAC@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c8
Changeset: r78244:bd31c1ba8c8b
Date: 2015-06-22 15:45 +0200
http://bitbucket.org/pypy/pypy/changeset/bd31c1ba8c8b/

Log:	hg merge release-2.6.x

diff too long, truncating to 2000 out of 54702 lines

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -3,16 +3,15 @@
 d8ac7d23d3ec5f9a0fa1264972f74a010dbfd07f release-1.6
 ff4af8f318821f7f5ca998613a60fca09aa137da release-1.7
 07e08e9c885ca67d89bcc304e45a32346daea2fa release-2.0-beta-1
-9b623bc48b5950cf07184462a0e48f2c4df0d720 pypy-2.1-beta1-arm
-9b623bc48b5950cf07184462a0e48f2c4df0d720 pypy-2.1-beta1-arm
 ab0dd631c22015ed88e583d9fdd4c43eebf0be21 pypy-2.1-beta1-arm
 20e51c4389ed4469b66bb9d6289ce0ecfc82c4b9 release-2.3.0
 394146e9bb673514c61f0150ab2013ccf78e8de7 release-2.3
 32f35069a16d819b58c1b6efb17c44e3e53397b2 release-2.2=3.1
 32f35069a16d819b58c1b6efb17c44e3e53397b2 release-2.3.1
 10f1b29a2bd21f837090286174a9ca030b8680b2 release-2.5.0
-8e24dac0b8e2db30d46d59f2c4daa3d4aaab7861 release-2.5.1
-8e24dac0b8e2db30d46d59f2c4daa3d4aaab7861 release-2.5.1
-0000000000000000000000000000000000000000 release-2.5.1
-0000000000000000000000000000000000000000 release-2.5.1
-e3d046c43451403f5969580fc1c41d5df6c4082a release-2.5.1
+9c4588d731b7fe0b08669bd732c2b676cb0a8233 release-2.5.1
+fcdb941565156385cbac04cfb891f8f4c7a92ef6 release-2.6.0
+fcdb941565156385cbac04cfb891f8f4c7a92ef6 release-2.6.0
+e03971291f3a0729ecd3ee7fae7ddb0bb82d476c release-2.6.0
+e03971291f3a0729ecd3ee7fae7ddb0bb82d476c release-2.6.0
+295ee98b69288471b0fcf2e0ede82ce5209eb90b release-2.6.0
diff --git a/.tddium.requirements.txt b/.tddium.requirements.txt
deleted file mode 100644
--- a/.tddium.requirements.txt
+++ /dev/null
@@ -1,1 +0,0 @@
-pytest
diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -38,8 +38,8 @@
   Armin Rigo
   Maciej Fijalkowski
   Carl Friedrich Bolz
+  Amaury Forgeot d'Arc
   Antonio Cuni
-  Amaury Forgeot d'Arc
   Samuele Pedroni
   Alex Gaynor
   Brian Kearns
@@ -50,9 +50,9 @@
   Holger Krekel
   Christian Tismer
   Hakan Ardo
-  Benjamin Peterson
   Manuel Jacob
   Ronan Lamy
+  Benjamin Peterson
   Anders Chrigstrom
   Eric van Riet Paap
   Wim Lavrijsen
@@ -63,8 +63,8 @@
   Sven Hager
   Anders Lehmann
   Aurelien Campeas
+  Remi Meier
   Niklaus Haldimann
-  Remi Meier
   Camillo Bruni
   Laura Creighton
   Toon Verwaest
@@ -76,10 +76,10 @@
   David Edelsohn
   Anders Hammarquist
   Jakub Gustak
+  Gregor Wegberg
   Guido Wesdorp
   Lawrence Oluyede
   Bartosz Skowron
-  Gregor Wegberg
   Daniel Roberts
   Niko Matsakis
   Adrien Di Mascio
@@ -87,10 +87,11 @@
   Ludovic Aubry
   Jacob Hallen
   Jason Creighton
+  Richard Plangger
   Alex Martelli
   Michal Bendowski
+  stian
   Jan de Mooij
-  stian
   Tyler Wade
   Michael Foord
   Stephan Diehl
@@ -133,15 +134,15 @@
   Georg Brandl
   Bert Freudenberg
   Stian Andreassen
+  Edd Barrett
   Wanja Saatkamp
   Gerald Klix
   Mike Blume
+  Tobias Pape
   Oscar Nierstrasz
   Stefan H. Muller
-  Edd Barrett
   Jeremy Thurgood
   Rami Chowdhury
-  Tobias Pape
   Eugene Oden
   Henry Mason
   Vasily Kuznetsov
@@ -167,11 +168,13 @@
   Michael Twomey
   Lucian Branescu Mihaila
   Yichao Yu
+  Anton Gulenko
   Gabriel Lavoie
   Olivier Dormond
   Jared Grubb
   Karl Bartel
   Wouter van Heyst
+  Sebastian Pawlu&#347;
   Brian Dorsey
   Victor Stinner
   Andrews Medina
@@ -188,6 +191,7 @@
   Neil Shepperd
   Stanislaw Halik
   Mikael Sch&#246;nenberg
+  Berkin Ilbeyi
   Elmo M?ntynen
   Jonathan David Riehl
   Anders Qvist
@@ -211,11 +215,11 @@
   Carl Meyer
   Karl Ramm
   Pieter Zieschang
-  Sebastian Pawlu&#347;
   Gabriel
   Lukas Vacek
   Andrew Dalke
   Sylvain Thenault
+  Jakub Stasiak
   Nathan Taylor
   Vladimir Kryachko
   Jacek Generowicz
@@ -242,6 +246,7 @@
   Tomo Cocoa
   Toni Mattis
   Lucas Stadler
+  Julian Berman
   roberto at goyle
   Yury V. Zaytsev
   Anna Katrina Dominguez
@@ -253,6 +258,8 @@
   Ben Darnell
   Roberto De Ioris
   Juan Francisco Cantero Hurtado
+  Ruochen Huang
+  Jeong YunWon
   Godefroid Chappelle
   Joshua Gilbert
   Dan Colish
@@ -271,6 +278,7 @@
   Christian Muirhead
   Berker Peksag
   James Lan
+  Volodymyr Vladymyrov
   shoma hosaka
   Daniel Neuh&#228;user
   Ben Mather
@@ -316,6 +324,7 @@
   yasirs
   Michael Chermside
   Anna Ravencroft
+  Andrey Churin
   Dan Crosta
   Julien Phalip
   Roman Podoliaka
@@ -420,3 +429,10 @@
 the terms of the GPL license version 2 or any later version.  Thus the
 gdbm module, provided in the file lib_pypy/gdbm.py, is redistributed
 under the terms of the GPL license as well.
+
+License for 'pypy/module/_vmprof/src'
+--------------------------------------
+
+The code is based on gperftools. You may see a copy of the License for it at
+
+    https://code.google.com/p/gperftools/source/browse/COPYING
diff --git a/lib-python/2.7/socket.py b/lib-python/2.7/socket.py
--- a/lib-python/2.7/socket.py
+++ b/lib-python/2.7/socket.py
@@ -145,6 +145,34 @@
             name = hostname
     return name
 
+class RefCountingWarning(UserWarning):
+    pass
+
+def _do_reuse_or_drop(socket, methname):
+    try:
+        method = getattr(socket, methname)
+    except (AttributeError, TypeError):
+        warnings.warn("""'%s' object has no _reuse/_drop methods
+{{
+    You make use (or a library you are using makes use) of the internal
+    classes '_socketobject' and '_fileobject' in socket.py, initializing
+    them with custom objects.  On PyPy, these custom objects need two
+    extra methods, _reuse() and _drop(), that maintain an explicit
+    reference counter.  When _drop() has been called as many times as
+    _reuse(), then the object should be freed.
+
+    Without these methods, you get the warning here.  This is to
+    prevent the following situation: if your (or the library's) code
+    relies on reference counting for prompt closing, then on PyPy, the
+    __del__ method will be called later than on CPython.  You can
+    easily end up in a situation where you open and close a lot of
+    (high-level) '_socketobject' or '_fileobject', but the (low-level)
+    custom objects will accumulate before their __del__ are called.
+    You quickly risk running out of file descriptors, for example.
+}}""" % (socket.__class__.__name__,), RefCountingWarning, stacklevel=3)
+    else:
+        method()
+
 
 _socketmethods = (
     'bind', 'connect', 'connect_ex', 'fileno', 'listen',
@@ -182,19 +210,7 @@
         if _sock is None:
             _sock = _realsocket(family, type, proto)
         else:
-            # PyPy note about refcounting: implemented with _reuse()/_drop()
-            # on the class '_socket.socket'.  Python 3 did it differently
-            # with a reference counter on this class 'socket._socketobject'
-            # instead, but it is a less compatible change.
-            
-            # Note that a few libraries (like eventlet) poke at the
-            # private implementation of socket.py, passing custom
-            # objects to _socketobject().  These libraries need the
-            # following fix for use on PyPy: the custom objects need
-            # methods _reuse() and _drop() that maintains an explicit
-            # reference counter, starting at 0.  When it drops back to
-            # zero, close() must be called.
-            _sock._reuse()
+            _do_reuse_or_drop(_sock, '_reuse')
 
         self._sock = _sock
 
@@ -228,13 +244,13 @@
     def close(self):
         s = self._sock
         self._sock = _closedsocket()
-        s._drop()
+        _do_reuse_or_drop(s, '_drop')
     close.__doc__ = _realsocket.close.__doc__
 
     def accept(self):
         sock, addr = self._sock.accept()
         sockobj = _socketobject(_sock=sock)
-        sock._drop()    # already a copy in the _socketobject()
+        _do_reuse_or_drop(sock, '_drop') # already a copy in the _socketobject()
         return sockobj, addr
     accept.__doc__ = _realsocket.accept.__doc__
 
@@ -290,14 +306,7 @@
                  "_close"]
 
     def __init__(self, sock, mode='rb', bufsize=-1, close=False):
-        # Note that a few libraries (like eventlet) poke at the
-        # private implementation of socket.py, passing custom
-        # objects to _fileobject().  These libraries need the
-        # following fix for use on PyPy: the custom objects need
-        # methods _reuse() and _drop() that maintains an explicit
-        # reference counter, starting at 0.  When it drops back to
-        # zero, close() must be called.
-        sock._reuse()
+        _do_reuse_or_drop(sock, '_reuse')
         self._sock = sock
         self.mode = mode # Not actually used in this version
         if bufsize < 0:
@@ -338,7 +347,7 @@
                 if self._close:
                     s.close()
                 else:
-                    s._drop()
+                    _do_reuse_or_drop(s, '_drop')
 
     def __del__(self):
         try:
diff --git a/lib-python/2.7/test/test_urllib2net.py b/lib-python/2.7/test/test_urllib2net.py
--- a/lib-python/2.7/test/test_urllib2net.py
+++ b/lib-python/2.7/test/test_urllib2net.py
@@ -102,11 +102,8 @@
 
     def test_ftp(self):
         urls = [
-            'ftp://ftp.kernel.org/pub/linux/kernel/README',
-            'ftp://ftp.kernel.org/pub/linux/kernel/non-existent-file',
-            #'ftp://ftp.kernel.org/pub/leenox/kernel/test',
-            'ftp://gatekeeper.research.compaq.com/pub/DEC/SRC'
-                '/research-reports/00README-Legal-Rules-Regs',
+            'ftp://ftp.debian.org/debian/README',
+            'ftp://ftp.debian.org/debian/non-existent-file',
             ]
         self._test_urls(urls, self._extra_handlers())
 
@@ -255,6 +252,7 @@
         with test_support.transient_internet(url, timeout=None):
             u = _urlopen_with_retry(url)
             self.assertIsNone(u.fp._sock.fp._sock.gettimeout())
+            u.close()
 
     def test_http_default_timeout(self):
         self.assertIsNone(socket.getdefaulttimeout())
@@ -266,6 +264,7 @@
             finally:
                 socket.setdefaulttimeout(None)
             self.assertEqual(u.fp._sock.fp._sock.gettimeout(), 60)
+            u.close()
 
     def test_http_no_timeout(self):
         self.assertIsNone(socket.getdefaulttimeout())
@@ -277,20 +276,23 @@
             finally:
                 socket.setdefaulttimeout(None)
             self.assertIsNone(u.fp._sock.fp._sock.gettimeout())
+            u.close()
 
     def test_http_timeout(self):
         url = "http://www.example.com"
         with test_support.transient_internet(url):
             u = _urlopen_with_retry(url, timeout=120)
             self.assertEqual(u.fp._sock.fp._sock.gettimeout(), 120)
+            u.close()
 
-    FTP_HOST = "ftp://ftp.mirror.nl/pub/gnu/"
+    FTP_HOST = 'ftp://ftp.debian.org/debian/'
 
     def test_ftp_basic(self):
         self.assertIsNone(socket.getdefaulttimeout())
         with test_support.transient_internet(self.FTP_HOST, timeout=None):
             u = _urlopen_with_retry(self.FTP_HOST)
             self.assertIsNone(u.fp.fp._sock.gettimeout())
+            u.close()
 
     def test_ftp_default_timeout(self):
         self.assertIsNone(socket.getdefaulttimeout())
@@ -301,6 +303,7 @@
             finally:
                 socket.setdefaulttimeout(None)
             self.assertEqual(u.fp.fp._sock.gettimeout(), 60)
+            u.close()
 
     def test_ftp_no_timeout(self):
         self.assertIsNone(socket.getdefaulttimeout(),)
@@ -311,11 +314,16 @@
             finally:
                 socket.setdefaulttimeout(None)
             self.assertIsNone(u.fp.fp._sock.gettimeout())
+            u.close()
 
     def test_ftp_timeout(self):
         with test_support.transient_internet(self.FTP_HOST):
-            u = _urlopen_with_retry(self.FTP_HOST, timeout=60)
+            try:
+                u = _urlopen_with_retry(self.FTP_HOST, timeout=60)
+            except:
+                raise
             self.assertEqual(u.fp.fp._sock.gettimeout(), 60)
+            u.close()
 
 
 def test_main():
diff --git a/lib_pypy/_audioop_build.py b/lib_pypy/_audioop_build.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_audioop_build.py
@@ -0,0 +1,621 @@
+from cffi import FFI
+
+ffi = FFI()
+ffi.cdef("""
+typedef short PyInt16;
+
+int ratecv(char* rv, char* cp, size_t len, int size,
+           int nchannels, int inrate, int outrate,
+           int* state_d, int* prev_i, int* cur_i,
+           int weightA, int weightB);
+
+void tostereo(char* rv, char* cp, size_t len, int size,
+              double fac1, double fac2);
+void add(char* rv, char* cp1, char* cp2, size_t len1, int size);
+
+/* 2's complement (14-bit range) */
+unsigned char
+st_14linear2ulaw(PyInt16 pcm_val);
+PyInt16 st_ulaw2linear16(unsigned char);
+
+/* 2's complement (13-bit range) */
+unsigned char
+st_linear2alaw(PyInt16 pcm_val);
+PyInt16 st_alaw2linear16(unsigned char);
+
+
+void lin2adcpm(unsigned char* rv, unsigned char* cp, size_t len,
+               size_t size, int* state);
+void adcpm2lin(unsigned char* rv, unsigned char* cp, size_t len,
+               size_t size, int* state);
+""")
+
+# This code is directly copied from CPython file: Modules/audioop.c
+_AUDIOOP_C_MODULE = r"""
+typedef short PyInt16;
+typedef int Py_Int32;
+
+/* Code shamelessly stolen from sox, 12.17.7, g711.c
+** (c) Craig Reese, Joe Campbell and Jeff Poskanzer 1989 */
+
+/* From g711.c:
+ *
+ * December 30, 1994:
+ * Functions linear2alaw, linear2ulaw have been updated to correctly
+ * convert unquantized 16 bit values.
+ * Tables for direct u- to A-law and A- to u-law conversions have been
+ * corrected.
+ * Borge Lindberg, Center for PersonKommunikation, Aalborg University.
+ * bli at cpk.auc.dk
+ *
+ */
+#define BIAS 0x84   /* define the add-in bias for 16 bit samples */
+#define CLIP 32635
+#define SIGN_BIT        (0x80)          /* Sign bit for a A-law byte. */
+#define QUANT_MASK      (0xf)           /* Quantization field mask. */
+#define SEG_SHIFT       (4)             /* Left shift for segment number. */
+#define SEG_MASK        (0x70)          /* Segment field mask. */
+
+static PyInt16 seg_aend[8] = {0x1F, 0x3F, 0x7F, 0xFF,
+                              0x1FF, 0x3FF, 0x7FF, 0xFFF};
+static PyInt16 seg_uend[8] = {0x3F, 0x7F, 0xFF, 0x1FF,
+                              0x3FF, 0x7FF, 0xFFF, 0x1FFF};
+
+static PyInt16
+search(PyInt16 val, PyInt16 *table, int size)
+{
+    int i;
+
+    for (i = 0; i < size; i++) {
+        if (val <= *table++)
+            return (i);
+    }
+    return (size);
+}
+#define st_ulaw2linear16(uc) (_st_ulaw2linear16[uc])
+#define st_alaw2linear16(uc) (_st_alaw2linear16[uc])
+
+static PyInt16 _st_ulaw2linear16[256] = {
+    -32124,  -31100,  -30076,  -29052,  -28028,  -27004,  -25980,
+    -24956,  -23932,  -22908,  -21884,  -20860,  -19836,  -18812,
+    -17788,  -16764,  -15996,  -15484,  -14972,  -14460,  -13948,
+    -13436,  -12924,  -12412,  -11900,  -11388,  -10876,  -10364,
+     -9852,   -9340,   -8828,   -8316,   -7932,   -7676,   -7420,
+     -7164,   -6908,   -6652,   -6396,   -6140,   -5884,   -5628,
+     -5372,   -5116,   -4860,   -4604,   -4348,   -4092,   -3900,
+     -3772,   -3644,   -3516,   -3388,   -3260,   -3132,   -3004,
+     -2876,   -2748,   -2620,   -2492,   -2364,   -2236,   -2108,
+     -1980,   -1884,   -1820,   -1756,   -1692,   -1628,   -1564,
+     -1500,   -1436,   -1372,   -1308,   -1244,   -1180,   -1116,
+     -1052,    -988,    -924,    -876,    -844,    -812,    -780,
+      -748,    -716,    -684,    -652,    -620,    -588,    -556,
+      -524,    -492,    -460,    -428,    -396,    -372,    -356,
+      -340,    -324,    -308,    -292,    -276,    -260,    -244,
+      -228,    -212,    -196,    -180,    -164,    -148,    -132,
+      -120,    -112,    -104,     -96,     -88,     -80,     -72,
+       -64,     -56,     -48,     -40,     -32,     -24,     -16,
+    -8,       0,   32124,   31100,   30076,   29052,   28028,
+     27004,   25980,   24956,   23932,   22908,   21884,   20860,
+     19836,   18812,   17788,   16764,   15996,   15484,   14972,
+     14460,   13948,   13436,   12924,   12412,   11900,   11388,
+     10876,   10364,    9852,    9340,    8828,    8316,    7932,
+      7676,    7420,    7164,    6908,    6652,    6396,    6140,
+      5884,    5628,    5372,    5116,    4860,    4604,    4348,
+      4092,    3900,    3772,    3644,    3516,    3388,    3260,
+      3132,    3004,    2876,    2748,    2620,    2492,    2364,
+      2236,    2108,    1980,    1884,    1820,    1756,    1692,
+      1628,    1564,    1500,    1436,    1372,    1308,    1244,
+      1180,    1116,    1052,     988,     924,     876,     844,
+       812,     780,     748,     716,     684,     652,     620,
+       588,     556,     524,     492,     460,     428,     396,
+       372,     356,     340,     324,     308,     292,     276,
+       260,     244,     228,     212,     196,     180,     164,
+       148,     132,     120,     112,     104,      96,      88,
+    80,      72,      64,      56,      48,      40,      32,
+    24,      16,       8,       0
+};
+
+/*
+ * linear2ulaw() accepts a 14-bit signed integer and encodes it as u-law data
+ * stored in a unsigned char.  This function should only be called with
+ * the data shifted such that it only contains information in the lower
+ * 14-bits.
+ *
+ * In order to simplify the encoding process, the original linear magnitude
+ * is biased by adding 33 which shifts the encoding range from (0 - 8158) to
+ * (33 - 8191). The result can be seen in the following encoding table:
+ *
+ *      Biased Linear Input Code        Compressed Code
+ *      ------------------------        ---------------
+ *      00000001wxyza                   000wxyz
+ *      0000001wxyzab                   001wxyz
+ *      000001wxyzabc                   010wxyz
+ *      00001wxyzabcd                   011wxyz
+ *      0001wxyzabcde                   100wxyz
+ *      001wxyzabcdef                   101wxyz
+ *      01wxyzabcdefg                   110wxyz
+ *      1wxyzabcdefgh                   111wxyz
+ *
+ * Each biased linear code has a leading 1 which identifies the segment
+ * number. The value of the segment number is equal to 7 minus the number
+ * of leading 0's. The quantization interval is directly available as the
+ * four bits wxyz.  * The trailing bits (a - h) are ignored.
+ *
+ * Ordinarily the complement of the resulting code word is used for
+ * transmission, and so the code word is complemented before it is returned.
+ *
+ * For further information see John C. Bellamy's Digital Telephony, 1982,
+ * John Wiley & Sons, pps 98-111 and 472-476.
+ */
+static unsigned char
+st_14linear2ulaw(PyInt16 pcm_val)       /* 2's complement (14-bit range) */
+{
+    PyInt16         mask;
+    PyInt16         seg;
+    unsigned char   uval;
+
+    /* The original sox code does this in the calling function, not here */
+    pcm_val = pcm_val >> 2;
+
+    /* u-law inverts all bits */
+    /* Get the sign and the magnitude of the value. */
+    if (pcm_val < 0) {
+        pcm_val = -pcm_val;
+        mask = 0x7F;
+    } else {
+        mask = 0xFF;
+    }
+    if ( pcm_val > CLIP ) pcm_val = CLIP;           /* clip the magnitude */
+    pcm_val += (BIAS >> 2);
+
+    /* Convert the scaled magnitude to segment number. */
+    seg = search(pcm_val, seg_uend, 8);
+
+    /*
+     * Combine the sign, segment, quantization bits;
+     * and complement the code word.
+     */
+    if (seg >= 8)           /* out of range, return maximum value. */
+        return (unsigned char) (0x7F ^ mask);
+    else {
+        uval = (unsigned char) (seg << 4) | ((pcm_val >> (seg + 1)) & 0xF);
+        return (uval ^ mask);
+    }
+
+}
+
+static PyInt16 _st_alaw2linear16[256] = {
+     -5504,   -5248,   -6016,   -5760,   -4480,   -4224,   -4992,
+     -4736,   -7552,   -7296,   -8064,   -7808,   -6528,   -6272,
+     -7040,   -6784,   -2752,   -2624,   -3008,   -2880,   -2240,
+     -2112,   -2496,   -2368,   -3776,   -3648,   -4032,   -3904,
+     -3264,   -3136,   -3520,   -3392,  -22016,  -20992,  -24064,
+    -23040,  -17920,  -16896,  -19968,  -18944,  -30208,  -29184,
+    -32256,  -31232,  -26112,  -25088,  -28160,  -27136,  -11008,
+    -10496,  -12032,  -11520,   -8960,   -8448,   -9984,   -9472,
+    -15104,  -14592,  -16128,  -15616,  -13056,  -12544,  -14080,
+    -13568,    -344,    -328,    -376,    -360,    -280,    -264,
+      -312,    -296,    -472,    -456,    -504,    -488,    -408,
+      -392,    -440,    -424,     -88,     -72,    -120,    -104,
+       -24,      -8,     -56,     -40,    -216,    -200,    -248,
+      -232,    -152,    -136,    -184,    -168,   -1376,   -1312,
+     -1504,   -1440,   -1120,   -1056,   -1248,   -1184,   -1888,
+     -1824,   -2016,   -1952,   -1632,   -1568,   -1760,   -1696,
+      -688,    -656,    -752,    -720,    -560,    -528,    -624,
+      -592,    -944,    -912,   -1008,    -976,    -816,    -784,
+      -880,    -848,    5504,    5248,    6016,    5760,    4480,
+      4224,    4992,    4736,    7552,    7296,    8064,    7808,
+      6528,    6272,    7040,    6784,    2752,    2624,    3008,
+      2880,    2240,    2112,    2496,    2368,    3776,    3648,
+      4032,    3904,    3264,    3136,    3520,    3392,   22016,
+     20992,   24064,   23040,   17920,   16896,   19968,   18944,
+     30208,   29184,   32256,   31232,   26112,   25088,   28160,
+     27136,   11008,   10496,   12032,   11520,    8960,    8448,
+      9984,    9472,   15104,   14592,   16128,   15616,   13056,
+     12544,   14080,   13568,     344,     328,     376,     360,
+       280,     264,     312,     296,     472,     456,     504,
+       488,     408,     392,     440,     424,      88,      72,
+       120,     104,      24,       8,      56,      40,     216,
+       200,     248,     232,     152,     136,     184,     168,
+      1376,    1312,    1504,    1440,    1120,    1056,    1248,
+      1184,    1888,    1824,    2016,    1952,    1632,    1568,
+      1760,    1696,     688,     656,     752,     720,     560,
+       528,     624,     592,     944,     912,    1008,     976,
+       816,     784,     880,     848
+};
+
+/*
+ * linear2alaw() accepts an 13-bit signed integer and encodes it as A-law data
+ * stored in a unsigned char.  This function should only be called with
+ * the data shifted such that it only contains information in the lower
+ * 13-bits.
+ *
+ *              Linear Input Code       Compressed Code
+ *      ------------------------        ---------------
+ *      0000000wxyza                    000wxyz
+ *      0000001wxyza                    001wxyz
+ *      000001wxyzab                    010wxyz
+ *      00001wxyzabc                    011wxyz
+ *      0001wxyzabcd                    100wxyz
+ *      001wxyzabcde                    101wxyz
+ *      01wxyzabcdef                    110wxyz
+ *      1wxyzabcdefg                    111wxyz
+ *
+ * For further information see John C. Bellamy's Digital Telephony, 1982,
+ * John Wiley & Sons, pps 98-111 and 472-476.
+ */
+static unsigned char
+st_linear2alaw(PyInt16 pcm_val) /* 2's complement (13-bit range) */
+{
+    PyInt16         mask;
+    short           seg;
+    unsigned char   aval;
+
+    /* The original sox code does this in the calling function, not here */
+    pcm_val = pcm_val >> 3;
+
+    /* A-law using even bit inversion */
+    if (pcm_val >= 0) {
+        mask = 0xD5;            /* sign (7th) bit = 1 */
+    } else {
+        mask = 0x55;            /* sign bit = 0 */
+        pcm_val = -pcm_val - 1;
+    }
+
+    /* Convert the scaled magnitude to segment number. */
+    seg = search(pcm_val, seg_aend, 8);
+
+    /* Combine the sign, segment, and quantization bits. */
+
+    if (seg >= 8)           /* out of range, return maximum value. */
+        return (unsigned char) (0x7F ^ mask);
+    else {
+        aval = (unsigned char) seg << SEG_SHIFT;
+        if (seg < 2)
+            aval |= (pcm_val >> 1) & QUANT_MASK;
+        else
+            aval |= (pcm_val >> seg) & QUANT_MASK;
+        return (aval ^ mask);
+    }
+}
+/* End of code taken from sox */
+
+/* Intel ADPCM step variation table */
+static int indexTable[16] = {
+    -1, -1, -1, -1, 2, 4, 6, 8,
+    -1, -1, -1, -1, 2, 4, 6, 8,
+};
+
+static int stepsizeTable[89] = {
+    7, 8, 9, 10, 11, 12, 13, 14, 16, 17,
+    19, 21, 23, 25, 28, 31, 34, 37, 41, 45,
+    50, 55, 60, 66, 73, 80, 88, 97, 107, 118,
+    130, 143, 157, 173, 190, 209, 230, 253, 279, 307,
+    337, 371, 408, 449, 494, 544, 598, 658, 724, 796,
+    876, 963, 1060, 1166, 1282, 1411, 1552, 1707, 1878, 2066,
+    2272, 2499, 2749, 3024, 3327, 3660, 4026, 4428, 4871, 5358,
+    5894, 6484, 7132, 7845, 8630, 9493, 10442, 11487, 12635, 13899,
+    15289, 16818, 18500, 20350, 22385, 24623, 27086, 29794, 32767
+};
+
+#define CHARP(cp, i) ((signed char *)(cp+i))
+#define SHORTP(cp, i) ((short *)(cp+i))
+#define LONGP(cp, i) ((Py_Int32 *)(cp+i))
+"""
+
+C_SOURCE = _AUDIOOP_C_MODULE + r"""
+#include <math.h>
+
+static const int maxvals[] = {0, 0x7F, 0x7FFF, 0x7FFFFF, 0x7FFFFFFF};
+/* -1 trick is needed on Windows to support -0x80000000 without a warning */
+static const int minvals[] = {0, -0x80, -0x8000, -0x800000, -0x7FFFFFFF-1};
+
+static int
+fbound(double val, double minval, double maxval)
+{
+    if (val > maxval)
+        val = maxval;
+    else if (val < minval + 1)
+        val = minval;
+    return val;
+}
+
+static int
+gcd(int a, int b)
+{
+    while (b > 0) {
+        int tmp = a % b;
+        a = b;
+        b = tmp;
+    }
+    return a;
+}
+
+int ratecv(char* rv, char* cp, size_t len, int size,
+           int nchannels, int inrate, int outrate,
+           int* state_d, int* prev_i, int* cur_i,
+           int weightA, int weightB)
+{
+    char *ncp = rv;
+    int d, chan;
+
+    /* divide inrate and outrate by their greatest common divisor */
+    d = gcd(inrate, outrate);
+    inrate /= d;
+    outrate /= d;
+    /* divide weightA and weightB by their greatest common divisor */
+    d = gcd(weightA, weightB);
+    weightA /= d;
+    weightA /= d;
+
+    d = *state_d;
+
+    for (;;) {
+        while (d < 0) {
+            if (len == 0) {
+                *state_d = d;
+                return ncp - rv;
+            }
+            for (chan = 0; chan < nchannels; chan++) {
+                prev_i[chan] = cur_i[chan];
+                if (size == 1)
+                    cur_i[chan] = ((int)*CHARP(cp, 0)) << 24;
+                else if (size == 2)
+                    cur_i[chan] = ((int)*SHORTP(cp, 0)) << 16;
+                else if (size == 4)
+                    cur_i[chan] = (int)*LONGP(cp, 0);
+                cp += size;
+                /* implements a simple digital filter */
+                cur_i[chan] = (int)(
+                    ((double)weightA * (double)cur_i[chan] +
+                     (double)weightB * (double)prev_i[chan]) /
+                    ((double)weightA + (double)weightB));
+            }
+            len--;
+            d += outrate;
+        }
+        while (d >= 0) {
+            for (chan = 0; chan < nchannels; chan++) {
+                int cur_o;
+                cur_o = (int)(((double)prev_i[chan] * (double)d +
+                         (double)cur_i[chan] * (double)(outrate - d)) /
+                    (double)outrate);
+                if (size == 1)
+                    *CHARP(ncp, 0) = (signed char)(cur_o >> 24);
+                else if (size == 2)
+                    *SHORTP(ncp, 0) = (short)(cur_o >> 16);
+                else if (size == 4)
+                    *LONGP(ncp, 0) = (Py_Int32)(cur_o);
+                ncp += size;
+            }
+            d -= inrate;
+        }
+    }
+}
+
+void tostereo(char* rv, char* cp, size_t len, int size,
+              double fac1, double fac2)
+{
+    int val1, val2, val = 0;
+    double fval, maxval, minval;
+    char *ncp = rv;
+    int i;
+
+    maxval = (double) maxvals[size];
+    minval = (double) minvals[size];
+
+    for ( i=0; i < len; i += size ) {
+        if ( size == 1 )      val = (int)*CHARP(cp, i);
+        else if ( size == 2 ) val = (int)*SHORTP(cp, i);
+        else if ( size == 4 ) val = (int)*LONGP(cp, i);
+
+        fval = (double)val*fac1;
+        val1 = (int)floor(fbound(fval, minval, maxval));
+
+        fval = (double)val*fac2;
+        val2 = (int)floor(fbound(fval, minval, maxval));
+
+        if ( size == 1 )      *CHARP(ncp, i*2) = (signed char)val1;
+        else if ( size == 2 ) *SHORTP(ncp, i*2) = (short)val1;
+        else if ( size == 4 ) *LONGP(ncp, i*2) = (Py_Int32)val1;
+
+        if ( size == 1 )      *CHARP(ncp, i*2+1) = (signed char)val2;
+        else if ( size == 2 ) *SHORTP(ncp, i*2+2) = (short)val2;
+        else if ( size == 4 ) *LONGP(ncp, i*2+4) = (Py_Int32)val2;
+    }
+}
+
+void add(char* rv, char* cp1, char* cp2, size_t len1, int size)
+{
+    int i;
+    int val1 = 0, val2 = 0, minval, maxval, newval;
+    char* ncp = rv;
+
+    maxval = maxvals[size];
+    minval = minvals[size];
+
+    for ( i=0; i < len1; i += size ) {
+        if ( size == 1 )      val1 = (int)*CHARP(cp1, i);
+        else if ( size == 2 ) val1 = (int)*SHORTP(cp1, i);
+        else if ( size == 4 ) val1 = (int)*LONGP(cp1, i);
+
+        if ( size == 1 )      val2 = (int)*CHARP(cp2, i);
+        else if ( size == 2 ) val2 = (int)*SHORTP(cp2, i);
+        else if ( size == 4 ) val2 = (int)*LONGP(cp2, i);
+
+        if (size < 4) {
+            newval = val1 + val2;
+            /* truncate in case of overflow */
+            if (newval > maxval)
+                newval = maxval;
+            else if (newval < minval)
+                newval = minval;
+        }
+        else {
+            double fval = (double)val1 + (double)val2;
+            /* truncate in case of overflow */
+            newval = (int)floor(fbound(fval, minval, maxval));
+        }
+
+        if ( size == 1 )      *CHARP(ncp, i) = (signed char)newval;
+        else if ( size == 2 ) *SHORTP(ncp, i) = (short)newval;
+        else if ( size == 4 ) *LONGP(ncp, i) = (Py_Int32)newval;
+    }
+}
+
+void lin2adcpm(unsigned char* ncp, unsigned char* cp, size_t len,
+               size_t size, int* state)
+{
+    int step, outputbuffer = 0, bufferstep;
+    int val = 0;
+    int diff, vpdiff, sign, delta;
+    size_t i;
+    int valpred = state[0];
+    int index = state[1];
+
+    step = stepsizeTable[index];
+    bufferstep = 1;
+
+    for ( i=0; i < len; i += size ) {
+        if ( size == 1 )      val = ((int)*CHARP(cp, i)) << 8;
+        else if ( size == 2 ) val = (int)*SHORTP(cp, i);
+        else if ( size == 4 ) val = ((int)*LONGP(cp, i)) >> 16;
+
+        /* Step 1 - compute difference with previous value */
+        diff = val - valpred;
+        sign = (diff < 0) ? 8 : 0;
+        if ( sign ) diff = (-diff);
+
+        /* Step 2 - Divide and clamp */
+        /* Note:
+        ** This code *approximately* computes:
+        **    delta = diff*4/step;
+        **    vpdiff = (delta+0.5)*step/4;
+        ** but in shift step bits are dropped. The net result of this
+        ** is that even if you have fast mul/div hardware you cannot
+        ** put it to good use since the fixup would be too expensive.
+        */
+        delta = 0;
+        vpdiff = (step >> 3);
+
+        if ( diff >= step ) {
+            delta = 4;
+            diff -= step;
+            vpdiff += step;
+        }
+        step >>= 1;
+        if ( diff >= step  ) {
+            delta |= 2;
+            diff -= step;
+            vpdiff += step;
+        }
+        step >>= 1;
+        if ( diff >= step ) {
+            delta |= 1;
+            vpdiff += step;
+        }
+
+        /* Step 3 - Update previous value */
+        if ( sign )
+            valpred -= vpdiff;
+        else
+            valpred += vpdiff;
+
+        /* Step 4 - Clamp previous value to 16 bits */
+        if ( valpred > 32767 )
+            valpred = 32767;
+        else if ( valpred < -32768 )
+            valpred = -32768;
+
+        /* Step 5 - Assemble value, update index and step values */
+        delta |= sign;
+
+        index += indexTable[delta];
+        if ( index < 0 ) index = 0;
+        if ( index > 88 ) index = 88;
+        step = stepsizeTable[index];
+
+        /* Step 6 - Output value */
+        if ( bufferstep ) {
+            outputbuffer = (delta << 4) & 0xf0;
+        } else {
+            *ncp++ = (delta & 0x0f) | outputbuffer;
+        }
+        bufferstep = !bufferstep;
+    }
+    state[0] = valpred;
+    state[1] = index;
+}
+
+
+void adcpm2lin(unsigned char* ncp, unsigned char* cp, size_t len,
+               size_t size, int* state)
+{
+    int step, inputbuffer = 0, bufferstep;
+    int val = 0;
+    int diff, vpdiff, sign, delta;
+    size_t i;
+    int valpred = state[0];
+    int index = state[1];
+
+    step = stepsizeTable[index];
+    bufferstep = 0;
+
+    for ( i=0; i < len*size*2; i += size ) {
+        /* Step 1 - get the delta value and compute next index */
+        if ( bufferstep ) {
+            delta = inputbuffer & 0xf;
+        } else {
+            inputbuffer = *cp++;
+            delta = (inputbuffer >> 4) & 0xf;
+        }
+
+        bufferstep = !bufferstep;
+
+        /* Step 2 - Find new index value (for later) */
+        index += indexTable[delta];
+        if ( index < 0 ) index = 0;
+        if ( index > 88 ) index = 88;
+
+        /* Step 3 - Separate sign and magnitude */
+        sign = delta & 8;
+        delta = delta & 7;
+
+        /* Step 4 - Compute difference and new predicted value */
+        /*
+        ** Computes 'vpdiff = (delta+0.5)*step/4', but see comment
+        ** in adpcm_coder.
+        */
+        vpdiff = step >> 3;
+        if ( delta & 4 ) vpdiff += step;
+        if ( delta & 2 ) vpdiff += step>>1;
+        if ( delta & 1 ) vpdiff += step>>2;
+
+        if ( sign )
+            valpred -= vpdiff;
+        else
+            valpred += vpdiff;
+
+        /* Step 5 - clamp output value */
+        if ( valpred > 32767 )
+            valpred = 32767;
+        else if ( valpred < -32768 )
+            valpred = -32768;
+
+        /* Step 6 - Update step value */
+        step = stepsizeTable[index];
+
+        /* Step 6 - Output value */
+        if ( size == 1 ) *CHARP(ncp, i) = (signed char)(valpred >> 8);
+        else if ( size == 2 ) *SHORTP(ncp, i) = (short)(valpred);
+        else if ( size == 4 ) *LONGP(ncp, i) = (Py_Int32)(valpred<<16);
+    }
+    state[0] = valpred;
+    state[1] = index;
+}
+"""
+
+ffi.set_source("_audioop_cffi", C_SOURCE)
+
+if __name__ == "__main__":
+    ffi.compile()
diff --git a/lib_pypy/_ctypes/function.py b/lib_pypy/_ctypes/function.py
--- a/lib_pypy/_ctypes/function.py
+++ b/lib_pypy/_ctypes/function.py
@@ -276,7 +276,11 @@
             if argtypes:
                 args = [argtype._CData_retval(argtype.from_address(arg)._buffer)
                         for argtype, arg in zip(argtypes, args)]
-            return to_call(*args)
+            try:
+                return to_call(*args)
+            except SystemExit as e:
+                handle_system_exit(e)
+                raise
         return f
 
     def __call__(self, *args, **kwargs):
@@ -302,10 +306,14 @@
 
             try:
                 newargs = self._convert_args_for_callback(argtypes, args)
-            except (UnicodeError, TypeError, ValueError), e:
+            except (UnicodeError, TypeError, ValueError) as e:
                 raise ArgumentError(str(e))
             try:
-                res = self.callable(*newargs)
+                try:
+                    res = self.callable(*newargs)
+                except SystemExit as e:
+                    handle_system_exit(e)
+                    raise
             except:
                 exc_info = sys.exc_info()
                 traceback.print_tb(exc_info[2], file=sys.stderr)
@@ -567,7 +575,7 @@
             for i, argtype in enumerate(argtypes):
                 try:
                     keepalive, newarg, newargtype = self._conv_param(argtype, args[i])
-                except (UnicodeError, TypeError, ValueError), e:
+                except (UnicodeError, TypeError, ValueError) as e:
                     raise ArgumentError(str(e))
                 keepalives.append(keepalive)
                 newargs.append(newarg)
@@ -578,7 +586,7 @@
             for i, arg in enumerate(extra):
                 try:
                     keepalive, newarg, newargtype = self._conv_param(None, arg)
-                except (UnicodeError, TypeError, ValueError), e:
+                except (UnicodeError, TypeError, ValueError) as e:
                     raise ArgumentError(str(e))
                 keepalives.append(keepalive)
                 newargs.append(newarg)
@@ -715,3 +723,22 @@
     make_fastpath_subclass.memo[CFuncPtr] = CFuncPtrFast
     return CFuncPtrFast
 make_fastpath_subclass.memo = {}
+
+
+def handle_system_exit(e):
+    # issue #1194: if we get SystemExit here, then exit the interpreter.
+    # Highly obscure imho but some people seem to depend on it.
+    if sys.flags.inspect:
+        return   # Don't exit if -i flag was given.
+    else:
+        code = e.code
+        if isinstance(code, int):
+            exitcode = code
+        else:
+            f = getattr(sys, 'stderr', None)
+            if f is None:
+                f = sys.__stderr__
+            print >> f, code
+            exitcode = 1
+
+        _rawffi.exit(exitcode)
diff --git a/lib_pypy/_curses.py b/lib_pypy/_curses.py
--- a/lib_pypy/_curses.py
+++ b/lib_pypy/_curses.py
@@ -6,326 +6,7 @@
     raise ImportError('No module named _curses')
 from functools import wraps
 
-from cffi import FFI
-
-ffi = FFI()
-
-ffi.cdef("""
-typedef ... WINDOW;
-typedef ... SCREEN;
-typedef unsigned long mmask_t;
-typedef unsigned char bool;
-typedef unsigned long chtype;
-typedef chtype attr_t;
-
-typedef struct
-{
-    short id;           /* ID to distinguish multiple devices */
-    int x, y, z;        /* event coordinates (character-cell) */
-    mmask_t bstate;     /* button state bits */
-}
-MEVENT;
-
-static const int ERR, OK;
-static const int TRUE, FALSE;
-static const int KEY_MIN, KEY_MAX;
-
-static const int COLOR_BLACK;
-static const int COLOR_RED;
-static const int COLOR_GREEN;
-static const int COLOR_YELLOW;
-static const int COLOR_BLUE;
-static const int COLOR_MAGENTA;
-static const int COLOR_CYAN;
-static const int COLOR_WHITE;
-
-static const chtype A_ATTRIBUTES;
-static const chtype A_NORMAL;
-static const chtype A_STANDOUT;
-static const chtype A_UNDERLINE;
-static const chtype A_REVERSE;
-static const chtype A_BLINK;
-static const chtype A_DIM;
-static const chtype A_BOLD;
-static const chtype A_ALTCHARSET;
-static const chtype A_INVIS;
-static const chtype A_PROTECT;
-static const chtype A_CHARTEXT;
-static const chtype A_COLOR;
-
-static const int BUTTON1_RELEASED;
-static const int BUTTON1_PRESSED;
-static const int BUTTON1_CLICKED;
-static const int BUTTON1_DOUBLE_CLICKED;
-static const int BUTTON1_TRIPLE_CLICKED;
-static const int BUTTON2_RELEASED;
-static const int BUTTON2_PRESSED;
-static const int BUTTON2_CLICKED;
-static const int BUTTON2_DOUBLE_CLICKED;
-static const int BUTTON2_TRIPLE_CLICKED;
-static const int BUTTON3_RELEASED;
-static const int BUTTON3_PRESSED;
-static const int BUTTON3_CLICKED;
-static const int BUTTON3_DOUBLE_CLICKED;
-static const int BUTTON3_TRIPLE_CLICKED;
-static const int BUTTON4_RELEASED;
-static const int BUTTON4_PRESSED;
-static const int BUTTON4_CLICKED;
-static const int BUTTON4_DOUBLE_CLICKED;
-static const int BUTTON4_TRIPLE_CLICKED;
-static const int BUTTON_SHIFT;
-static const int BUTTON_CTRL;
-static const int BUTTON_ALT;
-static const int ALL_MOUSE_EVENTS;
-static const int REPORT_MOUSE_POSITION;
-
-int setupterm(char *, int, int *);
-
-WINDOW *stdscr;
-int COLORS;
-int COLOR_PAIRS;
-int COLS;
-int LINES;
-
-int baudrate(void);
-int beep(void);
-int box(WINDOW *, chtype, chtype);
-bool can_change_color(void);
-int cbreak(void);
-int clearok(WINDOW *, bool);
-int color_content(short, short*, short*, short*);
-int copywin(const WINDOW*, WINDOW*, int, int, int, int, int, int, int);
-int curs_set(int);
-int def_prog_mode(void);
-int def_shell_mode(void);
-int delay_output(int);
-int delwin(WINDOW *);
-WINDOW * derwin(WINDOW *, int, int, int, int);
-int doupdate(void);
-int echo(void);
-int endwin(void);
-char erasechar(void);
-void filter(void);
-int flash(void);
-int flushinp(void);
-chtype getbkgd(WINDOW *);
-WINDOW * getwin(FILE *);
-int halfdelay(int);
-bool has_colors(void);
-bool has_ic(void);
-bool has_il(void);
-void idcok(WINDOW *, bool);
-int idlok(WINDOW *, bool);
-void immedok(WINDOW *, bool);
-WINDOW * initscr(void);
-int init_color(short, short, short, short);
-int init_pair(short, short, short);
-int intrflush(WINDOW *, bool);
-bool isendwin(void);
-bool is_linetouched(WINDOW *, int);
-bool is_wintouched(WINDOW *);
-const char * keyname(int);
-int keypad(WINDOW *, bool);
-char killchar(void);
-int leaveok(WINDOW *, bool);
-char * longname(void);
-int meta(WINDOW *, bool);
-int mvderwin(WINDOW *, int, int);
-int mvwaddch(WINDOW *, int, int, const chtype);
-int mvwaddnstr(WINDOW *, int, int, const char *, int);
-int mvwaddstr(WINDOW *, int, int, const char *);
-int mvwchgat(WINDOW *, int, int, int, attr_t, short, const void *);
-int mvwdelch(WINDOW *, int, int);
-int mvwgetch(WINDOW *, int, int);
-int mvwgetnstr(WINDOW *, int, int, char *, int);
-int mvwin(WINDOW *, int, int);
-chtype mvwinch(WINDOW *, int, int);
-int mvwinnstr(WINDOW *, int, int, char *, int);
-int mvwinsch(WINDOW *, int, int, chtype);
-int mvwinsnstr(WINDOW *, int, int, const char *, int);
-int mvwinsstr(WINDOW *, int, int, const char *);
-int napms(int);
-WINDOW * newpad(int, int);
-WINDOW * newwin(int, int, int, int);
-int nl(void);
-int nocbreak(void);
-int nodelay(WINDOW *, bool);
-int noecho(void);
-int nonl(void);
-void noqiflush(void);
-int noraw(void);
-int notimeout(WINDOW *, bool);
-int overlay(const WINDOW*, WINDOW *);
-int overwrite(const WINDOW*, WINDOW *);
-int pair_content(short, short*, short*);
-int pechochar(WINDOW *, const chtype);
-int pnoutrefresh(WINDOW*, int, int, int, int, int, int);
-int prefresh(WINDOW *, int, int, int, int, int, int);
-int putwin(WINDOW *, FILE *);
-void qiflush(void);
-int raw(void);
-int redrawwin(WINDOW *);
-int resetty(void);
-int reset_prog_mode(void);
-int reset_shell_mode(void);
-int savetty(void);
-int scroll(WINDOW *);
-int scrollok(WINDOW *, bool);
-int start_color(void);
-WINDOW * subpad(WINDOW *, int, int, int, int);
-WINDOW * subwin(WINDOW *, int, int, int, int);
-int syncok(WINDOW *, bool);
-chtype termattrs(void);
-char * termname(void);
-int touchline(WINDOW *, int, int);
-int touchwin(WINDOW *);
-int typeahead(int);
-int ungetch(int);
-int untouchwin(WINDOW *);
-void use_env(bool);
-int waddch(WINDOW *, const chtype);
-int waddnstr(WINDOW *, const char *, int);
-int waddstr(WINDOW *, const char *);
-int wattron(WINDOW *, int);
-int wattroff(WINDOW *, int);
-int wattrset(WINDOW *, int);
-int wbkgd(WINDOW *, chtype);
-void wbkgdset(WINDOW *, chtype);
-int wborder(WINDOW *, chtype, chtype, chtype, chtype,
-            chtype, chtype, chtype, chtype);
-int wchgat(WINDOW *, int, attr_t, short, const void *);
-int wclear(WINDOW *);
-int wclrtobot(WINDOW *);
-int wclrtoeol(WINDOW *);
-void wcursyncup(WINDOW *);
-int wdelch(WINDOW *);
-int wdeleteln(WINDOW *);
-int wechochar(WINDOW *, const chtype);
-int werase(WINDOW *);
-int wgetch(WINDOW *);
-int wgetnstr(WINDOW *, char *, int);
-int whline(WINDOW *, chtype, int);
-chtype winch(WINDOW *);
-int winnstr(WINDOW *, char *, int);
-int winsch(WINDOW *, chtype);
-int winsdelln(WINDOW *, int);
-int winsertln(WINDOW *);
-int winsnstr(WINDOW *, const char *, int);
-int winsstr(WINDOW *, const char *);
-int wmove(WINDOW *, int, int);
-int wresize(WINDOW *, int, int);
-int wnoutrefresh(WINDOW *);
-int wredrawln(WINDOW *, int, int);
-int wrefresh(WINDOW *);
-int wscrl(WINDOW *, int);
-int wsetscrreg(WINDOW *, int, int);
-int wstandout(WINDOW *);
-int wstandend(WINDOW *);
-void wsyncdown(WINDOW *);
-void wsyncup(WINDOW *);
-void wtimeout(WINDOW *, int);
-int wtouchln(WINDOW *, int, int, int);
-int wvline(WINDOW *, chtype, int);
-int tigetflag(char *);
-int tigetnum(char *);
-char * tigetstr(char *);
-int putp(const char *);
-char * tparm(const char *, ...);
-int getattrs(const WINDOW *);
-int getcurx(const WINDOW *);
-int getcury(const WINDOW *);
-int getbegx(const WINDOW *);
-int getbegy(const WINDOW *);
-int getmaxx(const WINDOW *);
-int getmaxy(const WINDOW *);
-int getparx(const WINDOW *);
-int getpary(const WINDOW *);
-
-int getmouse(MEVENT *);
-int ungetmouse(MEVENT *);
-mmask_t mousemask(mmask_t, mmask_t *);
-bool wenclose(const WINDOW *, int, int);
-int mouseinterval(int);
-
-void setsyx(int y, int x);
-const char *unctrl(chtype);
-int use_default_colors(void);
-
-int has_key(int);
-bool is_term_resized(int, int);
-
-#define _m_STRICT_SYSV_CURSES ...
-#define _m_NCURSES_MOUSE_VERSION ...
-#define _m_NetBSD ...
-int _m_ispad(WINDOW *);
-
-chtype acs_map[];
-
-// For _curses_panel:
-
-typedef ... PANEL;
-
-WINDOW *panel_window(const PANEL *);
-void update_panels(void);
-int hide_panel(PANEL *);
-int show_panel(PANEL *);
-int del_panel(PANEL *);
-int top_panel(PANEL *);
-int bottom_panel(PANEL *);
-PANEL *new_panel(WINDOW *);
-PANEL *panel_above(const PANEL *);
-PANEL *panel_below(const PANEL *);
-int set_panel_userptr(PANEL *, void *);
-const void *panel_userptr(const PANEL *);
-int move_panel(PANEL *, int, int);
-int replace_panel(PANEL *,WINDOW *);
-int panel_hidden(const PANEL *);
-
-void _m_getsyx(int *yx);
-""")
-
-
-lib = ffi.verify("""
-#ifdef __APPLE__
-/* the following define is necessary for OS X 10.6+; without it, the
-   Apple-supplied ncurses.h sets NCURSES_OPAQUE to 1, and then Python
-   can't get at the WINDOW flags field. */
-#define NCURSES_OPAQUE 0
-#endif
-
-#include <ncurses.h>
-#include <panel.h>
-#include <term.h>
-
-#if defined STRICT_SYSV_CURSES
-#define _m_STRICT_SYSV_CURSES TRUE
-#else
-#define _m_STRICT_SYSV_CURSES FALSE
-#endif
-
-#if defined NCURSES_MOUSE_VERSION
-#define _m_NCURSES_MOUSE_VERSION TRUE
-#else
-#define _m_NCURSES_MOUSE_VERSION FALSE
-#endif
-
-#if defined __NetBSD__
-#define _m_NetBSD TRUE
-#else
-#define _m_NetBSD FALSE
-#endif
-
-int _m_ispad(WINDOW *win) {
-    // <curses.h> may not have _flags (and possibly _ISPAD),
-    // but for now let's assume that <ncurses.h> always has it
-    return (win->_flags & _ISPAD);
-}
-
-void _m_getsyx(int *yx) {
-    getsyx(yx[0], yx[1]);
-}
-""", libraries=['ncurses', 'panel'])
-
+from _curses_cffi import ffi, lib
 
 def _copy_to_globals(name):
     globals()[name] = getattr(lib, name)
diff --git a/lib_pypy/_curses_build.py b/lib_pypy/_curses_build.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_curses_build.py
@@ -0,0 +1,323 @@
+from cffi import FFI
+
+ffi = FFI()
+
+ffi.set_source("_curses_cffi", """
+#ifdef __APPLE__
+/* the following define is necessary for OS X 10.6+; without it, the
+   Apple-supplied ncurses.h sets NCURSES_OPAQUE to 1, and then Python
+   can't get at the WINDOW flags field. */
+#define NCURSES_OPAQUE 0
+#endif
+
+#include <ncurses.h>
+#include <panel.h>
+#include <term.h>
+
+#if defined STRICT_SYSV_CURSES
+#define _m_STRICT_SYSV_CURSES TRUE
+#else
+#define _m_STRICT_SYSV_CURSES FALSE
+#endif
+
+#if defined NCURSES_MOUSE_VERSION
+#define _m_NCURSES_MOUSE_VERSION TRUE
+#else
+#define _m_NCURSES_MOUSE_VERSION FALSE
+#endif
+
+#if defined __NetBSD__
+#define _m_NetBSD TRUE
+#else
+#define _m_NetBSD FALSE
+#endif
+
+int _m_ispad(WINDOW *win) {
+    // <curses.h> may not have _flags (and possibly _ISPAD),
+    // but for now let's assume that <ncurses.h> always has it
+    return (win->_flags & _ISPAD);
+}
+
+void _m_getsyx(int *yx) {
+    getsyx(yx[0], yx[1]);
+}
+""", libraries=['ncurses', 'panel'])
+
+
+ffi.cdef("""
+typedef ... WINDOW;
+typedef ... SCREEN;
+typedef unsigned long... mmask_t;
+typedef unsigned char bool;
+typedef unsigned long... chtype;
+typedef chtype attr_t;
+
+typedef struct
+{
+    short id;           /* ID to distinguish multiple devices */
+    int x, y, z;        /* event coordinates (character-cell) */
+    mmask_t bstate;     /* button state bits */
+}
+MEVENT;
+
+static const int ERR, OK;
+static const int TRUE, FALSE;
+static const int KEY_MIN, KEY_MAX;
+
+static const int COLOR_BLACK;
+static const int COLOR_RED;
+static const int COLOR_GREEN;
+static const int COLOR_YELLOW;
+static const int COLOR_BLUE;
+static const int COLOR_MAGENTA;
+static const int COLOR_CYAN;
+static const int COLOR_WHITE;
+
+static const chtype A_ATTRIBUTES;
+static const chtype A_NORMAL;
+static const chtype A_STANDOUT;
+static const chtype A_UNDERLINE;
+static const chtype A_REVERSE;
+static const chtype A_BLINK;
+static const chtype A_DIM;
+static const chtype A_BOLD;
+static const chtype A_ALTCHARSET;
+static const chtype A_INVIS;
+static const chtype A_PROTECT;
+static const chtype A_CHARTEXT;
+static const chtype A_COLOR;
+
+static const int BUTTON1_RELEASED;
+static const int BUTTON1_PRESSED;
+static const int BUTTON1_CLICKED;
+static const int BUTTON1_DOUBLE_CLICKED;
+static const int BUTTON1_TRIPLE_CLICKED;
+static const int BUTTON2_RELEASED;
+static const int BUTTON2_PRESSED;
+static const int BUTTON2_CLICKED;
+static const int BUTTON2_DOUBLE_CLICKED;
+static const int BUTTON2_TRIPLE_CLICKED;
+static const int BUTTON3_RELEASED;
+static const int BUTTON3_PRESSED;
+static const int BUTTON3_CLICKED;
+static const int BUTTON3_DOUBLE_CLICKED;
+static const int BUTTON3_TRIPLE_CLICKED;
+static const int BUTTON4_RELEASED;
+static const int BUTTON4_PRESSED;
+static const int BUTTON4_CLICKED;
+static const int BUTTON4_DOUBLE_CLICKED;
+static const int BUTTON4_TRIPLE_CLICKED;
+static const int BUTTON_SHIFT;
+static const int BUTTON_CTRL;
+static const int BUTTON_ALT;
+static const int ALL_MOUSE_EVENTS;
+static const int REPORT_MOUSE_POSITION;
+
+int setupterm(char *, int, int *);
+
+WINDOW *stdscr;
+int COLORS;
+int COLOR_PAIRS;
+int COLS;
+int LINES;
+
+int baudrate(void);
+int beep(void);
+int box(WINDOW *, chtype, chtype);
+bool can_change_color(void);
+int cbreak(void);
+int clearok(WINDOW *, bool);
+int color_content(short, short*, short*, short*);
+int copywin(const WINDOW*, WINDOW*, int, int, int, int, int, int, int);
+int curs_set(int);
+int def_prog_mode(void);
+int def_shell_mode(void);
+int delay_output(int);
+int delwin(WINDOW *);
+WINDOW * derwin(WINDOW *, int, int, int, int);
+int doupdate(void);
+int echo(void);
+int endwin(void);
+char erasechar(void);
+void filter(void);
+int flash(void);
+int flushinp(void);
+chtype getbkgd(WINDOW *);
+WINDOW * getwin(FILE *);
+int halfdelay(int);
+bool has_colors(void);
+bool has_ic(void);
+bool has_il(void);
+void idcok(WINDOW *, bool);
+int idlok(WINDOW *, bool);
+void immedok(WINDOW *, bool);
+WINDOW * initscr(void);
+int init_color(short, short, short, short);
+int init_pair(short, short, short);
+int intrflush(WINDOW *, bool);
+bool isendwin(void);
+bool is_linetouched(WINDOW *, int);
+bool is_wintouched(WINDOW *);
+const char * keyname(int);
+int keypad(WINDOW *, bool);
+char killchar(void);
+int leaveok(WINDOW *, bool);
+char * longname(void);
+int meta(WINDOW *, bool);
+int mvderwin(WINDOW *, int, int);
+int mvwaddch(WINDOW *, int, int, const chtype);
+int mvwaddnstr(WINDOW *, int, int, const char *, int);
+int mvwaddstr(WINDOW *, int, int, const char *);
+int mvwchgat(WINDOW *, int, int, int, attr_t, short, const void *);
+int mvwdelch(WINDOW *, int, int);
+int mvwgetch(WINDOW *, int, int);
+int mvwgetnstr(WINDOW *, int, int, char *, int);
+int mvwin(WINDOW *, int, int);
+chtype mvwinch(WINDOW *, int, int);
+int mvwinnstr(WINDOW *, int, int, char *, int);
+int mvwinsch(WINDOW *, int, int, chtype);
+int mvwinsnstr(WINDOW *, int, int, const char *, int);
+int mvwinsstr(WINDOW *, int, int, const char *);
+int napms(int);
+WINDOW * newpad(int, int);
+WINDOW * newwin(int, int, int, int);
+int nl(void);
+int nocbreak(void);
+int nodelay(WINDOW *, bool);
+int noecho(void);
+int nonl(void);
+void noqiflush(void);
+int noraw(void);
+int notimeout(WINDOW *, bool);
+int overlay(const WINDOW*, WINDOW *);
+int overwrite(const WINDOW*, WINDOW *);
+int pair_content(short, short*, short*);
+int pechochar(WINDOW *, const chtype);
+int pnoutrefresh(WINDOW*, int, int, int, int, int, int);
+int prefresh(WINDOW *, int, int, int, int, int, int);
+int putwin(WINDOW *, FILE *);
+void qiflush(void);
+int raw(void);
+int redrawwin(WINDOW *);
+int resetty(void);
+int reset_prog_mode(void);
+int reset_shell_mode(void);
+int savetty(void);
+int scroll(WINDOW *);
+int scrollok(WINDOW *, bool);
+int start_color(void);
+WINDOW * subpad(WINDOW *, int, int, int, int);
+WINDOW * subwin(WINDOW *, int, int, int, int);
+int syncok(WINDOW *, bool);
+chtype termattrs(void);
+char * termname(void);
+int touchline(WINDOW *, int, int);
+int touchwin(WINDOW *);
+int typeahead(int);
+int ungetch(int);
+int untouchwin(WINDOW *);
+void use_env(bool);
+int waddch(WINDOW *, const chtype);
+int waddnstr(WINDOW *, const char *, int);
+int waddstr(WINDOW *, const char *);
+int wattron(WINDOW *, int);
+int wattroff(WINDOW *, int);
+int wattrset(WINDOW *, int);
+int wbkgd(WINDOW *, chtype);
+void wbkgdset(WINDOW *, chtype);
+int wborder(WINDOW *, chtype, chtype, chtype, chtype,
+            chtype, chtype, chtype, chtype);
+int wchgat(WINDOW *, int, attr_t, short, const void *);
+int wclear(WINDOW *);
+int wclrtobot(WINDOW *);
+int wclrtoeol(WINDOW *);
+void wcursyncup(WINDOW *);
+int wdelch(WINDOW *);
+int wdeleteln(WINDOW *);
+int wechochar(WINDOW *, const chtype);
+int werase(WINDOW *);
+int wgetch(WINDOW *);
+int wgetnstr(WINDOW *, char *, int);
+int whline(WINDOW *, chtype, int);
+chtype winch(WINDOW *);
+int winnstr(WINDOW *, char *, int);
+int winsch(WINDOW *, chtype);
+int winsdelln(WINDOW *, int);
+int winsertln(WINDOW *);
+int winsnstr(WINDOW *, const char *, int);
+int winsstr(WINDOW *, const char *);
+int wmove(WINDOW *, int, int);
+int wresize(WINDOW *, int, int);
+int wnoutrefresh(WINDOW *);
+int wredrawln(WINDOW *, int, int);
+int wrefresh(WINDOW *);
+int wscrl(WINDOW *, int);
+int wsetscrreg(WINDOW *, int, int);
+int wstandout(WINDOW *);
+int wstandend(WINDOW *);
+void wsyncdown(WINDOW *);
+void wsyncup(WINDOW *);
+void wtimeout(WINDOW *, int);
+int wtouchln(WINDOW *, int, int, int);
+int wvline(WINDOW *, chtype, int);
+int tigetflag(char *);
+int tigetnum(char *);
+char * tigetstr(char *);
+int putp(const char *);
+char * tparm(const char *, ...);
+int getattrs(const WINDOW *);
+int getcurx(const WINDOW *);
+int getcury(const WINDOW *);
+int getbegx(const WINDOW *);
+int getbegy(const WINDOW *);
+int getmaxx(const WINDOW *);
+int getmaxy(const WINDOW *);
+int getparx(const WINDOW *);
+int getpary(const WINDOW *);
+
+int getmouse(MEVENT *);
+int ungetmouse(MEVENT *);
+mmask_t mousemask(mmask_t, mmask_t *);
+bool wenclose(const WINDOW *, int, int);
+int mouseinterval(int);
+
+void setsyx(int y, int x);
+const char *unctrl(chtype);
+int use_default_colors(void);
+
+int has_key(int);
+bool is_term_resized(int, int);
+
+#define _m_STRICT_SYSV_CURSES ...
+#define _m_NCURSES_MOUSE_VERSION ...
+#define _m_NetBSD ...
+int _m_ispad(WINDOW *);
+
+chtype acs_map[];
+
+// For _curses_panel:
+
+typedef ... PANEL;
+
+WINDOW *panel_window(const PANEL *);
+void update_panels(void);
+int hide_panel(PANEL *);
+int show_panel(PANEL *);
+int del_panel(PANEL *);
+int top_panel(PANEL *);
+int bottom_panel(PANEL *);
+PANEL *new_panel(WINDOW *);
+PANEL *panel_above(const PANEL *);
+PANEL *panel_below(const PANEL *);
+int set_panel_userptr(PANEL *, void *);
+const void *panel_userptr(const PANEL *);
+int move_panel(PANEL *, int, int);
+int replace_panel(PANEL *,WINDOW *);
+int panel_hidden(const PANEL *);
+
+void _m_getsyx(int *yx);
+""")
+
+
+if __name__ == "__main__":
+    ffi.compile()
diff --git a/lib_pypy/_functools.py b/lib_pypy/_functools.py
--- a/lib_pypy/_functools.py
+++ b/lib_pypy/_functools.py
@@ -8,16 +8,16 @@
     partial(func, *args, **keywords) - new function with partial application
     of the given arguments and keywords.
     """
-
-    def __init__(self, *args, **keywords):
-        if not args:
-            raise TypeError('__init__() takes at least 2 arguments (1 given)')
-        func, args = args[0], args[1:]
+    def __init__(*args, **keywords):
+        if len(args) < 2:
+            raise TypeError('__init__() takes at least 2 arguments (%d given)'
+                            % len(args))
+        self, func, args = args[0], args[1], args[2:]
         if not callable(func):
             raise TypeError("the first argument must be callable")
         self._func = func
         self._args = args
-        self._keywords = keywords or None
+        self._keywords = keywords
 
     def __delattr__(self, key):
         if key == '__dict__':
@@ -37,19 +37,22 @@
         return self._keywords
 
     def __call__(self, *fargs, **fkeywords):
-        if self.keywords is not None:
-            fkeywords = dict(self.keywords, **fkeywords)
-        return self.func(*(self.args + fargs), **fkeywords)
+        if self._keywords:
+            fkeywords = dict(self._keywords, **fkeywords)
+        return self._func(*(self._args + fargs), **fkeywords)
 
     def __reduce__(self):
         d = dict((k, v) for k, v in self.__dict__.iteritems() if k not in
                 ('_func', '_args', '_keywords'))
         if len(d) == 0:
             d = None
-        return (type(self), (self.func,),
-                (self.func, self.args, self.keywords, d))
+        return (type(self), (self._func,),
+                (self._func, self._args, self._keywords, d))
 
     def __setstate__(self, state):
-        self._func, self._args, self._keywords, d = state
+        func, args, keywords, d = state
         if d is not None:
             self.__dict__.update(d)
+        self._func = func
+        self._args = args
+        self._keywords = keywords
diff --git a/lib_pypy/_gdbm_build.py b/lib_pypy/_gdbm_build.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_gdbm_build.py
@@ -0,0 +1,65 @@
+import cffi, os, sys
+
+ffi = cffi.FFI()
+ffi.cdef('''
+#define GDBM_READER ...
+#define GDBM_WRITER ...
+#define GDBM_WRCREAT ...
+#define GDBM_NEWDB ...
+#define GDBM_FAST ...
+#define GDBM_SYNC ...
+#define GDBM_NOLOCK ...
+#define GDBM_REPLACE ...
+
+void* gdbm_open(char *, int, int, int, void (*)());
+void gdbm_close(void*);
+
+typedef struct {
+    char *dptr;
+    int   dsize;
+} datum;
+
+datum gdbm_fetch(void*, datum);
+datum pygdbm_fetch(void*, char*, int);
+int gdbm_delete(void*, datum);
+int gdbm_store(void*, datum, datum, int);
+int gdbm_exists(void*, datum);
+int pygdbm_exists(void*, char*, int);
+
+int gdbm_reorganize(void*);
+
+datum gdbm_firstkey(void*);
+datum gdbm_nextkey(void*, datum);
+void gdbm_sync(void*);
+
+char* gdbm_strerror(int);
+int gdbm_errno;
+
+void free(void*);
+''')
+
+
+kwds = {}
+if sys.platform.startswith('freebsd'):
+    _localbase = os.environ.get('LOCALBASE', '/usr/local')
+    kwds['include_dirs'] = [os.path.join(_localbase, 'include')]
+    kwds['library_dirs'] = [os.path.join(_localbase, 'lib')]
+
+ffi.set_source("_gdbm_cffi", '''
+#include <stdlib.h>
+#include "gdbm.h"
+
+static datum pygdbm_fetch(GDBM_FILE gdbm_file, char *dptr, int dsize) {
+    datum key = {dptr, dsize};
+    return gdbm_fetch(gdbm_file, key);
+}
+
+static int pygdbm_exists(GDBM_FILE gdbm_file, char *dptr, int dsize) {
+    datum key = {dptr, dsize};
+    return gdbm_exists(gdbm_file, key);
+}
+''', libraries=['gdbm'], **kwds)
+
+
+if __name__ == '__main__':
+    ffi.compile()
diff --git a/lib_pypy/_pwdgrp_build.py b/lib_pypy/_pwdgrp_build.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_pwdgrp_build.py
@@ -0,0 +1,53 @@
+from cffi import FFI
+
+ffi = FFI()
+
+ffi.set_source("_pwdgrp_cffi", """
+#include <sys/types.h>
+#include <pwd.h>
+#include <grp.h>
+""")
+
+
+ffi.cdef("""
+
+typedef int... uid_t;
+typedef int... gid_t;
+
+struct passwd {
+    char *pw_name;
+    char *pw_passwd;
+    uid_t pw_uid;
+    gid_t pw_gid;
+    char *pw_gecos;
+    char *pw_dir;
+    char *pw_shell;
+    ...;
+};
+
+struct group {
+    char *gr_name;       /* group name */
+    char *gr_passwd;     /* group password */
+    gid_t gr_gid;        /* group ID */
+    char **gr_mem;        /* group members */
+};
+
+struct passwd *getpwuid(uid_t uid);
+struct passwd *getpwnam(const char *name);
+
+struct passwd *getpwent(void);
+void setpwent(void);
+void endpwent(void);
+
+struct group *getgrgid(gid_t gid);
+struct group *getgrnam(const char *name);
+
+struct group *getgrent(void);
+void setgrent(void);
+void endgrent(void);
+
+""")
+
+
+if __name__ == "__main__":
+    ffi.compile()
diff --git a/lib_pypy/_sqlite3.py b/lib_pypy/_sqlite3.py
--- a/lib_pypy/_sqlite3.py
+++ b/lib_pypy/_sqlite3.py
@@ -47,243 +47,7 @@
 else:
     _BLOB_TYPE = buffer
 
-from cffi import FFI as _FFI
-
-_ffi = _FFI()
-
-_ffi.cdef("""
-#define SQLITE_OK ...
-#define SQLITE_ERROR ...
-#define SQLITE_INTERNAL ...
-#define SQLITE_PERM ...
-#define SQLITE_ABORT ...
-#define SQLITE_BUSY ...
-#define SQLITE_LOCKED ...
-#define SQLITE_NOMEM ...
-#define SQLITE_READONLY ...
-#define SQLITE_INTERRUPT ...
-#define SQLITE_IOERR ...
-#define SQLITE_CORRUPT ...
-#define SQLITE_NOTFOUND ...
-#define SQLITE_FULL ...
-#define SQLITE_CANTOPEN ...
-#define SQLITE_PROTOCOL ...
-#define SQLITE_EMPTY ...
-#define SQLITE_SCHEMA ...
-#define SQLITE_TOOBIG ...
-#define SQLITE_CONSTRAINT ...
-#define SQLITE_MISMATCH ...
-#define SQLITE_MISUSE ...
-#define SQLITE_NOLFS ...
-#define SQLITE_AUTH ...
-#define SQLITE_FORMAT ...
-#define SQLITE_RANGE ...
-#define SQLITE_NOTADB ...
-#define SQLITE_ROW ...
-#define SQLITE_DONE ...
-#define SQLITE_INTEGER ...
-#define SQLITE_FLOAT ...
-#define SQLITE_BLOB ...
-#define SQLITE_NULL ...
-#define SQLITE_TEXT ...
-#define SQLITE3_TEXT ...
-
-#define SQLITE_TRANSIENT ...
-#define SQLITE_UTF8 ...
-
-#define SQLITE_DENY ...
-#define SQLITE_IGNORE ...
-
-#define SQLITE_CREATE_INDEX ...
-#define SQLITE_CREATE_TABLE ...
-#define SQLITE_CREATE_TEMP_INDEX ...
-#define SQLITE_CREATE_TEMP_TABLE ...
-#define SQLITE_CREATE_TEMP_TRIGGER ...
-#define SQLITE_CREATE_TEMP_VIEW ...
-#define SQLITE_CREATE_TRIGGER ...
-#define SQLITE_CREATE_VIEW ...
-#define SQLITE_DELETE ...
-#define SQLITE_DROP_INDEX ...
-#define SQLITE_DROP_TABLE ...
-#define SQLITE_DROP_TEMP_INDEX ...
-#define SQLITE_DROP_TEMP_TABLE ...
-#define SQLITE_DROP_TEMP_TRIGGER ...
-#define SQLITE_DROP_TEMP_VIEW ...
-#define SQLITE_DROP_TRIGGER ...
-#define SQLITE_DROP_VIEW ...
-#define SQLITE_INSERT ...
-#define SQLITE_PRAGMA ...
-#define SQLITE_READ ...
-#define SQLITE_SELECT ...
-#define SQLITE_TRANSACTION ...
-#define SQLITE_UPDATE ...
-#define SQLITE_ATTACH ...
-#define SQLITE_DETACH ...
-#define SQLITE_ALTER_TABLE ...
-#define SQLITE_REINDEX ...
-#define SQLITE_ANALYZE ...
-#define SQLITE_CREATE_VTABLE ...
-#define SQLITE_DROP_VTABLE ...
-#define SQLITE_FUNCTION ...
-

From noreply at buildbot.pypy.org  Sun Jun 21 22:09:40 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 21 Jun 2015 22:09:40 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c8: Try to add 'unsafe_write_int32' to
	experiment with it
Message-ID: <20150621200940.70A941C120E@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c8
Changeset: r78237:c75627b708d5
Date: 2015-06-21 22:09 +0200
http://bitbucket.org/pypy/pypy/changeset/c75627b708d5/

Log:	Try to add 'unsafe_write_int32' to experiment with it

diff --git a/pypy/module/pypystm/__init__.py b/pypy/module/pypystm/__init__.py
--- a/pypy/module/pypystm/__init__.py
+++ b/pypy/module/pypystm/__init__.py
@@ -29,4 +29,6 @@
         'stmdict': 'stmdict.W_STMDict',
         'queue': 'queue.W_Queue',
         'Empty': 'space.fromcache(queue.Cache).w_Empty',
+
+        'unsafe_write_int32': 'unsafe_op.unsafe_write_int32',
     }
diff --git a/pypy/module/pypystm/unsafe_op.py b/pypy/module/pypystm/unsafe_op.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/pypystm/unsafe_op.py
@@ -0,0 +1,13 @@
+from pypy.interpreter.gateway import unwrap_spec
+from pypy.module._cffi_backend import cdataobj
+from rpython.rlib.rstm import stm_ignored
+from rpython.rtyper.lltypesystem import rffi
+
+
+ at unwrap_spec(w_cdata=cdataobj.W_CData, index=int, value='c_int')
+def unsafe_write_int32(space, w_cdata, index, value):
+    with w_cdata as ptr:
+        ptr = rffi.cast(rffi.INTP, rffi.ptradd(ptr, index * 4))
+        value = rffi.cast(rffi.INT, value)
+        with stm_ignored:
+            ptr[0] = value

From noreply at buildbot.pypy.org  Sun Jun 21 09:27:41 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 21 Jun 2015 09:27:41 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c8: in-progress
Message-ID: <20150621072741.E3FC71C1302@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c8
Changeset: r78220:2d9ef7ec5527
Date: 2015-06-21 09:27 +0200
http://bitbucket.org/pypy/pypy/changeset/2d9ef7ec5527/

Log:	in-progress

diff --git a/TODO b/TODO
--- a/TODO
+++ b/TODO
@@ -1,3 +1,8 @@
+------------------------------------------------------------
+
+ll_math_modf(), say, causes stm_become_inevitable()
+because of the raw array read "intpart_p[0]"
+
 ------------------------------------------------------------
 
 fuse the two 32bit setfield_gc for stmflags & tid in the jit
diff --git a/pypy/stm/print_stm_log.py b/pypy/stm/print_stm_log.py
--- a/pypy/stm/print_stm_log.py
+++ b/pypy/stm/print_stm_log.py
@@ -111,6 +111,7 @@
     def progress(self, now, new_state):
         prev_time, prev_state = self._prev
         add_time = now - prev_time
+        add_time = abs(add_time) #XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
         assert add_time >= 0.0
         if prev_state == "run":
             self._transaction_cpu_time += add_time
@@ -226,7 +227,7 @@
 
 def print_marker(marker):
     s = '  %s' % marker
-    match = r_marker.match(marker)
+    match = r_marker.search(marker)
     if match:
         filename = match.group(1)
         if not (filename.endswith('.pyc') or filename.endswith('.pyo')):
@@ -254,6 +255,7 @@
                 print >> sys.stderr, '%.0f%%' % (entry.frac * 100.0,),
         cnt += 1
         #
+        #print entry
         t = threads.get(entry.threadnum)
         if t is None:
             t = threads[entry.threadnum] = ThreadState(entry.threadnum)

From noreply at buildbot.pypy.org  Fri Jun 19 00:16:45 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 19 Jun 2015 00:16:45 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c8: import stmgc/277dd2ad5226 and fix
 test_transaction. Now it seems to work
Message-ID: <20150618221645.B852C1C1FDD@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c8
Changeset: r78196:408cbce24e97
Date: 2015-06-18 23:17 +0100
http://bitbucket.org/pypy/pypy/changeset/408cbce24e97/

Log:	import stmgc/277dd2ad5226 and fix test_transaction. Now it seems to
	work

diff --git a/lib_pypy/pypy_test/test_transaction.py b/lib_pypy/pypy_test/test_transaction.py
--- a/lib_pypy/pypy_test/test_transaction.py
+++ b/lib_pypy/pypy_test/test_transaction.py
@@ -66,14 +66,12 @@
     for x in range(N):
         lsts = ([], [], [], [], [], [], [], [], [], [])
         def do_stuff(i, j):
-            print 'do_stuff', i, j
             lsts[i].append(j)
             j += 1
             if j < 5:
                 tq.add(do_stuff, i, j)
             else:
                 lsts[i].append('foo')
-                print 'raising FooError!'
                 raise FooError
         tq = transaction.TransactionQueue()
         for i in range(10):
@@ -138,7 +136,7 @@
     tq.run()
     assert tq.number_of_transactions_executed() == 1111
 
-def test_unexecuted_transactions_after_exception():
+def DONT_test_unexecuted_transactions_after_exception():
     class FooError(Exception):
         pass
     class BarError(Exception):
@@ -211,7 +209,8 @@
 def test_stmdict():
     d = transaction.stmdict()
     d["abc"] = "def"
-    assert list(d.iterkeys()) == ["abc"]
+    #assert list(d.iterkeys()) == ["abc"]
+    assert list(d) == ["abc"]
 
 def test_stmset():
     d = transaction.stmset()
diff --git a/rpython/translator/stm/src_stm/revision b/rpython/translator/stm/src_stm/revision
--- a/rpython/translator/stm/src_stm/revision
+++ b/rpython/translator/stm/src_stm/revision
@@ -1,1 +1,1 @@
-0a10e04f2119
+277dd2ad5226
diff --git a/rpython/translator/stm/src_stm/stm/nursery.c b/rpython/translator/stm/src_stm/stm/nursery.c
--- a/rpython/translator/stm/src_stm/stm/nursery.c
+++ b/rpython/translator/stm/src_stm/stm/nursery.c
@@ -553,6 +553,9 @@
     if (STM_PSEGMENT->finalizers != NULL)
         collect_objs_still_young_but_with_finalizers();
 
+    if (STM_PSEGMENT->active_queues != NULL)
+        collect_active_queues();
+
     collect_oldrefs_to_nursery();
     assert(list_is_empty(STM_PSEGMENT->old_objects_with_cards_set));
 
diff --git a/rpython/translator/stm/src_stm/stm/queue.c b/rpython/translator/stm/src_stm/stm/queue.c
--- a/rpython/translator/stm/src_stm/stm/queue.c
+++ b/rpython/translator/stm/src_stm/stm/queue.c
@@ -39,8 +39,10 @@
        and the 'segs' is an array of items 64 bytes each */
     stm_queue_segment_t segs[STM_NB_SEGMENTS];
 
-    /* a chained list of old entries in the queue */
-    queue_entry_t *volatile old_entries;
+    /* a chained list of old entries in the queue; modified only
+       with the lock */
+    queue_entry_t *old_entries;
+    uint8_t old_entries_lock;
 
     /* total of 'unfinished_tasks_in_this_transaction' for all
        committed transactions */
@@ -74,17 +76,6 @@
     for (i = 0; i < STM_NB_SEGMENTS; i++) {
         stm_queue_segment_t *seg = &queue->segs[i];
 
-        /* it is possible that queues_deactivate_all() runs in parallel,
-           but it should not be possible at this point for another thread
-           to change 'active' from false to true.  if it is false, then
-           that's it */
-        if (!seg->active) {
-            assert(!seg->added_in_this_transaction);
-            assert(!seg->added_young_limit);
-            assert(!seg->old_objects_popped);
-            continue;
-        }
-
         struct stm_priv_segment_info_s *pseg = get_priv_segment(i + 1);
         spinlock_acquire(pseg->active_queues_lock);
 
@@ -94,10 +85,16 @@
             assert(ok);
             (void)ok;
         }
+        else {
+            assert(!seg->added_in_this_transaction);
+            assert(!seg->added_young_limit);
+            assert(!seg->old_objects_popped);
+        }
+
+        spinlock_release(pseg->active_queues_lock);
+
         queue_free_entries(seg->added_in_this_transaction);
         queue_free_entries(seg->old_objects_popped);
-
-        spinlock_release(pseg->active_queues_lock);
     }
     free(queue);
 }
@@ -113,9 +110,9 @@
     spinlock_release(get_priv_segment(num)->active_queues_lock);
 }
 
-static void queue_activate(stm_queue_t *queue)
+static void queue_activate(stm_queue_t *queue, stm_queue_segment_t *seg)
 {
-    stm_queue_segment_t *seg = &queue->segs[STM_SEGMENT->segment_num - 1];
+    assert(seg == &queue->segs[STM_SEGMENT->segment_num - 1]);
 
     if (!seg->active) {
         queue_lock_acquire();
@@ -168,14 +165,19 @@
         if (head != NULL) {
             queue_entry_t *old;
             queue_entry_t *tail = head;
-            while (tail->next != NULL)
+            assert(!_is_in_nursery(head->object));
+            while (tail->next != NULL) {
                 tail = tail->next;
+                assert(!_is_in_nursery(tail->object));
+            }
             dprintf(("items move to old_entries in queue %p\n", queue));
-         retry:
+
+            spinlock_acquire(queue->old_entries_lock);
             old = queue->old_entries;
             tail->next = old;
-            if (!__sync_bool_compare_and_swap(&queue->old_entries, old, head))
-                goto retry;
+            queue->old_entries = head;
+            spinlock_release(queue->old_entries_lock);
+
             added_any_old_entries = true;
         }
 
@@ -204,21 +206,20 @@
        delays or transaction breaks.  you need to push roots!
     */
     stm_queue_segment_t *seg = &queue->segs[STM_SEGMENT->segment_num - 1];
+    queue_activate(queue, seg);
+
     queue_entry_t *entry = malloc(sizeof(queue_entry_t));
     assert(entry);
     entry->object = newitem;
     entry->next = seg->added_in_this_transaction;
     seg->added_in_this_transaction = entry;
+    seg->unfinished_tasks_in_this_transaction++;
+}
 
-    queue_activate(queue);
-    seg->unfinished_tasks_in_this_transaction++;
-
-    /* add qobj to 'objects_pointing_to_nursery' if it has the
-       WRITE_BARRIER flag */
-    if (qobj->stm_flags & GCFLAG_WRITE_BARRIER) {
-        qobj->stm_flags &= ~GCFLAG_WRITE_BARRIER;
-        LIST_APPEND(STM_PSEGMENT->objects_pointing_to_nursery, qobj);
-    }
+static void queue_check_entry(queue_entry_t *entry)
+{
+    assert(entry->object != NULL);
+    assert(((TLPREFIX int *)entry->object)[1] != 0);   /* userdata != 0 */
 }
 
 object_t *stm_queue_get(object_t *qobj, stm_queue_t *queue, double timeout,
@@ -239,26 +240,45 @@
         seg->added_in_this_transaction = entry->next;
         if (entry == seg->added_young_limit)
             seg->added_young_limit = entry->next;
+        queue_check_entry(entry);
         result = entry->object;
-        assert(result != NULL);
         free(entry);
         return result;
     }
 
  retry:
+    /* careful, STM_SEGMENT->segment_num may change here because
+       we're starting new transactions below! */
+    seg = &queue->segs[STM_SEGMENT->segment_num - 1];
+    assert(!seg->added_in_this_transaction);
+
+    /* can't easily use compare_and_swap here.  The issue is that
+       if we do "compare_and_swap(&old_entry, entry, entry->next)",
+       then we need to read entry->next, but a parallel thread
+       could have grabbed the same entry and already freed it.
+       More subtly, there is also an ABA problem: even if we
+       read the correct entry->next, maybe a parallel thread
+       can free and reuse this entry.  Then the compare_and_swap
+       succeeds, but the value written is outdated nonsense.
+    */
+    spinlock_acquire(queue->old_entries_lock);
     entry = queue->old_entries;
+    if (entry != NULL)
+        queue->old_entries = entry->next;
+    spinlock_release(queue->old_entries_lock);
+
     if (entry != NULL) {
-        if (!__sync_bool_compare_and_swap(&queue->old_entries,
-                                          entry, entry->next))
-            goto retry;
+        /* successfully popped the old 'entry'.  It remains in the
+           'old_objects_popped' list for now.  From now on, this entry
+           "belongs" to this segment and should never be read by
+           another segment. */
+        queue_activate(queue, seg);
 
-        /* successfully popped the old 'entry'.  It remains in the
-           'old_objects_popped' list for now. */
+        queue_check_entry(entry);
+        assert(!_is_in_nursery(entry->object));
+
         entry->next = seg->old_objects_popped;
         seg->old_objects_popped = entry;
-
-        queue_activate(queue);
-        assert(entry->object != NULL);
         return entry->object;
     }
     else {
@@ -268,7 +288,9 @@
 #endif
         if (timeout == 0.0) {
             if (!stm_is_inevitable(tl)) {
+                STM_PUSH_ROOT(*tl, qobj);
                 stm_become_inevitable(tl, "stm_queue_get");
+                STM_POP_ROOT(*tl, qobj);
                 goto retry;
             }
             else
@@ -304,8 +326,8 @@
 
 void stm_queue_task_done(stm_queue_t *queue)
 {
-    queue_activate(queue);
     stm_queue_segment_t *seg = &queue->segs[STM_SEGMENT->segment_num - 1];
+    queue_activate(queue, seg);
     seg->unfinished_tasks_in_this_transaction--;
 }
 
@@ -358,14 +380,30 @@
         }
         queue_trace_list(queue->old_entries, trace, NULL);
     }
-    else {
-        /* for minor collections: it is enough to trace the objects
-           added in the current transaction.  All other objects are
-           old (or, worse, belong to a parallel thread and must not
-           be traced). */
+    /* for minor collections, done differently.
+       see collect_active_queues() below */
+}
+
+static void collect_active_queues(void)
+{
+    wlog_t *item;
+    TREE_LOOP_FORWARD(STM_PSEGMENT->active_queues, item) {
+        /* it is enough to trace the objects added in the current
+           transaction.  All other objects reachable from the queue
+           are old (or, worse, belong to a parallel thread and must
+           not be traced).  Performance note: this is linear in the
+           total number of active queues, but at least each queue that
+           has not been touched for a while in a long transaction is
+           handled very cheaply.
+        */
+        stm_queue_t *queue = (stm_queue_t *)item->addr;
         stm_queue_segment_t *seg = &queue->segs[STM_SEGMENT->segment_num - 1];
-        queue_trace_list(seg->added_in_this_transaction, trace,
-                         seg->added_young_limit);
-        seg->added_young_limit = seg->added_in_this_transaction;
-    }
+        if (seg->added_young_limit != seg->added_in_this_transaction) {
+            dprintf(("minor collection trace queue %p\n", queue));
+            queue_trace_list(seg->added_in_this_transaction,
+                             &minor_trace_if_young,
+                             seg->added_young_limit);
+            seg->added_young_limit = seg->added_in_this_transaction;
+        }
+    } TREE_LOOP_END;
 }
diff --git a/rpython/translator/stm/src_stm/stm/queue.h b/rpython/translator/stm/src_stm/stm/queue.h
--- a/rpython/translator/stm/src_stm/stm/queue.h
+++ b/rpython/translator/stm/src_stm/stm/queue.h
@@ -1,2 +1,3 @@
 /* Imported by rpython/translator/stm/import_stmgc.py */
 static void queues_deactivate_all(bool at_commit);
+static void collect_active_queues(void);

From noreply at buildbot.pypy.org  Sun Jun 21 23:25:47 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 21 Jun 2015 23:25:47 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c8: translation fix
Message-ID: <20150621212547.38DB71C1F6A@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c8
Changeset: r78238:7260a61ff88a
Date: 2015-06-21 22:27 +0100
http://bitbucket.org/pypy/pypy/changeset/7260a61ff88a/

Log:	translation fix

diff --git a/pypy/module/pypystm/unsafe_op.py b/pypy/module/pypystm/unsafe_op.py
--- a/pypy/module/pypystm/unsafe_op.py
+++ b/pypy/module/pypystm/unsafe_op.py
@@ -1,13 +1,18 @@
 from pypy.interpreter.gateway import unwrap_spec
 from pypy.module._cffi_backend import cdataobj
 from rpython.rlib.rstm import stm_ignored
+from rpython.rlib.jit import dont_look_inside
 from rpython.rtyper.lltypesystem import rffi
 
 
+ at dont_look_inside
+def unsafe_write(ptr, value):
+    with stm_ignored:
+        ptr[0] = value
+
 @unwrap_spec(w_cdata=cdataobj.W_CData, index=int, value='c_int')
 def unsafe_write_int32(space, w_cdata, index, value):
     with w_cdata as ptr:
         ptr = rffi.cast(rffi.INTP, rffi.ptradd(ptr, index * 4))
         value = rffi.cast(rffi.INT, value)
-        with stm_ignored:
-            ptr[0] = value
+        unsafe_write(ptr, value)

From noreply at buildbot.pypy.org  Thu Jun 18 19:38:59 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 18 Jun 2015 19:38:59 +0200 (CEST)
Subject: [pypy-commit] stmgc queue: hg merge default
Message-ID: <20150618173859.40E601C1FD6@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: queue
Changeset: r1866:950c6296eab7
Date: 2015-06-18 18:39 +0100
http://bitbucket.org/pypy/stmgc/changeset/950c6296eab7/

Log:	hg merge default

diff --git a/c7/gdb/gdb_stm.py b/c7/gdb/gdb_stm.py
--- a/c7/gdb/gdb_stm.py
+++ b/c7/gdb/gdb_stm.py
@@ -34,6 +34,12 @@
                 raise
     Func(func.__name__)
 
+def int_(x):
+    if isinstance(x, gdb.Value):
+        T = gdb.lookup_type('long')
+        x = x.cast(T)
+    return int(x)
+
 # -------------------------------------------------------
 
 _nb_segments = None
@@ -43,26 +49,26 @@
 def get_nb_segments():
     global _nb_segments
     if _nb_segments is None:
-        _nb_segments = int(gdb.parse_and_eval('_stm_nb_segments'))
+        _nb_segments = int_(gdb.parse_and_eval('_stm_nb_segments'))
         assert 1 < _nb_segments <= 240
     return _nb_segments
 
 def get_segment_size():
     global _segment_size
     if _segment_size is None:
-        nb_pages = int(gdb.parse_and_eval('_stm_segment_nb_pages'))
+        nb_pages = int_(gdb.parse_and_eval('_stm_segment_nb_pages'))
         _segment_size = nb_pages * 4096
     return _segment_size
 
 def get_psegment_ofs():
     global _psegment_ofs
     if _psegment_ofs is None:
-        _psegment_ofs = int(gdb.parse_and_eval('_stm_psegment_ofs'))
+        _psegment_ofs = int_(gdb.parse_and_eval('_stm_psegment_ofs'))
     return _psegment_ofs
 
 def get_segment_base(segment_id):
     assert 0 <= segment_id <= get_nb_segments()
-    base = int(gdb.parse_and_eval('stm_object_pages'))
+    base = int_(gdb.parse_and_eval('stm_object_pages'))
     return base + get_segment_size() * segment_id
 
 def get_psegment(segment_id, field=''):
@@ -72,13 +78,13 @@
         % (get_segment_size() * segment_id + get_psegment_ofs(), field))
 
 def thread_to_segment_id(thread_id):
-    base = int(gdb.parse_and_eval('stm_object_pages'))
+    base = int_(gdb.parse_and_eval('stm_object_pages'))
     for j in range(1, get_nb_segments() + 1):
         #ti = get_psegment(j, '->pub.running_thread->creating_pthread[0]')
         ti = get_psegment(j, '->running_pthread')
-        if int(ti) == thread_id:
+        if int_(ti) == thread_id:
             ts = get_psegment(j, '->transaction_state')
-            if int(ts) == 0:
+            if int_(ts) == 0:
                 print >> sys.stderr, "note: transaction_state == 0"
             return j
     raise Exception("thread not found: %r" % (thread_id,))
@@ -94,10 +100,10 @@
         thread_id = int(fields[2], 16)
         segment_id = thread_to_segment_id(thread_id)
     elif thread.type.code == gdb.TYPE_CODE_INT:
-        if 0 <= int(thread) < 256:
-            segment_id = int(thread)
+        if 0 <= int_(thread) < 256:
+            segment_id = int_(thread)
         else:
-            thread_id = int(thread)
+            thread_id = int_(thread)
             segment_id = thread_to_segment_id(thread_id)
     else:
         raise TypeError("'thread' argument must be an int or not given")
@@ -107,12 +113,12 @@
 def gc(p=None, thread=None):
     sb = interactive_segment_base(thread)
     if p is not None and p.type.code == gdb.TYPE_CODE_PTR:
-        return gdb.Value(sb + int(p)).cast(p.type).dereference()
+        return gdb.Value(sb + int_(p)).cast(p.type).dereference()
     else:
         if p is None:
             p = 0
         else:
-            p = int(p)
+            p = int_(p)
         T = gdb.lookup_type('char').pointer()
         return gdb.Value(sb + p).cast(T)
 

From noreply at buildbot.pypy.org  Sun Jun 21 09:27:39 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 21 Jun 2015 09:27:39 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c8: Keep the threads around for the next
	call to run()
Message-ID: <20150621072739.9CC8F1C11FB@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c8
Changeset: r78218:e92765580f57
Date: 2015-06-20 07:40 +0200
http://bitbucket.org/pypy/pypy/changeset/e92765580f57/

Log:	Keep the threads around for the next call to run()

diff --git a/lib_pypy/pypy_test/test_transaction.py b/lib_pypy/pypy_test/test_transaction.py
--- a/lib_pypy/pypy_test/test_transaction.py
+++ b/lib_pypy/pypy_test/test_transaction.py
@@ -1,5 +1,8 @@
 import py
-from lib_pypy import transaction
+try:
+    from lib_pypy import transaction
+except ImportError:
+    import transaction
 
 N = 1000
 VERBOSE = False
diff --git a/lib_pypy/transaction.py b/lib_pypy/transaction.py
--- a/lib_pypy/transaction.py
+++ b/lib_pypy/transaction.py
@@ -118,6 +118,8 @@
     finished.  The run() call only returns when the queue is completely
     empty.
     """
+    _nb_threads = 0
+    _thread_queue = queue()
 
     def __init__(self):
         self._queue = queue()
@@ -150,10 +152,16 @@
                 "TransactionQueue.run() cannot be called in an atomic context")
         if nb_segments <= 0:
             nb_segments = getsegmentlimit()
-
+        while TransactionQueue._nb_threads < nb_segments:
+            with atomic:
+                if TransactionQueue._nb_threads >= nb_segments:
+                    break
+                TransactionQueue._nb_threads += 1
+            thread.start_new_thread(TransactionQueue._thread_runner, ())
+        #
         self._exception = []
         for i in range(nb_segments):
-            thread.start_new_thread(self._thread_runner, ())
+            TransactionQueue._thread_queue.put((self._queue, self._exception))
         #
         # The threads run here until queue.join() returns, i.e. until
         # all add()ed transactions are executed.
@@ -164,29 +172,30 @@
         #
         if self._exception:
             exc_type, exc_value, exc_traceback = self._exception
-            self._exception = None
+            del self._exception
             raise exc_type, exc_value, exc_traceback
 
     #def number_of_transactions_executed(self):
     #    disabled for now
 
-    def _thread_runner(self):
-        queue = self._queue
-        exception = self._exception
+    @staticmethod
+    def _thread_runner():
         while True:
-            f, args, kwds = queue.get()
-            try:
-                if args is None:
-                    break
-                with atomic:
-                    if not exception:
-                        try:
-                            with signals_enabled:
-                                f(*args, **kwds)
-                        except:
-                            exception.extend(sys.exc_info())
-            finally:
-                queue.task_done()
+            queue, exception = TransactionQueue._thread_queue.get()
+            while True:
+                f, args, kwds = queue.get()
+                try:
+                    if args is None:
+                        break
+                    with atomic:
+                        if not exception:
+                            try:
+                                with signals_enabled:
+                                    f(*args, **kwds)
+                            except:
+                                exception.extend(sys.exc_info())
+                finally:
+                    queue.task_done()
 
 
 # ____________________________________________________________

From noreply at buildbot.pypy.org  Thu Jun 18 19:38:58 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 18 Jun 2015 19:38:58 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Next version of gdb breaks existing
	scripts, like usual
Message-ID: <20150618173858.2DADA1C1FD1@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r1865:c3a059fb73bc
Date: 2015-06-18 18:39 +0100
http://bitbucket.org/pypy/stmgc/changeset/c3a059fb73bc/

Log:	Next version of gdb breaks existing scripts, like usual

diff --git a/c7/gdb/gdb_stm.py b/c7/gdb/gdb_stm.py
--- a/c7/gdb/gdb_stm.py
+++ b/c7/gdb/gdb_stm.py
@@ -34,6 +34,12 @@
                 raise
     Func(func.__name__)
 
+def int_(x):
+    if isinstance(x, gdb.Value):
+        T = gdb.lookup_type('long')
+        x = x.cast(T)
+    return int(x)
+
 # -------------------------------------------------------
 
 _nb_segments = None
@@ -43,26 +49,26 @@
 def get_nb_segments():
     global _nb_segments
     if _nb_segments is None:
-        _nb_segments = int(gdb.parse_and_eval('_stm_nb_segments'))
+        _nb_segments = int_(gdb.parse_and_eval('_stm_nb_segments'))
         assert 1 < _nb_segments <= 240
     return _nb_segments
 
 def get_segment_size():
     global _segment_size
     if _segment_size is None:
-        nb_pages = int(gdb.parse_and_eval('_stm_segment_nb_pages'))
+        nb_pages = int_(gdb.parse_and_eval('_stm_segment_nb_pages'))
         _segment_size = nb_pages * 4096
     return _segment_size
 
 def get_psegment_ofs():
     global _psegment_ofs
     if _psegment_ofs is None:
-        _psegment_ofs = int(gdb.parse_and_eval('_stm_psegment_ofs'))
+        _psegment_ofs = int_(gdb.parse_and_eval('_stm_psegment_ofs'))
     return _psegment_ofs
 
 def get_segment_base(segment_id):
     assert 0 <= segment_id <= get_nb_segments()
-    base = int(gdb.parse_and_eval('stm_object_pages'))
+    base = int_(gdb.parse_and_eval('stm_object_pages'))
     return base + get_segment_size() * segment_id
 
 def get_psegment(segment_id, field=''):
@@ -72,13 +78,13 @@
         % (get_segment_size() * segment_id + get_psegment_ofs(), field))
 
 def thread_to_segment_id(thread_id):
-    base = int(gdb.parse_and_eval('stm_object_pages'))
+    base = int_(gdb.parse_and_eval('stm_object_pages'))
     for j in range(1, get_nb_segments() + 1):
         #ti = get_psegment(j, '->pub.running_thread->creating_pthread[0]')
         ti = get_psegment(j, '->running_pthread')
-        if int(ti) == thread_id:
+        if int_(ti) == thread_id:
             ts = get_psegment(j, '->transaction_state')
-            if int(ts) == 0:
+            if int_(ts) == 0:
                 print >> sys.stderr, "note: transaction_state == 0"
             return j
     raise Exception("thread not found: %r" % (thread_id,))
@@ -94,10 +100,10 @@
         thread_id = int(fields[2], 16)
         segment_id = thread_to_segment_id(thread_id)
     elif thread.type.code == gdb.TYPE_CODE_INT:
-        if 0 <= int(thread) < 256:
-            segment_id = int(thread)
+        if 0 <= int_(thread) < 256:
+            segment_id = int_(thread)
         else:
-            thread_id = int(thread)
+            thread_id = int_(thread)
             segment_id = thread_to_segment_id(thread_id)
     else:
         raise TypeError("'thread' argument must be an int or not given")
@@ -107,12 +113,12 @@
 def gc(p=None, thread=None):
     sb = interactive_segment_base(thread)
     if p is not None and p.type.code == gdb.TYPE_CODE_PTR:
-        return gdb.Value(sb + int(p)).cast(p.type).dereference()
+        return gdb.Value(sb + int_(p)).cast(p.type).dereference()
     else:
         if p is None:
             p = 0
         else:
-            p = int(p)
+            p = int_(p)
         T = gdb.lookup_type('char').pointer()
         return gdb.Value(sb + p).cast(T)
 

From noreply at buildbot.pypy.org  Sun Jun 21 09:27:40 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 21 Jun 2015 09:27:40 +0200 (CEST)
Subject: [pypy-commit] pypy default: Test and fix for a better error message
 on failing imports
Message-ID: <20150621072740.C0D121C124A@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78219:fcb2ba48f07c
Date: 2015-06-21 09:26 +0200
http://bitbucket.org/pypy/pypy/changeset/fcb2ba48f07c/

Log:	Test and fix for a better error message on failing imports

diff --git a/rpython/flowspace/flowcontext.py b/rpython/flowspace/flowcontext.py
--- a/rpython/flowspace/flowcontext.py
+++ b/rpython/flowspace/flowcontext.py
@@ -1207,7 +1207,8 @@
     def nomoreblocks(self, ctx):
         w_exc = self.w_exc
         if w_exc.w_type == const(ImportError):
-            msg = 'import statement always raises %s' % self
+            msg = 'ImportError is raised in RPython: %s' % (
+                getattr(w_exc.w_value, 'value', '<not a constant message>'),)
             raise ImportError(msg)
         link = Link([w_exc.w_type, w_exc.w_value], ctx.graph.exceptblock)
         ctx.recorder.crnt_block.closeblock(link)
diff --git a/rpython/flowspace/test/cant_import.py b/rpython/flowspace/test/cant_import.py
new file mode 100644
--- /dev/null
+++ b/rpython/flowspace/test/cant_import.py
@@ -0,0 +1,1 @@
+raise ImportError("some explanation here")
diff --git a/rpython/flowspace/test/test_objspace.py b/rpython/flowspace/test/test_objspace.py
--- a/rpython/flowspace/test/test_objspace.py
+++ b/rpython/flowspace/test/test_objspace.py
@@ -816,6 +816,12 @@
             from rpython import this_does_not_exist
         py.test.raises(ImportError, 'self.codetest(f)')
 
+    def test_importerror_3(self):
+        def f():
+            import rpython.flowspace.test.cant_import
+        e = py.test.raises(ImportError, 'self.codetest(f)')
+        assert "some explanation here" in str(e.value)
+
     def test_relative_import(self):
         def f():
             from ..objspace import build_flow

From noreply at buildbot.pypy.org  Thu Jun 18 14:56:28 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 18 Jun 2015 14:56:28 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c8: task_done(), join() on stm.queue
Message-ID: <20150618125628.0E8471C1FC6@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c8
Changeset: r78178:5a114ddbc08d
Date: 2015-06-18 14:56 +0200
http://bitbucket.org/pypy/pypy/changeset/5a114ddbc08d/

Log:	task_done(), join() on stm.queue

diff --git a/pypy/module/pypystm/queue.py b/pypy/module/pypystm/queue.py
--- a/pypy/module/pypystm/queue.py
+++ b/pypy/module/pypystm/queue.py
@@ -32,14 +32,8 @@
     @unwrap_spec(block=int)
     def get_w(self, space, block=1, w_timeout=None):
         """Remove and return an item from the queue.
-
-        If optional args 'block' is true and 'timeout' is None (the default),
-        block if necessary until an item is available. If 'timeout' is
-        a non-negative number, it blocks at most 'timeout' seconds and raises
-        the Empty exception if no item was available within that time.
-        Otherwise ('block' is false), return an item if one is immediately
-        available, else raise the Empty exception ('timeout' is ignored
-        in that case).
+        The 'block' and 'timeout' arguments are like Queue.Queue.get().
+        Note that using them is inefficient so far.
         """
         if block == 0:
             timeout = 0.0     # 'w_timeout' ignored in this case
@@ -62,6 +56,28 @@
                                  space.w_None)
         return cast_gcref_to_instance(W_Root, gcref)
 
+    def task_done_w(self, space):
+        """Indicate that a formerly enqueued task is complete.
+        See Queue.Queue.task_done().
+
+        Note that we cannot easily detect if task_done() is called more
+        times than there were items placed in the queue.  This situation
+        is detect by join() instead.
+        """
+        self.q.task_done()
+
+    def join_w(self, space):
+        """Blocks until all items in the Queue have been gotten and processed.
+        See Queue.Queue.join().
+
+        Raises ValueError if we detect that task_done() has been called
+        more times than there were items placed in the queue.
+        """
+        res = self.q.join()
+        if res != 0:
+            raise oefmt('task_done() called too many times (%d more than '
+                        'there were items placed in the queue)', -res)
+
 
 def W_Queue___new__(space, w_subtype):
     r = space.allocate_instance(W_Queue, w_subtype)
@@ -73,4 +89,6 @@
     __new__ = interp2app(W_Queue___new__),
     get = interp2app(W_Queue.get_w),
     put = interp2app(W_Queue.put_w),
+    task_done = interp2app(W_Queue.task_done_w),
+    join = interp2app(W_Queue.join_w),
 )
diff --git a/pypy/module/pypystm/test/test_queue.py b/pypy/module/pypystm/test/test_queue.py
--- a/pypy/module/pypystm/test/test_queue.py
+++ b/pypy/module/pypystm/test/test_queue.py
@@ -18,3 +18,12 @@
         q.put(obj)
         obj1 = q.get(timeout=0.01)
         assert obj1 is obj
+
+    def test_task_done(self):
+        import pypystm
+        q = pypystm.queue()
+        q.put([])
+        q.get()
+        # --q.join() here would cause deadlock, but hard to test--
+        q.task_done()
+        q.join()

From noreply at buildbot.pypy.org  Thu Jun 18 23:06:53 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Thu, 18 Jun 2015 23:06:53 +0200 (CEST)
Subject: [pypy-commit] pypy py3.3: Port _decimal to CFFI 1.0.
Message-ID: <20150618210653.11ADE1C1FE6@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob <me at manueljacob.de>
Branch: py3.3
Changeset: r78193:3c835daef6cc
Date: 2015-06-18 23:05 +0200
http://bitbucket.org/pypy/pypy/changeset/3c835daef6cc/

Log:	Port _decimal to CFFI 1.0.

diff --git a/.hgignore b/.hgignore
--- a/.hgignore
+++ b/.hgignore
@@ -75,6 +75,7 @@
 ^lib_pypy/__pycache__$
 ^lib_pypy/ctypes_config_cache/_.+_cache\.py$
 ^lib_pypy/ctypes_config_cache/_.+_.+_\.py$
+^lib_pypy/_libmpdec/.+.o$
 ^rpython/translator/cli/query-descriptions$
 ^pypy/doc/discussion/.+\.html$
 ^include/.+\.h$
diff --git a/lib_pypy/_decimal.py b/lib_pypy/_decimal.py
--- a/lib_pypy/_decimal.py
+++ b/lib_pypy/_decimal.py
@@ -1,11 +1,12 @@
 # Implementation of the "decimal" module, based on libmpdec library.
 
-from cffi import FFI as _FFI
 import collections as _collections
 import math as _math
 import numbers as _numbers
 import sys as _sys
 
+from _decimal_cffi import ffi as _ffi, lib as _mpdec
+
 # Compatibility with the C version
 HAVE_THREADS = True
 if _sys.maxsize == 2**63-1:
@@ -94,278 +95,6 @@
 class FloatOperation(DecimalException, TypeError):
     __module__ = 'decimal'
 
-# Bindings to the libmpdec library
-
-_ffi = _FFI()
-_ffi.cdef("""
-typedef size_t mpd_size_t; /* unsigned size type */
-typedef ssize_t mpd_ssize_t; /* signed size type */
-typedef size_t mpd_uint_t;
-#define MPD_SIZE_MAX ...
-#define MPD_SSIZE_MIN ...
-#define MPD_SSIZE_MAX ...
-
-const char *mpd_version(void);
-void mpd_free(void *ptr);
-
-typedef struct mpd_context_t {
-    mpd_ssize_t prec;   /* precision */
-    mpd_ssize_t emax;   /* max positive exp */
-    mpd_ssize_t emin;   /* min negative exp */
-    uint32_t traps;     /* status events that should be trapped */
-    uint32_t status;    /* status flags */
-    uint32_t newtrap;   /* set by mpd_addstatus_raise() */
-    int      round;     /* rounding mode */
-    int      clamp;     /* clamp mode */
-    int      allcr;     /* all functions correctly rounded */
-} mpd_context_t;
-
-enum {
-    MPD_ROUND_UP,          /* round away from 0               */
-    MPD_ROUND_DOWN,        /* round toward 0 (truncate)       */
-    MPD_ROUND_CEILING,     /* round toward +infinity          */
-    MPD_ROUND_FLOOR,       /* round toward -infinity          */
-    MPD_ROUND_HALF_UP,     /* 0.5 is rounded up               */
-    MPD_ROUND_HALF_DOWN,   /* 0.5 is rounded down             */
-    MPD_ROUND_HALF_EVEN,   /* 0.5 is rounded to even          */
-    MPD_ROUND_05UP,        /* round zero or five away from 0  */
-    MPD_ROUND_TRUNC,       /* truncate, but set infinity      */
-    MPD_ROUND_GUARD
-};
-
-#define MPD_Clamped             ...
-#define MPD_Conversion_syntax   ...
-#define MPD_Division_by_zero    ...
-#define MPD_Division_impossible ...
-#define MPD_Division_undefined  ...
-#define MPD_Fpu_error           ...
-#define MPD_Inexact             ...
-#define MPD_Invalid_context     ...
-#define MPD_Invalid_operation   ...
-#define MPD_Malloc_error        ...
-#define MPD_Not_implemented     ...
-#define MPD_Overflow            ...
-#define MPD_Rounded             ...
-#define MPD_Subnormal           ...
-#define MPD_Underflow           ...
-#define MPD_Max_status          ...
-/* Conditions that result in an IEEE 754 exception */
-#define MPD_IEEE_Invalid_operation ...
-/* Errors that require the result of an operation to be set to NaN */
-#define MPD_Errors              ...
-
-
-
-void mpd_maxcontext(mpd_context_t *ctx);
-int mpd_qsetprec(mpd_context_t *ctx, mpd_ssize_t prec);
-int mpd_qsetemax(mpd_context_t *ctx, mpd_ssize_t emax);
-int mpd_qsetemin(mpd_context_t *ctx, mpd_ssize_t emin);
-int mpd_qsetround(mpd_context_t *ctx, int newround);
-int mpd_qsettraps(mpd_context_t *ctx, uint32_t flags);
-int mpd_qsetstatus(mpd_context_t *ctx, uint32_t flags);
-int mpd_qsetclamp(mpd_context_t *ctx, int c);
-
-
-
-
-typedef struct mpd_t {
-    uint8_t flags;
-    mpd_ssize_t exp;
-    mpd_ssize_t digits;
-    mpd_ssize_t len;
-    mpd_ssize_t alloc;
-    mpd_uint_t *data;
-} mpd_t;
-
-#define MPD_POS                 ...
-#define MPD_NEG                 ...
-#define MPD_INF                 ...
-#define MPD_NAN                 ...
-#define MPD_SNAN                ...
-#define MPD_SPECIAL             ...
-#define MPD_STATIC              ...
-#define MPD_STATIC_DATA         ...
-#define MPD_SHARED_DATA         ...
-#define MPD_CONST_DATA          ...
-#define MPD_DATAFLAGS           ...
-
-
-mpd_t *mpd_qnew(void);
-void mpd_del(mpd_t *dec);
-
-
-/* Operations */
-void mpd_qabs(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, uint32_t *status);
-void mpd_qplus(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, uint32_t *status);
-void mpd_qminus(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, uint32_t *status);
-void mpd_qsqrt(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, uint32_t *status);
-void mpd_qexp(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, uint32_t *status);
-void mpd_qln(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, uint32_t *status);
-void mpd_qlog10(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, uint32_t *status);
-void mpd_qlogb(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, uint32_t *status);
-void mpd_qinvert(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, uint32_t *status);
-
-void mpd_qmax(mpd_t *result, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status);
-void mpd_qmax_mag(mpd_t *result, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status);
-void mpd_qmin(mpd_t *result, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status);
-void mpd_qmin_mag(mpd_t *result, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status);
-
-void mpd_qadd(mpd_t *result, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status);
-void mpd_qsub(mpd_t *result, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status);
-void mpd_qmul(mpd_t *result, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status);
-void mpd_qdiv(mpd_t *q, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status);
-void mpd_qdivint(mpd_t *q, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status);
-void mpd_qfma(mpd_t *result, const mpd_t *a, const mpd_t *b, const mpd_t *c, const mpd_context_t *ctx, uint32_t *status);
-void mpd_qrem(mpd_t *r, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status);
-void mpd_qrem_near(mpd_t *r, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status);
-void mpd_qpow(mpd_t *result, const mpd_t *base, const mpd_t *exp, const mpd_context_t *ctx, uint32_t *status);
-void mpd_qpowmod(mpd_t *result, const mpd_t *base, const mpd_t *exp, const mpd_t *mod, const mpd_context_t *ctx, uint32_t *status);
-int mpd_qcopy_sign(mpd_t *result, const mpd_t *a, const mpd_t *b, uint32_t *status);
-int mpd_qcopy_abs(mpd_t *result, const mpd_t *a, uint32_t *status);
-int mpd_qcopy_negate(mpd_t *result, const mpd_t *a, uint32_t *status);
-void mpd_qdivmod(mpd_t *q, mpd_t *r, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status);
-void mpd_qand(mpd_t *result, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status);
-void mpd_qor(mpd_t *result, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status);
-void mpd_qxor(mpd_t *result, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status);
-int mpd_same_quantum(const mpd_t *a, const mpd_t *b);
-
-void mpd_qround_to_intx(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, uint32_t *status);
-void mpd_qround_to_int(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, uint32_t *status);
-int mpd_qcopy(mpd_t *result, const mpd_t *a,  uint32_t *status);
-
-int mpd_qcmp(const mpd_t *a, const mpd_t *b, uint32_t *status);
-int mpd_qcompare(mpd_t *result, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status);
-int mpd_qcompare_signal(mpd_t *result, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status);
-int mpd_compare_total(mpd_t *result, const mpd_t *a, const mpd_t *b);
-int mpd_compare_total_mag(mpd_t *result, const mpd_t *a, const mpd_t *b);
-void mpd_qnext_toward(mpd_t *result, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status);
-void mpd_qnext_minus(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, uint32_t *status);
-void mpd_qnext_plus(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, uint32_t *status);
-void mpd_qquantize(mpd_t *result, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status);
-
-void mpd_qrotate(mpd_t *result, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status);
-void mpd_qscaleb(mpd_t *result, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status);
-void mpd_qshift(mpd_t *result, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status);
-void mpd_qreduce(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, uint32_t *status);
-
-/* Get attributes */
-uint8_t mpd_sign(const mpd_t *dec);
-int mpd_isnegative(const mpd_t *dec);
-int mpd_ispositive(const mpd_t *dec);
-int mpd_iszero(const mpd_t *dec);
-int mpd_isfinite(const mpd_t *dec);
-int mpd_isinfinite(const mpd_t *dec);
-int mpd_issigned(const mpd_t *dec);
-int mpd_isnan(const mpd_t *dec);
-int mpd_issnan(const mpd_t *dec);
-int mpd_isspecial(const mpd_t *dec);
-int mpd_isqnan(const mpd_t *dec);
-int mpd_isnormal(const mpd_t *dec, const mpd_context_t *ctx);
-int mpd_issubnormal(const mpd_t *dec, const mpd_context_t *ctx);
-mpd_ssize_t mpd_adjexp(const mpd_t *dec);
-mpd_ssize_t mpd_etiny(const mpd_context_t *ctx);
-mpd_ssize_t mpd_etop(const mpd_context_t *ctx);
-
-mpd_t *mpd_qncopy(const mpd_t *a);
-
-/* Set attributes */
-void mpd_set_sign(mpd_t *result, uint8_t sign);
-void mpd_set_positive(mpd_t *result);
-void mpd_clear_flags(mpd_t *result);
-void mpd_seterror(mpd_t *result, uint32_t flags, uint32_t *status);
-void mpd_setspecial(mpd_t *dec, uint8_t sign, uint8_t type);
-
-/* I/O */
-void mpd_qimport_u16(mpd_t *result, const uint16_t *srcdata, size_t srclen,
-                     uint8_t srcsign, uint32_t srcbase,
-                     const mpd_context_t *ctx, uint32_t *status);
-size_t mpd_qexport_u16(uint16_t **rdata, size_t rlen, uint32_t base,
-                       const mpd_t *src, uint32_t *status);
-void mpd_qset_string(mpd_t *dec, const char *s, const mpd_context_t *ctx, uint32_t *status);
-void mpd_qset_uint(mpd_t *result, mpd_uint_t a, const mpd_context_t *ctx, uint32_t *status);
-void mpd_qset_ssize(mpd_t *result, mpd_ssize_t a, const mpd_context_t *ctx, uint32_t *status);
-void mpd_qsset_ssize(mpd_t *result, mpd_ssize_t a, const mpd_context_t *ctx, uint32_t *status);
-mpd_ssize_t mpd_qget_ssize(const mpd_t *dec, uint32_t *status);
-int mpd_lsnprint_signals(char *dest, int nmemb, uint32_t flags, const char *signal_string[]);
-#define MPD_MAX_SIGNAL_LIST ...
-const char *dec_signal_string[];
-
-void mpd_qfinalize(mpd_t *result, const mpd_context_t *ctx, uint32_t *status);
-const char *mpd_class(const mpd_t *a, const mpd_context_t *ctx);
-
-/* format specification */
-typedef struct mpd_spec_t {
-    mpd_ssize_t min_width; /* minimum field width */
-    mpd_ssize_t prec;      /* fraction digits or significant digits */
-    char type;             /* conversion specifier */
-    char align;            /* alignment */
-    char sign;             /* sign printing/alignment */
-    char fill[5];          /* fill character */
-    const char *dot;       /* decimal point */
-    const char *sep;       /* thousands separator */
-    const char *grouping;  /* grouping of digits */
-} mpd_spec_t;
-
-char *mpd_to_sci(const mpd_t *dec, int fmt);
-char *mpd_to_eng(const mpd_t *dec, int fmt);
-int mpd_parse_fmt_str(mpd_spec_t *spec, const char *fmt, int caps);
-int mpd_validate_lconv(mpd_spec_t *spec);
-char *mpd_qformat_spec(const mpd_t *dec, const mpd_spec_t *spec, const mpd_context_t *ctx, uint32_t *status);
-
-""")
-
-import os
-
-_libdir = os.path.join(os.path.dirname(__file__), '_libmpdec')
-_mpdec = _ffi.verify(
-    """
-#include "mpdecimal.h"
-
-const char *dec_signal_string[MPD_NUM_FLAGS] = {
-    "Clamped",
-    "InvalidOperation",
-    "DivisionByZero",
-    "InvalidOperation",
-    "InvalidOperation",
-    "InvalidOperation",
-    "Inexact",
-    "InvalidOperation",
-    "InvalidOperation",
-    "InvalidOperation",
-    "FloatOperation",
-    "Overflow",
-    "Rounded",
-    "Subnormal",
-    "Underflow",
-};
-""",
-    sources=[os.path.join(_libdir, 'mpdecimal.c'),
-             os.path.join(_libdir, 'basearith.c'),
-             os.path.join(_libdir, 'convolute.c'),
-             os.path.join(_libdir, 'constants.c'),
-             os.path.join(_libdir, 'context.c'),
-             os.path.join(_libdir, 'io.c'),
-             os.path.join(_libdir, 'fourstep.c'),
-             os.path.join(_libdir, 'sixstep.c'),
-             os.path.join(_libdir, 'transpose.c'),
-             os.path.join(_libdir, 'difradix2.c'),
-             os.path.join(_libdir, 'numbertheory.c'),
-             os.path.join(_libdir, 'fnt.c'),
-             os.path.join(_libdir, 'crt.c'),
-             os.path.join(_libdir, 'memory.c'),
-         ],
-    include_dirs=[_libdir],
-    extra_compile_args=[
-        "-DANSI",
-        "-DHAVE_STDINT_H",
-        "-DHAVE_INTTYPES_H",
-        "-DCONFIG_64" if _sys.maxsize > 1 << 32 else "-DCONFIG_32",
-    ],
-)
-
-del os
-
-_mpdec.MPD_Float_operation = _mpdec.MPD_Not_implemented
 
 __version__ = "1.70"
 __libmpdec_version__ = _ffi.string(_mpdec.mpd_version())
diff --git a/lib_pypy/_decimal_build.py b/lib_pypy/_decimal_build.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_decimal_build.py
@@ -0,0 +1,277 @@
+import os
+import sys
+
+from cffi import FFI
+
+
+ffi = FFI()
+ffi.cdef("""
+typedef size_t mpd_size_t; /* unsigned size type */
+typedef ssize_t mpd_ssize_t; /* signed size type */
+typedef size_t mpd_uint_t;
+#define MPD_SIZE_MAX ...
+#define MPD_SSIZE_MIN ...
+#define MPD_SSIZE_MAX ...
+
+const char *mpd_version(void);
+void mpd_free(void *ptr);
+
+typedef struct mpd_context_t {
+    mpd_ssize_t prec;   /* precision */
+    mpd_ssize_t emax;   /* max positive exp */
+    mpd_ssize_t emin;   /* min negative exp */
+    uint32_t traps;     /* status events that should be trapped */
+    uint32_t status;    /* status flags */
+    uint32_t newtrap;   /* set by mpd_addstatus_raise() */
+    int      round;     /* rounding mode */
+    int      clamp;     /* clamp mode */
+    int      allcr;     /* all functions correctly rounded */
+} mpd_context_t;
+
+enum {
+    MPD_ROUND_UP,          /* round away from 0               */
+    MPD_ROUND_DOWN,        /* round toward 0 (truncate)       */
+    MPD_ROUND_CEILING,     /* round toward +infinity          */
+    MPD_ROUND_FLOOR,       /* round toward -infinity          */
+    MPD_ROUND_HALF_UP,     /* 0.5 is rounded up               */
+    MPD_ROUND_HALF_DOWN,   /* 0.5 is rounded down             */
+    MPD_ROUND_HALF_EVEN,   /* 0.5 is rounded to even          */
+    MPD_ROUND_05UP,        /* round zero or five away from 0  */
+    MPD_ROUND_TRUNC,       /* truncate, but set infinity      */
+    MPD_ROUND_GUARD
+};
+
+#define MPD_Clamped             ...
+#define MPD_Conversion_syntax   ...
+#define MPD_Division_by_zero    ...
+#define MPD_Division_impossible ...
+#define MPD_Division_undefined  ...
+#define MPD_Float_operation     ...
+#define MPD_Fpu_error           ...
+#define MPD_Inexact             ...
+#define MPD_Invalid_context     ...
+#define MPD_Invalid_operation   ...
+#define MPD_Malloc_error        ...
+#define MPD_Not_implemented     ...
+#define MPD_Overflow            ...
+#define MPD_Rounded             ...
+#define MPD_Subnormal           ...
+#define MPD_Underflow           ...
+#define MPD_Max_status          ...
+/* Conditions that result in an IEEE 754 exception */
+#define MPD_IEEE_Invalid_operation ...
+/* Errors that require the result of an operation to be set to NaN */
+#define MPD_Errors              ...
+
+
+
+void mpd_maxcontext(mpd_context_t *ctx);
+int mpd_qsetprec(mpd_context_t *ctx, mpd_ssize_t prec);
+int mpd_qsetemax(mpd_context_t *ctx, mpd_ssize_t emax);
+int mpd_qsetemin(mpd_context_t *ctx, mpd_ssize_t emin);
+int mpd_qsetround(mpd_context_t *ctx, int newround);
+int mpd_qsettraps(mpd_context_t *ctx, uint32_t flags);
+int mpd_qsetstatus(mpd_context_t *ctx, uint32_t flags);
+int mpd_qsetclamp(mpd_context_t *ctx, int c);
+
+
+
+
+typedef struct mpd_t {
+    uint8_t flags;
+    mpd_ssize_t exp;
+    mpd_ssize_t digits;
+    mpd_ssize_t len;
+    mpd_ssize_t alloc;
+    mpd_uint_t *data;
+} mpd_t;
+
+#define MPD_POS                 ...
+#define MPD_NEG                 ...
+#define MPD_INF                 ...
+#define MPD_NAN                 ...
+#define MPD_SNAN                ...
+#define MPD_SPECIAL             ...
+#define MPD_STATIC              ...
+#define MPD_STATIC_DATA         ...
+#define MPD_SHARED_DATA         ...
+#define MPD_CONST_DATA          ...
+#define MPD_DATAFLAGS           ...
+
+
+mpd_t *mpd_qnew(void);
+void mpd_del(mpd_t *dec);
+
+
+/* Operations */
+void mpd_qabs(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, uint32_t *status);
+void mpd_qplus(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, uint32_t *status);
+void mpd_qminus(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, uint32_t *status);
+void mpd_qsqrt(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, uint32_t *status);
+void mpd_qexp(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, uint32_t *status);
+void mpd_qln(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, uint32_t *status);
+void mpd_qlog10(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, uint32_t *status);
+void mpd_qlogb(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, uint32_t *status);
+void mpd_qinvert(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, uint32_t *status);
+
+void mpd_qmax(mpd_t *result, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status);
+void mpd_qmax_mag(mpd_t *result, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status);
+void mpd_qmin(mpd_t *result, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status);
+void mpd_qmin_mag(mpd_t *result, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status);
+
+void mpd_qadd(mpd_t *result, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status);
+void mpd_qsub(mpd_t *result, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status);
+void mpd_qmul(mpd_t *result, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status);
+void mpd_qdiv(mpd_t *q, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status);
+void mpd_qdivint(mpd_t *q, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status);
+void mpd_qfma(mpd_t *result, const mpd_t *a, const mpd_t *b, const mpd_t *c, const mpd_context_t *ctx, uint32_t *status);
+void mpd_qrem(mpd_t *r, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status);
+void mpd_qrem_near(mpd_t *r, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status);
+void mpd_qpow(mpd_t *result, const mpd_t *base, const mpd_t *exp, const mpd_context_t *ctx, uint32_t *status);
+void mpd_qpowmod(mpd_t *result, const mpd_t *base, const mpd_t *exp, const mpd_t *mod, const mpd_context_t *ctx, uint32_t *status);
+int mpd_qcopy_sign(mpd_t *result, const mpd_t *a, const mpd_t *b, uint32_t *status);
+int mpd_qcopy_abs(mpd_t *result, const mpd_t *a, uint32_t *status);
+int mpd_qcopy_negate(mpd_t *result, const mpd_t *a, uint32_t *status);
+void mpd_qdivmod(mpd_t *q, mpd_t *r, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status);
+void mpd_qand(mpd_t *result, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status);
+void mpd_qor(mpd_t *result, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status);
+void mpd_qxor(mpd_t *result, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status);
+int mpd_same_quantum(const mpd_t *a, const mpd_t *b);
+
+void mpd_qround_to_intx(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, uint32_t *status);
+void mpd_qround_to_int(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, uint32_t *status);
+int mpd_qcopy(mpd_t *result, const mpd_t *a,  uint32_t *status);
+
+int mpd_qcmp(const mpd_t *a, const mpd_t *b, uint32_t *status);
+int mpd_qcompare(mpd_t *result, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status);
+int mpd_qcompare_signal(mpd_t *result, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status);
+int mpd_compare_total(mpd_t *result, const mpd_t *a, const mpd_t *b);
+int mpd_compare_total_mag(mpd_t *result, const mpd_t *a, const mpd_t *b);
+void mpd_qnext_toward(mpd_t *result, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status);
+void mpd_qnext_minus(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, uint32_t *status);
+void mpd_qnext_plus(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, uint32_t *status);
+void mpd_qquantize(mpd_t *result, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status);
+
+void mpd_qrotate(mpd_t *result, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status);
+void mpd_qscaleb(mpd_t *result, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status);
+void mpd_qshift(mpd_t *result, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status);
+void mpd_qreduce(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, uint32_t *status);
+
+/* Get attributes */
+uint8_t mpd_sign(const mpd_t *dec);
+int mpd_isnegative(const mpd_t *dec);
+int mpd_ispositive(const mpd_t *dec);
+int mpd_iszero(const mpd_t *dec);
+int mpd_isfinite(const mpd_t *dec);
+int mpd_isinfinite(const mpd_t *dec);
+int mpd_issigned(const mpd_t *dec);
+int mpd_isnan(const mpd_t *dec);
+int mpd_issnan(const mpd_t *dec);
+int mpd_isspecial(const mpd_t *dec);
+int mpd_isqnan(const mpd_t *dec);
+int mpd_isnormal(const mpd_t *dec, const mpd_context_t *ctx);
+int mpd_issubnormal(const mpd_t *dec, const mpd_context_t *ctx);
+mpd_ssize_t mpd_adjexp(const mpd_t *dec);
+mpd_ssize_t mpd_etiny(const mpd_context_t *ctx);
+mpd_ssize_t mpd_etop(const mpd_context_t *ctx);
+
+mpd_t *mpd_qncopy(const mpd_t *a);
+
+/* Set attributes */
+void mpd_set_sign(mpd_t *result, uint8_t sign);
+void mpd_set_positive(mpd_t *result);
+void mpd_clear_flags(mpd_t *result);
+void mpd_seterror(mpd_t *result, uint32_t flags, uint32_t *status);
+void mpd_setspecial(mpd_t *dec, uint8_t sign, uint8_t type);
+
+/* I/O */
+void mpd_qimport_u16(mpd_t *result, const uint16_t *srcdata, size_t srclen,
+                     uint8_t srcsign, uint32_t srcbase,
+                     const mpd_context_t *ctx, uint32_t *status);
+size_t mpd_qexport_u16(uint16_t **rdata, size_t rlen, uint32_t base,
+                       const mpd_t *src, uint32_t *status);
+void mpd_qset_string(mpd_t *dec, const char *s, const mpd_context_t *ctx, uint32_t *status);
+void mpd_qset_uint(mpd_t *result, mpd_uint_t a, const mpd_context_t *ctx, uint32_t *status);
+void mpd_qset_ssize(mpd_t *result, mpd_ssize_t a, const mpd_context_t *ctx, uint32_t *status);
+void mpd_qsset_ssize(mpd_t *result, mpd_ssize_t a, const mpd_context_t *ctx, uint32_t *status);
+mpd_ssize_t mpd_qget_ssize(const mpd_t *dec, uint32_t *status);
+int mpd_lsnprint_signals(char *dest, int nmemb, uint32_t flags, const char *signal_string[]);
+#define MPD_MAX_SIGNAL_LIST ...
+const char *dec_signal_string[];
+
+void mpd_qfinalize(mpd_t *result, const mpd_context_t *ctx, uint32_t *status);
+const char *mpd_class(const mpd_t *a, const mpd_context_t *ctx);
+
+/* format specification */
+typedef struct mpd_spec_t {
+    mpd_ssize_t min_width; /* minimum field width */
+    mpd_ssize_t prec;      /* fraction digits or significant digits */
+    char type;             /* conversion specifier */
+    char align;            /* alignment */
+    char sign;             /* sign printing/alignment */
+    char fill[5];          /* fill character */
+    const char *dot;       /* decimal point */
+    const char *sep;       /* thousands separator */
+    const char *grouping;  /* grouping of digits */
+} mpd_spec_t;
+
+char *mpd_to_sci(const mpd_t *dec, int fmt);
+char *mpd_to_eng(const mpd_t *dec, int fmt);
+int mpd_parse_fmt_str(mpd_spec_t *spec, const char *fmt, int caps);
+int mpd_validate_lconv(mpd_spec_t *spec);
+char *mpd_qformat_spec(const mpd_t *dec, const mpd_spec_t *spec, const mpd_context_t *ctx, uint32_t *status);
+
+""")
+
+_libdir = os.path.join(os.path.dirname(__file__), '_libmpdec')
+ffi.set_source('_decimal_cffi',
+    """
+#include "mpdecimal.h"
+
+#define MPD_Float_operation MPD_Not_implemented
+
+const char *dec_signal_string[MPD_NUM_FLAGS] = {
+    "Clamped",
+    "InvalidOperation",
+    "DivisionByZero",
+    "InvalidOperation",
+    "InvalidOperation",
+    "InvalidOperation",
+    "Inexact",
+    "InvalidOperation",
+    "InvalidOperation",
+    "InvalidOperation",
+    "FloatOperation",
+    "Overflow",
+    "Rounded",
+    "Subnormal",
+    "Underflow",
+};
+""",
+    sources=[os.path.join(_libdir, 'mpdecimal.c'),
+             os.path.join(_libdir, 'basearith.c'),
+             os.path.join(_libdir, 'convolute.c'),
+             os.path.join(_libdir, 'constants.c'),
+             os.path.join(_libdir, 'context.c'),
+             os.path.join(_libdir, 'io.c'),
+             os.path.join(_libdir, 'fourstep.c'),
+             os.path.join(_libdir, 'sixstep.c'),
+             os.path.join(_libdir, 'transpose.c'),
+             os.path.join(_libdir, 'difradix2.c'),
+             os.path.join(_libdir, 'numbertheory.c'),
+             os.path.join(_libdir, 'fnt.c'),
+             os.path.join(_libdir, 'crt.c'),
+             os.path.join(_libdir, 'memory.c'),
+         ],
+    include_dirs=[_libdir],
+    extra_compile_args=[
+        "-DANSI",
+        "-DHAVE_STDINT_H",
+        "-DHAVE_INTTYPES_H",
+        "-DCONFIG_64" if sys.maxsize > 1 << 32 else "-DCONFIG_32",
+    ],
+)
+
+
+if __name__ == '__main__':
+    ffi.compile()
diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py
--- a/pypy/tool/release/package.py
+++ b/pypy/tool/release/package.py
@@ -62,7 +62,7 @@
     "_gdbm": "_gdbm_build.py"  if sys.platform != "win32" else None,
     "pwdgrp": "_pwdgrp_build.py" if sys.platform != "win32" else None,
     "lzma": "_lzma_build.py",
-    "_decimal": None, # XXX change _decimal to use CFFI 1.0
+    "_decimal": "_decimal_build.py",
     "xx": None,    # for testing: 'None' should be completely ignored
     }
 

From noreply at buildbot.pypy.org  Sun Jun 21 09:27:43 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 21 Jun 2015 09:27:43 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c8: merge heads
Message-ID: <20150621072743.266EA1C130F@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c8
Changeset: r78221:e4e69af5598c
Date: 2015-06-21 09:27 +0200
http://bitbucket.org/pypy/pypy/changeset/e4e69af5598c/

Log:	merge heads

diff --git a/rpython/translator/stm/src_stm/extracode.h b/rpython/translator/stm/src_stm/extracode.h
--- a/rpython/translator/stm/src_stm/extracode.h
+++ b/rpython/translator/stm/src_stm/extracode.h
@@ -133,12 +133,13 @@
         }
     }
 
+    uintptr_t next_instr = marker->odd_number >> 1;
+
     ll = _fetch_strlen(segment_base, co_lnotab);
     if (ll > 0) {
         long lnotablen = ll;
         unsigned char *lnotab = (unsigned char *)_fetch_stritems(segment_base,
                                                                  co_lnotab);
-        uintptr_t next_instr = marker->odd_number >> 1;
         line = co_firstlineno;
         uintptr_t ii, curaddr = 0;
         for (ii = 0; ii < lnotablen; ii += 2) {
@@ -151,8 +152,9 @@
 
     int result;
     result = snprintf(outputbuf, outputbufsize,
-                      "File \"%s%.*s\", line %ld, in %.*s%s",
-                      fntrunc, (int)fnlen, fn, line, (int)nlen, name, ntrunc);
+                      "File \"%s%.*s\", line %ld, in %.*s%s (#%ld)",
+                      fntrunc, (int)fnlen, fn, line, (int)nlen,
+                      name, ntrunc, next_instr);
     if (result >= outputbufsize)
         result = outputbufsize - 1;
     if (result < 0)
diff --git a/rpython/translator/stm/src_stm/revision b/rpython/translator/stm/src_stm/revision
--- a/rpython/translator/stm/src_stm/revision
+++ b/rpython/translator/stm/src_stm/revision
@@ -1,1 +1,1 @@
-9cb167448d92
+9ffba4fe03df
diff --git a/rpython/translator/stm/src_stm/stm/core.c b/rpython/translator/stm/src_stm/stm/core.c
--- a/rpython/translator/stm/src_stm/stm/core.c
+++ b/rpython/translator/stm/src_stm/stm/core.c
@@ -1365,7 +1365,8 @@
     }
 
     if (STM_PSEGMENT->active_queues)
-        queues_deactivate_all(/*at_commit=*/true);
+        queues_deactivate_all(get_priv_segment(STM_SEGMENT->segment_num),
+                              /*at_commit=*/true);
 
     invoke_and_clear_user_callbacks(0);   /* for commit */
 
@@ -1476,6 +1477,9 @@
 #endif
     tl->thread_local_obj = pseg->threadlocal_at_start_of_transaction;
 
+    if (pseg->active_queues)
+        queues_deactivate_all(pseg, /*at_commit=*/false);
+
 
     /* Set the next nursery_mark: first compute the value that
        nursery_mark must have had at the start of the aborted transaction */
@@ -1521,9 +1525,6 @@
     if (tl->mem_clear_on_abort)
         memset(tl->mem_clear_on_abort, 0, tl->mem_bytes_to_clear_on_abort);
 
-    if (STM_PSEGMENT->active_queues)
-        queues_deactivate_all(/*at_commit=*/false);
-
     invoke_and_clear_user_callbacks(1);   /* for abort */
 
     if (is_abort(STM_SEGMENT->nursery_end)) {
@@ -1570,6 +1571,8 @@
 {
     int num_waits = 0;
 
+    timing_become_inevitable();
+
  retry_from_start:
     assert(STM_PSEGMENT->transaction_state == TS_REGULAR);
     _stm_collectable_safe_point();
@@ -1619,7 +1622,6 @@
         if (!_validate_and_turn_inevitable())
             return;
     }
-    timing_become_inevitable();
     soon_finished_or_inevitable_thread_segment();
     STM_PSEGMENT->transaction_state = TS_INEVITABLE;
 
diff --git a/rpython/translator/stm/src_stm/stm/queue.c b/rpython/translator/stm/src_stm/stm/queue.c
--- a/rpython/translator/stm/src_stm/stm/queue.c
+++ b/rpython/translator/stm/src_stm/stm/queue.c
@@ -126,16 +126,21 @@
     }
 }
 
-static void queues_deactivate_all(bool at_commit)
+static void queues_deactivate_all(struct stm_priv_segment_info_s *pseg,
+                                  bool at_commit)
 {
-    queue_lock_acquire();
+#pragma push_macro("STM_PSEGMENT")
+#pragma push_macro("STM_SEGMENT")
+#undef STM_PSEGMENT
+#undef STM_SEGMENT
+    spinlock_acquire(pseg->active_queues_lock);
 
     bool added_any_old_entries = false;
     bool finished_more_tasks = false;
     wlog_t *item;
-    TREE_LOOP_FORWARD(STM_PSEGMENT->active_queues, item) {
+    TREE_LOOP_FORWARD(pseg->active_queues, item) {
         stm_queue_t *queue = (stm_queue_t *)item->addr;
-        stm_queue_segment_t *seg = &queue->segs[STM_SEGMENT->segment_num - 1];
+        stm_queue_segment_t *seg = &queue->segs[pseg->pub.segment_num - 1];
         queue_entry_t *head, *freehead;
 
         if (at_commit) {
@@ -188,16 +193,17 @@
 
     } TREE_LOOP_END;
 
-    tree_free(STM_PSEGMENT->active_queues);
-    STM_PSEGMENT->active_queues = NULL;
+    tree_free(pseg->active_queues);
+    pseg->active_queues = NULL;
 
-    queue_lock_release();
+    spinlock_release(pseg->active_queues_lock);
 
-    assert(_has_mutex());
     if (added_any_old_entries)
         cond_broadcast(C_QUEUE_OLD_ENTRIES);
     if (finished_more_tasks)
         cond_broadcast(C_QUEUE_FINISHED_MORE_TASKS);
+#pragma pop_macro("STM_SEGMENT")
+#pragma pop_macro("STM_PSEGMENT")
 }
 
 void stm_queue_put(object_t *qobj, stm_queue_t *queue, object_t *newitem)
diff --git a/rpython/translator/stm/src_stm/stm/queue.h b/rpython/translator/stm/src_stm/stm/queue.h
--- a/rpython/translator/stm/src_stm/stm/queue.h
+++ b/rpython/translator/stm/src_stm/stm/queue.h
@@ -1,4 +1,5 @@
 /* Imported by rpython/translator/stm/import_stmgc.py */
-static void queues_deactivate_all(bool at_commit);
+static void queues_deactivate_all(struct stm_priv_segment_info_s *pseg,
+                                  bool at_commit);
 static void collect_active_queues(void);           /* minor collections */
 static void mark_visit_from_active_queues(void);   /* major collections */

From noreply at buildbot.pypy.org  Thu Jun 18 13:50:40 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 18 Jun 2015 13:50:40 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: scarily enough to start
	passing unrolling tests
Message-ID: <20150618115040.B4B481C1FC7@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult-unroll
Changeset: r78176:ab923482ef81
Date: 2015-06-18 13:50 +0200
http://bitbucket.org/pypy/pypy/changeset/ab923482ef81/

Log:	scarily enough to start passing unrolling tests

diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -279,39 +279,16 @@
         self.metainterp_sd.profiler.count(jitprof.Counters.OPT_FORCINGS)
         self.resumedata_memo.forget_numberings()
 
-    def getinterned(self, box):
-        constbox = self.get_constant_box(box)
-        if constbox is None:
-            return box
-        if constbox.type == REF:
-            value = constbox.getref_base()
-            if not value:
-                return box
-            return self.interned_refs.setdefault(value, box)
-        #elif constbox.type == INT:
-        #    value = constbox.getint()
-        #    return self.interned_ints.setdefault(value, box)
+    def getinfo(self, op):
+        if op.type == 'r':
+            return self.getptrinfo(op)
+        elif op.type == 'i':
+            return self.getintbound(op)
         else:
-            return box
+            zzz
 
-    ## def getinfo(self, op, create=False):
-    ##     xxx
-    ##     yyy
-
-    ##     XXX
-    ##     box = self.getinterned(box)
-    ##     try:
-    ##         value = self.values[box]
-    ##     except KeyError:
-    ##         if box.type == "r":
-    ##             value = self.values[box] = PtrOptValue(box)
-    ##         elif box.type == "i":
-    ##             value = self.values[box] = IntOptValue(box)
-    ##         else:
-    ##             assert box.type == "f"
-    ##             value = self.values[box] = OptValue(box)
-    ##     self.ensure_imported(value)
-    ##     return value
+    def setinfo_from_preamble(self, op, old_info):
+        pass # deal with later
 
     def get_box_replacement(self, op):
         if op is None:
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -179,9 +179,9 @@
 
         virtual_state = self.get_virtual_state(jump_args)
 
-        values = [self.getinfo(arg) for arg in jump_args]
-        inputargs = virtual_state.make_inputargs(values, self.optimizer)
-        short_inputargs = virtual_state.make_inputargs(values, self.optimizer, keyboxes=True)
+        inputargs = virtual_state.make_inputargs(jump_args, self.optimizer)
+        short_inputargs = virtual_state.make_inputargs(jump_args,
+                                            self.optimizer, keyboxes=True)
 
         if self.boxes_created_this_iteration is not None:
             for box in self.inputargs:
@@ -190,16 +190,22 @@
         short_boxes = ShortBoxes(self.optimizer, inputargs)
 
         self.optimizer.clear_newoperations()
-        for i in range(len(original_jump_args)):
-            srcbox = jump_args[i]
-            if values[i].is_virtual():
-                srcbox = values[i].force_box(self.optimizer)
-            if original_jump_args[i] is not srcbox:
-                opnum = OpHelpers.same_as_for_type(original_jump_args[i].type)
-                op = self.optimizer.replace_op_with(original_jump_args[i],
-                                                    opnum, [srcbox],
-                                                    descr=DONT_CHANGE)
-                self.optimizer.emit_operation(op)
+        # for i in range(len(original_jump_args)):
+        #     srcbox = jump_args[i]
+        #     if srcbox is not original_jump_args[i]:
+        #         xxx
+        #     if srcbox.type != 'r':
+        #         continue
+        #     info = self.optimizer.getptrinfo(srcbox)
+        #     if info and info.is_virtual():
+        #         xxx
+        #         srcbox = values[i].force_box(self.optimizer)
+        #     if original_jump_args[i] is not srcbox:
+        #         opnum = OpHelpers.same_as_for_type(original_jump_args[i].type)
+        #         op = self.optimizer.replace_op_with(original_jump_args[i],
+        #                                             opnum, [srcbox],
+        #                                             descr=DONT_CHANGE)
+        #         self.optimizer.emit_operation(op)
         inputarg_setup_ops = self.optimizer.get_newoperations()
 
         target_token = targetop.getdescr()
@@ -210,10 +216,10 @@
 
         exported_values = {}
         for box in inputargs:
-            exported_values[box] = self.optimizer.getvalue(box)
+            exported_values[box] = self.optimizer.getinfo(box)
         for op in short_boxes.operations():
             if op and op.type != 'v':
-                exported_values[op] = self.optimizer.getvalue(op)
+                exported_values[op] = self.optimizer.getinfo(op)
 
         return ExportedState(short_boxes, inputarg_setup_ops, exported_values)
 
@@ -240,9 +246,8 @@
         self.initial_virtual_state = target_token.virtual_state
 
         for box in self.inputargs:
-            preamble_value = exported_state.exported_values[box]
-            value = self.optimizer.getvalue(box)
-            value.import_from(preamble_value, self.optimizer)
+            preamble_info = exported_state.exported_values[box]
+            self.optimizer.setinfo_from_preamble(box, preamble_info)
 
         # Setup the state of the new optimizer by emiting the
         # short operations and discarding the result
@@ -255,6 +260,7 @@
             self.ensure_short_op_emitted(op, self.optimizer, seen)
             if op and op.type != 'v':
                 preamble_value = exported_state.exported_values[op]
+                continue
                 value = self.optimizer.getvalue(op)
                 if not value.is_virtual() and not value.is_constant():
                     imp = ValueImporter(self, preamble_value, op)
@@ -306,9 +312,10 @@
 
         # Construct jumpargs from the virtual state
         original_jumpargs = jumpop.getarglist()[:]
-        values = [self.getvalue(arg) for arg in jumpop.getarglist()]
+        jump_boxes = [self.get_box_replacement(arg) for arg in
+                      jumpop.getarglist()]
         try:
-            jumpargs = virtual_state.make_inputargs(values, self.optimizer)
+            jumpargs = virtual_state.make_inputargs(jump_boxes, self.optimizer)
         except BadVirtualState:
             raise InvalidLoop('The state of the optimizer at the end of ' +
                               'peeled loop is inconsistent with the ' +
@@ -317,7 +324,7 @@
         jumpop.initarglist(jumpargs)
 
         # Inline the short preamble at the end of the loop
-        jmp_to_short_args = virtual_state.make_inputargs(values,
+        jmp_to_short_args = virtual_state.make_inputargs(jump_boxes,
                                                          self.optimizer,
                                                          keyboxes=True)
         assert len(short_inputargs) == len(jmp_to_short_args)
diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py
@@ -9,6 +9,8 @@
 from rpython.rlib.debug import debug_start, debug_stop, debug_print
 from rpython.rlib.objectmodel import we_are_translated
 
+LEVEL_UNKNOWN = '\x00'
+LEVEL_CONSTANT = '\x01'
 
 class BadVirtualState(Exception):
     pass
@@ -276,9 +278,14 @@
     def debug_header(self, indent):
         debug_print(indent + 'VArrayStructStateInfo(%d):' % self.position)
 
-
 class NotVirtualStateInfo(AbstractVirtualStateInfo):
-    def __init__(self, value, is_opaque=False):
+    lenbound = None
+    intbound = None
+    
+    def __init__(self, box, is_opaque=False):
+        self.level = LEVEL_UNKNOWN
+        return
+        xxx
         self.is_opaque = is_opaque
         self.known_class = value.get_known_class()
         self.level = value.getlevel()
@@ -385,6 +392,8 @@
         assert 0, "unreachable"
 
     def _generate_guards_intbounds(self, other, box, extra_guards):
+        if self.intbound is None:
+            return
         if self.intbound.contains_bound(other.intbound):
             return
         xxx
@@ -450,6 +459,15 @@
         debug_print(indent + mark + 'NotVirtualInfo(%d' % self.position +
                     ', ' + l + ', ' + self.intbound.__repr__() + lb + ')')
 
+class IntNotVirtualStateInfo(NotVirtualStateInfo):
+    def __init__(self, intbound):
+        # XXX do we care about non null?
+        self.intbound = intbound
+        if intbound.is_constant():
+            self.level = LEVEL_CONSTANT
+        else:
+            self.level = LEVEL_UNKNOWN
+
 
 class VirtualState(object):
     def __init__(self, state):
@@ -478,10 +496,11 @@
                                           state)
         return state
 
-    def make_inputargs(self, values, optimizer, keyboxes=False):
+    def make_inputargs(self, inputargs, optimizer, keyboxes=False):
         if optimizer.optearlyforce:
             optimizer = optimizer.optearlyforce
-        assert len(values) == len(self.state)
+        assert len(inputargs) == len(self.state)
+        return inputargs
         inputargs = [None] * self.numnotvirtuals
 
         # We try twice. The first time around we allow boxes to be forced
@@ -555,20 +574,32 @@
             opt = self.optimizer.optearlyforce
         else:
             opt = self.optimizer
+        state = []
         for box in jump_args:
+            box = opt.get_box_replacement(box)
             if box.type == 'r':
-                zxsadsadsa
+                info = opt.getptrinfo(box)
+                if info is not None and info.is_virtual():
+                    xxx
+                else:
+                    state.append(self.visit_not_virtual(box))
+            elif box.type == 'i':
+                intbound = opt.getintbound(box)
+                state.append(self.visit_not_ptr(box, intbound))
+            else:
+                xxx
         #values = [self.getvalue(box).force_at_end_of_preamble(already_forced,
         #                                                      opt)
         #          for box in jump_args]
 
-        #for value in values:
-        #    value.visitor_walk_recursive(self)
-        return VirtualState([self.state(box) for box in jump_args])
+        return VirtualState(state)
 
-    def visit_not_virtual(self, value):
-        is_opaque = value in self.optimizer.opaque_pointers
-        return NotVirtualStateInfo(value, is_opaque)
+    def visit_not_ptr(self, box, intbound):
+        return IntNotVirtualStateInfo(intbound=intbound)
+    
+    def visit_not_virtual(self, box):
+        is_opaque = box in self.optimizer.opaque_pointers
+        return NotVirtualStateInfo(box, is_opaque)
 
     def visit_virtual(self, known_class, fielddescrs):
         return VirtualStateInfo(known_class, fielddescrs)
@@ -643,10 +674,6 @@
         return alts
 
     def add_to_short(self, box, op):
-        xxx
-        #if op:
-        #    op = op.clone(self.memo)
-        #    op.is_source_op = True
         if box in self.short_boxes:
             xxx
             return # XXX avoid those corner cases

From noreply at buildbot.pypy.org  Sun Jun 21 14:10:34 2015
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sun, 21 Jun 2015 14:10:34 +0200 (CEST)
Subject: [pypy-commit] pypy default: Fix test_tcl on 32bit machines.
Message-ID: <20150621121034.0A71C1C066E@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: 
Changeset: r78228:6453697f3a30
Date: 2015-06-21 14:09 +0200
http://bitbucket.org/pypy/pypy/changeset/6453697f3a30/

Log:	Fix test_tcl on 32bit machines.

diff --git a/lib_pypy/_tkinter/tklib_build.py b/lib_pypy/_tkinter/tklib_build.py
--- a/lib_pypy/_tkinter/tklib_build.py
+++ b/lib_pypy/_tkinter/tklib_build.py
@@ -179,6 +179,7 @@
 typedef int... Tcl_WideInt;
 
 int Tcl_GetWideIntFromObj(Tcl_Interp *interp, Tcl_Obj *obj, Tcl_WideInt *value);
+Tcl_Obj *Tcl_NewWideIntObj(Tcl_WideInt value);
 """)
 
 if HAVE_LIBTOMMATH:

From noreply at buildbot.pypy.org  Fri Jun 19 00:27:42 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 19 Jun 2015 00:27:42 +0200 (CEST)
Subject: [pypy-commit] stmgc queue: oups
Message-ID: <20150618222742.844B51C1FEF@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: queue
Changeset: r1873:0ef378d2da3d
Date: 2015-06-19 00:28 +0200
http://bitbucket.org/pypy/stmgc/changeset/0ef378d2da3d/

Log:	oups

diff --git a/c8/stm/queue.c b/c8/stm/queue.c
--- a/c8/stm/queue.c
+++ b/c8/stm/queue.c
@@ -387,6 +387,7 @@
 static void collect_active_queues(void)
 {
     wlog_t *item;
+    queue_lock_acquire();
     TREE_LOOP_FORWARD(STM_PSEGMENT->active_queues, item) {
         /* it is enough to trace the objects added in the current
            transaction.  All other objects reachable from the queue
@@ -406,4 +407,5 @@
             seg->added_young_limit = seg->added_in_this_transaction;
         }
     } TREE_LOOP_END;
+    queue_lock_release();
 }

From noreply at buildbot.pypy.org  Fri Jun 19 02:46:55 2015
From: noreply at buildbot.pypy.org (pjenvey)
Date: Fri, 19 Jun 2015 02:46:55 +0200 (CEST)
Subject: [pypy-commit] pypy default: hide more init functions
Message-ID: <20150619004655.483231C1FF2@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: 
Changeset: r78201:47da843f46fc
Date: 2015-06-18 17:46 -0700
http://bitbucket.org/pypy/pypy/changeset/47da843f46fc/

Log:	hide more init functions

diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py
--- a/pypy/interpreter/app_main.py
+++ b/pypy/interpreter/app_main.py
@@ -515,6 +515,7 @@
 
     return options
 
+ at hidden_applevel
 def run_command_line(interactive,
                      inspect,
                      run_command,
@@ -752,6 +753,7 @@
     # This is important for py3k
     sys.executable = executable
 
+ at hidden_applevel
 def entry_point(executable, argv):
     # note that before calling setup_bootstrap_path, we are limited because we
     # cannot import stdlib modules. In particular, we cannot use unicode

From noreply at buildbot.pypy.org  Thu Jun 18 18:35:51 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 18 Jun 2015 18:35:51 +0200 (CEST)
Subject: [pypy-commit] stmgc default: tweaks
Message-ID: <20150618163551.417A41C1FDA@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r1861:5b75b3f3a9b7
Date: 2015-06-18 17:01 +0200
http://bitbucket.org/pypy/stmgc/changeset/5b75b3f3a9b7/

Log:	tweaks

diff --git a/c8/stm/core.c b/c8/stm/core.c
--- a/c8/stm/core.c
+++ b/c8/stm/core.c
@@ -529,7 +529,7 @@
             enter_safe_point_if_requested();
         }
         else if (STM_PSEGMENT->last_commit_log_entry->next == INEV_RUNNING) {
-            cond_wait(C_SEGMENT_FREE_OR_SAFE_POINT);
+            cond_wait(C_SEGMENT_FREE_OR_SAFE_POINT_REQ);
         }
         s_mutex_unlock();
         goto retry_from_start;   /* redo _stm_validate() now */
@@ -1119,7 +1119,7 @@
 {
     assert(!_stm_in_transaction(tl));
 
-    while (!acquire_thread_segment(tl)) {}
+    acquire_thread_segment(tl);
     /* GS invalid before this point! */
 
     assert(STM_PSEGMENT->safe_point == SP_NO_TRANSACTION);
@@ -1567,7 +1567,7 @@
             s_mutex_lock();
             if (any_soon_finished_or_inevitable_thread_segment() &&
                     !safe_point_requested()) {
-                cond_wait(C_SEGMENT_FREE_OR_SAFE_POINT);
+                cond_wait(C_SEGMENT_FREE_OR_SAFE_POINT_REQ);
             }
             s_mutex_unlock();
             num_waits++;
diff --git a/c8/stm/sync.c b/c8/stm/sync.c
--- a/c8/stm/sync.c
+++ b/c8/stm/sync.c
@@ -188,10 +188,11 @@
 #endif
 
 
-static bool acquire_thread_segment(stm_thread_local_t *tl)
+static void acquire_thread_segment(stm_thread_local_t *tl)
 {
     /* This function acquires a segment for the currently running thread,
        and set up the GS register if it changed. */
+ retry_from_start:
     assert(_has_mutex());
     assert(_is_tl_registered(tl));
 
@@ -225,13 +226,13 @@
         }
     }
     /* No segment available.  Wait until release_thread_segment()
-       signals that one segment has been freed. */
+       signals that one segment has been freed.  Note that we prefer
+       waiting rather than detaching an inevitable transaction, here. */
     timing_event(tl, STM_WAIT_FREE_SEGMENT);
     cond_wait(C_SEGMENT_FREE);
     timing_event(tl, STM_WAIT_DONE);
 
-    /* Return false to the caller, which will call us again */
-    return false;
+    goto retry_from_start;
 
  got_num:
     OPT_ASSERT(num >= 0 && num < NB_SEGMENTS-1);
@@ -242,7 +243,6 @@
     assert(!in_transaction(tl));
     STM_SEGMENT->running_thread = tl;
     assert(in_transaction(tl));
-    return true;
 }
 
 static void release_thread_segment(stm_thread_local_t *tl)
@@ -251,7 +251,7 @@
     assert(_has_mutex());
 
     cond_signal(C_SEGMENT_FREE);
-    cond_broadcast(C_SEGMENT_FREE_OR_SAFE_POINT);   /* often no listener */
+    cond_broadcast(C_SEGMENT_FREE_OR_SAFE_POINT_REQ);  /* often no listener */
 
     assert(STM_SEGMENT->running_thread == tl);
     segnum = STM_SEGMENT->segment_num;
@@ -347,7 +347,7 @@
     assert(!pause_signalled);
     pause_signalled = true;
     dprintf(("request to pause\n"));
-    cond_broadcast(C_SEGMENT_FREE_OR_SAFE_POINT);
+    cond_broadcast(C_SEGMENT_FREE_OR_SAFE_POINT_REQ);
 }
 
 static inline long count_other_threads_sp_running(void)
diff --git a/c8/stm/sync.h b/c8/stm/sync.h
--- a/c8/stm/sync.h
+++ b/c8/stm/sync.h
@@ -5,7 +5,7 @@
     C_AT_SAFE_POINT,
     C_REQUEST_REMOVED,
     C_SEGMENT_FREE,
-    C_SEGMENT_FREE_OR_SAFE_POINT,
+    C_SEGMENT_FREE_OR_SAFE_POINT_REQ,
     _C_TOTAL
 };
 
@@ -23,7 +23,7 @@
 
 /* acquire and release one of the segments for running the given thread
    (must have the mutex acquired!) */
-static bool acquire_thread_segment(stm_thread_local_t *tl);
+static void acquire_thread_segment(stm_thread_local_t *tl);
 static void release_thread_segment(stm_thread_local_t *tl);
 static void soon_finished_or_inevitable_thread_segment(void);
 static bool any_soon_finished_or_inevitable_thread_segment(void);

From noreply at buildbot.pypy.org  Fri Jun 19 02:46:51 2015
From: noreply at buildbot.pypy.org (pjenvey)
Date: Fri, 19 Jun 2015 02:46:51 +0200 (CEST)
Subject: [pypy-commit] pypy default: hide app_main's frames. this breaks
 sys.exc_info but py3 offers a workaround
Message-ID: <20150619004651.BD13C1C1FDC@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: 
Changeset: r78198:04afcba2a748
Date: 2013-02-24 10:51 -0800
http://bitbucket.org/pypy/pypy/changeset/04afcba2a748/

Log:	hide app_main's frames. this breaks sys.exc_info but py3 offers a
	workaround (grafted from 894b0fa3245b1584e5f8b7404d8c3206f7ab9f2d)

diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py
--- a/pypy/interpreter/app_main.py
+++ b/pypy/interpreter/app_main.py
@@ -40,6 +40,10 @@
 PYPYLOG: If set to a non-empty value, enable logging.
 """
 
+try:
+    from __pypy__ import hidden_applevel
+except ImportError:
+    hidden_applevel = lambda f: f
 import sys
 
 DEBUG = False       # dump exceptions before calling the except hook
@@ -63,6 +67,7 @@
             exitcode = 1
     raise SystemExit(exitcode)
 
+ at hidden_applevel
 def run_toplevel(f, *fargs, **fkwds):
     """Calls f() and handles all OperationErrors.
     Intended use is to run the main program or one interactive statement.
@@ -87,13 +92,13 @@
 
     except SystemExit as e:
         handle_sys_exit(e)
-    except:
-        display_exception()
+    except BaseException as e:
+        display_exception(e)
         return False
     return True   # success
 
-def display_exception():
-    etype, evalue, etraceback = sys.exc_info()
+def display_exception(e):
+    etype, evalue, etraceback = type(e), e, e.__traceback__
     try:
         # extra debugging info in case the code below goes very wrong
         if DEBUG and hasattr(sys, 'stderr'):
@@ -119,11 +124,11 @@
         hook(etype, evalue, etraceback)
         return # done
 
-    except:
+    except BaseException as e:
         try:
             stderr = sys.stderr
             print >> stderr, 'Error calling sys.excepthook:'
-            originalexcepthook(*sys.exc_info())
+            originalexcepthook(type(e), e, e.__traceback__)
             print >> stderr
             print >> stderr, 'Original exception was:'
         except:
@@ -597,6 +602,7 @@
             # Put '' on sys.path
             sys.path.insert(0, '')
 
+            @hidden_applevel
             def run_it():
                 exec run_command in mainmodule.__dict__
             success = run_toplevel(run_it)
@@ -634,6 +640,7 @@
                         print >> sys.stderr, "Could not open PYTHONSTARTUP"
                         print >> sys.stderr, "IOError:", e
                     else:
+                        @hidden_applevel
                         def run_it():
                             co_python_startup = compile(startup,
                                                         python_startup,
@@ -650,6 +657,7 @@
                 inspect = True
             else:
                 # If not interactive, just read and execute stdin normally.
+                @hidden_applevel
                 def run_it():
                     co_stdin = compile(sys.stdin.read(), '<stdin>', 'exec',
                                        PyCF_ACCEPT_NULL_BYTES)
diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py
--- a/pypy/module/__pypy__/__init__.py
+++ b/pypy/module/__pypy__/__init__.py
@@ -71,6 +71,7 @@
         'debug_print_once'          : 'interp_debug.debug_print_once',
         'debug_flush'               : 'interp_debug.debug_flush',
         'builtinify'                : 'interp_magic.builtinify',
+        'hidden_applevel'           : 'interp_magic.hidden_applevel',
         'lookup_special'            : 'interp_magic.lookup_special',
         'do_what_I_mean'            : 'interp_magic.do_what_I_mean',
         'validate_fd'               : 'interp_magic.validate_fd',
diff --git a/pypy/module/__pypy__/interp_magic.py b/pypy/module/__pypy__/interp_magic.py
--- a/pypy/module/__pypy__/interp_magic.py
+++ b/pypy/module/__pypy__/interp_magic.py
@@ -59,6 +59,13 @@
     bltn = BuiltinFunction(func)
     return space.wrap(bltn)
 
+def hidden_applevel(space, w_func):
+    """Decorator that hides a function's frame from app-level"""
+    from pypy.interpreter.function import Function
+    func = space.interp_w(Function, w_func)
+    func.getcode().hidden_applevel = True
+    return w_func
+
 @unwrap_spec(meth=str)
 def lookup_special(space, w_obj, meth):
     """Lookup up a special method on an object."""
diff --git a/pypy/module/__pypy__/test/test_special.py b/pypy/module/__pypy__/test/test_special.py
--- a/pypy/module/__pypy__/test/test_special.py
+++ b/pypy/module/__pypy__/test/test_special.py
@@ -27,6 +27,39 @@
         assert A.a is not A.__dict__['a']
         assert A.b is A.__dict__['b']
 
+    def test_hidden_applevel(self):
+        import __pypy__
+        import sys
+
+        @__pypy__.hidden_applevel
+        def sneak(): (lambda: 1/0)()
+        try:
+            sneak()
+        except ZeroDivisionError as e:
+            tb = e.__traceback__
+            assert tb.tb_frame == sys._getframe()
+            assert tb.tb_next.tb_frame.f_code.co_name == '<lambda>'
+        else:
+            assert False, 'Expected ZeroDivisionError'
+
+    def test_hidden_applevel_frames(self):
+        import __pypy__
+        import sys
+
+        @__pypy__.hidden_applevel
+        def test_hidden():
+            assert sys._getframe().f_code.co_name != 'test_hidden'
+            def e(): 1/0
+            try: e()
+            except ZeroDivisionError as e:
+                assert sys.exc_info() == (None, None, None)
+                frame = e.__traceback__.tb_frame
+                assert frame != sys._getframe()
+                assert frame.f_code.co_name == 'e'
+            else: assert False
+            return 2
+        assert test_hidden() == 2
+
     def test_lookup_special(self):
         from __pypy__ import lookup_special
         class X(object):

From noreply at buildbot.pypy.org  Thu Jun 18 18:35:52 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 18 Jun 2015 18:35:52 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Tweak tweak tweak: a deadlock was
	possible there
Message-ID: <20150618163552.4CD3A1C1FDB@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r1862:ee0e63d791cb
Date: 2015-06-18 18:36 +0200
http://bitbucket.org/pypy/stmgc/changeset/ee0e63d791cb/

Log:	Tweak tweak tweak: a deadlock was possible there

diff --git a/c8/stm/core.c b/c8/stm/core.c
--- a/c8/stm/core.c
+++ b/c8/stm/core.c
@@ -495,6 +495,33 @@
 static void readd_wb_executed_flags(void);
 static void check_all_write_barrier_flags(char *segbase, struct list_s *list);
 
+static void wait_for_inevitable(void)
+{
+    intptr_t detached = 0;
+
+    s_mutex_lock();
+    if (safe_point_requested()) {
+        /* XXXXXX if the safe point below aborts, in
+           _validate_and_attach(), 'new' leaks */
+        enter_safe_point_if_requested();
+    }
+    else if (STM_PSEGMENT->last_commit_log_entry->next == INEV_RUNNING) {
+        /* loop until C_SEGMENT_FREE_OR_SAFE_POINT_REQ is signalled, but
+           try to detach an inevitable transaction regularly */
+        while (1) {
+            detached = fetch_detached_transaction();
+            if (detached != 0)
+                break;
+            if (cond_wait_timeout(C_SEGMENT_FREE_OR_SAFE_POINT_REQ, 0.00001))
+                break;
+        }
+    }
+    s_mutex_unlock();
+
+    if (detached != 0)
+        commit_fetched_detached_transaction(detached);
+}
+
 /* This is called to do stm_validate() and then attach 'new' at the
    head of the 'commit_log_root' chained list.  This function sleeps
    and retries until it succeeds or aborts.
@@ -523,24 +550,10 @@
 #endif
 
     if (STM_PSEGMENT->last_commit_log_entry->next == INEV_RUNNING) {
-        s_mutex_lock();
-        if (safe_point_requested()) {
-            /* XXXXXX if the safe point below aborts, 'new' leaks */
-            enter_safe_point_if_requested();
-        }
-        else if (STM_PSEGMENT->last_commit_log_entry->next == INEV_RUNNING) {
-            cond_wait(C_SEGMENT_FREE_OR_SAFE_POINT_REQ);
-        }
-        s_mutex_unlock();
+        wait_for_inevitable();
         goto retry_from_start;   /* redo _stm_validate() now */
     }
 
-    intptr_t detached = fetch_detached_transaction();
-    if (detached != 0) {
-        commit_fetched_detached_transaction(detached);
-        goto retry_from_start;
-    }
-
     /* we must not remove the WB_EXECUTED flags before validation as
        it is part of a condition in import_objects() called by
        copy_bk_objs_in_page_from to not overwrite our modifications.
@@ -1564,12 +1577,31 @@
             timing_become_inevitable(); /* for tests: another transaction */
             stm_abort_transaction();    /* is already inevitable, abort   */
 #endif
+
+            intptr_t detached = 0;
+
             s_mutex_lock();
             if (any_soon_finished_or_inevitable_thread_segment() &&
                     !safe_point_requested()) {
-                cond_wait(C_SEGMENT_FREE_OR_SAFE_POINT_REQ);
+
+                /* wait until C_SEGMENT_FREE_OR_SAFE_POINT_REQ is signalled */
+                while (!cond_wait_timeout(C_SEGMENT_FREE_OR_SAFE_POINT_REQ,
+                                          0.000054321)) {
+                    /* try to detach another inevitable transaction, but
+                       only after waiting a bit.  This is necessary to avoid
+                       deadlocks in some situations, which are hopefully
+                       not too common.  We don't want two threads constantly
+                       detaching each other. */
+                    detached = fetch_detached_transaction();
+                    if (detached != 0)
+                        break;
+                }
             }
             s_mutex_unlock();
+
+            if (detached != 0)
+                commit_fetched_detached_transaction(detached);
+
             num_waits++;
             goto retry_from_start;
         }

From noreply at buildbot.pypy.org  Thu Jun 18 18:35:50 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 18 Jun 2015 18:35:50 +0200 (CEST)
Subject: [pypy-commit] stmgc queue: Clarify the return value of join()
Message-ID: <20150618163550.2A3D11C1FD9@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: queue
Changeset: r1860:d083e426a17d
Date: 2015-06-18 14:45 +0200
http://bitbucket.org/pypy/stmgc/changeset/d083e426a17d/

Log:	Clarify the return value of join()

diff --git a/c8/stm/queue.c b/c8/stm/queue.c
--- a/c8/stm/queue.c
+++ b/c8/stm/queue.c
@@ -309,7 +309,7 @@
     seg->unfinished_tasks_in_this_transaction--;
 }
 
-int stm_queue_join(object_t *qobj, stm_queue_t *queue, stm_thread_local_t *tl)
+long stm_queue_join(object_t *qobj, stm_queue_t *queue, stm_thread_local_t *tl)
 {
     int64_t result;
 
@@ -317,8 +317,7 @@
     result = queue->unfinished_tasks;   /* can't wait in tests */
     result += (queue->segs[STM_SEGMENT->segment_num - 1]
                .unfinished_tasks_in_this_transaction);
-    if (result > 0)
-        return 42;
+    return result;
 #else
     STM_PUSH_ROOT(*tl, qobj);
     _stm_commit_transaction();
@@ -333,8 +332,9 @@
     STM_POP_ROOT(*tl, qobj);   /* 'queue' should stay alive until here */
 #endif
 
-    /* returns 1 for 'ok', or 0 for error: negative 'unfinished_tasks' */
-    return (result == 0);
+    /* returns 0 for 'ok', or negative if there was more task_done()
+       than put() so far */
+    return result;
 }
 
 static void queue_trace_list(queue_entry_t *entry, void trace(object_t **),
diff --git a/c8/stmgc.h b/c8/stmgc.h
--- a/c8/stmgc.h
+++ b/c8/stmgc.h
@@ -751,7 +751,7 @@
 void stm_queue_task_done(stm_queue_t *queue);
 /* join() commits and waits outside a transaction (so push roots).
    Unsuitable if the current transaction is atomic! */
-int stm_queue_join(object_t *qobj, stm_queue_t *queue, stm_thread_local_t *tl);
+long stm_queue_join(object_t *qobj, stm_queue_t *queue, stm_thread_local_t *tl);
 void stm_queue_tracefn(stm_queue_t *queue, void trace(object_t **));
 
 
diff --git a/c8/test/test_queue.py b/c8/test/test_queue.py
--- a/c8/test/test_queue.py
+++ b/c8/test/test_queue.py
@@ -59,12 +59,12 @@
     def join(self, obj):
         q = get_queue(obj)
         res = lib.stm_queue_join(obj, q, self.tls[self.current_thread]);
-        if res == 1:
+        if res == 0:
             return
-        elif res == 42:
+        elif res > 0:
             raise Conflict("join() cannot wait in tests")
         else:
-            raise AssertionError("stm_queue_join error")
+            raise AssertionError("too much task_done()!")
 
 
 class TestQueue(BaseTestQueue):

From noreply at buildbot.pypy.org  Thu Jun 18 22:08:19 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Thu, 18 Jun 2015 22:08:19 +0200 (CEST)
Subject: [pypy-commit] pypy py3.3: Re-add has_so_extension() to
 module/imp/importing.py (it was removed during the last merge because I
 thought it's dead code).
Message-ID: <20150618200819.336D01C1FD7@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob <me at manueljacob.de>
Branch: py3.3
Changeset: r78191:f02eca6ea2cb
Date: 2015-06-18 20:25 +0200
http://bitbucket.org/pypy/pypy/changeset/f02eca6ea2cb/

Log:	Re-add has_so_extension() to module/imp/importing.py (it was removed
	during the last merge because I thought it's dead code).

diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py
--- a/pypy/module/imp/importing.py
+++ b/pypy/module/imp/importing.py
@@ -52,6 +52,10 @@
 
     return '.' + soabi + SO
 
+def has_so_extension(space):
+    return (space.config.objspace.usemodules.cpyext or
+            space.config.objspace.usemodules._cffi_backend)
+
 def check_sys_modules(space, w_modulename):
     return space.finditem(space.sys.get('modules'), w_modulename)
 

From noreply at buildbot.pypy.org  Thu Jun 18 16:19:13 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Thu, 18 Jun 2015 16:19:13 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Fix CFFI test.
Message-ID: <20150618141913.187AC1C1FCA@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob <me at manueljacob.de>
Branch: py3k
Changeset: r78181:961d5a41abc1
Date: 2015-06-18 14:17 +0200
http://bitbucket.org/pypy/pypy/changeset/961d5a41abc1/

Log:	Fix CFFI test.

diff --git a/pypy/module/_cffi_backend/test/test_ffi_obj.py b/pypy/module/_cffi_backend/test/test_ffi_obj.py
--- a/pypy/module/_cffi_backend/test/test_ffi_obj.py
+++ b/pypy/module/_cffi_backend/test/test_ffi_obj.py
@@ -183,7 +183,7 @@
                                 "       ^")
         e = raises(ffi.error, ffi.cast, "\t\n\x01\x1f~\x7f\x80\xff", 0)
         assert str(e.value) == ("identifier expected\n"
-                                "  ??~???\n"
+                                "  ??~?????\n"
                                 "  ^")
         e = raises(ffi.error, ffi.cast, "X" * 600, 0)
         assert str(e.value) == ("undefined type name")

From noreply at buildbot.pypy.org  Fri Jun 19 00:28:44 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 19 Jun 2015 00:28:44 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c8: import stmgc/0ef378d2da3d
Message-ID: <20150618222844.2BBDE1C1FF0@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c8
Changeset: r78197:8f3dbb28a3a3
Date: 2015-06-19 00:29 +0200
http://bitbucket.org/pypy/pypy/changeset/8f3dbb28a3a3/

Log:	import stmgc/0ef378d2da3d

diff --git a/rpython/translator/stm/src_stm/revision b/rpython/translator/stm/src_stm/revision
--- a/rpython/translator/stm/src_stm/revision
+++ b/rpython/translator/stm/src_stm/revision
@@ -1,1 +1,1 @@
-277dd2ad5226
+0ef378d2da3d
diff --git a/rpython/translator/stm/src_stm/stm/queue.c b/rpython/translator/stm/src_stm/stm/queue.c
--- a/rpython/translator/stm/src_stm/stm/queue.c
+++ b/rpython/translator/stm/src_stm/stm/queue.c
@@ -387,6 +387,7 @@
 static void collect_active_queues(void)
 {
     wlog_t *item;
+    queue_lock_acquire();
     TREE_LOOP_FORWARD(STM_PSEGMENT->active_queues, item) {
         /* it is enough to trace the objects added in the current
            transaction.  All other objects reachable from the queue
@@ -406,4 +407,5 @@
             seg->added_young_limit = seg->added_in_this_transaction;
         }
     } TREE_LOOP_END;
+    queue_lock_release();
 }

From noreply at buildbot.pypy.org  Fri Jun 19 02:46:54 2015
From: noreply at buildbot.pypy.org (pjenvey)
Date: Fri, 19 Jun 2015 02:46:54 +0200 (CEST)
Subject: [pypy-commit] pypy default: add __pypy__.get_hidden_tb so pypy2's
 app_main can grab the current traceback
Message-ID: <20150619004654.2AA861C1FF1@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: 
Changeset: r78200:8e8a73a3fdfa
Date: 2015-06-18 17:45 -0700
http://bitbucket.org/pypy/pypy/changeset/8e8a73a3fdfa/

Log:	add __pypy__.get_hidden_tb so pypy2's app_main can grab the current
	traceback from within a hidden frame (pypy3 can already get to it
	via __traceback__)

diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py
--- a/pypy/interpreter/app_main.py
+++ b/pypy/interpreter/app_main.py
@@ -41,8 +41,9 @@
 """
 
 try:
-    from __pypy__ import hidden_applevel
+    from __pypy__ import get_hidden_tb, hidden_applevel
 except ImportError:
+    get_hidden_tb = lambda: sys.exc_info()[2]
     hidden_applevel = lambda f: f
 import sys
 
@@ -98,7 +99,7 @@
     return True   # success
 
 def display_exception(e):
-    etype, evalue, etraceback = type(e), e, e.__traceback__
+    etype, evalue, etraceback = type(e), e, get_hidden_tb()
     try:
         # extra debugging info in case the code below goes very wrong
         if DEBUG and hasattr(sys, 'stderr'):
@@ -697,7 +698,7 @@
     except SystemExit as e:
         status = e.code
         if inspect_requested():
-            display_exception()
+            display_exception(e)
     else:
         status = not success
 
diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -1,6 +1,7 @@
 import sys
 from pypy.interpreter.error import OperationError, get_cleared_operation_error
 from rpython.rlib.unroll import unrolling_iterable
+from rpython.rlib.objectmodel import specialize
 from rpython.rlib import jit
 
 TICK_COUNTER_STEP = 100
@@ -214,13 +215,21 @@
             self._trace(frame, 'exception', None, operationerr)
         #operationerr.print_detailed_traceback(self.space)
 
-    def sys_exc_info(self): # attn: the result is not the wrapped sys.exc_info() !!!
+    @specialize.arg(1)
+    def sys_exc_info(self, for_hidden=False):
         """Implements sys.exc_info().
-        Return an OperationError instance or None."""
+        Return an OperationError instance or None.
+
+        Ignores exceptions within hidden frames unless for_hidden=True
+        is specified.
+
+        # NOTE: the result is not the wrapped sys.exc_info() !!!
+
+        """
         frame = self.gettopframe()
         while frame:
             if frame.last_exception is not None:
-                if (not frame.hide() or
+                if ((for_hidden or not frame.hide()) or
                         frame.last_exception is
                             get_cleared_operation_error(self.space)):
                     return frame.last_exception
diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py
--- a/pypy/module/__pypy__/__init__.py
+++ b/pypy/module/__pypy__/__init__.py
@@ -72,6 +72,7 @@
         'debug_flush'               : 'interp_debug.debug_flush',
         'builtinify'                : 'interp_magic.builtinify',
         'hidden_applevel'           : 'interp_magic.hidden_applevel',
+        'get_hidden_tb'             : 'interp_magic.get_hidden_tb',
         'lookup_special'            : 'interp_magic.lookup_special',
         'do_what_I_mean'            : 'interp_magic.do_what_I_mean',
         'validate_fd'               : 'interp_magic.validate_fd',
diff --git a/pypy/module/__pypy__/interp_magic.py b/pypy/module/__pypy__/interp_magic.py
--- a/pypy/module/__pypy__/interp_magic.py
+++ b/pypy/module/__pypy__/interp_magic.py
@@ -66,6 +66,13 @@
     func.getcode().hidden_applevel = True
     return w_func
 
+def get_hidden_tb(space):
+    """Return the traceback of the current exception being handled by a
+    frame hidden from applevel.
+    """
+    operr = space.getexecutioncontext().sys_exc_info(for_hidden=True)
+    return space.w_None if operr is None else space.wrap(operr.get_traceback())
+
 @unwrap_spec(meth=str)
 def lookup_special(space, w_obj, meth):
     """Lookup up a special method on an object."""
diff --git a/pypy/module/__pypy__/test/test_special.py b/pypy/module/__pypy__/test/test_special.py
--- a/pypy/module/__pypy__/test/test_special.py
+++ b/pypy/module/__pypy__/test/test_special.py
@@ -57,6 +57,22 @@
             return 2
         assert test_hidden() == 2
 
+    def test_get_hidden_tb(self):
+        import __pypy__
+        import sys
+
+        @__pypy__.hidden_applevel
+        def test_hidden_with_tb():
+            def not_hidden(): 1/0
+            try: not_hidden()
+            except ZeroDivisionError as e:
+                assert sys.exc_info() == (None, None, None)
+                tb = __pypy__.get_hidden_tb()
+                assert tb.tb_frame.f_code.co_name == 'not_hidden'
+                return True
+            else: return False
+        assert test_hidden_with_tb()
+
     def test_lookup_special(self):
         from __pypy__ import lookup_special
         class X(object):

From noreply at buildbot.pypy.org  Fri Jun 19 02:46:53 2015
From: noreply at buildbot.pypy.org (pjenvey)
Date: Fri, 19 Jun 2015 02:46:53 +0200 (CEST)
Subject: [pypy-commit] pypy default: adapt to py2.7
Message-ID: <20150619004653.038E31C1FE3@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: 
Changeset: r78199:74cadf90d4ec
Date: 2015-06-18 15:57 -0700
http://bitbucket.org/pypy/pypy/changeset/74cadf90d4ec/

Log:	adapt to py2.7

diff --git a/pypy/module/__pypy__/test/test_special.py b/pypy/module/__pypy__/test/test_special.py
--- a/pypy/module/__pypy__/test/test_special.py
+++ b/pypy/module/__pypy__/test/test_special.py
@@ -36,7 +36,7 @@
         try:
             sneak()
         except ZeroDivisionError as e:
-            tb = e.__traceback__
+            tb = sys.exc_info()[2]
             assert tb.tb_frame == sys._getframe()
             assert tb.tb_next.tb_frame.f_code.co_name == '<lambda>'
         else:
@@ -53,9 +53,6 @@
             try: e()
             except ZeroDivisionError as e:
                 assert sys.exc_info() == (None, None, None)
-                frame = e.__traceback__.tb_frame
-                assert frame != sys._getframe()
-                assert frame.f_code.co_name == 'e'
             else: assert False
             return 2
         assert test_hidden() == 2

From noreply at buildbot.pypy.org  Thu Jun 18 11:36:30 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 18 Jun 2015 11:36:30 +0200 (CEST)
Subject: [pypy-commit] stmgc queue: - fix timespec_delay() to accept values
 of 'incr' larger than 1.0
Message-ID: <20150618093630.C55D31C1F76@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: queue
Changeset: r1857:7c61144ee23d
Date: 2015-06-18 11:36 +0200
http://bitbucket.org/pypy/stmgc/changeset/7c61144ee23d/

Log:	- fix timespec_delay() to accept values of 'incr' larger than 1.0

	- pthread_cond_timedwait() can return EINTR

diff --git a/c8/stm/sync.c b/c8/stm/sync.c
--- a/c8/stm/sync.c
+++ b/c8/stm/sync.c
@@ -115,12 +115,15 @@
     t->tv_sec = tv.tv_sec;
     t->tv_nsec = tv.tv_usec * 1000 + 999;
 #endif
-    /* assumes that "incr" is not too large, less than 1 second */
+
+    long integral_part = (long)incr;
+    t->tv_sec += integral_part;
+    incr -= integral_part;
+
     long nsec = t->tv_nsec + (long)(incr * 1000000000.0);
-    if (nsec >= 1000000000) {
+    while (nsec >= 1000000000) {
         t->tv_sec += 1;
         nsec -= 1000000000;
-        assert(nsec < 1000000000);
     }
     t->tv_nsec = nsec;
 }
@@ -131,15 +134,21 @@
     stm_fatalerror("*** cond_wait/%d called!", (int)ctype);
 #endif
 
+ retry:
     assert(_has_mutex_here);
 
     int err = pthread_cond_timedwait(&sync_ctl.cond[ctype],
                                      &sync_ctl.global_mutex, pt);
-    if (err == 0)
+    switch (err) {
+    case 0:
         return true;     /* success */
-    if (LIKELY(err == ETIMEDOUT))
+    case ETIMEDOUT:
         return false;    /* timeout */
-    stm_fatalerror("pthread_cond_timedwait/%d: %d", (int)ctype, err);
+    case EINTR:
+        goto retry;
+    default:
+        stm_fatalerror("pthread_cond_timedwait/%d: %d", (int)ctype, err);
+    }
 }
 
 static bool cond_wait_timeout(enum cond_type_e ctype, double delay)

From noreply at buildbot.pypy.org  Thu Jun 18 16:19:14 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Thu, 18 Jun 2015 16:19:14 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Fix translation.
Message-ID: <20150618141914.6D75E1C1FCC@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob <me at manueljacob.de>
Branch: py3k
Changeset: r78182:ae79f5787d65
Date: 2015-06-18 16:18 +0200
http://bitbucket.org/pypy/pypy/changeset/ae79f5787d65/

Log:	Fix translation.

diff --git a/pypy/module/micronumpy/ctors.py b/pypy/module/micronumpy/ctors.py
--- a/pypy/module/micronumpy/ctors.py
+++ b/pypy/module/micronumpy/ctors.py
@@ -142,7 +142,7 @@
     else:
         # not an array
         shape, elems_w = strides.find_shape_and_elems(space, w_object, dtype)
-    if dtype is None and space.isinstance_w(w_object, space.w_buffer):
+    if dtype is None and space.isinstance_w(w_object, space.w_memoryview):
         dtype = descriptor.get_dtype_cache(space).w_uint8dtype
     if dtype is None or (dtype.is_str_or_unicode() and dtype.elsize < 1):
         dtype = find_dtype_for_seq(space, elems_w, dtype)

From noreply at buildbot.pypy.org  Thu Jun 18 22:08:18 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Thu, 18 Jun 2015 22:08:18 +0200 (CEST)
Subject: [pypy-commit] pypy py3.3: hg merge py3k
Message-ID: <20150618200818.06DE71C1FD0@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob <me at manueljacob.de>
Branch: py3.3
Changeset: r78190:d55a565a0496
Date: 2015-06-18 18:57 +0200
http://bitbucket.org/pypy/pypy/changeset/d55a565a0496/

Log:	hg merge py3k

diff too long, truncating to 2000 out of 74512 lines

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -3,11 +3,15 @@
 d8ac7d23d3ec5f9a0fa1264972f74a010dbfd07f release-1.6
 ff4af8f318821f7f5ca998613a60fca09aa137da release-1.7
 07e08e9c885ca67d89bcc304e45a32346daea2fa release-2.0-beta-1
-9b623bc48b5950cf07184462a0e48f2c4df0d720 pypy-2.1-beta1-arm
-9b623bc48b5950cf07184462a0e48f2c4df0d720 pypy-2.1-beta1-arm
 ab0dd631c22015ed88e583d9fdd4c43eebf0be21 pypy-2.1-beta1-arm
 20e51c4389ed4469b66bb9d6289ce0ecfc82c4b9 release-2.3.0
 394146e9bb673514c61f0150ab2013ccf78e8de7 release-2.3
 32f35069a16d819b58c1b6efb17c44e3e53397b2 release-2.2=3.1
 32f35069a16d819b58c1b6efb17c44e3e53397b2 release-2.3.1
 10f1b29a2bd21f837090286174a9ca030b8680b2 release-2.5.0
+9c4588d731b7fe0b08669bd732c2b676cb0a8233 release-2.5.1
+fcdb941565156385cbac04cfb891f8f4c7a92ef6 release-2.6.0
+fcdb941565156385cbac04cfb891f8f4c7a92ef6 release-2.6.0
+e03971291f3a0729ecd3ee7fae7ddb0bb82d476c release-2.6.0
+e03971291f3a0729ecd3ee7fae7ddb0bb82d476c release-2.6.0
+295ee98b69288471b0fcf2e0ede82ce5209eb90b release-2.6.0
diff --git a/.tddium.requirements.txt b/.tddium.requirements.txt
deleted file mode 100644
--- a/.tddium.requirements.txt
+++ /dev/null
@@ -1,1 +0,0 @@
-pytest
diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -38,8 +38,8 @@
   Armin Rigo
   Maciej Fijalkowski
   Carl Friedrich Bolz
+  Amaury Forgeot d'Arc
   Antonio Cuni
-  Amaury Forgeot d'Arc
   Samuele Pedroni
   Alex Gaynor
   Brian Kearns
@@ -50,9 +50,9 @@
   Holger Krekel
   Christian Tismer
   Hakan Ardo
-  Benjamin Peterson
   Manuel Jacob
   Ronan Lamy
+  Benjamin Peterson
   Anders Chrigstrom
   Eric van Riet Paap
   Wim Lavrijsen
@@ -63,8 +63,8 @@
   Sven Hager
   Anders Lehmann
   Aurelien Campeas
+  Remi Meier
   Niklaus Haldimann
-  Remi Meier
   Camillo Bruni
   Laura Creighton
   Toon Verwaest
@@ -76,10 +76,10 @@
   David Edelsohn
   Anders Hammarquist
   Jakub Gustak
+  Gregor Wegberg
   Guido Wesdorp
   Lawrence Oluyede
   Bartosz Skowron
-  Gregor Wegberg
   Daniel Roberts
   Niko Matsakis
   Adrien Di Mascio
@@ -87,10 +87,11 @@
   Ludovic Aubry
   Jacob Hallen
   Jason Creighton
+  Richard Plangger
   Alex Martelli
   Michal Bendowski
+  stian
   Jan de Mooij
-  stian
   Tyler Wade
   Michael Foord
   Stephan Diehl
@@ -133,15 +134,15 @@
   Georg Brandl
   Bert Freudenberg
   Stian Andreassen
+  Edd Barrett
   Wanja Saatkamp
   Gerald Klix
   Mike Blume
+  Tobias Pape
   Oscar Nierstrasz
   Stefan H. Muller
-  Edd Barrett
   Jeremy Thurgood
   Rami Chowdhury
-  Tobias Pape
   Eugene Oden
   Henry Mason
   Vasily Kuznetsov
@@ -167,11 +168,13 @@
   Michael Twomey
   Lucian Branescu Mihaila
   Yichao Yu
+  Anton Gulenko
   Gabriel Lavoie
   Olivier Dormond
   Jared Grubb
   Karl Bartel
   Wouter van Heyst
+  Sebastian Pawlu&#347;
   Brian Dorsey
   Victor Stinner
   Andrews Medina
@@ -188,6 +191,7 @@
   Neil Shepperd
   Stanislaw Halik
   Mikael Sch&#246;nenberg
+  Berkin Ilbeyi
   Elmo M?ntynen
   Jonathan David Riehl
   Anders Qvist
@@ -211,11 +215,11 @@
   Carl Meyer
   Karl Ramm
   Pieter Zieschang
-  Sebastian Pawlu&#347;
   Gabriel
   Lukas Vacek
   Andrew Dalke
   Sylvain Thenault
+  Jakub Stasiak
   Nathan Taylor
   Vladimir Kryachko
   Jacek Generowicz
@@ -242,6 +246,7 @@
   Tomo Cocoa
   Toni Mattis
   Lucas Stadler
+  Julian Berman
   roberto at goyle
   Yury V. Zaytsev
   Anna Katrina Dominguez
@@ -253,6 +258,8 @@
   Ben Darnell
   Roberto De Ioris
   Juan Francisco Cantero Hurtado
+  Ruochen Huang
+  Jeong YunWon
   Godefroid Chappelle
   Joshua Gilbert
   Dan Colish
@@ -271,6 +278,7 @@
   Christian Muirhead
   Berker Peksag
   James Lan
+  Volodymyr Vladymyrov
   shoma hosaka
   Daniel Neuh&#228;user
   Ben Mather
@@ -316,6 +324,7 @@
   yasirs
   Michael Chermside
   Anna Ravencroft
+  Andrey Churin
   Dan Crosta
   Julien Phalip
   Roman Podoliaka
@@ -421,6 +430,13 @@
 _gdbm module, provided in the file lib_pypy/_gdbm.py, is redistributed
 under the terms of the GPL license as well.
 
+License for 'pypy/module/_vmprof/src'
+--------------------------------------
+
+The code is based on gperftools. You may see a copy of the License for it at
+
+    https://code.google.com/p/gperftools/source/browse/COPYING
+
 License for 'liblzma and 'lzmaffi'
 ----------------------------------
 
diff --git a/lib-python/2.7/Cookie.py b/lib-python/2.7/Cookie.py
--- a/lib-python/2.7/Cookie.py
+++ b/lib-python/2.7/Cookie.py
@@ -528,12 +528,13 @@
 # result, the parsing rules here are less strict.
 #
 
-_LegalCharsPatt  = r"[\w\d!#%&'~_`><@,:/\$\*\+\-\.\^\|\)\(\?\}\{\=]"
+_LegalKeyChars  = r"\w\d!#%&'~_`><@,:/\$\*\+\-\.\^\|\)\(\?\}\{\="
+_LegalValueChars = _LegalKeyChars + r"\[\]"
 _CookiePattern = re.compile(
     r"(?x)"                       # This is a Verbose pattern
     r"\s*"                        # Optional whitespace at start of cookie
     r"(?P<key>"                   # Start of group 'key'
-    ""+ _LegalCharsPatt +"+?"     # Any word of at least one letter, nongreedy
+    "["+ _LegalKeyChars +"]+?"     # Any word of at least one letter, nongreedy
     r")"                          # End of group 'key'
     r"("                          # Optional group: there may not be a value.
     r"\s*=\s*"                    # Equal Sign
@@ -542,7 +543,7 @@
     r"|"                            # or
     r"\w{3},\s[\s\w\d-]{9,11}\s[\d:]{8}\sGMT" # Special case for "expires" attr
     r"|"                            # or
-    ""+ _LegalCharsPatt +"*"        # Any word or empty string
+    "["+ _LegalValueChars +"]*"        # Any word or empty string
     r")"                          # End of group 'val'
     r")?"                         # End of optional value group
     r"\s*"                        # Any number of spaces.
diff --git a/lib-python/2.7/SimpleHTTPServer.py b/lib-python/2.7/SimpleHTTPServer.py
--- a/lib-python/2.7/SimpleHTTPServer.py
+++ b/lib-python/2.7/SimpleHTTPServer.py
@@ -14,6 +14,7 @@
 import posixpath
 import BaseHTTPServer
 import urllib
+import urlparse
 import cgi
 import sys
 import shutil
@@ -68,10 +69,14 @@
         path = self.translate_path(self.path)
         f = None
         if os.path.isdir(path):
-            if not self.path.endswith('/'):
+            parts = urlparse.urlsplit(self.path)
+            if not parts.path.endswith('/'):
                 # redirect browser - doing basically what apache does
                 self.send_response(301)
-                self.send_header("Location", self.path + "/")
+                new_parts = (parts[0], parts[1], parts[2] + '/',
+                             parts[3], parts[4])
+                new_url = urlparse.urlunsplit(new_parts)
+                self.send_header("Location", new_url)
                 self.end_headers()
                 return None
             for index in "index.html", "index.htm":
diff --git a/lib-python/2.7/_LWPCookieJar.py b/lib-python/2.7/_LWPCookieJar.py
--- a/lib-python/2.7/_LWPCookieJar.py
+++ b/lib-python/2.7/_LWPCookieJar.py
@@ -18,7 +18,7 @@
                        iso2time, time2isoz)
 
 def lwp_cookie_str(cookie):
-    """Return string representation of Cookie in an the LWP cookie file format.
+    """Return string representation of Cookie in the LWP cookie file format.
 
     Actually, the format is extended a bit -- see module docstring.
 
diff --git a/lib-python/2.7/_abcoll.py b/lib-python/2.7/_abcoll.py
--- a/lib-python/2.7/_abcoll.py
+++ b/lib-python/2.7/_abcoll.py
@@ -548,23 +548,25 @@
             If E present and lacks .keys() method, does:     for (k, v) in E: D[k] = v
             In either case, this is followed by: for k, v in F.items(): D[k] = v
         '''
-        if len(args) > 2:
-            raise TypeError("update() takes at most 2 positional "
-                            "arguments ({} given)".format(len(args)))
-        elif not args:
-            raise TypeError("update() takes at least 1 argument (0 given)")
+        if not args:
+            raise TypeError("descriptor 'update' of 'MutableMapping' object "
+                            "needs an argument")
         self = args[0]
-        other = args[1] if len(args) >= 2 else ()
-
-        if isinstance(other, Mapping):
-            for key in other:
-                self[key] = other[key]
-        elif hasattr(other, "keys"):
-            for key in other.keys():
-                self[key] = other[key]
-        else:
-            for key, value in other:
-                self[key] = value
+        args = args[1:]
+        if len(args) > 1:
+            raise TypeError('update expected at most 1 arguments, got %d' %
+                            len(args))
+        if args:
+            other = args[0]
+            if isinstance(other, Mapping):
+                for key in other:
+                    self[key] = other[key]
+            elif hasattr(other, "keys"):
+                for key in other.keys():
+                    self[key] = other[key]
+            else:
+                for key, value in other:
+                    self[key] = value
         for key, value in kwds.items():
             self[key] = value
 
diff --git a/lib-python/2.7/_pyio.py b/lib-python/2.7/_pyio.py
--- a/lib-python/2.7/_pyio.py
+++ b/lib-python/2.7/_pyio.py
@@ -25,8 +25,8 @@
 DEFAULT_BUFFER_SIZE = 8 * 1024  # bytes
 
 # NOTE: Base classes defined here are registered with the "official" ABCs
-# defined in io.py. We don't use real inheritance though, because we don't
-# want to inherit the C implementations.
+# defined in io.py. We don't use real inheritance though, because we don't want
+# to inherit the C implementations.
 
 
 class BlockingIOError(IOError):
@@ -775,7 +775,7 @@
         clsname = self.__class__.__name__
         try:
             name = self.name
-        except AttributeError:
+        except Exception:
             return "<_pyio.{0}>".format(clsname)
         else:
             return "<_pyio.{0} name={1!r}>".format(clsname, name)
@@ -1216,8 +1216,10 @@
         return self.writer.flush()
 
     def close(self):
-        self.writer.close()
-        self.reader.close()
+        try:
+            self.writer.close()
+        finally:
+            self.reader.close()
 
     def isatty(self):
         return self.reader.isatty() or self.writer.isatty()
@@ -1538,7 +1540,7 @@
     def __repr__(self):
         try:
             name = self.name
-        except AttributeError:
+        except Exception:
             return "<_pyio.TextIOWrapper encoding='{0}'>".format(self.encoding)
         else:
             return "<_pyio.TextIOWrapper name={0!r} encoding='{1}'>".format(
diff --git a/lib-python/2.7/_strptime.py b/lib-python/2.7/_strptime.py
--- a/lib-python/2.7/_strptime.py
+++ b/lib-python/2.7/_strptime.py
@@ -335,9 +335,9 @@
     # though
     week_of_year = -1
     week_of_year_start = -1
-    # weekday and julian defaulted to -1 so as to signal need to calculate
+    # weekday and julian defaulted to None so as to signal need to calculate
     # values
-    weekday = julian = -1
+    weekday = julian = None
     found_dict = found.groupdict()
     for group_key in found_dict.iterkeys():
         # Directives not explicitly handled below:
@@ -434,14 +434,14 @@
         year = 1900
     # If we know the week of the year and what day of that week, we can figure
     # out the Julian day of the year.
-    if julian == -1 and week_of_year != -1 and weekday != -1:
+    if julian is None and week_of_year != -1 and weekday is not None:
         week_starts_Mon = True if week_of_year_start == 0 else False
         julian = _calc_julian_from_U_or_W(year, week_of_year, weekday,
                                             week_starts_Mon)
     # Cannot pre-calculate datetime_date() since can change in Julian
     # calculation and thus could have different value for the day of the week
     # calculation.
-    if julian == -1:
+    if julian is None:
         # Need to add 1 to result since first day of the year is 1, not 0.
         julian = datetime_date(year, month, day).toordinal() - \
                   datetime_date(year, 1, 1).toordinal() + 1
@@ -451,7 +451,7 @@
         year = datetime_result.year
         month = datetime_result.month
         day = datetime_result.day
-    if weekday == -1:
+    if weekday is None:
         weekday = datetime_date(year, month, day).weekday()
     if leap_year_fix:
         # the caller didn't supply a year but asked for Feb 29th. We couldn't
diff --git a/lib-python/2.7/aifc.py b/lib-python/2.7/aifc.py
--- a/lib-python/2.7/aifc.py
+++ b/lib-python/2.7/aifc.py
@@ -357,10 +357,13 @@
         self._soundpos = 0
 
     def close(self):
-        if self._decomp:
-            self._decomp.CloseDecompressor()
-            self._decomp = None
-        self._file.close()
+        decomp = self._decomp
+        try:
+            if decomp:
+                self._decomp = None
+                decomp.CloseDecompressor()
+        finally:
+            self._file.close()
 
     def tell(self):
         return self._soundpos
diff --git a/lib-python/2.7/binhex.py b/lib-python/2.7/binhex.py
--- a/lib-python/2.7/binhex.py
+++ b/lib-python/2.7/binhex.py
@@ -32,7 +32,8 @@
     pass
 
 # States (what have we written)
-[_DID_HEADER, _DID_DATA, _DID_RSRC] = range(3)
+_DID_HEADER = 0
+_DID_DATA = 1
 
 # Various constants
 REASONABLY_LARGE=32768  # Minimal amount we pass the rle-coder
@@ -235,17 +236,22 @@
         self._write(data)
 
     def close(self):
-        if self.state < _DID_DATA:
-            self.close_data()
-        if self.state != _DID_DATA:
-            raise Error, 'Close at the wrong time'
-        if self.rlen != 0:
-            raise Error, \
-                  "Incorrect resource-datasize, diff=%r" % (self.rlen,)
-        self._writecrc()
-        self.ofp.close()
-        self.state = None
-        del self.ofp
+        if self.state is None:
+            return
+        try:
+            if self.state < _DID_DATA:
+                self.close_data()
+            if self.state != _DID_DATA:
+                raise Error, 'Close at the wrong time'
+            if self.rlen != 0:
+                raise Error, \
+                    "Incorrect resource-datasize, diff=%r" % (self.rlen,)
+            self._writecrc()
+        finally:
+            self.state = None
+            ofp = self.ofp
+            del self.ofp
+            ofp.close()
 
 def binhex(inp, out):
     """(infilename, outfilename) - Create binhex-encoded copy of a file"""
@@ -463,11 +469,15 @@
         return self._read(n)
 
     def close(self):
-        if self.rlen:
-            dummy = self.read_rsrc(self.rlen)
-        self._checkcrc()
-        self.state = _DID_RSRC
-        self.ifp.close()
+        if self.state is None:
+            return
+        try:
+            if self.rlen:
+                dummy = self.read_rsrc(self.rlen)
+            self._checkcrc()
+        finally:
+            self.state = None
+            self.ifp.close()
 
 def hexbin(inp, out):
     """(infilename, outfilename) - Decode binhexed file"""
diff --git a/lib-python/2.7/bsddb/test/test_all.py b/lib-python/2.7/bsddb/test/test_all.py
--- a/lib-python/2.7/bsddb/test/test_all.py
+++ b/lib-python/2.7/bsddb/test/test_all.py
@@ -412,9 +412,6 @@
         def get_dbp(self) :
             return self._db
 
-    import string
-    string.letters=[chr(i) for i in xrange(65,91)]
-
     bsddb._db.DBEnv_orig = bsddb._db.DBEnv
     bsddb._db.DB_orig = bsddb._db.DB
     if bsddb.db.version() <= (4, 3) :
diff --git a/lib-python/2.7/bsddb/test/test_basics.py b/lib-python/2.7/bsddb/test/test_basics.py
--- a/lib-python/2.7/bsddb/test/test_basics.py
+++ b/lib-python/2.7/bsddb/test/test_basics.py
@@ -999,7 +999,7 @@
         for x in "The quick brown fox jumped over the lazy dog".split():
             d2.put(x, self.makeData(x))
 
-        for x in string.letters:
+        for x in string.ascii_letters:
             d3.put(x, x*70)
 
         d1.sync()
@@ -1047,7 +1047,7 @@
             if verbose:
                 print rec
             rec = c3.next()
-        self.assertEqual(count, len(string.letters))
+        self.assertEqual(count, len(string.ascii_letters))
 
 
         c1.close()
diff --git a/lib-python/2.7/bsddb/test/test_dbshelve.py b/lib-python/2.7/bsddb/test/test_dbshelve.py
--- a/lib-python/2.7/bsddb/test/test_dbshelve.py
+++ b/lib-python/2.7/bsddb/test/test_dbshelve.py
@@ -59,7 +59,7 @@
             return bytes(key, "iso8859-1")  # 8 bits
 
     def populateDB(self, d):
-        for x in string.letters:
+        for x in string.ascii_letters:
             d[self.mk('S' + x)] = 10 * x           # add a string
             d[self.mk('I' + x)] = ord(x)           # add an integer
             d[self.mk('L' + x)] = [x] * 10         # add a list
diff --git a/lib-python/2.7/bsddb/test/test_get_none.py b/lib-python/2.7/bsddb/test/test_get_none.py
--- a/lib-python/2.7/bsddb/test/test_get_none.py
+++ b/lib-python/2.7/bsddb/test/test_get_none.py
@@ -26,14 +26,14 @@
         d.open(self.filename, db.DB_BTREE, db.DB_CREATE)
         d.set_get_returns_none(1)
 
-        for x in string.letters:
+        for x in string.ascii_letters:
             d.put(x, x * 40)
 
         data = d.get('bad key')
         self.assertEqual(data, None)
 
-        data = d.get(string.letters[0])
-        self.assertEqual(data, string.letters[0]*40)
+        data = d.get(string.ascii_letters[0])
+        self.assertEqual(data, string.ascii_letters[0]*40)
 
         count = 0
         c = d.cursor()
@@ -43,7 +43,7 @@
             rec = c.next()
 
         self.assertEqual(rec, None)
-        self.assertEqual(count, len(string.letters))
+        self.assertEqual(count, len(string.ascii_letters))
 
         c.close()
         d.close()
@@ -54,14 +54,14 @@
         d.open(self.filename, db.DB_BTREE, db.DB_CREATE)
         d.set_get_returns_none(0)
 
-        for x in string.letters:
+        for x in string.ascii_letters:
             d.put(x, x * 40)
 
         self.assertRaises(db.DBNotFoundError, d.get, 'bad key')
         self.assertRaises(KeyError, d.get, 'bad key')
 
-        data = d.get(string.letters[0])
-        self.assertEqual(data, string.letters[0]*40)
+        data = d.get(string.ascii_letters[0])
+        self.assertEqual(data, string.ascii_letters[0]*40)
 
         count = 0
         exceptionHappened = 0
@@ -77,7 +77,7 @@
 
         self.assertNotEqual(rec, None)
         self.assertTrue(exceptionHappened)
-        self.assertEqual(count, len(string.letters))
+        self.assertEqual(count, len(string.ascii_letters))
 
         c.close()
         d.close()
diff --git a/lib-python/2.7/bsddb/test/test_queue.py b/lib-python/2.7/bsddb/test/test_queue.py
--- a/lib-python/2.7/bsddb/test/test_queue.py
+++ b/lib-python/2.7/bsddb/test/test_queue.py
@@ -10,7 +10,6 @@
 
 #----------------------------------------------------------------------
 
- at unittest.skip("fails on Windows; see issue 22943")
 class SimpleQueueTestCase(unittest.TestCase):
     def setUp(self):
         self.filename = get_new_database_path()
@@ -37,17 +36,17 @@
             print "before appends" + '-' * 30
             pprint(d.stat())
 
-        for x in string.letters:
+        for x in string.ascii_letters:
             d.append(x * 40)
 
-        self.assertEqual(len(d), len(string.letters))
+        self.assertEqual(len(d), len(string.ascii_letters))
 
         d.put(100, "some more data")
         d.put(101, "and some more ")
         d.put(75,  "out of order")
         d.put(1,   "replacement data")
 
-        self.assertEqual(len(d), len(string.letters)+3)
+        self.assertEqual(len(d), len(string.ascii_letters)+3)
 
         if verbose:
             print "before close" + '-' * 30
@@ -108,17 +107,17 @@
             print "before appends" + '-' * 30
             pprint(d.stat())
 
-        for x in string.letters:
+        for x in string.ascii_letters:
             d.append(x * 40)
 
-        self.assertEqual(len(d), len(string.letters))
+        self.assertEqual(len(d), len(string.ascii_letters))
 
         d.put(100, "some more data")
         d.put(101, "and some more ")
         d.put(75,  "out of order")
         d.put(1,   "replacement data")
 
-        self.assertEqual(len(d), len(string.letters)+3)
+        self.assertEqual(len(d), len(string.ascii_letters)+3)
 
         if verbose:
             print "before close" + '-' * 30
diff --git a/lib-python/2.7/bsddb/test/test_recno.py b/lib-python/2.7/bsddb/test/test_recno.py
--- a/lib-python/2.7/bsddb/test/test_recno.py
+++ b/lib-python/2.7/bsddb/test/test_recno.py
@@ -4,12 +4,11 @@
 import os, sys
 import errno
 from pprint import pprint
+import string
 import unittest
 
 from test_all import db, test_support, verbose, get_new_environment_path, get_new_database_path
 
-letters = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
-
 
 #----------------------------------------------------------------------
 
@@ -39,7 +38,7 @@
 
         d.open(self.filename, db.DB_RECNO, db.DB_CREATE)
 
-        for x in letters:
+        for x in string.ascii_letters:
             recno = d.append(x * 60)
             self.assertIsInstance(recno, int)
             self.assertGreaterEqual(recno, 1)
@@ -270,7 +269,7 @@
         d.set_re_pad(45)  # ...test both int and char
         d.open(self.filename, db.DB_RECNO, db.DB_CREATE)
 
-        for x in letters:
+        for x in string.ascii_letters:
             d.append(x * 35)    # These will be padded
 
         d.append('.' * 40)      # this one will be exact
diff --git a/lib-python/2.7/chunk.py b/lib-python/2.7/chunk.py
--- a/lib-python/2.7/chunk.py
+++ b/lib-python/2.7/chunk.py
@@ -85,8 +85,10 @@
 
     def close(self):
         if not self.closed:
-            self.skip()
-            self.closed = True
+            try:
+                self.skip()
+            finally:
+                self.closed = True
 
     def isatty(self):
         if self.closed:
diff --git a/lib-python/2.7/codecs.py b/lib-python/2.7/codecs.py
--- a/lib-python/2.7/codecs.py
+++ b/lib-python/2.7/codecs.py
@@ -20,8 +20,14 @@
            "BOM_LE", "BOM32_BE", "BOM32_LE", "BOM64_BE", "BOM64_LE",
            "BOM_UTF8", "BOM_UTF16", "BOM_UTF16_LE", "BOM_UTF16_BE",
            "BOM_UTF32", "BOM_UTF32_LE", "BOM_UTF32_BE",
+           "CodecInfo", "Codec", "IncrementalEncoder", "IncrementalDecoder",
+           "StreamReader", "StreamWriter",
+           "StreamReaderWriter", "StreamRecoder",
+           "getencoder", "getdecoder", "getincrementalencoder",
+           "getincrementaldecoder", "getreader", "getwriter",
+           "encode", "decode", "iterencode", "iterdecode",
            "strict_errors", "ignore_errors", "replace_errors",
-           "xmlcharrefreplace_errors",
+           "xmlcharrefreplace_errors", "backslashreplace_errors",
            "register_error", "lookup_error"]
 
 ### Constants
@@ -1051,7 +1057,7 @@
         during translation.
 
         One example where this happens is cp875.py which decodes
-        multiple character to \u001a.
+        multiple character to \\u001a.
 
     """
     m = {}
diff --git a/lib-python/2.7/collections.py b/lib-python/2.7/collections.py
--- a/lib-python/2.7/collections.py
+++ b/lib-python/2.7/collections.py
@@ -330,7 +330,7 @@
     #   http://code.activestate.com/recipes/259174/
     #   Knuth, TAOCP Vol. II section 4.6.3
 
-    def __init__(self, iterable=None, **kwds):
+    def __init__(*args, **kwds):
         '''Create a new, empty Counter object.  And if given, count elements
         from an input iterable.  Or, initialize the count from another mapping
         of elements to their counts.
@@ -341,8 +341,15 @@
         >>> c = Counter(a=4, b=2)                   # a new counter from keyword args
 
         '''
+        if not args:
+            raise TypeError("descriptor '__init__' of 'Counter' object "
+                            "needs an argument")
+        self = args[0]
+        args = args[1:]
+        if len(args) > 1:
+            raise TypeError('expected at most 1 arguments, got %d' % len(args))
         super(Counter, self).__init__()
-        self.update(iterable, **kwds)
+        self.update(*args, **kwds)
 
     def __missing__(self, key):
         'The count of elements not in the Counter is zero.'
@@ -393,7 +400,7 @@
         raise NotImplementedError(
             'Counter.fromkeys() is undefined.  Use Counter(iterable) instead.')
 
-    def update(self, iterable=None, **kwds):
+    def update(*args, **kwds):
         '''Like dict.update() but add counts instead of replacing them.
 
         Source can be an iterable, a dictionary, or another Counter instance.
@@ -413,6 +420,14 @@
         # contexts.  Instead, we implement straight-addition.  Both the inputs
         # and outputs are allowed to contain zero and negative counts.
 
+        if not args:
+            raise TypeError("descriptor 'update' of 'Counter' object "
+                            "needs an argument")
+        self = args[0]
+        args = args[1:]
+        if len(args) > 1:
+            raise TypeError('expected at most 1 arguments, got %d' % len(args))
+        iterable = args[0] if args else None
         if iterable is not None:
             if isinstance(iterable, Mapping):
                 if self:
@@ -428,7 +443,7 @@
         if kwds:
             self.update(kwds)
 
-    def subtract(self, iterable=None, **kwds):
+    def subtract(*args, **kwds):
         '''Like dict.update() but subtracts counts instead of replacing them.
         Counts can be reduced below zero.  Both the inputs and outputs are
         allowed to contain zero and negative counts.
@@ -444,6 +459,14 @@
         -1
 
         '''
+        if not args:
+            raise TypeError("descriptor 'subtract' of 'Counter' object "
+                            "needs an argument")
+        self = args[0]
+        args = args[1:]
+        if len(args) > 1:
+            raise TypeError('expected at most 1 arguments, got %d' % len(args))
+        iterable = args[0] if args else None
         if iterable is not None:
             self_get = self.get
             if isinstance(iterable, Mapping):
diff --git a/lib-python/2.7/cookielib.py b/lib-python/2.7/cookielib.py
--- a/lib-python/2.7/cookielib.py
+++ b/lib-python/2.7/cookielib.py
@@ -464,26 +464,42 @@
     for ns_header in ns_headers:
         pairs = []
         version_set = False
-        for ii, param in enumerate(re.split(r";\s*", ns_header)):
-            param = param.rstrip()
-            if param == "": continue
-            if "=" not in param:
-                k, v = param, None
-            else:
-                k, v = re.split(r"\s*=\s*", param, 1)
-                k = k.lstrip()
+
+        # XXX: The following does not strictly adhere to RFCs in that empty
+        # names and values are legal (the former will only appear once and will
+        # be overwritten if multiple occurrences are present). This is
+        # mostly to deal with backwards compatibility.
+        for ii, param in enumerate(ns_header.split(';')):
+            param = param.strip()
+
+            key, sep, val = param.partition('=')
+            key = key.strip()
+
+            if not key:
+                if ii == 0:
+                    break
+                else:
+                    continue
+
+            # allow for a distinction between present and empty and missing
+            # altogether
+            val = val.strip() if sep else None
+
             if ii != 0:
-                lc = k.lower()
+                lc = key.lower()
                 if lc in known_attrs:
-                    k = lc
-                if k == "version":
+                    key = lc
+
+                if key == "version":
                     # This is an RFC 2109 cookie.
-                    v = _strip_quotes(v)
+                    if val is not None:
+                        val = _strip_quotes(val)
                     version_set = True
-                if k == "expires":
+                elif key == "expires":
                     # convert expires date to seconds since epoch
-                    v = http2time(_strip_quotes(v))  # None if invalid
-            pairs.append((k, v))
+                    if val is not None:
+                        val = http2time(_strip_quotes(val))  # None if invalid
+            pairs.append((key, val))
 
         if pairs:
             if not version_set:
diff --git a/lib-python/2.7/ctypes/macholib/fetch_macholib.bat b/lib-python/2.7/ctypes/macholib/fetch_macholib.bat
--- a/lib-python/2.7/ctypes/macholib/fetch_macholib.bat
+++ b/lib-python/2.7/ctypes/macholib/fetch_macholib.bat
@@ -1,1 +1,1 @@
-svn export --force http://svn.red-bean.com/bob/macholib/trunk/macholib/ .
+svn export --force http://svn.red-bean.com/bob/macholib/trunk/macholib/ .
diff --git a/lib-python/2.7/ctypes/test/test_find.py b/lib-python/2.7/ctypes/test/test_find.py
--- a/lib-python/2.7/ctypes/test/test_find.py
+++ b/lib-python/2.7/ctypes/test/test_find.py
@@ -32,15 +32,24 @@
     def setUp(self):
         self.gl = self.glu = self.gle = None
         if lib_gl:
-            self.gl = CDLL(lib_gl, mode=RTLD_GLOBAL)
+            try:
+                self.gl = CDLL(lib_gl, mode=RTLD_GLOBAL)
+            except OSError:
+                pass
         if lib_glu:
-            self.glu = CDLL(lib_glu, RTLD_GLOBAL)
+            try:
+                self.glu = CDLL(lib_glu, RTLD_GLOBAL)
+            except OSError:
+                pass
         if lib_gle:
             try:
                 self.gle = CDLL(lib_gle)
             except OSError:
                 pass
 
+    def tearDown(self):
+        self.gl = self.glu = self.gle = None
+
     @unittest.skipUnless(lib_gl, 'lib_gl not available')
     def test_gl(self):
         if self.gl:
diff --git a/lib-python/2.7/ctypes/test/test_pickling.py b/lib-python/2.7/ctypes/test/test_pickling.py
--- a/lib-python/2.7/ctypes/test/test_pickling.py
+++ b/lib-python/2.7/ctypes/test/test_pickling.py
@@ -15,9 +15,9 @@
 class Y(X):
     _fields_ = [("str", c_char_p)]
 
-class PickleTest(unittest.TestCase):
+class PickleTest:
     def dumps(self, item):
-        return pickle.dumps(item)
+        return pickle.dumps(item, self.proto)
 
     def loads(self, item):
         return pickle.loads(item)
@@ -72,17 +72,15 @@
 
     @xfail
     def test_wchar(self):
-        pickle.dumps(c_char("x"))
+        self.dumps(c_char(b"x"))
         # Issue 5049
-        pickle.dumps(c_wchar(u"x"))
+        self.dumps(c_wchar(u"x"))
 
-class PickleTest_1(PickleTest):
-    def dumps(self, item):
-        return pickle.dumps(item, 1)
-
-class PickleTest_2(PickleTest):
-    def dumps(self, item):
-        return pickle.dumps(item, 2)
+for proto in range(pickle.HIGHEST_PROTOCOL + 1):
+    name = 'PickleTest_%s' % proto
+    globals()[name] = type(name,
+                           (PickleTest, unittest.TestCase),
+                           {'proto': proto})
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/lib-python/2.7/ctypes/test/test_pointers.py b/lib-python/2.7/ctypes/test/test_pointers.py
--- a/lib-python/2.7/ctypes/test/test_pointers.py
+++ b/lib-python/2.7/ctypes/test/test_pointers.py
@@ -7,8 +7,6 @@
                  c_long, c_ulong, c_longlong, c_ulonglong, c_double, c_float]
 python_types = [int, int, int, int, int, long,
                 int, long, long, long, float, float]
-LargeNamedType = type('T' * 2 ** 25, (Structure,), {})
-large_string = 'T' * 2 ** 25
 
 class PointersTestCase(unittest.TestCase):
 
@@ -191,9 +189,11 @@
             self.assertEqual(bool(mth), True)
 
     def test_pointer_type_name(self):
+        LargeNamedType = type('T' * 2 ** 25, (Structure,), {})
         self.assertTrue(POINTER(LargeNamedType))
 
     def test_pointer_type_str_name(self):
+        large_string = 'T' * 2 ** 25
         self.assertTrue(POINTER(large_string))
 
 if __name__ == '__main__':
diff --git a/lib-python/2.7/ctypes/util.py b/lib-python/2.7/ctypes/util.py
--- a/lib-python/2.7/ctypes/util.py
+++ b/lib-python/2.7/ctypes/util.py
@@ -178,7 +178,7 @@
             res = re.findall(expr, data)
             if not res:
                 return _get_soname(_findLib_gcc(name))
-            res.sort(cmp= lambda x,y: cmp(_num_version(x), _num_version(y)))
+            res.sort(key=_num_version)
             return res[-1]
 
     elif sys.platform == "sunos5":
diff --git a/lib-python/2.7/distutils/__init__.py b/lib-python/2.7/distutils/__init__.py
--- a/lib-python/2.7/distutils/__init__.py
+++ b/lib-python/2.7/distutils/__init__.py
@@ -15,5 +15,5 @@
 # Updated automatically by the Python release process.
 #
 #--start constants--
-__version__ = "2.7.9"
+__version__ = "2.7.10"
 #--end constants--
diff --git a/lib-python/2.7/distutils/command/check.py b/lib-python/2.7/distutils/command/check.py
--- a/lib-python/2.7/distutils/command/check.py
+++ b/lib-python/2.7/distutils/command/check.py
@@ -126,7 +126,7 @@
         """Returns warnings when the provided data doesn't compile."""
         source_path = StringIO()
         parser = Parser()
-        settings = frontend.OptionParser().get_default_values()
+        settings = frontend.OptionParser(components=(Parser,)).get_default_values()
         settings.tab_width = 4
         settings.pep_references = None
         settings.rfc_references = None
@@ -142,8 +142,8 @@
         document.note_source(source_path, -1)
         try:
             parser.parse(data, document)
-        except AttributeError:
-            reporter.messages.append((-1, 'Could not finish the parsing.',
-                                      '', {}))
+        except AttributeError as e:
+            reporter.messages.append(
+                (-1, 'Could not finish the parsing: %s.' % e, '', {}))
 
         return reporter.messages
diff --git a/lib-python/2.7/distutils/dir_util.py b/lib-python/2.7/distutils/dir_util.py
--- a/lib-python/2.7/distutils/dir_util.py
+++ b/lib-python/2.7/distutils/dir_util.py
@@ -83,7 +83,7 @@
     """Create all the empty directories under 'base_dir' needed to put 'files'
     there.
 
-    'base_dir' is just the a name of a directory which doesn't necessarily
+    'base_dir' is just the name of a directory which doesn't necessarily
     exist yet; 'files' is a list of filenames to be interpreted relative to
     'base_dir'.  'base_dir' + the directory portion of every file in 'files'
     will be created if it doesn't already exist.  'mode', 'verbose' and
diff --git a/lib-python/2.7/distutils/tests/test_check.py b/lib-python/2.7/distutils/tests/test_check.py
--- a/lib-python/2.7/distutils/tests/test_check.py
+++ b/lib-python/2.7/distutils/tests/test_check.py
@@ -1,5 +1,6 @@
 # -*- encoding: utf8 -*-
 """Tests for distutils.command.check."""
+import textwrap
 import unittest
 from test.test_support import run_unittest
 
@@ -93,6 +94,36 @@
         cmd = self._run(metadata, strict=1, restructuredtext=1)
         self.assertEqual(cmd._warnings, 0)
 
+    @unittest.skipUnless(HAS_DOCUTILS, "won't test without docutils")
+    def test_check_restructuredtext_with_syntax_highlight(self):
+        # Don't fail if there is a `code` or `code-block` directive
+
+        example_rst_docs = []
+        example_rst_docs.append(textwrap.dedent("""\
+            Here's some code:
+
+            .. code:: python
+
+                def foo():
+                    pass
+            """))
+        example_rst_docs.append(textwrap.dedent("""\
+            Here's some code:
+
+            .. code-block:: python
+
+                def foo():
+                    pass
+            """))
+
+        for rest_with_code in example_rst_docs:
+            pkg_info, dist = self.create_dist(long_description=rest_with_code)
+            cmd = check(dist)
+            cmd.check_restructuredtext()
+            self.assertEqual(cmd._warnings, 0)
+            msgs = cmd._check_rst_data(rest_with_code)
+            self.assertEqual(len(msgs), 0)
+
     def test_check_all(self):
 
         metadata = {'url': 'xxx', 'author': 'xxx'}
diff --git a/lib-python/2.7/distutils/text_file.py b/lib-python/2.7/distutils/text_file.py
--- a/lib-python/2.7/distutils/text_file.py
+++ b/lib-python/2.7/distutils/text_file.py
@@ -124,11 +124,11 @@
     def close (self):
         """Close the current file and forget everything we know about it
            (filename, current line number)."""
-
-        self.file.close ()
+        file = self.file
         self.file = None
         self.filename = None
         self.current_line = None
+        file.close()
 
 
     def gen_error (self, msg, line=None):
diff --git a/lib-python/2.7/dumbdbm.py b/lib-python/2.7/dumbdbm.py
--- a/lib-python/2.7/dumbdbm.py
+++ b/lib-python/2.7/dumbdbm.py
@@ -21,6 +21,7 @@
 
 """
 
+import ast as _ast
 import os as _os
 import __builtin__
 import UserDict
@@ -85,7 +86,7 @@
             with f:
                 for line in f:
                     line = line.rstrip()
-                    key, pos_and_siz_pair = eval(line)
+                    key, pos_and_siz_pair = _ast.literal_eval(line)
                     self._index[key] = pos_and_siz_pair
 
     # Write the index dict to the directory file.  The original directory
@@ -208,8 +209,10 @@
         return len(self._index)
 
     def close(self):
-        self._commit()
-        self._index = self._datfile = self._dirfile = self._bakfile = None
+        try:
+            self._commit()
+        finally:
+            self._index = self._datfile = self._dirfile = self._bakfile = None
 
     __del__ = close
 
diff --git a/lib-python/2.7/encodings/uu_codec.py b/lib-python/2.7/encodings/uu_codec.py
--- a/lib-python/2.7/encodings/uu_codec.py
+++ b/lib-python/2.7/encodings/uu_codec.py
@@ -84,7 +84,7 @@
             data = a2b_uu(s)
         except binascii.Error, v:
             # Workaround for broken uuencoders by /Fredrik Lundh
-            nbytes = (((ord(s[0])-32) & 63) * 4 + 5) / 3
+            nbytes = (((ord(s[0])-32) & 63) * 4 + 5) // 3
             data = a2b_uu(s[:nbytes])
             #sys.stderr.write("Warning: %s\n" % str(v))
         write(data)
diff --git a/lib-python/2.7/ensurepip/__init__.py b/lib-python/2.7/ensurepip/__init__.py
--- a/lib-python/2.7/ensurepip/__init__.py
+++ b/lib-python/2.7/ensurepip/__init__.py
@@ -12,9 +12,9 @@
 __all__ = ["version", "bootstrap"]
 
 
-_SETUPTOOLS_VERSION = "7.0"
+_SETUPTOOLS_VERSION = "15.2"
 
-_PIP_VERSION = "1.5.6"
+_PIP_VERSION = "6.1.1"
 
 # pip currently requires ssl support, so we try to provide a nicer
 # error message when that is missing (http://bugs.python.org/issue19744)
diff --git a/lib-python/2.7/ensurepip/_bundled/pip-1.5.6-py2.py3-none-any.whl b/lib-python/2.7/ensurepip/_bundled/pip-1.5.6-py2.py3-none-any.whl
deleted file mode 100644
Binary file lib-python/2.7/ensurepip/_bundled/pip-1.5.6-py2.py3-none-any.whl has changed
diff --git a/lib-python/2.7/ensurepip/_bundled/pip-6.1.1-py2.py3-none-any.whl b/lib-python/2.7/ensurepip/_bundled/pip-6.1.1-py2.py3-none-any.whl
new file mode 100644
index 0000000000000000000000000000000000000000..e59694a019051d58b9a378a1adfc9461b8cec9c3
GIT binary patch

[cut]

diff --git a/lib-python/2.7/ensurepip/_bundled/setuptools-15.2-py2.py3-none-any.whl b/lib-python/2.7/ensurepip/_bundled/setuptools-15.2-py2.py3-none-any.whl
new file mode 100644
index 0000000000000000000000000000000000000000..f153ed376684275e08fcfebdb2de8352fb074171
GIT binary patch

[cut]

diff --git a/lib-python/2.7/ensurepip/_bundled/setuptools-7.0-py2.py3-none-any.whl b/lib-python/2.7/ensurepip/_bundled/setuptools-7.0-py2.py3-none-any.whl
deleted file mode 100644
Binary file lib-python/2.7/ensurepip/_bundled/setuptools-7.0-py2.py3-none-any.whl has changed
diff --git a/lib-python/2.7/fileinput.py b/lib-python/2.7/fileinput.py
--- a/lib-python/2.7/fileinput.py
+++ b/lib-python/2.7/fileinput.py
@@ -233,8 +233,10 @@
         self.close()
 
     def close(self):
-        self.nextfile()
-        self._files = ()
+        try:
+            self.nextfile()
+        finally:
+            self._files = ()
 
     def __iter__(self):
         return self
@@ -270,23 +272,25 @@
 
         output = self._output
         self._output = 0
-        if output:
-            output.close()
+        try:
+            if output:
+                output.close()
+        finally:
+            file = self._file
+            self._file = 0
+            try:
+                if file and not self._isstdin:
+                    file.close()
+            finally:
+                backupfilename = self._backupfilename
+                self._backupfilename = 0
+                if backupfilename and not self._backup:
+                    try: os.unlink(backupfilename)
+                    except OSError: pass
 
-        file = self._file
-        self._file = 0
-        if file and not self._isstdin:
-            file.close()
-
-        backupfilename = self._backupfilename
-        self._backupfilename = 0
-        if backupfilename and not self._backup:
-            try: os.unlink(backupfilename)
-            except OSError: pass
-
-        self._isstdin = False
-        self._buffer = []
-        self._bufindex = 0
+                self._isstdin = False
+                self._buffer = []
+                self._bufindex = 0
 
     def readline(self):
         try:
diff --git a/lib-python/2.7/fnmatch.py b/lib-python/2.7/fnmatch.py
--- a/lib-python/2.7/fnmatch.py
+++ b/lib-python/2.7/fnmatch.py
@@ -47,12 +47,14 @@
     import os,posixpath
     result=[]
     pat=os.path.normcase(pat)
-    if not pat in _cache:
+    try:
+        re_pat = _cache[pat]
+    except KeyError:
         res = translate(pat)
         if len(_cache) >= _MAXCACHE:
             _cache.clear()
-        _cache[pat] = re.compile(res)
-    match=_cache[pat].match
+        _cache[pat] = re_pat = re.compile(res)
+    match = re_pat.match
     if os.path is posixpath:
         # normcase on posix is NOP. Optimize it away from the loop.
         for name in names:
@@ -71,12 +73,14 @@
     its arguments.
     """
 
-    if not pat in _cache:
+    try:
+        re_pat = _cache[pat]
+    except KeyError:
         res = translate(pat)
         if len(_cache) >= _MAXCACHE:
             _cache.clear()
-        _cache[pat] = re.compile(res)
-    return _cache[pat].match(name) is not None
+        _cache[pat] = re_pat = re.compile(res)
+    return re_pat.match(name) is not None
 
 def translate(pat):
     """Translate a shell PATTERN to a regular expression.
diff --git a/lib-python/2.7/ftplib.py b/lib-python/2.7/ftplib.py
--- a/lib-python/2.7/ftplib.py
+++ b/lib-python/2.7/ftplib.py
@@ -594,11 +594,16 @@
 
     def close(self):
         '''Close the connection without assuming anything about it.'''
-        if self.file is not None:
-            self.file.close()
-        if self.sock is not None:
-            self.sock.close()
-        self.file = self.sock = None
+        try:
+            file = self.file
+            self.file = None
+            if file is not None:
+                file.close()
+        finally:
+            sock = self.sock
+            self.sock = None
+            if sock is not None:
+                sock.close()
 
 try:
     import ssl
@@ -638,12 +643,24 @@
         '221 Goodbye.'
         >>>
         '''
-        ssl_version = ssl.PROTOCOL_TLSv1
+        ssl_version = ssl.PROTOCOL_SSLv23
 
         def __init__(self, host='', user='', passwd='', acct='', keyfile=None,
-                     certfile=None, timeout=_GLOBAL_DEFAULT_TIMEOUT):
+                     certfile=None, context=None,
+                     timeout=_GLOBAL_DEFAULT_TIMEOUT, source_address=None):
+            if context is not None and keyfile is not None:
+                raise ValueError("context and keyfile arguments are mutually "
+                                 "exclusive")
+            if context is not None and certfile is not None:
+                raise ValueError("context and certfile arguments are mutually "
+                                 "exclusive")
             self.keyfile = keyfile
             self.certfile = certfile
+            if context is None:
+                context = ssl._create_stdlib_context(self.ssl_version,
+                                                     certfile=certfile,
+                                                     keyfile=keyfile)
+            self.context = context
             self._prot_p = False
             FTP.__init__(self, host, user, passwd, acct, timeout)
 
@@ -656,12 +673,12 @@
             '''Set up secure control connection by using TLS/SSL.'''
             if isinstance(self.sock, ssl.SSLSocket):
                 raise ValueError("Already using TLS")
-            if self.ssl_version == ssl.PROTOCOL_TLSv1:
+            if self.ssl_version >= ssl.PROTOCOL_SSLv23:
                 resp = self.voidcmd('AUTH TLS')
             else:
                 resp = self.voidcmd('AUTH SSL')
-            self.sock = ssl.wrap_socket(self.sock, self.keyfile, self.certfile,
-                                        ssl_version=self.ssl_version)
+            self.sock = self.context.wrap_socket(self.sock,
+                                                 server_hostname=self.host)
             self.file = self.sock.makefile(mode='rb')
             return resp
 
@@ -692,8 +709,8 @@
         def ntransfercmd(self, cmd, rest=None):
             conn, size = FTP.ntransfercmd(self, cmd, rest)
             if self._prot_p:
-                conn = ssl.wrap_socket(conn, self.keyfile, self.certfile,
-                                       ssl_version=self.ssl_version)
+                conn = self.context.wrap_socket(conn,
+                                                server_hostname=self.host)
             return conn, size
 
         def retrbinary(self, cmd, callback, blocksize=8192, rest=None):
diff --git a/lib-python/2.7/genericpath.py b/lib-python/2.7/genericpath.py
--- a/lib-python/2.7/genericpath.py
+++ b/lib-python/2.7/genericpath.py
@@ -10,6 +10,14 @@
            'getsize', 'isdir', 'isfile']
 
 
+try:
+    _unicode = unicode
+except NameError:
+    # If Python is built without Unicode support, the unicode type
+    # will not exist. Fake one.
+    class _unicode(object):
+        pass
+
 # Does a path exist?
 # This is false for dangling symbolic links on systems that support them.
 def exists(path):
diff --git a/lib-python/2.7/gettext.py b/lib-python/2.7/gettext.py
--- a/lib-python/2.7/gettext.py
+++ b/lib-python/2.7/gettext.py
@@ -52,7 +52,9 @@
 
 __all__ = ['NullTranslations', 'GNUTranslations', 'Catalog',
            'find', 'translation', 'install', 'textdomain', 'bindtextdomain',
-           'dgettext', 'dngettext', 'gettext', 'ngettext',
+           'bind_textdomain_codeset',
+           'dgettext', 'dngettext', 'gettext', 'lgettext', 'ldgettext',
+           'ldngettext', 'lngettext', 'ngettext',
            ]
 
 _default_localedir = os.path.join(sys.prefix, 'share', 'locale')
@@ -294,11 +296,12 @@
             # See if we're looking at GNU .mo conventions for metadata
             if mlen == 0:
                 # Catalog description
-                lastk = k = None
+                lastk = None
                 for item in tmsg.splitlines():
                     item = item.strip()
                     if not item:
                         continue
+                    k = v = None
                     if ':' in item:
                         k, v = item.split(':', 1)
                         k = k.strip().lower()
diff --git a/lib-python/2.7/gzip.py b/lib-python/2.7/gzip.py
--- a/lib-python/2.7/gzip.py
+++ b/lib-python/2.7/gzip.py
@@ -238,9 +238,9 @@
             data = data.tobytes()
 
         if len(data) > 0:
-            self.size = self.size + len(data)
+            self.fileobj.write(self.compress.compress(data))
+            self.size += len(data)
             self.crc = zlib.crc32(data, self.crc) & 0xffffffffL
-            self.fileobj.write( self.compress.compress(data) )
             self.offset += len(data)
 
         return len(data)
@@ -369,19 +369,21 @@
         return self.fileobj is None
 
     def close(self):
-        if self.fileobj is None:
+        fileobj = self.fileobj
+        if fileobj is None:
             return
-        if self.mode == WRITE:
-            self.fileobj.write(self.compress.flush())
-            write32u(self.fileobj, self.crc)
-            # self.size may exceed 2GB, or even 4GB
-            write32u(self.fileobj, self.size & 0xffffffffL)
-            self.fileobj = None
-        elif self.mode == READ:
-            self.fileobj = None
-        if self.myfileobj:
-            self.myfileobj.close()
-            self.myfileobj = None
+        self.fileobj = None
+        try:
+            if self.mode == WRITE:
+                fileobj.write(self.compress.flush())
+                write32u(fileobj, self.crc)
+                # self.size may exceed 2GB, or even 4GB
+                write32u(fileobj, self.size & 0xffffffffL)
+        finally:
+            myfileobj = self.myfileobj
+            if myfileobj:
+                self.myfileobj = None
+                myfileobj.close()
 
     def flush(self,zlib_mode=zlib.Z_SYNC_FLUSH):
         self._check_closed()
diff --git a/lib-python/2.7/hashlib.py b/lib-python/2.7/hashlib.py
--- a/lib-python/2.7/hashlib.py
+++ b/lib-python/2.7/hashlib.py
@@ -187,7 +187,7 @@
 
         def prf(msg, inner=inner, outer=outer):
             # PBKDF2_HMAC uses the password as key. We can re-use the same
-            # digest objects and and just update copies to skip initialization.
+            # digest objects and just update copies to skip initialization.
             icpy = inner.copy()
             ocpy = outer.copy()
             icpy.update(msg)
diff --git a/lib-python/2.7/htmlentitydefs.py b/lib-python/2.7/htmlentitydefs.py
--- a/lib-python/2.7/htmlentitydefs.py
+++ b/lib-python/2.7/htmlentitydefs.py
@@ -1,6 +1,6 @@
 """HTML character entity references."""
 
-# maps the HTML entity name to the Unicode codepoint
+# maps the HTML entity name to the Unicode code point
 name2codepoint = {
     'AElig':    0x00c6, # latin capital letter AE = latin capital ligature AE, U+00C6 ISOlat1
     'Aacute':   0x00c1, # latin capital letter A with acute, U+00C1 ISOlat1
@@ -256,7 +256,7 @@
     'zwnj':     0x200c, # zero width non-joiner, U+200C NEW RFC 2070
 }
 
-# maps the Unicode codepoint to the HTML entity name
+# maps the Unicode code point to the HTML entity name
 codepoint2name = {}
 
 # maps the HTML entity name to the character
diff --git a/lib-python/2.7/httplib.py b/lib-python/2.7/httplib.py
--- a/lib-python/2.7/httplib.py
+++ b/lib-python/2.7/httplib.py
@@ -68,6 +68,7 @@
 
 from array import array
 import os
+import re
 import socket
 from sys import py3kwarning
 from urlparse import urlsplit
@@ -218,6 +219,38 @@
 # maximum amount of headers accepted
 _MAXHEADERS = 100
 
+# Header name/value ABNF (http://tools.ietf.org/html/rfc7230#section-3.2)
+#
+# VCHAR          = %x21-7E
+# obs-text       = %x80-FF
+# header-field   = field-name ":" OWS field-value OWS
+# field-name     = token
+# field-value    = *( field-content / obs-fold )
+# field-content  = field-vchar [ 1*( SP / HTAB ) field-vchar ]
+# field-vchar    = VCHAR / obs-text
+#
+# obs-fold       = CRLF 1*( SP / HTAB )
+#                ; obsolete line folding
+#                ; see Section 3.2.4
+
+# token          = 1*tchar
+#
+# tchar          = "!" / "#" / "$" / "%" / "&" / "'" / "*"
+#                / "+" / "-" / "." / "^" / "_" / "`" / "|" / "~"
+#                / DIGIT / ALPHA
+#                ; any VCHAR, except delimiters
+#
+# VCHAR defined in http://tools.ietf.org/html/rfc5234#appendix-B.1
+
+# the patterns for both name and value are more leniant than RFC
+# definitions to allow for backwards compatibility
+_is_legal_header_name = re.compile(r'\A[^:\s][^:\r\n]*\Z').match
+_is_illegal_header_value = re.compile(r'\n(?![ \t])|\r(?![ \t\n])').search
+
+# We always set the Content-Length header for these methods because some
+# servers will otherwise respond with a 411
+_METHODS_EXPECTING_BODY = {'PATCH', 'POST', 'PUT'}
+
 
 class HTTPMessage(mimetools.Message):
 
@@ -313,6 +346,11 @@
                 hlist.append(line)
                 self.addheader(headerseen, line[len(headerseen)+1:].strip())
                 continue
+            elif headerseen is not None:
+                # An empty header name. These aren't allowed in HTTP, but it's
+                # probably a benign mistake. Don't add the header, just keep
+                # going.
+                continue
             else:
                 # It's not a header line; throw it back and stop here.
                 if not self.dict:
@@ -522,9 +560,10 @@
         return True
 
     def close(self):
-        if self.fp:
-            self.fp.close()
+        fp = self.fp
+        if fp:
             self.fp = None
+            fp.close()
 
     def isclosed(self):
         # NOTE: it is possible that we will not ever call self.close(). This
@@ -723,7 +762,7 @@
         endpoint passed to set_tunnel. This is done by sending a HTTP CONNECT
         request to the proxy server when the connection is established.
 
-        This method must be called before the HTML connection has been
+        This method must be called before the HTTP connection has been
         established.
 
         The headers argument should be a mapping of extra HTTP headers
@@ -797,13 +836,17 @@
 
     def close(self):
         """Close the connection to the HTTP server."""
-        if self.sock:
-            self.sock.close()   # close it manually... there may be other refs
-            self.sock = None
-        if self.__response:
-            self.__response.close()
-            self.__response = None
         self.__state = _CS_IDLE
+        try:
+            sock = self.sock
+            if sock:
+                self.sock = None
+                sock.close()   # close it manually... there may be other refs
+        finally:
+            response = self.__response
+            if response:
+                self.__response = None
+                response.close()
 
     def send(self, data):
         """Send `data' to the server."""
@@ -978,7 +1021,16 @@
         if self.__state != _CS_REQ_STARTED:
             raise CannotSendHeader()
 
-        hdr = '%s: %s' % (header, '\r\n\t'.join([str(v) for v in values]))
+        header = '%s' % header
+        if not _is_legal_header_name(header):
+            raise ValueError('Invalid header name %r' % (header,))
+
+        values = [str(v) for v in values]
+        for one_value in values:
+            if _is_illegal_header_value(one_value):
+                raise ValueError('Invalid header value %r' % (one_value,))
+
+        hdr = '%s: %s' % (header, '\r\n\t'.join(values))
         self._output(hdr)
 
     def endheaders(self, message_body=None):
@@ -1000,19 +1052,25 @@
         """Send a complete request to the server."""
         self._send_request(method, url, body, headers)
 
-    def _set_content_length(self, body):
-        # Set the content-length based on the body.
+    def _set_content_length(self, body, method):
+        # Set the content-length based on the body. If the body is "empty", we
+        # set Content-Length: 0 for methods that expect a body (RFC 7230,
+        # Section 3.3.2). If the body is set for other methods, we set the
+        # header provided we can figure out what the length is.
         thelen = None
-        try:
-            thelen = str(len(body))
-        except TypeError, te:
-            # If this is a file-like object, try to
-            # fstat its file descriptor
+        if body is None and method.upper() in _METHODS_EXPECTING_BODY:
+            thelen = '0'
+        elif body is not None:
             try:
-                thelen = str(os.fstat(body.fileno()).st_size)
-            except (AttributeError, OSError):
-                # Don't send a length if this failed
-                if self.debuglevel > 0: print "Cannot stat!!"
+                thelen = str(len(body))
+            except TypeError:
+                # If this is a file-like object, try to
+                # fstat its file descriptor
+                try:
+                    thelen = str(os.fstat(body.fileno()).st_size)
+                except (AttributeError, OSError):
+                    # Don't send a length if this failed
+                    if self.debuglevel > 0: print "Cannot stat!!"
 
         if thelen is not None:
             self.putheader('Content-Length', thelen)
@@ -1028,8 +1086,8 @@
 
         self.putrequest(method, url, **skips)
 
-        if body is not None and 'content-length' not in header_names:
-            self._set_content_length(body)
+        if 'content-length' not in header_names:
+            self._set_content_length(body, method)
         for hdr, value in headers.iteritems():
             self.putheader(hdr, value)
         self.endheaders(body)
@@ -1072,20 +1130,20 @@
 
         try:
             response.begin()
+            assert response.will_close != _UNKNOWN
+            self.__state = _CS_IDLE
+
+            if response.will_close:
+                # this effectively passes the connection to the response
+                self.close()
+            else:
+                # remember this, so we can tell when it is complete
+                self.__response = response
+
+            return response
         except:
             response.close()
             raise
-        assert response.will_close != _UNKNOWN
-        self.__state = _CS_IDLE
-
-        if response.will_close:
-            # this effectively passes the connection to the response
-            self.close()
-        else:
-            # remember this, so we can tell when it is complete
-            self.__response = response
-
-        return response
 
 
 class HTTP:
@@ -1129,7 +1187,7 @@
         "Accept arguments to set the host/port, since the superclass doesn't."
 
         if host is not None:
-            self._conn._set_hostport(host, port)
+            (self._conn.host, self._conn.port) = self._conn._get_hostport(host, port)
         self._conn.connect()
 
     def getfile(self):
diff --git a/lib-python/2.7/idlelib/CodeContext.py b/lib-python/2.7/idlelib/CodeContext.py
--- a/lib-python/2.7/idlelib/CodeContext.py
+++ b/lib-python/2.7/idlelib/CodeContext.py
@@ -15,8 +15,8 @@
 from sys import maxint as INFINITY
 from idlelib.configHandler import idleConf
 
-BLOCKOPENERS = set(["class", "def", "elif", "else", "except", "finally", "for",
-                    "if", "try", "while", "with"])
+BLOCKOPENERS = {"class", "def", "elif", "else", "except", "finally", "for",
+                    "if", "try", "while", "with"}
 UPDATEINTERVAL = 100 # millisec
 FONTUPDATEINTERVAL = 1000 # millisec
 
diff --git a/lib-python/2.7/idlelib/EditorWindow.py b/lib-python/2.7/idlelib/EditorWindow.py
--- a/lib-python/2.7/idlelib/EditorWindow.py
+++ b/lib-python/2.7/idlelib/EditorWindow.py
@@ -469,13 +469,10 @@
         ("format", "F_ormat"),
         ("run", "_Run"),
         ("options", "_Options"),
-        ("windows", "_Windows"),
+        ("windows", "_Window"),
         ("help", "_Help"),
     ]
 
-    if sys.platform == "darwin":
-        menu_specs[-2] = ("windows", "_Window")
-
 
     def createmenubar(self):
         mbar = self.menubar
diff --git a/lib-python/2.7/idlelib/FormatParagraph.py b/lib-python/2.7/idlelib/FormatParagraph.py
--- a/lib-python/2.7/idlelib/FormatParagraph.py
+++ b/lib-python/2.7/idlelib/FormatParagraph.py
@@ -44,9 +44,11 @@
 
         The length limit parameter is for testing with a known value.
         """
-        if limit == None:
+        if limit is None:
+            # The default length limit is that defined by pep8
             limit = idleConf.GetOption(
-                    'main', 'FormatParagraph', 'paragraph', type='int')
+                'extensions', 'FormatParagraph', 'max-width',
+                type='int', default=72)
         text = self.editwin.text
         first, last = self.editwin.get_selection_indices()
         if first and last:
diff --git a/lib-python/2.7/idlelib/PyShell.py b/lib-python/2.7/idlelib/PyShell.py
--- a/lib-python/2.7/idlelib/PyShell.py
+++ b/lib-python/2.7/idlelib/PyShell.py
@@ -871,13 +871,10 @@
         ("edit", "_Edit"),
         ("debug", "_Debug"),
         ("options", "_Options"),
-        ("windows", "_Windows"),
+        ("windows", "_Window"),
         ("help", "_Help"),
     ]
 
-    if sys.platform == "darwin":
-        menu_specs[-2] = ("windows", "_Window")
-
 
     # New classes
     from idlelib.IdleHistory import History
@@ -1350,7 +1347,7 @@
         if type(s) not in (unicode, str, bytearray):
             # See issue #19481
             if isinstance(s, unicode):
-                s = unicode.__getslice__(s, None, None)
+                s = unicode.__getitem__(s, slice(None))
             elif isinstance(s, str):
                 s = str.__str__(s)
             elif isinstance(s, bytearray):
diff --git a/lib-python/2.7/idlelib/SearchEngine.py b/lib-python/2.7/idlelib/SearchEngine.py
--- a/lib-python/2.7/idlelib/SearchEngine.py
+++ b/lib-python/2.7/idlelib/SearchEngine.py
@@ -191,7 +191,7 @@
 
     This is done by searching forwards until there is no match.
     Prog: compiled re object with a search method returning a match.
-    Chars: line of text, without \n.
+    Chars: line of text, without \\n.
     Col: stop index for the search; the limit for match.end().
     '''
     m = prog.search(chars)
diff --git a/lib-python/2.7/idlelib/config-extensions.def b/lib-python/2.7/idlelib/config-extensions.def
--- a/lib-python/2.7/idlelib/config-extensions.def
+++ b/lib-python/2.7/idlelib/config-extensions.def
@@ -66,6 +66,7 @@
 
 [FormatParagraph]
 enable=True
+max-width=72
 [FormatParagraph_cfgBindings]
 format-paragraph=<Alt-Key-q>
 
diff --git a/lib-python/2.7/idlelib/config-main.def b/lib-python/2.7/idlelib/config-main.def
--- a/lib-python/2.7/idlelib/config-main.def
+++ b/lib-python/2.7/idlelib/config-main.def
@@ -58,9 +58,6 @@
 font-bold= 0
 encoding= none
 
-[FormatParagraph]
-paragraph=72
-
 [Indent]
 use-spaces= 1
 num-spaces= 4
diff --git a/lib-python/2.7/idlelib/configDialog.py b/lib-python/2.7/idlelib/configDialog.py
--- a/lib-python/2.7/idlelib/configDialog.py
+++ b/lib-python/2.7/idlelib/configDialog.py
@@ -371,7 +371,6 @@
         parent = self.parent
         self.winWidth = StringVar(parent)
         self.winHeight = StringVar(parent)
-        self.paraWidth = StringVar(parent)
         self.startupEdit = IntVar(parent)
         self.autoSave = IntVar(parent)
         self.encoding = StringVar(parent)
@@ -387,7 +386,6 @@
         frameSave = LabelFrame(frame, borderwidth=2, relief=GROOVE,
                                text=' Autosave Preferences ')
         frameWinSize = Frame(frame, borderwidth=2, relief=GROOVE)
-        frameParaSize = Frame(frame, borderwidth=2, relief=GROOVE)
         frameEncoding = Frame(frame, borderwidth=2, relief=GROOVE)
         frameHelp = LabelFrame(frame, borderwidth=2, relief=GROOVE,
                                text=' Additional Help Sources ')
@@ -416,11 +414,6 @@
         labelWinHeightTitle = Label(frameWinSize, text='Height')
         entryWinHeight = Entry(
                 frameWinSize, textvariable=self.winHeight, width=3)
-        #paragraphFormatWidth
-        labelParaWidthTitle = Label(
-                frameParaSize, text='Paragraph reformat width (in characters)')
-        entryParaWidth = Entry(
-                frameParaSize, textvariable=self.paraWidth, width=3)
         #frameEncoding
         labelEncodingTitle = Label(
                 frameEncoding, text="Default Source Encoding")
@@ -458,7 +451,6 @@
         frameRun.pack(side=TOP, padx=5, pady=5, fill=X)
         frameSave.pack(side=TOP, padx=5, pady=5, fill=X)
         frameWinSize.pack(side=TOP, padx=5, pady=5, fill=X)
-        frameParaSize.pack(side=TOP, padx=5, pady=5, fill=X)
         frameEncoding.pack(side=TOP, padx=5, pady=5, fill=X)
         frameHelp.pack(side=TOP, padx=5, pady=5, expand=TRUE, fill=BOTH)
         #frameRun
@@ -475,9 +467,6 @@
         labelWinHeightTitle.pack(side=RIGHT, anchor=E, pady=5)
         entryWinWidth.pack(side=RIGHT, anchor=E, padx=10, pady=5)
         labelWinWidthTitle.pack(side=RIGHT, anchor=E, pady=5)
-        #paragraphFormatWidth
-        labelParaWidthTitle.pack(side=LEFT, anchor=W, padx=5, pady=5)
-        entryParaWidth.pack(side=RIGHT, anchor=E, padx=10, pady=5)
         #frameEncoding
         labelEncodingTitle.pack(side=LEFT, anchor=W, padx=5, pady=5)
         radioEncNone.pack(side=RIGHT, anchor=E, pady=5)
@@ -509,7 +498,6 @@
         self.keysAreBuiltin.trace_variable('w', self.VarChanged_keysAreBuiltin)
         self.winWidth.trace_variable('w', self.VarChanged_winWidth)
         self.winHeight.trace_variable('w', self.VarChanged_winHeight)
-        self.paraWidth.trace_variable('w', self.VarChanged_paraWidth)
         self.startupEdit.trace_variable('w', self.VarChanged_startupEdit)
         self.autoSave.trace_variable('w', self.VarChanged_autoSave)
         self.encoding.trace_variable('w', self.VarChanged_encoding)
@@ -594,10 +582,6 @@
         value = self.winHeight.get()
         self.AddChangedItem('main', 'EditorWindow', 'height', value)
 
-    def VarChanged_paraWidth(self, *params):
-        value = self.paraWidth.get()
-        self.AddChangedItem('main', 'FormatParagraph', 'paragraph', value)
-
     def VarChanged_startupEdit(self, *params):
         value = self.startupEdit.get()
         self.AddChangedItem('main', 'General', 'editor-on-startup', value)
@@ -1094,9 +1078,6 @@
                 'main', 'EditorWindow', 'width', type='int'))
         self.winHeight.set(idleConf.GetOption(
                 'main', 'EditorWindow', 'height', type='int'))
-        #initial paragraph reformat size
-        self.paraWidth.set(idleConf.GetOption(
-                'main', 'FormatParagraph', 'paragraph', type='int'))
         # default source encoding
         self.encoding.set(idleConf.GetOption(
                 'main', 'EditorWindow', 'encoding', default='none'))
diff --git a/lib-python/2.7/idlelib/help.txt b/lib-python/2.7/idlelib/help.txt
--- a/lib-python/2.7/idlelib/help.txt
+++ b/lib-python/2.7/idlelib/help.txt
@@ -100,7 +100,7 @@
 			  which is scrolling off the top or the window.
 			  (Not present in Shell window.)
 
-Windows Menu:
+Window Menu:
 
 	Zoom Height -- toggles the window between configured size
 	and maximum height.
diff --git a/lib-python/2.7/idlelib/idle.bat b/lib-python/2.7/idlelib/idle.bat
--- a/lib-python/2.7/idlelib/idle.bat
+++ b/lib-python/2.7/idlelib/idle.bat
@@ -1,4 +1,4 @@
- at echo off
-rem Start IDLE using the appropriate Python interpreter
-set CURRDIR=%~dp0
-start "IDLE" "%CURRDIR%..\..\pythonw.exe" "%CURRDIR%idle.pyw" %1 %2 %3 %4 %5 %6 %7 %8 %9
+ at echo off
+rem Start IDLE using the appropriate Python interpreter
+set CURRDIR=%~dp0
+start "IDLE" "%CURRDIR%..\..\pythonw.exe" "%CURRDIR%idle.pyw" %1 %2 %3 %4 %5 %6 %7 %8 %9
diff --git a/lib-python/2.7/idlelib/idle_test/test_calltips.py b/lib-python/2.7/idlelib/idle_test/test_calltips.py
--- a/lib-python/2.7/idlelib/idle_test/test_calltips.py
+++ b/lib-python/2.7/idlelib/idle_test/test_calltips.py
@@ -55,7 +55,8 @@
         def gtest(obj, out):
             self.assertEqual(signature(obj), out)
 
-        gtest(List, '()\n' + List.__doc__)
+        if List.__doc__ is not None:
+            gtest(List, '()\n' + List.__doc__)
         gtest(list.__new__,
                'T.__new__(S, ...) -> a new object with type S, a subtype of T')
         gtest(list.__init__,
@@ -70,7 +71,8 @@
 
     def test_signature_wrap(self):
         # This is also a test of an old-style class
-        self.assertEqual(signature(textwrap.TextWrapper), '''\
+        if textwrap.TextWrapper.__doc__ is not None:
+            self.assertEqual(signature(textwrap.TextWrapper), '''\
 (width=70, initial_indent='', subsequent_indent='', expand_tabs=True,
     replace_whitespace=True, fix_sentence_endings=False, break_long_words=True,
     drop_whitespace=True, break_on_hyphens=True)''')
@@ -106,20 +108,23 @@
         def t5(a, b=None, *args, **kwds): 'doc'
         t5.tip = "(a, b=None, *args, **kwargs)"
 
+        doc = '\ndoc' if t1.__doc__ is not None else ''
         for func in (t1, t2, t3, t4, t5, TC):
-            self.assertEqual(signature(func), func.tip + '\ndoc')
+            self.assertEqual(signature(func), func.tip + doc)
 
     def test_methods(self):
+        doc = '\ndoc' if TC.__doc__ is not None else ''
         for meth in (TC.t1, TC.t2, TC.t3, TC.t4, TC.t5, TC.t6, TC.__call__):
-            self.assertEqual(signature(meth), meth.tip + "\ndoc")
-        self.assertEqual(signature(TC.cm), "(a)\ndoc")
-        self.assertEqual(signature(TC.sm), "(b)\ndoc")
+            self.assertEqual(signature(meth), meth.tip + doc)
+        self.assertEqual(signature(TC.cm), "(a)" + doc)
+        self.assertEqual(signature(TC.sm), "(b)" + doc)
 
     def test_bound_methods(self):
         # test that first parameter is correctly removed from argspec
+        doc = '\ndoc' if TC.__doc__ is not None else ''
         for meth, mtip  in ((tc.t1, "()"), (tc.t4, "(*args)"), (tc.t6, "(self)"),
                             (tc.__call__, '(ci)'), (tc, '(ci)'), (TC.cm, "(a)"),):
-            self.assertEqual(signature(meth), mtip + "\ndoc")
+            self.assertEqual(signature(meth), mtip + doc)
 
     def test_starred_parameter(self):
         # test that starred first parameter is *not* removed from argspec
diff --git a/lib-python/2.7/idlelib/idle_test/test_io.py b/lib-python/2.7/idlelib/idle_test/test_io.py
new file mode 100644
--- /dev/null
+++ b/lib-python/2.7/idlelib/idle_test/test_io.py
@@ -0,0 +1,267 @@
+import unittest
+import io
+from idlelib.PyShell import PseudoInputFile, PseudoOutputFile
+from test import test_support as support
+
+
+class Base(object):
+    def __str__(self):
+        return '%s:str' % type(self).__name__
+    def __unicode__(self):
+        return '%s:unicode' % type(self).__name__
+    def __len__(self):
+        return 3
+    def __iter__(self):
+        return iter('abc')
+    def __getitem__(self, *args):
+        return '%s:item' % type(self).__name__
+    def __getslice__(self, *args):
+        return '%s:slice' % type(self).__name__
+
+class S(Base, str):
+    pass
+
+class U(Base, unicode):
+    pass
+
+class BA(Base, bytearray):
+    pass
+
+class MockShell:
+    def __init__(self):
+        self.reset()
+
+    def write(self, *args):
+        self.written.append(args)
+
+    def readline(self):
+        return self.lines.pop()
+
+    def close(self):
+        pass
+
+    def reset(self):
+        self.written = []
+
+    def push(self, lines):
+        self.lines = list(lines)[::-1]
+
+
+class PseudeOutputFilesTest(unittest.TestCase):
+    def test_misc(self):
+        shell = MockShell()
+        f = PseudoOutputFile(shell, 'stdout', 'utf-8')
+        self.assertIsInstance(f, io.TextIOBase)
+        self.assertEqual(f.encoding, 'utf-8')
+        self.assertIsNone(f.errors)
+        self.assertIsNone(f.newlines)
+        self.assertEqual(f.name, '<stdout>')
+        self.assertFalse(f.closed)
+        self.assertTrue(f.isatty())
+        self.assertFalse(f.readable())
+        self.assertTrue(f.writable())
+        self.assertFalse(f.seekable())
+

From noreply at buildbot.pypy.org  Thu Jun 18 11:42:40 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 18 Jun 2015 11:42:40 +0200 (CEST)
Subject: [pypy-commit] stmgc queue: Ah,
 this loop should never run more than once even now (re-add asserts)
Message-ID: <20150618094240.9292D1C1F77@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: queue
Changeset: r1858:7592a0f11ac2
Date: 2015-06-18 11:43 +0200
http://bitbucket.org/pypy/stmgc/changeset/7592a0f11ac2/

Log:	Ah, this loop should never run more than once even now (re-add
	asserts)

diff --git a/c8/stm/sync.c b/c8/stm/sync.c
--- a/c8/stm/sync.c
+++ b/c8/stm/sync.c
@@ -119,11 +119,13 @@
     long integral_part = (long)incr;
     t->tv_sec += integral_part;
     incr -= integral_part;
+    assert(incr >= 0.0 && incr <= 1.0);
 
     long nsec = t->tv_nsec + (long)(incr * 1000000000.0);
-    while (nsec >= 1000000000) {
+    if (nsec >= 1000000000) {
         t->tv_sec += 1;
         nsec -= 1000000000;
+        assert(nsec < 1000000000);
     }
     t->tv_nsec = nsec;
 }

From noreply at buildbot.pypy.org  Thu Jun 18 11:41:42 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Thu, 18 Jun 2015 11:41:42 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: remembering the position of the guard
 exit to resume the regallocator at a guard exit
Message-ID: <20150618094142.9D8111C0FE0@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r78169:6466422700f1
Date: 2015-06-18 11:41 +0200
http://bitbucket.org/pypy/pypy/changeset/6466422700f1/

Log:	remembering the position of the guard exit to resume the
	regallocator at a guard exit

diff --git a/rpython/jit/backend/arm/assembler.py b/rpython/jit/backend/arm/assembler.py
--- a/rpython/jit/backend/arm/assembler.py
+++ b/rpython/jit/backend/arm/assembler.py
@@ -269,7 +269,7 @@
         #
         self._push_all_regs_to_jitframe(mc, [], self.cpu.supports_floats, callee_only)
         ## args are in their respective positions
-        mc.PUSH([r.ip.value, r.lr.value])
+        mlc.PUSH([r.ip.value, r.lr.value])
         mc.BLX(r.r4.value)
         self._reload_frame_if_necessary(mc)
         self._pop_all_regs_from_jitframe(mc, [], supports_floats,
@@ -930,6 +930,7 @@
         while regalloc.position() < len(operations) - 1:
             regalloc.next_instruction()
             i = regalloc.position()
+            self.position = i
             op = operations[i]
             self.mc.mark_op(op)
             opnum = op.getopnum()
diff --git a/rpython/jit/backend/arm/opassembler.py b/rpython/jit/backend/arm/opassembler.py
--- a/rpython/jit/backend/arm/opassembler.py
+++ b/rpython/jit/backend/arm/opassembler.py
@@ -37,10 +37,10 @@
 
 
 class ArmGuardToken(GuardToken):
-    def __init__(self, cpu, gcmap, faildescr, failargs, fail_locs,
+    def __init__(self, cpu, pos, gcmap, faildescr, failargs, fail_locs,
                  offset, exc, frame_depth, is_guard_not_invalidated=False,
                  is_guard_not_forced=False, fcond=c.AL):
-        GuardToken.__init__(self, cpu, gcmap, faildescr, failargs, fail_locs,
+        GuardToken.__init__(self, cpu, pos, gcmap, faildescr, failargs, fail_locs,
                             exc, frame_depth, is_guard_not_invalidated,
                             is_guard_not_forced)
         self.fcond = fcond
@@ -211,16 +211,16 @@
         assert isinstance(descr, AbstractFailDescr)
 
         gcmap = allocate_gcmap(self, frame_depth, JITFRAME_FIXED_SIZE)
-        token = ArmGuardToken(self.cpu, gcmap,
-                                    descr,
-                                    failargs=op.getfailargs(),
-                                    fail_locs=arglocs,
-                                    offset=offset,
-                                    exc=save_exc,
-                                    frame_depth=frame_depth,
-                                    is_guard_not_invalidated=is_guard_not_invalidated,
-                                    is_guard_not_forced=is_guard_not_forced,
-                                    fcond=fcond)
+        token = ArmGuardToken(self.cpu, self.position, gcmap,
+                              descr,
+                              failargs=op.getfailargs(),
+                              fail_locs=arglocs,
+                              offset=offset,
+                              exc=save_exc,
+                              frame_depth=frame_depth,
+                              is_guard_not_invalidated=is_guard_not_invalidated,
+                              is_guard_not_forced=is_guard_not_forced,
+                              fcond=fcond)
         return token
 
     def _emit_guard(self, op, arglocs, fcond, save_exc,
diff --git a/rpython/jit/backend/llsupport/assembler.py b/rpython/jit/backend/llsupport/assembler.py
--- a/rpython/jit/backend/llsupport/assembler.py
+++ b/rpython/jit/backend/llsupport/assembler.py
@@ -22,10 +22,11 @@
 )
 
 class GuardToken(object):
-    def __init__(self, cpu, gcmap, faildescr, failargs, fail_locs, exc,
+    def __init__(self, cpu, pos, gcmap, faildescr, failargs, fail_locs, exc,
                  frame_depth, is_guard_not_invalidated, is_guard_not_forced):
         assert isinstance(faildescr, AbstractFailDescr)
         self.cpu = cpu
+        self.position = pos
         self.faildescr = faildescr
         self.failargs = failargs
         self.fail_locs = fail_locs
@@ -62,6 +63,7 @@
 
     def __init__(self, cpu, translate_support_code=False):
         self.cpu = cpu
+        self.position = 0
         self.memcpy_addr = 0
         self.memset_addr = 0
         self.rtyper = cpu.rtyper
@@ -127,6 +129,7 @@
         self.gcmap_for_finish[0] = r_uint(1)
 
     def setup(self, looptoken):
+        self.position = 0
         if self.cpu.HAS_CODEMAP:
             self.codemap_builder = CodemapBuilder()
         self._finish_gcmap = lltype.nullptr(jitframe.GCMAP)
diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -591,6 +591,7 @@
         # for each pending guard, generate the code of the recovery stub
         # at the end of self.mc.
         for tok in self.pending_guard_tokens:
+            regalloc.position = tok.position
             tok.pos_recovery_stub = self.generate_quick_failure(tok, regalloc)
         if WORD == 8 and len(self.pending_memoryerror_trampoline_from) > 0:
             self.error_trampoline_64 = self.generate_propagate_error_64()
@@ -1794,7 +1795,7 @@
         is_guard_not_invalidated = guard_opnum == rop.GUARD_NOT_INVALIDATED
         is_guard_not_forced = guard_opnum == rop.GUARD_NOT_FORCED
         gcmap = allocate_gcmap(self, frame_depth, JITFRAME_FIXED_SIZE)
-        return GuardToken(self.cpu, gcmap, faildescr, failargs,
+        return GuardToken(self.cpu, self.position, gcmap, faildescr, failargs,
                           fail_locs, exc, frame_depth,
                           is_guard_not_invalidated, is_guard_not_forced)
 
@@ -2483,14 +2484,18 @@
         for i,arg in enumerate(fail_args):
             if arg is None:
                 continue
+            assert arg.scalar_var is not None
             if isinstance(arg, BoxVectorAccum):
                 loc = fail_locs[i]
+                assert isinstance(loc, RegLoc)
+                assert loc.is_xmm
                 tgtloc = regalloc.force_allocate_reg(arg.scalar_var, fail_args)
+                assert tgtloc is not None
                 if arg.operator == '+':
                     # reduction using plus
                     self._accum_reduce_sum(arg, loc, tgtloc)
                     fail_locs[i] = tgtloc
-                    self._regalloc.possibly_free_var(arg)
+                    regalloc.possibly_free_var(arg)
                     fail_args[i] = arg.scalar_var
                 else:
                     raise NotImplementedError("accum operator %s not implemented" %
diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -341,6 +341,7 @@
             op = operations[i]
             self.assembler.mc.mark_op(op)
             assert self.assembler.mc._frame_size == DEFAULT_FRAME_BYTES
+            self.assembler.position = i
             self.rm.position = i
             self.xrm.position = i
             if op.has_no_side_effect() and op.result not in self.longevity:

From noreply at buildbot.pypy.org  Sat Jun 20 17:21:22 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 20 Jun 2015 17:21:22 +0200 (CEST)
Subject: [pypy-commit] stmgc default: fix: must deactivate queues earlier,
 else major collection will try to look
Message-ID: <20150620152122.E9EBB1C1F66@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r1881:9ffba4fe03df
Date: 2015-06-20 17:22 +0200
http://bitbucket.org/pypy/stmgc/changeset/9ffba4fe03df/

Log:	fix: must deactivate queues earlier, else major collection will try
	to look inside them even if the transaction is aborting

diff --git a/c8/stm/core.c b/c8/stm/core.c
--- a/c8/stm/core.c
+++ b/c8/stm/core.c
@@ -1365,7 +1365,8 @@
     }
 
     if (STM_PSEGMENT->active_queues)
-        queues_deactivate_all(/*at_commit=*/true);
+        queues_deactivate_all(get_priv_segment(STM_SEGMENT->segment_num),
+                              /*at_commit=*/true);
 
     invoke_and_clear_user_callbacks(0);   /* for commit */
 
@@ -1476,6 +1477,9 @@
 #endif
     tl->thread_local_obj = pseg->threadlocal_at_start_of_transaction;
 
+    if (pseg->active_queues)
+        queues_deactivate_all(pseg, /*at_commit=*/false);
+
 
     /* Set the next nursery_mark: first compute the value that
        nursery_mark must have had at the start of the aborted transaction */
@@ -1521,9 +1525,6 @@
     if (tl->mem_clear_on_abort)
         memset(tl->mem_clear_on_abort, 0, tl->mem_bytes_to_clear_on_abort);
 
-    if (STM_PSEGMENT->active_queues)
-        queues_deactivate_all(/*at_commit=*/false);
-
     invoke_and_clear_user_callbacks(1);   /* for abort */
 
     if (is_abort(STM_SEGMENT->nursery_end)) {
diff --git a/c8/stm/queue.c b/c8/stm/queue.c
--- a/c8/stm/queue.c
+++ b/c8/stm/queue.c
@@ -126,16 +126,21 @@
     }
 }
 
-static void queues_deactivate_all(bool at_commit)
+static void queues_deactivate_all(struct stm_priv_segment_info_s *pseg,
+                                  bool at_commit)
 {
-    queue_lock_acquire();
+#pragma push_macro("STM_PSEGMENT")
+#pragma push_macro("STM_SEGMENT")
+#undef STM_PSEGMENT
+#undef STM_SEGMENT
+    spinlock_acquire(pseg->active_queues_lock);
 
     bool added_any_old_entries = false;
     bool finished_more_tasks = false;
     wlog_t *item;
-    TREE_LOOP_FORWARD(STM_PSEGMENT->active_queues, item) {
+    TREE_LOOP_FORWARD(pseg->active_queues, item) {
         stm_queue_t *queue = (stm_queue_t *)item->addr;
-        stm_queue_segment_t *seg = &queue->segs[STM_SEGMENT->segment_num - 1];
+        stm_queue_segment_t *seg = &queue->segs[pseg->pub.segment_num - 1];
         queue_entry_t *head, *freehead;
 
         if (at_commit) {
@@ -188,16 +193,17 @@
 
     } TREE_LOOP_END;
 
-    tree_free(STM_PSEGMENT->active_queues);
-    STM_PSEGMENT->active_queues = NULL;
+    tree_free(pseg->active_queues);
+    pseg->active_queues = NULL;
 
-    queue_lock_release();
+    spinlock_release(pseg->active_queues_lock);
 
-    assert(_has_mutex());
     if (added_any_old_entries)
         cond_broadcast(C_QUEUE_OLD_ENTRIES);
     if (finished_more_tasks)
         cond_broadcast(C_QUEUE_FINISHED_MORE_TASKS);
+#pragma pop_macro("STM_SEGMENT")
+#pragma pop_macro("STM_PSEGMENT")
 }
 
 void stm_queue_put(object_t *qobj, stm_queue_t *queue, object_t *newitem)
diff --git a/c8/stm/queue.h b/c8/stm/queue.h
--- a/c8/stm/queue.h
+++ b/c8/stm/queue.h
@@ -1,3 +1,4 @@
-static void queues_deactivate_all(bool at_commit);
+static void queues_deactivate_all(struct stm_priv_segment_info_s *pseg,
+                                  bool at_commit);
 static void collect_active_queues(void);           /* minor collections */
 static void mark_visit_from_active_queues(void);   /* major collections */

From noreply at buildbot.pypy.org  Thu Jun 18 11:42:55 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Thu, 18 Jun 2015 11:42:55 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: typo in arm assembler.py
Message-ID: <20150618094255.EA3691C1F79@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r78170:e5cf1e51db67
Date: 2015-06-18 11:43 +0200
http://bitbucket.org/pypy/pypy/changeset/e5cf1e51db67/

Log:	typo in arm assembler.py

diff --git a/rpython/jit/backend/arm/assembler.py b/rpython/jit/backend/arm/assembler.py
--- a/rpython/jit/backend/arm/assembler.py
+++ b/rpython/jit/backend/arm/assembler.py
@@ -269,7 +269,7 @@
         #
         self._push_all_regs_to_jitframe(mc, [], self.cpu.supports_floats, callee_only)
         ## args are in their respective positions
-        mlc.PUSH([r.ip.value, r.lr.value])
+        mc.PUSH([r.ip.value, r.lr.value])
         mc.BLX(r.r4.value)
         self._reload_frame_if_necessary(mc)
         self._pop_all_regs_from_jitframe(mc, [], supports_floats,

From noreply at buildbot.pypy.org  Sat Jun 20 17:21:21 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 20 Jun 2015 17:21:21 +0200 (CEST)
Subject: [pypy-commit] stmgc default: fixed one TODO, added another
Message-ID: <20150620152121.C12101C1EF8@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r1880:09ca23bd1e4a
Date: 2015-06-20 17:21 +0200
http://bitbucket.org/pypy/stmgc/changeset/09ca23bd1e4a/

Log:	fixed one TODO, added another

diff --git a/c8/TODO b/c8/TODO
--- a/c8/TODO
+++ b/c8/TODO
@@ -1,14 +1,11 @@
+
+- fix markers (e.g. become_inevitable doesn't seem to show up)
 
 - improve sync of small objs on commit (see FLAG_SYNC_LARGE in nursery.c)
 
 - reshare pages:
   make seg0 MAP_SHARED in order to re-share private pages during major GC
 
-- avoid usleep(10) when waiting for an inevitable transaction:
-  we do this sleep when we try to commit and another inev transaction is
-  currently running. idea: signal the inev transaction to do the commit
-  for us
-
 - maybe re-implement the "please commit soon" signal
 
 - the highest_overflow_number can overflow after 2**30 non-collect-time

From noreply at buildbot.pypy.org  Tue Jun 23 12:02:17 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 23 Jun 2015 12:02:17 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c8: Add _weakref.weakkeyiddict(),
 a generally useful weak-keyed-dict with
Message-ID: <20150623100217.81F4E1C0354@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c8
Changeset: r78252:f5af62298413
Date: 2015-06-23 11:41 +0200
http://bitbucket.org/pypy/pypy/changeset/f5af62298413/

Log:	Add _weakref.weakkeyiddict(), a generally useful weak-keyed-dict
	with identity instead of equality. It is useful to add more
	attributes to existing objects, for example, without attaching them
	to the objects themselves.

diff --git a/pypy/module/_weakref/__init__.py b/pypy/module/_weakref/__init__.py
--- a/pypy/module/_weakref/__init__.py
+++ b/pypy/module/_weakref/__init__.py
@@ -10,5 +10,8 @@
         'ReferenceType': 'interp__weakref.W_Weakref',
         'ProxyType': 'interp__weakref.W_Proxy', 
         'CallableProxyType': 'interp__weakref.W_CallableProxy',
-        'proxy': 'interp__weakref.proxy'
+        'proxy': 'interp__weakref.proxy',
+
+        # PyPy extension
+        'weakkeyiddict': 'weakkeyiddict.W_WeakKeyIdDict',
     }
diff --git a/pypy/module/_weakref/test/test_weakkeyiddict.py b/pypy/module/_weakref/test/test_weakkeyiddict.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_weakref/test/test_weakkeyiddict.py
@@ -0,0 +1,59 @@
+class AppTestWeakKeyIdDict(object):
+    spaceconfig = dict(usemodules=('_weakref',))
+
+    def test_simple(self):
+        import _weakref
+        class A(object):
+            pass
+        d = _weakref.weakkeyiddict()
+        a1 = A()
+        a2 = A()
+        d[a1] = 11
+        d[a2] = 22.5
+        assert d[a1] == 11
+        assert d[a2] == 22.5
+        assert d.get(a2, 5) == 22.5
+        del d[a2]
+        raises(KeyError, "d[a2]")
+        assert d.get(a2, 5) == 5
+        assert a1 in d
+        assert a2 not in d
+        assert d.setdefault(a1, 82) == 11
+        assert d[a1] == 11
+        assert d.setdefault(a2, 83) == 83
+        assert d[a2] == 83
+
+    def test_nonhashable_key(self):
+        import _weakref
+        d = _weakref.weakkeyiddict()
+        lst = []
+        lst2 = []
+        d[lst] = 84
+        assert lst in d
+        assert lst2 not in d
+        assert d.pop(lst) == 84
+        assert lst not in d
+        assert d.pop(lst, 85) == 85
+
+    def test_collect(self):
+        import _weakref
+        gone = []
+        class A(object):
+            def __del__(self):
+                gone.append(True)
+        d = _weakref.weakkeyiddict()
+        a1 = A()
+        a2 = A()
+        d[a1] = -42
+        d[a2] = 83
+        assert gone == []
+        #
+        del a1
+        tries = 0
+        while not gone:
+            tries += 1
+            if tries > 5:
+                raise AssertionError("a1 doesn't disappear")
+            gc.collect()
+        assert gone == [True]
+        assert d[a2] == 83
diff --git a/pypy/module/_weakref/weakkeyiddict.py b/pypy/module/_weakref/weakkeyiddict.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_weakref/weakkeyiddict.py
@@ -0,0 +1,82 @@
+#
+# This is _weakref.weakkeyiddict(), a generally useful weak-keyed-dict
+# with identity instead of equality. It is useful to add more
+# attributes to existing objects, for example, without attaching
+# them to the objects themselves.  It can be emulated in pure Python,
+# of course, but given that we already have a class in rlib.rweakref
+# that is doing exactly that in a cheap way, it is far more efficient
+# this way.
+#
+
+from pypy.interpreter.baseobjspace import W_Root
+from pypy.interpreter.typedef import TypeDef
+from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault
+
+from rpython.rlib.rweakref import RWeakKeyDictionary
+
+
+class W_WeakKeyIdDict(W_Root):
+    def __init__(self):
+        self.data = RWeakKeyDictionary(W_Root, W_Root)
+
+    def getitem_w(self, space, w_key):
+        w_value = self.data.get(w_key)
+        if w_value is None:
+            space.raise_key_error(w_key)
+        return w_value
+
+    def setitem_w(self, space, w_key, w_value):
+        self.data.set(w_key, w_value)
+
+    def delitem_w(self, space, w_key):
+        if self.data.get(w_key) is None:
+            space.raise_key_error(w_key)
+        self.data.set(w_key, None)
+
+    def contains_w(self, space, w_key):
+        return space.wrap(self.data.get(w_key) is not None)
+
+    @unwrap_spec(w_default=WrappedDefault(None))
+    def get_w(self, space, w_key, w_default):
+        w_value = self.data.get(w_key)
+        if w_value is not None:
+            return w_value
+        else:
+            return w_default
+
+    def pop_w(self, space, w_key, w_default=None):
+        w_value = self.data.get(w_key)
+        if w_value is not None:
+            self.data.set(w_key, None)
+            return w_value
+        elif w_default is not None:
+            return w_default
+        else:
+            space.raise_key_error(w_key)
+
+    @unwrap_spec(w_default=WrappedDefault(None))
+    def setdefault_w(self, space, w_key, w_default):
+        w_value = self.data.get(w_key)
+        if w_value is not None:
+            return w_value
+        else:
+            self.data.set(w_key, w_default)
+            return w_default
+
+
+def W_WeakKeyIdDict___new__(space, w_subtype):
+    r = space.allocate_instance(W_WeakKeyIdDict, w_subtype)
+    r.__init__()
+    return space.wrap(r)
+
+W_WeakKeyIdDict.typedef = TypeDef(
+    '_weakref.weakkeyiddict',
+    __new__ = interp2app(W_WeakKeyIdDict___new__),
+    __getitem__ = interp2app(W_WeakKeyIdDict.getitem_w),
+    __setitem__ = interp2app(W_WeakKeyIdDict.setitem_w),
+    __delitem__ = interp2app(W_WeakKeyIdDict.delitem_w),
+    __contains__ = interp2app(W_WeakKeyIdDict.contains_w),
+    get = interp2app(W_WeakKeyIdDict.get_w),
+    pop = interp2app(W_WeakKeyIdDict.pop_w),
+    setdefault = interp2app(W_WeakKeyIdDict.setdefault_w),
+)

From noreply at buildbot.pypy.org  Tue Jun 23 12:02:18 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 23 Jun 2015 12:02:18 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c8: Change the implementation method of
 threadlocalproperty().
Message-ID: <20150623100218.C35241C0354@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c8
Changeset: r78253:d3e128694145
Date: 2015-06-23 12:02 +0200
http://bitbucket.org/pypy/pypy/changeset/d3e128694145/

Log:	Change the implementation method of threadlocalproperty(). Now it
	shouldn't give conflicts the first time the property is accessed on
	a given object.

diff --git a/lib_pypy/_weakkeyiddict.py b/lib_pypy/_weakkeyiddict.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_weakkeyiddict.py
@@ -0,0 +1,60 @@
+# Copied and adapted from lib-python/2.7/weakref.py.
+# In PyPy (at least -STM), this is not used: a more
+# efficient version is found in the _weakref module.
+
+import UserDict, weakref
+
+
+class idref(weakref.ref):
+
+    def __eq__(self, other):
+        if self is other:
+            return True
+        if not isinstance(other, idref):
+            return NotImplementedError
+        s = self()
+        o = other()
+        return s is o is not None
+
+    def __hash__(self):
+        try:
+            return self._hash_cache
+        except AttributeError:
+            self._hash_cache = id(self())
+            return self._hash_cache
+
+
+class weakkeyiddict(object):
+    """ Mapping class that references keys weakly.
+    In addition, uses the identity of the keys, rather than the equality.
+    (Only a subset of the dict interface is available.)
+    """
+
+    def __init__(self):
+        self.data = {}
+        def remove(k, selfref=weakref.ref(self)):
+            self = selfref()
+            if self is not None:
+                del self.data[k]
+        self._remove = remove
+
+    def __delitem__(self, key):
+        del self.data[idref(key)]
+
+    def __getitem__(self, key):
+        return self.data[idref(key)]
+
+    def __setitem__(self, key, value):
+        self.data[idref(key, self._remove)] = value
+
+    def get(self, key, default=None):
+        return self.data.get(idref(key),default)
+
+    def __contains__(self, key):
+        return idref(key) in self.data
+
+    def pop(self, key, *args):
+        return self.data.pop(idref(key), *args)
+
+    def setdefault(self, key, default=None):
+        return self.data.setdefault(idref(key, self._remove),default)
diff --git a/lib_pypy/transaction.py b/lib_pypy/transaction.py
--- a/lib_pypy/transaction.py
+++ b/lib_pypy/transaction.py
@@ -52,6 +52,12 @@
     from time import time, clock
 
 try:
+    from _weakref import weakkeyiddict
+except ImportError:
+    # Not a STM-enabled PyPy.
+    from _weakkeyiddict import weakkeyiddict
+
+try:
     from pypystm import queue, Empty
 except ImportError:
     from Queue import Queue as queue
@@ -232,29 +238,32 @@
 class threadlocalproperty(object):
     def __init__(self, default_factory=None):
         self.tl_default_factory = default_factory
-        self.tl_name = intern('tlprop.%d' % id(self))
+        self.tl_local = thread._local()
 
-    def tl_get(self, obj):
+    def tl_wrefs(self):
         try:
-            return obj._threadlocalproperties
+            return self.tl_local.wrefs
         except AttributeError:
-            return obj.__dict__.setdefault('_threadlocalproperties',
-                                           thread._local())
+            self.tl_local.wrefs = wrefs = weakkeyiddict()
+            return wrefs
 
     def __get__(self, obj, cls=None):
         if obj is None:
             return self
+        wrefs = self.tl_wrefs()
         try:
-            return getattr(self.tl_get(obj), self.tl_name)
-        except AttributeError:
+            return wrefs[obj]
+        except KeyError:
             if self.tl_default_factory is None:
-                raise
-            result = self.tl_default_factory()
-            setattr(self.tl_get(obj), self.tl_name, result)
+                raise AttributeError
+            wrefs[obj] = result = self.tl_default_factory()
             return result
 
     def __set__(self, obj, value):
-        setattr(self.tl_get(obj), self.tl_name, value)
+        self.tl_wrefs()[obj] = value
 
     def __delete__(self, obj):
-        delattr(self.tl_get(obj), self.tl_name)
+        try:
+            del self.tl_wrefs()[obj]
+        except KeyError:
+            raise AttributeError

From noreply at buildbot.pypy.org  Tue Jun 23 12:10:14 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Tue, 23 Jun 2015 12:10:14 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: timing the vecopt and printing it to the
	logfiles
Message-ID: <20150623101014.6534C1C02A3@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r78254:96394198b3ec
Date: 2015-06-23 10:47 +0200
http://bitbucket.org/pypy/pypy/changeset/96394198b3ec/

Log:	timing the vecopt and printing it to the logfiles

diff --git a/rpython/jit/metainterp/optimizeopt/guard.py b/rpython/jit/metainterp/optimizeopt/guard.py
--- a/rpython/jit/metainterp/optimizeopt/guard.py
+++ b/rpython/jit/metainterp/optimizeopt/guard.py
@@ -122,6 +122,7 @@
         self.index_vars = index_vars
         self._newoperations = []
         self._same_as = {}
+        self.strength_reduced = 0 # how many guards could be removed?
 
     def find_compare_guard_bool(self, boolarg, operations, index):
         i = index - 1
@@ -211,6 +212,7 @@
                 guard = guards.get(i, None)
                 if not guard or guard.implied:
                     # this guard is implied or marked as not emitted (= None)
+                    self.strength_reduced += 1
                     continue
                 if guard.stronger:
                     guard.emit_operations(self)
diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/vectorize.py
@@ -60,10 +60,23 @@
         debug_start("vec-opt-loop")
         metainterp_sd.logger_noopt.log_loop(loop.inputargs, loop.operations, -2, None, None, "pre vectorize")
         metainterp_sd.profiler.count(Counters.OPT_VECTORIZE_TRY)
+        start = time.clock()
         opt = VectorizingOptimizer(metainterp_sd, jitdriver_sd, loop, cost_threshold)
         opt.propagate_all_forward()
+        gso = GuardStrengthenOpt(opt.dependency_graph.index_vars)
+        gso.propagate_all_forward(opt.loop)
+        end = time.clock()
         metainterp_sd.profiler.count(Counters.OPT_VECTORIZED)
         metainterp_sd.logger_noopt.log_loop(loop.inputargs, loop.operations, -2, None, None, "post vectorize")
+        debug_start("vec-opt-clock")
+        debug_print("unroll: %d gso count: %d opcount: (%d -> %d) took %fns" % \
+                      (opt.unroll_count+1,
+                       gso.strength_reduced,
+                       len(orig_ops),
+                       len(loop.operations),
+                       (end-start)*10.0**9))
+        debug_stop("vec-opt-clock")
+
     except NotAVectorizeableLoop:
         # vectorization is not possible
         loop.operations = orig_ops
@@ -135,9 +148,6 @@
         if not self.costmodel.profitable():
             raise NotAProfitableLoop()
 
-        gso = GuardStrengthenOpt(self.dependency_graph.index_vars)
-        gso.propagate_all_forward(self.loop)
-
     def emit_operation(self, op):
         if op.getopnum() == rop.DEBUG_MERGE_POINT:
             return

From noreply at buildbot.pypy.org  Tue Jun 23 12:10:15 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Tue, 23 Jun 2015 12:10:15 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: added some temporary timing code for the
	trace execution
Message-ID: <20150623101015.A48FC1C02A3@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r78255:052ab2d7af35
Date: 2015-06-23 12:10 +0200
http://bitbucket.org/pypy/pypy/changeset/052ab2d7af35/

Log:	added some temporary timing code for the trace execution

diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -530,6 +530,10 @@
             self.status = hash & self.ST_SHIFT_MASK
 
     def handle_fail(self, deadframe, metainterp_sd, jitdriver_sd):
+        # XXX debug purpose only
+        from rpython.jit.metainterp.optimizeopt.vectorize import xxx_clock_stop
+        xxx_clock_stop(jitdriver_sd, fail=True)
+        # XXX debug purpose only end
         if self.must_compile(deadframe, metainterp_sd, jitdriver_sd):
             self.start_compiling()
             try:
diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/vectorize.py
@@ -28,13 +28,26 @@
 from rpython.rlib.jit import Counters
 from rpython.rtyper.lltypesystem import lltype, rffi
 
-class NotAVectorizeableLoop(JitException):
-    def __str__(self):
-        return 'NotAVectorizeableLoop()'
+import time
+XXXBench = {}
+def xxx_clock_start(jitdriver_sd):
+    XXXBench[jitdriver_sd] = time.clock()
+def xxx_clock_stop(jitdriver_sd, fail=False):
+    end = time.clock()
+    if jitdriver_sd not in XXXBench:
+        raise AssertionError("trying to stop clock but timing for jit driver sd has never started")
+    start = XXXBench[jitdriver_sd]
+    name = "<unkown>"
+    if jitdriver_sd.jitdriver:
+        name = jitdriver_sd.jitdriver.name
+    unique_id = jitdriver_sd 
+    ns = (end - start) * 10**9
+    debug_start("xxx-clock-stop")
+    debug_print("name: %s id(jdsd): %s exe time: %dns fail? %d vec? %d" % \
+            (name, unique_id, int(ns), int(fail), int(jitdriver_sd.vectorize)))
+    debug_stop("xxx-clock-stop")
 
-class NotAProfitableLoop(JitException):
-    def __str__(self):
-        return 'NotAProfitableLoop()'
+
 
 def debug_print_operations(loop):
     """ NOT_RPYTHON """
@@ -51,6 +64,15 @@
             else:
                 print ""
 
+
+class NotAVectorizeableLoop(JitException):
+    def __str__(self):
+        return 'NotAVectorizeableLoop()'
+
+class NotAProfitableLoop(JitException):
+    def __str__(self):
+        return 'NotAProfitableLoop()'
+
 def optimize_vector(metainterp_sd, jitdriver_sd, loop, optimizations,
                     inline_short_preamble, start_state, cost_threshold):
     optimize_unroll(metainterp_sd, jitdriver_sd, loop, optimizations,
@@ -68,13 +90,15 @@
         end = time.clock()
         metainterp_sd.profiler.count(Counters.OPT_VECTORIZED)
         metainterp_sd.logger_noopt.log_loop(loop.inputargs, loop.operations, -2, None, None, "post vectorize")
+        #
+        ns = int((end-start)*10.0**9)
         debug_start("vec-opt-clock")
         debug_print("unroll: %d gso count: %d opcount: (%d -> %d) took %fns" % \
                       (opt.unroll_count+1,
                        gso.strength_reduced,
                        len(orig_ops),
                        len(loop.operations),
-                       (end-start)*10.0**9))
+                       ns)
         debug_stop("vec-opt-clock")
 
     except NotAVectorizeableLoop:
diff --git a/rpython/jit/metainterp/warmspot.py b/rpython/jit/metainterp/warmspot.py
--- a/rpython/jit/metainterp/warmspot.py
+++ b/rpython/jit/metainterp/warmspot.py
@@ -829,6 +829,12 @@
                     else:
                         value = cast_base_ptr_to_instance(Exception, value)
                         raise Exception, value
+                finally:
+                    # XXX debug purpose only
+                    from rpython.jit.metainterp.optimizeopt.vectorize import xxx_clock_stop
+                    xxx_clock_stop(jd, fail=False)
+                    # XXX debug purpose only end
+
 
         def handle_jitexception(e):
             # XXX the bulk of this function is mostly a copy-paste from above
diff --git a/rpython/jit/metainterp/warmstate.py b/rpython/jit/metainterp/warmstate.py
--- a/rpython/jit/metainterp/warmstate.py
+++ b/rpython/jit/metainterp/warmstate.py
@@ -368,7 +368,11 @@
             if vinfo is not None:
                 virtualizable = args[index_of_virtualizable]
                 vinfo.clear_vable_token(virtualizable)
-            
+            # XXX debug purpose only
+            from rpython.jit.metainterp.optimizeopt.vectorize import xxx_clock_start
+            xxx_clock_start(jitdriver_sd)
+            # XXX debug purpose only end
+
             deadframe = func_execute_token(loop_token, *args)
             #
             # Record in the memmgr that we just ran this loop,

From noreply at buildbot.pypy.org  Tue Jun 23 12:26:33 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Tue, 23 Jun 2015 12:26:33 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: missing bracket
Message-ID: <20150623102633.422521C02A3@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r78256:7cdaa1548c93
Date: 2015-06-23 12:26 +0200
http://bitbucket.org/pypy/pypy/changeset/7cdaa1548c93/

Log:	missing bracket

diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/vectorize.py
@@ -48,7 +48,6 @@
     debug_stop("xxx-clock-stop")
 
 
-
 def debug_print_operations(loop):
     """ NOT_RPYTHON """
     if not we_are_translated():
@@ -98,7 +97,7 @@
                        gso.strength_reduced,
                        len(orig_ops),
                        len(loop.operations),
-                       ns)
+                       ns))
         debug_stop("vec-opt-clock")
 
     except NotAVectorizeableLoop:

From noreply at buildbot.pypy.org  Tue Jun 23 12:50:30 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 23 Jun 2015 12:50:30 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c8: Support the common primitive types in
	unsafe_write()
Message-ID: <20150623105030.C574A1C02A3@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c8
Changeset: r78257:350b6a396e7a
Date: 2015-06-23 12:50 +0200
http://bitbucket.org/pypy/pypy/changeset/350b6a396e7a/

Log:	Support the common primitive types in unsafe_write()

diff --git a/pypy/module/_cffi_backend/ctypeprim.py b/pypy/module/_cffi_backend/ctypeprim.py
--- a/pypy/module/_cffi_backend/ctypeprim.py
+++ b/pypy/module/_cffi_backend/ctypeprim.py
@@ -184,12 +184,16 @@
         value = misc.read_raw_signed_data(cdata, self.size)
         return self.space.wrap(value)    # r_longlong => on 32-bit, 'long'
 
+    def _convert_to_long(self, w_ob):
+        value = misc.as_long(self.space, w_ob)
+        if self.value_smaller_than_long:
+            if value != misc.signext(value, self.size):
+                self._overflow(w_ob)
+        return value
+
     def convert_from_object(self, cdata, w_ob):
         if self.value_fits_long:
-            value = misc.as_long(self.space, w_ob)
-            if self.value_smaller_than_long:
-                if value != misc.signext(value, self.size):
-                    self._overflow(w_ob)
+            value = self._convert_to_long(w_ob)
             misc.write_raw_signed_data(cdata, value, self.size)
         else:
             self._convert_from_object_longlong(cdata, w_ob)
@@ -261,12 +265,16 @@
     def cast_to_int(self, cdata):
         return self.convert_to_object(cdata)
 
+    def _convert_to_ulong(self, w_ob):
+        value = misc.as_unsigned_long(self.space, w_ob, strict=True)
+        if self.value_fits_long:
+            if value > self.vrangemax:
+                self._overflow(w_ob)
+        return value
+
     def convert_from_object(self, cdata, w_ob):
         if self.value_fits_ulong:
-            value = misc.as_unsigned_long(self.space, w_ob, strict=True)
-            if self.value_fits_long:
-                if value > self.vrangemax:
-                    self._overflow(w_ob)
+            value = self._convert_to_ulong(w_ob)
             misc.write_raw_unsigned_data(cdata, value, self.size)
         else:
             self._convert_from_object_longlong(cdata, w_ob)
@@ -373,9 +381,12 @@
         value = misc.read_raw_float_data(cdata, self.size)
         return self.space.wrap(value)
 
+    def _convert_to_double(self, w_ob):
+        space = self.space
+        return space.float_w(space.float(w_ob))
+
     def convert_from_object(self, cdata, w_ob):
-        space = self.space
-        value = space.float_w(space.float(w_ob))
+        value = self._convert_to_double(w_ob)
         misc.write_raw_float_data(cdata, value, self.size)
 
     def unpack_list_of_float_items(self, w_cdata):
diff --git a/pypy/module/pypystm/__init__.py b/pypy/module/pypystm/__init__.py
--- a/pypy/module/pypystm/__init__.py
+++ b/pypy/module/pypystm/__init__.py
@@ -30,5 +30,5 @@
         'queue': 'queue.W_Queue',
         'Empty': 'space.fromcache(queue.Cache).w_Empty',
 
-        'unsafe_write_int32': 'unsafe_op.unsafe_write_int32',
+        'unsafe_write': 'unsafe_op.unsafe_write',
     }
diff --git a/pypy/module/pypystm/test/test_unsafe_op.py b/pypy/module/pypystm/test/test_unsafe_op.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/pypystm/test/test_unsafe_op.py
@@ -0,0 +1,62 @@
+import pypy.module._cffi_backend.misc    # side-effects
+
+
+class AppTestUnsafeOp:
+    spaceconfig = dict(usemodules=['pypystm', '_cffi_backend'])
+
+    def test_unsafe_write_char(self):
+        import pypystm, _cffi_backend
+        BChar = _cffi_backend.new_primitive_type('char')
+        BCharP = _cffi_backend.new_pointer_type(BChar)
+        x = _cffi_backend.newp(_cffi_backend.new_array_type(BCharP, 2))
+        pypystm.unsafe_write(x, 0, 'A')
+        pypystm.unsafe_write(x, 1, '\xAA')
+        assert x[0] == 'A'
+        assert x[1] == '\xAA'
+
+    def test_unsafe_write_int32(self):
+        import pypystm, _cffi_backend
+        BInt32 = _cffi_backend.new_primitive_type('int32_t')
+        BInt32P = _cffi_backend.new_pointer_type(BInt32)
+        x = _cffi_backend.newp(_cffi_backend.new_array_type(BInt32P, 2))
+        pypystm.unsafe_write(x, 0, -0x01020304)
+        pypystm.unsafe_write(x, 1, -0x05060708)
+        assert x[0] == -0x01020304
+        assert x[1] == -0x05060708
+
+    def test_unsafe_write_uint64(self):
+        import pypystm, _cffi_backend
+        BUInt64 = _cffi_backend.new_primitive_type('uint64_t')
+        BUInt64P = _cffi_backend.new_pointer_type(BUInt64)
+        x = _cffi_backend.newp(_cffi_backend.new_array_type(BUInt64P, 2))
+        pypystm.unsafe_write(x, 0, 0x0102030411223344)
+        pypystm.unsafe_write(x, 1, 0xF506070855667788)
+        assert x[0] == 0x0102030411223344
+        assert x[1] == 0xF506070855667788
+
+    def test_unsafe_write_unsupported_case(self):
+        import pypystm, _cffi_backend
+        BUniChar = _cffi_backend.new_primitive_type('wchar_t')
+        BUniCharP = _cffi_backend.new_pointer_type(BUniChar)
+        x = _cffi_backend.newp(_cffi_backend.new_array_type(BUniCharP, 2))
+        raises(TypeError, pypystm.unsafe_write, x, 0, u'X')
+
+    def test_unsafe_write_float(self):
+        import pypystm, _cffi_backend
+        BFloat = _cffi_backend.new_primitive_type('float')
+        BFloatP = _cffi_backend.new_pointer_type(BFloat)
+        x = _cffi_backend.newp(_cffi_backend.new_array_type(BFloatP, 2))
+        pypystm.unsafe_write(x, 0, 12.25)
+        pypystm.unsafe_write(x, 1, -42.0)
+        assert x[0] == 12.25
+        assert x[1] == -42.0
+
+    def test_unsafe_write_double(self):
+        import pypystm, _cffi_backend
+        BDouble = _cffi_backend.new_primitive_type('double')
+        BDoubleP = _cffi_backend.new_pointer_type(BDouble)
+        x = _cffi_backend.newp(_cffi_backend.new_array_type(BDoubleP, 2))
+        pypystm.unsafe_write(x, 0, 12.25)
+        pypystm.unsafe_write(x, 1, -42.0)
+        assert x[0] == 12.25
+        assert x[1] == -42.0
diff --git a/pypy/module/pypystm/unsafe_op.py b/pypy/module/pypystm/unsafe_op.py
--- a/pypy/module/pypystm/unsafe_op.py
+++ b/pypy/module/pypystm/unsafe_op.py
@@ -1,15 +1,75 @@
 from pypy.interpreter.gateway import unwrap_spec
-from pypy.module._cffi_backend import cdataobj
+from pypy.interpreter.error import oefmt
+from pypy.module._cffi_backend import cdataobj, ctypeptr, ctypeprim, misc
 from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.rlib.objectmodel import specialize
 
 
-UNSAFE_INT = lltype.Struct('UNSAFE_INT', ('x', rffi.INT),
+ at specialize.memo()
+def get_unsafe_type_ptr(TP):
+    UNSAFE = lltype.Struct('UNSAFE', ('x', TP),
                            hints = {'stm_dont_track_raw_accesses': True})
-UNSAFE_INT_P = lltype.Ptr(UNSAFE_INT)
+    return rffi.CArrayPtr(UNSAFE)
 
 
- at unwrap_spec(w_cdata=cdataobj.W_CData, index=int, value='c_int')
-def unsafe_write_int32(space, w_cdata, index, value):
-    with w_cdata as ptr:
-        ptr = rffi.cast(UNSAFE_INT_P, rffi.ptradd(ptr, index * 4))
-        ptr.x = rffi.cast(rffi.INT, value)
+def unsafe_write_raw_signed_data(w_cdata, index, source, size):
+    with w_cdata as target:
+        for TP, _ in misc._prim_signed_types:
+            if size == rffi.sizeof(TP):
+                TPP = get_unsafe_type_ptr(TP)
+                rffi.cast(TPP, target)[index].x = rffi.cast(TP, source)
+                return
+    raise NotImplementedError("bad integer size")
+
+def unsafe_write_raw_unsigned_data(w_cdata, index, source, size):
+    with w_cdata as target:
+        for TP, _ in misc._prim_unsigned_types:
+            if size == rffi.sizeof(TP):
+                TPP = get_unsafe_type_ptr(TP)
+                rffi.cast(TPP, target)[index].x = rffi.cast(TP, source)
+                return
+    raise NotImplementedError("bad integer size")
+
+def unsafe_write_raw_float_data(w_cdata, index, source, size):
+    with w_cdata as target:
+        for TP, _ in misc._prim_float_types:
+            if size == rffi.sizeof(TP):
+                TPP = get_unsafe_type_ptr(TP)
+                rffi.cast(TPP, target)[index].x = rffi.cast(TP, source)
+                return
+    raise NotImplementedError("bad float size")
+
+
+ at unwrap_spec(w_cdata=cdataobj.W_CData, index=int)
+def unsafe_write(space, w_cdata, index, w_value):
+    ctype = w_cdata.ctype
+    if not isinstance(ctype, ctypeptr.W_CTypePtrOrArray):
+        raise oefmt(space.w_TypeError,
+                    "expected a cdata of type pointer or array")
+    ctitem = ctype.ctitem
+
+    if isinstance(ctitem, ctypeprim.W_CTypePrimitiveChar):
+        charvalue = ctitem._convert_to_char(w_value)
+        unsafe_write_raw_signed_data(w_cdata, index, ord(charvalue), size=1)
+        return
+
+    if isinstance(ctitem, ctypeprim.W_CTypePrimitiveSigned):
+        if ctitem.value_fits_long:
+            value = ctitem._convert_to_long(w_value)
+            unsafe_write_raw_signed_data(w_cdata, index, value, ctitem.size)
+            return
+
+    if isinstance(ctitem, ctypeprim.W_CTypePrimitiveUnsigned):
+        if ctitem.value_fits_ulong:
+            value = ctitem._convert_to_ulong(w_value)
+            unsafe_write_raw_unsigned_data(w_cdata, index, value, ctitem.size)
+            return
+
+    if isinstance(ctitem, ctypeprim.W_CTypePrimitiveFloat):
+        if not isinstance(ctitem, ctypeprim.W_CTypePrimitiveLongDouble):
+            value = ctitem._convert_to_double(w_value)
+            unsafe_write_raw_float_data(w_cdata, index, value, ctitem.size)
+            return
+
+    raise oefmt(space.w_TypeError, "unsupported type in unsafe_write(): '%s'",
+                ctitem.name)

From noreply at buildbot.pypy.org  Tue Jun 23 13:20:50 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 23 Jun 2015 13:20:50 +0200 (CEST)
Subject: [pypy-commit] pypy default: Test and fix for 'math.fsum([nan])'.
 It's a typo when copying from CPython's mathmodule.c.
Message-ID: <20150623112050.80FA21C02BB@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78258:459148974175
Date: 2015-06-23 13:20 +0200
http://bitbucket.org/pypy/pypy/changeset/459148974175/

Log:	Test and fix for 'math.fsum([nan])'. It's a typo when copying from
	CPython's mathmodule.c.

diff --git a/pypy/module/math/interp_math.py b/pypy/module/math/interp_math.py
--- a/pypy/module/math/interp_math.py
+++ b/pypy/module/math/interp_math.py
@@ -345,7 +345,7 @@
             else:
                 partials.append(v)
     if special_sum != 0.0:
-        if rfloat.isnan(special_sum):
+        if rfloat.isnan(inf_sum):
             raise OperationError(space.w_ValueError, space.wrap("-inf + inf"))
         return space.wrap(special_sum)
     hi = 0.0
diff --git a/pypy/module/math/test/test_math.py b/pypy/module/math/test/test_math.py
--- a/pypy/module/math/test/test_math.py
+++ b/pypy/module/math/test/test_math.py
@@ -1,5 +1,6 @@
 from __future__ import with_statement
 
+import py
 from pypy.interpreter.function import Function
 from pypy.interpreter.gateway import BuiltinCode
 from pypy.module.math.test import test_direct
@@ -96,6 +97,10 @@
             ([2.**n - 2.**(n+50) + 2.**(n+52) for n in range(-1074, 972, 2)] +
              [-2.**1022],
              float.fromhex('0x1.5555555555555p+970')),
+            # infinity and nans
+            ([float("inf")], float("inf")),
+            ([float("-inf")], float("-inf")),
+            ([float("nan")], float("nan")),
             ]
 
         for i, (vals, expected) in enumerate(test_values):
@@ -107,7 +112,8 @@
             except ValueError:
                 py.test.fail("test %d failed: got ValueError, expected %r "
                           "for math.fsum(%.100r)" % (i, expected, vals))
-            assert actual == expected
+            assert actual == expected or (
+                math.isnan(actual) and math.isnan(expected))
 
     def test_factorial(self):
         import math

From noreply at buildbot.pypy.org  Tue Jun 23 14:11:51 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 23 Jun 2015 14:11:51 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c8: With stm, disable this assert,
 which fails for now (we need to fix it
Message-ID: <20150623121151.47EB01C02BB@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c8
Changeset: r78259:a1a7539b21d4
Date: 2015-06-23 14:11 +0200
http://bitbucket.org/pypy/pypy/changeset/a1a7539b21d4/

Log:	With stm, disable this assert, which fails for now (we need to fix
	it for vmprof+stm, but it's unclear if it would work like this right
	now)

diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -321,7 +321,8 @@
         while i < len(operations):
             op = operations[i]
             self.assembler.mc.mark_op(op)
-            assert self.assembler.mc._frame_size == DEFAULT_FRAME_BYTES
+            if not self.assembler.cpu.gc_ll_descr.stm:   # incorrect with stm
+                assert self.assembler.mc._frame_size == DEFAULT_FRAME_BYTES
             self.rm.position = i
             self.xrm.position = i
             #

From noreply at buildbot.pypy.org  Tue Jun 23 14:32:11 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Tue, 23 Jun 2015 14:32:11 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: moved around the timing to be able to
 translate the project with it
Message-ID: <20150623123211.35F271C04C1@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r78260:5af70e37351c
Date: 2015-06-23 14:31 +0200
http://bitbucket.org/pypy/pypy/changeset/5af70e37351c/

Log:	moved around the timing to be able to translate the project with it

diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -531,8 +531,7 @@
 
     def handle_fail(self, deadframe, metainterp_sd, jitdriver_sd):
         # XXX debug purpose only
-        from rpython.jit.metainterp.optimizeopt.vectorize import xxx_clock_stop
-        xxx_clock_stop(jitdriver_sd, fail=True)
+        jitdriver_sd.xxxbench.xxx_clock_stop(fail=True)
         # XXX debug purpose only end
         if self.must_compile(deadframe, metainterp_sd, jitdriver_sd):
             self.start_compiling()
diff --git a/rpython/jit/metainterp/jitdriver.py b/rpython/jit/metainterp/jitdriver.py
--- a/rpython/jit/metainterp/jitdriver.py
+++ b/rpython/jit/metainterp/jitdriver.py
@@ -33,3 +33,4 @@
 
     def _freeze_(self):
         return True
+
diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/vectorize.py
@@ -6,6 +6,7 @@
 """
 
 import py
+import time
 
 from rpython.jit.metainterp.resume import Snapshot
 from rpython.jit.metainterp.jitexc import JitException
@@ -28,26 +29,6 @@
 from rpython.rlib.jit import Counters
 from rpython.rtyper.lltypesystem import lltype, rffi
 
-import time
-XXXBench = {}
-def xxx_clock_start(jitdriver_sd):
-    XXXBench[jitdriver_sd] = time.clock()
-def xxx_clock_stop(jitdriver_sd, fail=False):
-    end = time.clock()
-    if jitdriver_sd not in XXXBench:
-        raise AssertionError("trying to stop clock but timing for jit driver sd has never started")
-    start = XXXBench[jitdriver_sd]
-    name = "<unkown>"
-    if jitdriver_sd.jitdriver:
-        name = jitdriver_sd.jitdriver.name
-    unique_id = jitdriver_sd 
-    ns = (end - start) * 10**9
-    debug_start("xxx-clock-stop")
-    debug_print("name: %s id(jdsd): %s exe time: %dns fail? %d vec? %d" % \
-            (name, unique_id, int(ns), int(fail), int(jitdriver_sd.vectorize)))
-    debug_stop("xxx-clock-stop")
-
-
 def debug_print_operations(loop):
     """ NOT_RPYTHON """
     if not we_are_translated():
@@ -63,7 +44,6 @@
             else:
                 print ""
 
-
 class NotAVectorizeableLoop(JitException):
     def __str__(self):
         return 'NotAVectorizeableLoop()'
diff --git a/rpython/jit/metainterp/warmspot.py b/rpython/jit/metainterp/warmspot.py
--- a/rpython/jit/metainterp/warmspot.py
+++ b/rpython/jit/metainterp/warmspot.py
@@ -28,6 +28,33 @@
 from rpython.rlib.entrypoint import all_jit_entrypoints,\
      annotated_jit_entrypoints
 
+from rpython.rlib.debug import debug_print, debug_start, debug_stop
+import time
+
+# XXX XXX XXX
+class XXXBench(object):
+    def __init__(self, name, id, vec):
+        self.t = []
+        self.name = name
+        self.unique_id = id
+        self.vec = vec
+
+    def xxx_clock_start(self):
+        self.t.append(time.clock())
+
+    def xxx_clock_stop(self, fail=False):
+        end = time.clock()
+        if len(self.t) == 0:
+            raise AssertionError("trying to stop clock but timing for jit driver sd has never started")
+        start = self.t[-1]
+        del self.t[-1]
+        ns = (end - start) * 10**9
+        debug_start("xxx-clock-stop")
+        debug_print("name: %s id(jdsd): %s exe time: %dns fail? %d vec? %d" % \
+                    (self.name, self.unique_id, int(ns), int(fail), int(self.vec)))
+        debug_stop("xxx-clock-stop")
+
+
 
 # ____________________________________________________________
 # Bootstrapping
@@ -403,6 +430,9 @@
         jd.portal_runner_ptr = "<not set so far>"
         jd.result_type = history.getkind(jd.portal_graph.getreturnvar()
                                          .concretetype)[0]
+        # XXX XXX XXX
+        jd.xxxbench = XXXBench(jd.jitdriver.name, id(jd), jd.vectorize)
+        # XXX XXX XXX
         self.jitdrivers_sd.append(jd)
 
     def check_access_directly_sanity(self, graphs):
@@ -831,8 +861,7 @@
                         raise Exception, value
                 finally:
                     # XXX debug purpose only
-                    from rpython.jit.metainterp.optimizeopt.vectorize import xxx_clock_stop
-                    xxx_clock_stop(jd, fail=False)
+                    jd.xxxbench.xxx_clock_stop(fail=False)
                     # XXX debug purpose only end
 
 
diff --git a/rpython/jit/metainterp/warmstate.py b/rpython/jit/metainterp/warmstate.py
--- a/rpython/jit/metainterp/warmstate.py
+++ b/rpython/jit/metainterp/warmstate.py
@@ -369,8 +369,7 @@
                 virtualizable = args[index_of_virtualizable]
                 vinfo.clear_vable_token(virtualizable)
             # XXX debug purpose only
-            from rpython.jit.metainterp.optimizeopt.vectorize import xxx_clock_start
-            xxx_clock_start(jitdriver_sd)
+            jitdriver_sd.xxxbench.xxx_clock_start()
             # XXX debug purpose only end
 
             deadframe = func_execute_token(loop_token, *args)

From noreply at buildbot.pypy.org  Tue Jun 23 14:35:05 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 23 Jun 2015 14:35:05 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c8: More
Message-ID: <20150623123505.5C5281C04C1@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c8
Changeset: r78261:dbbc3ef7779c
Date: 2015-06-23 14:35 +0200
http://bitbucket.org/pypy/pypy/changeset/dbbc3ef7779c/

Log:	More

diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -321,7 +321,7 @@
         while i < len(operations):
             op = operations[i]
             self.assembler.mc.mark_op(op)
-            if not self.assembler.cpu.gc_ll_descr.stm:   # incorrect with stm
+            if not rgc.stm_is_enabled():   # incorrect with stm
                 assert self.assembler.mc._frame_size == DEFAULT_FRAME_BYTES
             self.rm.position = i
             self.xrm.position = i
diff --git a/rpython/jit/backend/x86/rx86.py b/rpython/jit/backend/x86/rx86.py
--- a/rpython/jit/backend/x86/rx86.py
+++ b/rpython/jit/backend/x86/rx86.py
@@ -3,6 +3,7 @@
 from rpython.rlib.objectmodel import specialize
 from rpython.rlib.unroll import unrolling_iterable
 from rpython.rlib.rarithmetic import intmask
+from rpython.rlib import rgc
 from rpython.rtyper.lltypesystem import rffi
 from rpython.jit.backend.x86.arch import IS_X86_64
 
@@ -504,12 +505,14 @@
     def stack_frame_size_delta(self, delta):
         "Called when we generate an instruction that changes the value of ESP"
         self._frame_size += delta
+        if rgc.stm_is_enabled():
+            return      # the rest is ignored with STM
         self.frame_positions.append(self.get_relative_pos()) 
         self.frame_assignments.append(self._frame_size)
         assert self._frame_size >= self.WORD
 
     def check_stack_size_at_ret(self):
-        if IS_X86_64:
+        if IS_X86_64 and not rgc.stm_is_enabled():
             assert self._frame_size == self.WORD
             if not we_are_translated():
                 self._frame_size = None

From noreply at buildbot.pypy.org  Tue Jun 23 14:46:53 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Tue, 23 Jun 2015 14:46:53 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: some changes after being tested with
	test_zjit
Message-ID: <20150623124653.9D4721C02BB@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r78262:fefc4fafe4f6
Date: 2015-06-23 14:47 +0200
http://bitbucket.org/pypy/pypy/changeset/fefc4fafe4f6/

Log:	some changes after being tested with test_zjit

diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/vectorize.py
@@ -72,7 +72,7 @@
         #
         ns = int((end-start)*10.0**9)
         debug_start("vec-opt-clock")
-        debug_print("unroll: %d gso count: %d opcount: (%d -> %d) took %fns" % \
+        debug_print("unroll: %d gso count: %d opcount: (%d -> %d) took %dns" % \
                       (opt.unroll_count+1,
                        gso.strength_reduced,
                        len(orig_ops),
diff --git a/rpython/jit/metainterp/warmspot.py b/rpython/jit/metainterp/warmspot.py
--- a/rpython/jit/metainterp/warmspot.py
+++ b/rpython/jit/metainterp/warmspot.py
@@ -36,22 +36,29 @@
     def __init__(self, name, id, vec):
         self.t = []
         self.name = name
-        self.unique_id = id
+        self.unique_id = hex(id)
         self.vec = vec
 
     def xxx_clock_start(self):
-        self.t.append(time.clock())
+        now = time.clock()
+        self.t.append(now)
+        debug_start("xxx-clock-start")
+        debug_print("name: %s id(jdsd): %s now: %dns" % \
+                (self.name, self.unique_id, int(now*10**9)))
+        debug_stop("xxx-clock-start")
 
     def xxx_clock_stop(self, fail=False):
         end = time.clock()
         if len(self.t) == 0:
-            raise AssertionError("trying to stop clock but timing for jit driver sd has never started")
+            return
+        assert len(self.t) == 1
         start = self.t[-1]
-        del self.t[-1]
+        if not fail:
+            del self.t[-1]
         ns = (end - start) * 10**9
         debug_start("xxx-clock-stop")
-        debug_print("name: %s id(jdsd): %s exe time: %dns fail? %d vec? %d" % \
-                    (self.name, self.unique_id, int(ns), int(fail), int(self.vec)))
+        debug_print("name: %s id(jdsd): %s now: %ns exe time: %dns fail? %d vec? %d" % \
+                    (self.name, self.unique_id, int(end*10**9), int(ns), int(fail), int(self.vec)))
         debug_stop("xxx-clock-stop")
 
 

From noreply at buildbot.pypy.org  Tue Jun 23 14:51:52 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Tue, 23 Jun 2015 14:51:52 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: missing format character
Message-ID: <20150623125152.923961C02FD@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r78263:58b4e8859909
Date: 2015-06-23 14:52 +0200
http://bitbucket.org/pypy/pypy/changeset/58b4e8859909/

Log:	missing format character

diff --git a/rpython/jit/metainterp/warmspot.py b/rpython/jit/metainterp/warmspot.py
--- a/rpython/jit/metainterp/warmspot.py
+++ b/rpython/jit/metainterp/warmspot.py
@@ -57,8 +57,8 @@
             del self.t[-1]
         ns = (end - start) * 10**9
         debug_start("xxx-clock-stop")
-        debug_print("name: %s id(jdsd): %s now: %ns exe time: %dns fail? %d vec? %d" % \
-                    (self.name, self.unique_id, int(end*10**9), int(ns), int(fail), int(self.vec)))
+        debug_print("name: %s id(jdsd): %s now: %dns exe time: %dns fail? %d vec? %d" % \
+                    (self.name, self.unique_id, int(end)*10**9, int(ns), int(fail), int(self.vec)))
         debug_stop("xxx-clock-stop")
 
 

From noreply at buildbot.pypy.org  Tue Jun 23 14:54:06 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 23 Jun 2015 14:54:06 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c8: bah
Message-ID: <20150623125406.E4A6A1C02FD@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c8
Changeset: r78264:6d905fef7cbf
Date: 2015-06-23 14:54 +0200
http://bitbucket.org/pypy/pypy/changeset/6d905fef7cbf/

Log:	bah

diff --git a/rpython/jit/backend/llsupport/stmrewrite.py b/rpython/jit/backend/llsupport/stmrewrite.py
--- a/rpython/jit/backend/llsupport/stmrewrite.py
+++ b/rpython/jit/backend/llsupport/stmrewrite.py
@@ -83,6 +83,7 @@
             return
         # ----------  other ignored ops  ----------
         if opnum in (rop.STM_SHOULD_BREAK_TRANSACTION, rop.FORCE_TOKEN,
+                     rop.ENTER_PORTAL_FRAME, rop.LEAVE_PORTAL_FRAME,
                      rop.MARK_OPAQUE_PTR,
                      rop.JIT_DEBUG, rop.KEEPALIVE,
                      rop.QUASIIMMUT_FIELD, rop.RECORD_KNOWN_CLASS,

From noreply at buildbot.pypy.org  Tue Jun 23 16:48:28 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Tue, 23 Jun 2015 16:48:28 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: added test case for conversion int64 <->
	int8
Message-ID: <20150623144828.1FE491C02FD@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r78265:77645ab90fd3
Date: 2015-06-23 15:34 +0200
http://bitbucket.org/pypy/pypy/changeset/77645ab90fd3/

Log:	added test case for conversion int64 <-> int8

diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -344,6 +344,28 @@
         assert int(result) == 7*7+8*8+11*11+12*12
         self.check_vectorized(2, 2)
 
+    def define_conversion():
+        return """
+        a = astype(|30|, int8)
+        b = astype(|30|, int)
+        c = a + b
+        sum(c)
+        """
+    def test_conversion(self):
+        result = self.run("conversion")
+        assert result == sum(range(30)) + sum(range(30))
+        self.check_vectorized(4, 2) # only sum and astype(int) succeed
+
+    def define_sum():
+        return """
+        a = |30|
+        sum(a)
+        """
+    def test_sum(self):
+        result = self.run("sum")
+        assert result == sum(range(30))
+        self.check_vectorized(1, 1)
+
     def define_sum():
         return """
         a = |30|
diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/vectorize.py
@@ -70,6 +70,7 @@
         metainterp_sd.profiler.count(Counters.OPT_VECTORIZED)
         metainterp_sd.logger_noopt.log_loop(loop.inputargs, loop.operations, -2, None, None, "post vectorize")
         #
+        # XXX
         ns = int((end-start)*10.0**9)
         debug_start("vec-opt-clock")
         debug_print("unroll: %d gso count: %d opcount: (%d -> %d) took %dns" % \

From noreply at buildbot.pypy.org  Tue Jun 23 16:48:29 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Tue, 23 Jun 2015 16:48:29 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: started to bail out for some operations
 that cannot be implemented efficiently in SSE4.1
Message-ID: <20150623144829.646D31C02FD@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r78266:c9e6e55b3900
Date: 2015-06-23 16:48 +0200
http://bitbucket.org/pypy/pypy/changeset/c9e6e55b3900/

Log:	started to bail out for some operations that cannot be implemented
	efficiently in SSE4.1

diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -2536,10 +2536,6 @@
                 self.mc.PHADDD(accumloc, accumloc)
                 self.mc.PEXTRD_rxi(targetloc.value, accumloc.value, 0)
                 return
-            if size == 2:
-                pass
-            if size == 1:
-                pass
 
         raise NotImplementedError("reduce sum for %s not impl." % vector_var)
 
diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/vectorize.py
@@ -657,6 +657,8 @@
         if packed.is_raw_array_access():
             if packed.getarg(1) == inquestion.result:
                 return True
+        if inquestion.casts_box():
+            pass
         return False
 
     def combine(self, i, j):
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -189,6 +189,12 @@
     def returns_bool_result(self):
         return self._cls_has_bool_result
 
+    def casts_box(self):
+        return self.getopnum() == rop.INT_SIGNEXT or \
+               rop.CAST_FLOAT_TO_INT <= opnum <= rop.CAST_SINGLEFLOAT_TO_FLOAT or \
+               rop._VEC_CAST_FIRST <= opnum <= rop._VEC_CAST_LAST or \
+               rop.CAST_PTR_TO_INT == opnum or \
+               rop.CAST_INT_TO_PTR == opnum
 
 # ===================
 # Top of the hierachy
@@ -472,6 +478,7 @@
     '_VEC_ARITHMETIC_LAST',
     'VEC_FLOAT_EQ/2',
 
+    '_VEC_CAST_FIRST',
     'VEC_INT_SIGNEXT/2',
     # double -> float: v2 = cast(v1, 2) equal to v2 = (v1[0], v1[1], X, X)
     'VEC_CAST_FLOAT_TO_SINGLEFLOAT/1',
@@ -479,6 +486,7 @@
     'VEC_CAST_SINGLEFLOAT_TO_FLOAT/1',
     'VEC_CAST_FLOAT_TO_INT/1',
     'VEC_CAST_INT_TO_FLOAT/1',
+    '_VEC_CAST_LAST',
 
     'VEC_FLOAT_UNPACK/3',        # iX|fX = VEC_FLOAT_UNPACK(vX, index, item_count)
     'VEC_FLOAT_PACK/4',          # VEC_FLOAT_PACK(vX, var/const, index, item_count)
diff --git a/rpython/jit/metainterp/warmspot.py b/rpython/jit/metainterp/warmspot.py
--- a/rpython/jit/metainterp/warmspot.py
+++ b/rpython/jit/metainterp/warmspot.py
@@ -40,18 +40,21 @@
         self.vec = vec
 
     def xxx_clock_start(self):
+        if not self.vec:
+            return
         now = time.clock()
         self.t.append(now)
         debug_start("xxx-clock-start")
         debug_print("name: %s id(jdsd): %s now: %dns" % \
-                (self.name, self.unique_id, int(now*10**9)))
+                (self.name, self.unique_id, int(now)*10**9) )
         debug_stop("xxx-clock-start")
 
     def xxx_clock_stop(self, fail=False):
+        if not self.vec:
+            return
         end = time.clock()
         if len(self.t) == 0:
             return
-        assert len(self.t) == 1
         start = self.t[-1]
         if not fail:
             del self.t[-1]

From noreply at buildbot.pypy.org  Tue Jun 23 18:06:18 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 23 Jun 2015 18:06:18 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c8: Be more explicit in this test: it
 avoids the case where we add some
Message-ID: <20150623160618.00D5C1C02FD@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c8
Changeset: r78267:8e20bc2bbb84
Date: 2015-06-23 16:09 +0200
http://bitbucket.org/pypy/pypy/changeset/8e20bc2bbb84/

Log:	Be more explicit in this test: it avoids the case where we add some
	minor resop and then pypy-stm inserts a "stm_become_inevitable"
	before it

diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py
--- a/rpython/jit/backend/llsupport/rewrite.py
+++ b/rpython/jit/backend/llsupport/rewrite.py
@@ -79,8 +79,6 @@
                 self.emit_pending_zeros()
             elif op.is_call():
                 self.emitting_an_operation_that_can_collect()
-            elif op.getopnum() == rop.DEBUG_MERGE_POINT:
-                continue # ignore debug_merge_points
             elif op.getopnum() == rop.LABEL:
                 self.emitting_an_operation_that_can_collect()
                 self.known_lengths.clear()
diff --git a/rpython/jit/backend/llsupport/test/test_stmrewrite.py b/rpython/jit/backend/llsupport/test/test_stmrewrite.py
--- a/rpython/jit/backend/llsupport/test/test_stmrewrite.py
+++ b/rpython/jit/backend/llsupport/test/test_stmrewrite.py
@@ -9,33 +9,54 @@
 
 
 def test_all_operations_with_gc_in_their_name():
-    # hack, but will fail if we add a new ResOperation called .._GC_..
+    # hack, but will fail if we add a new ResOperation that is not
+    # always pure or a guard, and we forget about it
     import os, re
-    r_gc = re.compile(r"(^|_)GC(_|$)")
     with open(os.path.join(os.path.dirname(
           os.path.dirname(os.path.abspath(__file__))), 'stmrewrite.py')) as f:
         source = f.read()
         words = re.split("\W", source)
-    # extra op names with GC in their name but where it's ok if stmrewrite
-    # doesn't mention them:
+    # op names where it's ok if stmrewrite doesn't mention
+    # them individually:
+    for opnum, name in resoperation.opname.items():
+        if (rop._ALWAYS_PURE_FIRST <= opnum <= rop._ALWAYS_PURE_LAST or
+            rop._CALL_FIRST <= opnum <= rop._CALL_LAST or
+            rop._GUARD_FIRST <= opnum <= rop._GUARD_LAST or
+            rop._OVF_FIRST <= opnum <= rop._OVF_LAST):
+            words.append(name)
+    # extra op names where it's ok if stmrewrite doesn't mention them:
     words.append('CALL_MALLOC_GC')
     words.append('COND_CALL_GC_WB')
     words.append('COND_CALL_GC_WB_ARRAY')
-    # these are pure, and can be done without any read barrier
-    words.append('ARRAYLEN_GC')
-    words.append('GETFIELD_GC_PURE')
-    words.append('GETARRAYITEM_GC_PURE')
-    # these are handled by rewrite.py
+    # these are handled by rewrite.py (sometimes with some overridden code
+    # in stmrewrite.py too)
+    words.append('DEBUG_MERGE_POINT')
     words.append('SETFIELD_GC')
     words.append('SETARRAYITEM_GC')
     words.append('SETINTERIORFIELD_GC')
+    words.append('NEW')
+    words.append('NEWSTR')
+    words.append('NEWUNICODE')
+    words.append('NEW_ARRAY')
+    words.append('NEW_ARRAY_CLEAR')
+    words.append('NEW_WITH_VTABLE')
+    words.append('ZERO_ARRAY')
+    words.append('ZERO_PTR_FIELD')
+    # these always turn inevitable
+    words.append('GETARRAYITEM_RAW')
+    words.append('SETARRAYITEM_RAW')
+    words.append('SETINTERIORFIELD_RAW')
+    words.append('RAW_LOAD')
+    words.append('RAW_STORE')
+    # these should be processed by the front-end and not reach this point
+    words.append('VIRTUAL_REF')
+    words.append('VIRTUAL_REF_FINISH')
     #
     words = set(words)
     missing = []
     for name in sorted(resoperation.opname.values()):
-        if r_gc.search(name):
-            if name not in words:
-                missing.append(name)
+        if name not in words:
+            missing.append(name)
     assert not missing
 
 

From noreply at buildbot.pypy.org  Tue Jun 23 18:06:19 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 23 Jun 2015 18:06:19 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c8: Fixes (sometimes of the code,
	sometimes of the test)
Message-ID: <20150623160619.4CBD01C02FD@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c8
Changeset: r78268:9c99314d9c90
Date: 2015-06-23 16:15 +0200
http://bitbucket.org/pypy/pypy/changeset/9c99314d9c90/

Log:	Fixes (sometimes of the code, sometimes of the test)

diff --git a/rpython/jit/backend/llsupport/stmrewrite.py b/rpython/jit/backend/llsupport/stmrewrite.py
--- a/rpython/jit/backend/llsupport/stmrewrite.py
+++ b/rpython/jit/backend/llsupport/stmrewrite.py
@@ -37,8 +37,7 @@
             self.newop(op)
             return
         # ----------  non-pure getfields  ----------
-        if opnum in (rop.GETFIELD_GC, rop.GETARRAYITEM_GC,
-                     rop.GETINTERIORFIELD_GC):
+        if opnum in (rop.GETARRAYITEM_GC, rop.GETINTERIORFIELD_GC):
             self.handle_getfields(op)
             return
         # ----------  calls  ----------
@@ -120,6 +119,10 @@
             self.newop(op1)
             self.read_barrier_applied[v_ptr] = None
 
+    def handle_getfield_gc(self, op):
+        self.emit_pending_zeros()
+        self.handle_getfields(op)
+
     def add_dummy_allocation(self):
         if not self.does_any_allocation:
             # do a fake allocation since this is needed to check
diff --git a/rpython/jit/backend/llsupport/test/test_stmrewrite.py b/rpython/jit/backend/llsupport/test/test_stmrewrite.py
--- a/rpython/jit/backend/llsupport/test/test_stmrewrite.py
+++ b/rpython/jit/backend/llsupport/test/test_stmrewrite.py
@@ -31,6 +31,7 @@
     # these are handled by rewrite.py (sometimes with some overridden code
     # in stmrewrite.py too)
     words.append('DEBUG_MERGE_POINT')
+    words.append('GETFIELD_GC')
     words.append('SETFIELD_GC')
     words.append('SETARRAYITEM_GC')
     words.append('SETINTERIORFIELD_GC')
@@ -268,8 +269,8 @@
             p3 = call_malloc_nursery(%(tdescr.size)d)
             setfield_gc(p3, 0, descr=stmflagsdescr)
             setfield_gc(p3, %(tdescr.tid)d, descr=tiddescr)
+            zero_ptr_field(p3, %(tdescr.gc_fielddescrs[0].offset)s)
             p4 = getfield_gc(p1, descr=tzdescr)
-            zero_ptr_field(p3, %(tdescr.gc_fielddescrs[0].offset)s)
             jump(p2)
         """)
 
@@ -299,8 +300,8 @@
             p2 = call_malloc_nursery(%(tdescr.size)d)
             setfield_gc(p2, 0, descr=stmflagsdescr)
             setfield_gc(p2, %(tdescr.tid)d, descr=tiddescr)
+            zero_ptr_field(p2, %(tdescr.gc_fielddescrs[0].offset)s)
             p1 = getfield_gc(p2, descr=tzdescr)
-            zero_ptr_field(p2, %(tdescr.gc_fielddescrs[0].offset)s)
             jump(p1)
         """)
 

From noreply at buildbot.pypy.org  Tue Jun 23 18:06:20 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 23 Jun 2015 18:06:20 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c8: Add missing operations
Message-ID: <20150623160620.68E601C02FD@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c8
Changeset: r78269:64a14ac8476f
Date: 2015-06-23 17:16 +0200
http://bitbucket.org/pypy/pypy/changeset/64a14ac8476f/

Log:	Add missing operations

diff --git a/rpython/translator/stm/inevitable.py b/rpython/translator/stm/inevitable.py
--- a/rpython/translator/stm/inevitable.py
+++ b/rpython/translator/stm/inevitable.py
@@ -11,6 +11,7 @@
     'cast_ptr_to_weakrefptr', 'cast_weakrefptr_to_ptr',
     'debug_print', 'debug_assert', 'debug_flush', 'debug_offset',
     'debug_start', 'debug_stop', 'have_debug_prints',
+    'have_debug_prints_for',
     'debug_catch_exception', 'debug_nonnull_pointer',
     'debug_record_traceback', 'debug_start_traceback',
     'debug_reraise_traceback',
@@ -88,6 +89,7 @@
     'gc_dump_rpy_heap', 'gc_thread_start', 'gc_thread_die',
     'raw_memclear', 'raw_memcopy', 'raw_memmove', 'raw_memset',
     'gc_thread_after_fork', 'gc_thread_before_fork',
+    'debug_forked',
     ])
 
 # ____________________________________________________________

From noreply at buildbot.pypy.org  Tue Jun 23 18:06:21 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 23 Jun 2015 18:06:21 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c8: updates
Message-ID: <20150623160621.7BDFD1C02FD@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c8
Changeset: r78270:8d3e51e892af
Date: 2015-06-23 17:25 +0200
http://bitbucket.org/pypy/pypy/changeset/8d3e51e892af/

Log:	updates

diff --git a/TODO b/TODO
--- a/TODO
+++ b/TODO
@@ -6,6 +6,9 @@
 ------------------------------------------------------------
 
 fuse the two 32bit setfield_gc for stmflags & tid in the jit
+(note: unclear if it makes a major difference: a single
+64bit setfield of an immediate that doesn't fit 32bit
+requires two instructions too)
 
 ------------------------------------------------------------
 
@@ -13,6 +16,10 @@
 rarely break if there are not threads running in parallel.
 But we need to break sometimes in order to run finalizers...
 
+IMPROVED, but we should check if we break often enough to run
+finaliers from time to time, or if we really make infinite
+transactions
+
 ------------------------------------------------------------
 
 maybe statically optimize away some stm_become_inevitable(), there
@@ -98,7 +105,8 @@
 
 ------------------------------------------------------------
 
-change the limit of 1 GB
+change the memory limit of a few GBs, try to replace it with something
+larger or even dynamic
 
 ------------------------------------------------------------
 
@@ -139,15 +147,6 @@
 
 ------------------------------------------------------------
 
-dicts: have an implementation that follows the principles in
-stmgc/hashtable/design.txt
-
-------------------------------------------------------------
-
-replace "atomic transactions" with better management of thread.locks.
-
-------------------------------------------------------------
-
 stm_read(p125)
 cond_call_gc_wb_array(p125...)    # don't need the stm_read maybe?
 

From noreply at buildbot.pypy.org  Tue Jun 23 18:06:22 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 23 Jun 2015 18:06:22 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c8: fix test
Message-ID: <20150623160622.9142E1C02FD@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c8
Changeset: r78271:4fc86fe2580c
Date: 2015-06-23 17:29 +0200
http://bitbucket.org/pypy/pypy/changeset/4fc86fe2580c/

Log:	fix test

diff --git a/rpython/translator/stm/test/test_ztranslated.py b/rpython/translator/stm/test/test_ztranslated.py
--- a/rpython/translator/stm/test/test_ztranslated.py
+++ b/rpython/translator/stm/test/test_ztranslated.py
@@ -491,32 +491,32 @@
         t, cbuilder = self.compile(main)
         data = cbuilder.cmdexec('')
         assert ('starting baz\n'
-                'File "/tmp/foobar.py", line 41, in baz\n'
-                'File "/tmp/foobar.py", line 41, in baz\n'
-                'File "/tmp/foobar.py", line 41, in baz\n'
-                'File "/tmp/foobar.py", line 41, in baz\n'
-                'File "/tmp/foobar.py", line 41, in baz\n'
-                'File "/tmp/foobar.py", line 42, in baz\n'
-                'File "/tmp/foobar.py", line 42, in baz\n'
-                'File "/tmp/foobar.py", line 42, in baz\n'
-                'File "/tmp/foobar.py", line 42, in baz\n'
-                'File "/tmp/foobar.py", line 42, in baz\n'
+                'File "/tmp/foobar.py", line 41, in baz (#0)\n'
+                'File "/tmp/foobar.py", line 41, in baz (#1)\n'
+                'File "/tmp/foobar.py", line 41, in baz (#2)\n'
+                'File "/tmp/foobar.py", line 41, in baz (#3)\n'
+                'File "/tmp/foobar.py", line 41, in baz (#4)\n'
+                'File "/tmp/foobar.py", line 42, in baz (#5)\n'
+                'File "/tmp/foobar.py", line 42, in baz (#6)\n'
+                'File "/tmp/foobar.py", line 42, in baz (#7)\n'
+                'File "/tmp/foobar.py", line 42, in baz (#8)\n'
+                'File "/tmp/foobar.py", line 42, in baz (#9)\n'
                 'stopping baz\n') in data
         assert ('starting bar\n'
-                'File "/tmp/foobaz.py", line 71, in bar\n'
-                'File "/tmp/foobaz.py", line 71, in bar\n'
-                'File "/tmp/foobaz.py", line 71, in bar\n'
-                'File "/tmp/foobaz.py", line 71, in bar\n'
-                'File "/tmp/foobaz.py", line 73, in bar\n'
-                'File "/tmp/foobaz.py", line 73, in bar\n'
-                'File "/tmp/foobaz.py", line 73, in bar\n'
-                'File "/tmp/foobaz.py", line 73, in bar\n'
-                'File "/tmp/foobaz.py", line 73, in bar\n'
-                'File "/tmp/foobaz.py", line 73, in bar\n'
+                'File "/tmp/foobaz.py", line 71, in bar (#0)\n'
+                'File "/tmp/foobaz.py", line 71, in bar (#1)\n'
+                'File "/tmp/foobaz.py", line 71, in bar (#2)\n'
+                'File "/tmp/foobaz.py", line 71, in bar (#3)\n'
+                'File "/tmp/foobaz.py", line 73, in bar (#4)\n'
+                'File "/tmp/foobaz.py", line 73, in bar (#5)\n'
+                'File "/tmp/foobaz.py", line 73, in bar (#6)\n'
+                'File "/tmp/foobaz.py", line 73, in bar (#7)\n'
+                'File "/tmp/foobaz.py", line 73, in bar (#8)\n'
+                'File "/tmp/foobaz.py", line 73, in bar (#9)\n'
                 'stopping bar\n') in data
         assert ('starting some_extremely_longish_and_boring_function_name\n'
                 'File "<bla/br/project/foobaz.py", line 81,'
-                ' in some_extremely_longish_a>\n') in data
+                ' in some_extremely_longish_a> (#0)\n') in data
 
     def test_finalizer(self):
         class Counter:

From noreply at buildbot.pypy.org  Tue Jun 23 18:06:23 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 23 Jun 2015 18:06:23 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c8: Fix and de-duplicate the test
Message-ID: <20150623160623.A2AF31C02FD@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c8
Changeset: r78272:af31e936b812
Date: 2015-06-23 17:34 +0200
http://bitbucket.org/pypy/pypy/changeset/af31e936b812/

Log:	Fix and de-duplicate the test

diff --git a/rpython/jit/backend/x86/test/test_zrpy_gc.py b/rpython/jit/backend/x86/test/test_zrpy_gc.py
--- a/rpython/jit/backend/x86/test/test_zrpy_gc.py
+++ b/rpython/jit/backend/x86/test/test_zrpy_gc.py
@@ -1,11 +1,6 @@
 from rpython.jit.backend.llsupport.test.zrpy_gc_test import CompileFrameworkTests
 
 
-class TestSTMShadowStack(CompileFrameworkTests):
-    gcrootfinder = "stm"
-    can_pin = False
-
-
 class TestShadowStack(CompileFrameworkTests):
     gcrootfinder = "shadowstack"
     gc = "incminimark"
diff --git a/rpython/jit/backend/x86/test/test_zrpy_stm.py b/rpython/jit/backend/x86/test/test_zrpy_stm.py
--- a/rpython/jit/backend/x86/test/test_zrpy_stm.py
+++ b/rpython/jit/backend/x86/test/test_zrpy_stm.py
@@ -3,3 +3,4 @@
 
 class TestSTMShadowStack(CompileFrameworkTests):
     gcrootfinder = "stm"
+    can_pin = False

From noreply at buildbot.pypy.org  Tue Jun 23 19:02:25 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 23 Jun 2015 19:02:25 +0200 (CEST)
Subject: [pypy-commit] pypy default: Double mention of DEBUG_MERGE_POINT
Message-ID: <20150623170225.C130E1C0354@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78273:6c9e9cfc6553
Date: 2015-06-23 16:03 +0200
http://bitbucket.org/pypy/pypy/changeset/6c9e9cfc6553/

Log:	Double mention of DEBUG_MERGE_POINT

diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py
--- a/rpython/jit/backend/llsupport/rewrite.py
+++ b/rpython/jit/backend/llsupport/rewrite.py
@@ -73,8 +73,6 @@
                 self.emit_pending_zeros()
             elif op.can_malloc():
                 self.emitting_an_operation_that_can_collect()
-            elif op.getopnum() == rop.DEBUG_MERGE_POINT:
-                continue # ignore debug_merge_points
             elif op.getopnum() == rop.LABEL:
                 self.emitting_an_operation_that_can_collect()
                 self.known_lengths.clear()

From noreply at buildbot.pypy.org  Tue Jun 23 19:30:26 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Tue, 23 Jun 2015 19:30:26 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: missing variable definition
Message-ID: <20150623173026.425661C02A3@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r78274:7e3a332fec89
Date: 2015-06-23 19:30 +0200
http://bitbucket.org/pypy/pypy/changeset/7e3a332fec89/

Log:	missing variable definition

diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/vectorize.py
@@ -658,6 +658,9 @@
             if packed.getarg(1) == inquestion.result:
                 return True
         if inquestion.casts_box():
+            #input_type = packed.output_type
+            #if not input_type:
+            #    return True
             pass
         return False
 
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -190,7 +190,8 @@
         return self._cls_has_bool_result
 
     def casts_box(self):
-        return self.getopnum() == rop.INT_SIGNEXT or \
+        opnum = self.getopnum()
+        return opnum == rop.INT_SIGNEXT or \
                rop.CAST_FLOAT_TO_INT <= opnum <= rop.CAST_SINGLEFLOAT_TO_FLOAT or \
                rop._VEC_CAST_FIRST <= opnum <= rop._VEC_CAST_LAST or \
                rop.CAST_PTR_TO_INT == opnum or \

From noreply at buildbot.pypy.org  Tue Jun 23 19:50:22 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 23 Jun 2015 19:50:22 +0200 (CEST)
Subject: [pypy-commit] cffi default: Fix tests for Python 3
Message-ID: <20150623175022.6D00A1C033F@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2192:a010585333d4
Date: 2015-06-23 19:51 +0200
http://bitbucket.org/cffi/cffi/changeset/a010585333d4/

Log:	Fix tests for Python 3

diff --git a/testing/cffi1/test_new_ffi_1.py b/testing/cffi1/test_new_ffi_1.py
--- a/testing/cffi1/test_new_ffi_1.py
+++ b/testing/cffi1/test_new_ffi_1.py
@@ -1714,6 +1714,6 @@
         assert myvar == -5     # but can't be changed, so not very useful
         py.test.raises(ImportError, "from _test_import_from_lib.lib import bar")
         d = {}
-        exec "from _test_import_from_lib.lib import *" in d
+        exec("from _test_import_from_lib.lib import *", d)
         assert (set(key for key in d if not key.startswith('_')) ==
                 set(['myfunc', 'MYFOO']))
diff --git a/testing/cffi1/test_zdist.py b/testing/cffi1/test_zdist.py
--- a/testing/cffi1/test_zdist.py
+++ b/testing/cffi1/test_zdist.py
@@ -67,6 +67,8 @@
                 name += '.SO'
             if name.startswith('pycparser') and name.endswith('.egg'):
                 continue    # no clue why this shows up sometimes and not others
+            if name == '.eggs':
+                continue    # seems new in 3.5, ignore it
             assert name in content, "found unexpected file %r" % (
                 os.path.join(curdir, name),)
             value = content.pop(name)

From noreply at buildbot.pypy.org  Tue Jun 23 22:21:21 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Tue, 23 Jun 2015 22:21:21 +0200 (CEST)
Subject: [pypy-commit] pypy run-create_cffi_imports: error message,
	update whatsnew
Message-ID: <20150623202121.1C6331C02A3@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: run-create_cffi_imports
Changeset: r78275:fb0d7cdb95e6
Date: 2015-06-23 23:18 +0300
http://bitbucket.org/pypy/pypy/changeset/fb0d7cdb95e6/

Log:	error message, update whatsnew

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -16,3 +16,9 @@
 
 .. branch: disable-unroll-for-short-loops
 The JIT no longer performs loop unrolling if the loop compiles to too much code.
+
+.. branch: run-create_cffi_imports
+
+Build cffi import libraries as part of translation by monkey-patching an 
+aditional task into translation
+
diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -330,6 +330,7 @@
             # XXX possibly adapt options using modules
             failures = create_cffi_import_libraries(exename, options, basedir)
             # if failures, they were already printed
+            print  >> sys.stderr, str(exename),'successfully built, but errors while building the above modules will be ignored'
         driver.task_build_cffi_imports = types.MethodType(task_build_cffi_imports, driver)
         driver.tasks['build_cffi_imports'] = driver.task_build_cffi_imports, ['compile_c']
         driver.default_goal = 'build_cffi_imports'

From noreply at buildbot.pypy.org  Tue Jun 23 22:21:22 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Tue, 23 Jun 2015 22:21:22 +0200 (CEST)
Subject: [pypy-commit] pypy run-create_cffi_imports: close branch to be
	merged
Message-ID: <20150623202122.4B4011C02A3@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: run-create_cffi_imports
Changeset: r78276:b4867bad65e7
Date: 2015-06-23 23:19 +0300
http://bitbucket.org/pypy/pypy/changeset/b4867bad65e7/

Log:	close branch to be merged


From noreply at buildbot.pypy.org  Tue Jun 23 22:21:23 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Tue, 23 Jun 2015 22:21:23 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge run-create_cffi_imports,
 which builds cffi import libraries as part of translation
Message-ID: <20150623202123.8F63A1C02A3@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: 
Changeset: r78277:53ae94237bb3
Date: 2015-06-23 23:19 +0300
http://bitbucket.org/pypy/pypy/changeset/53ae94237bb3/

Log:	merge run-create_cffi_imports, which builds cffi import libraries as
	part of translation

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -16,3 +16,9 @@
 
 .. branch: disable-unroll-for-short-loops
 The JIT no longer performs loop unrolling if the loop compiles to too much code.
+
+.. branch: run-create_cffi_imports
+
+Build cffi import libraries as part of translation by monkey-patching an 
+aditional task into translation
+
diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -1,6 +1,6 @@
 import py
 
-import os, sys
+import os, sys, subprocess
 
 import pypy
 from pypy.interpreter import gateway
@@ -298,6 +298,44 @@
         wrapstr = 'space.wrap(%r)' % (options)
         pypy.module.sys.Module.interpleveldefs['pypy_translation_info'] = wrapstr
 
+        # HACKHACKHACK
+        # ugly hack to modify target goal from compile_c to build_cffi_imports
+        # this should probably get cleaned up and merged with driver.create_exe
+        from rpython.translator.driver import taskdef
+        import types
+
+        class Options(object):
+            pass
+
+
+        def mkexename(name):
+            if sys.platform == 'win32':
+                name = name.new(ext='exe')
+            return name
+
+        @taskdef(['compile_c'], "Create cffi bindings for modules")
+        def task_build_cffi_imports(self):
+            from pypy.tool.build_cffi_imports import create_cffi_import_libraries
+            ''' Use cffi to compile cffi interfaces to modules'''
+            exename = mkexename(driver.compute_exe_name())
+            basedir = exename
+            while not basedir.join('include').exists():
+                _basedir = basedir.dirpath()
+                if _basedir == basedir:
+                    raise ValueError('interpreter %s not inside pypy repo', 
+                                     str(exename))
+                basedir = _basedir
+            modules = self.config.objspace.usemodules.getpaths()
+            options = Options()
+            # XXX possibly adapt options using modules
+            failures = create_cffi_import_libraries(exename, options, basedir)
+            # if failures, they were already printed
+            print  >> sys.stderr, str(exename),'successfully built, but errors while building the above modules will be ignored'
+        driver.task_build_cffi_imports = types.MethodType(task_build_cffi_imports, driver)
+        driver.tasks['build_cffi_imports'] = driver.task_build_cffi_imports, ['compile_c']
+        driver.default_goal = 'build_cffi_imports'
+        # HACKHACKHACK end
+
         return self.get_entry_point(config)
 
     def jitpolicy(self, driver):
diff --git a/pypy/tool/build_cffi_imports.py b/pypy/tool/build_cffi_imports.py
new file mode 100644
--- /dev/null
+++ b/pypy/tool/build_cffi_imports.py
@@ -0,0 +1,75 @@
+import sys, shutil
+from rpython.tool.runsubprocess import run_subprocess
+
+class MissingDependenciesError(Exception):
+    pass
+
+
+cffi_build_scripts = {
+    "sqlite3": "_sqlite3_build.py",
+    "audioop": "_audioop_build.py",
+    "tk": "_tkinter/tklib_build.py",
+    "curses": "_curses_build.py" if sys.platform != "win32" else None,
+    "syslog": "_syslog_build.py" if sys.platform != "win32" else None,
+    "gdbm": "_gdbm_build.py"  if sys.platform != "win32" else None,
+    "pwdgrp": "_pwdgrp_build.py" if sys.platform != "win32" else None,
+    "xx": None,    # for testing: 'None' should be completely ignored
+    }
+
+def create_cffi_import_libraries(pypy_c, options, basedir):
+    shutil.rmtree(str(basedir.join('lib_pypy', '__pycache__')),
+                  ignore_errors=True)
+    failures = []
+    for key, module in sorted(cffi_build_scripts.items()):
+        if module is None or getattr(options, 'no_' + key, False):
+            continue
+        if module.endswith('.py'):
+            args = [module]
+            cwd = str(basedir.join('lib_pypy'))
+        else:
+            args = ['-c', 'import ' + module]
+            cwd = None
+        print >> sys.stderr, '*', ' '.join(args)
+        try:
+            status, stdout, stderr = run_subprocess(str(pypy_c), args, cwd=cwd)
+            if status != 0:
+                print >> sys.stderr, stdout, stderr
+                failures.append((key, module))
+        except:
+            import traceback;traceback.print_exc()
+            failures.append((key, module))
+    return failures
+
+if __name__ == '__main__':
+    import py, os
+    if '__pypy__' not in sys.builtin_module_names:
+        print 'Call with a pypy interpreter'
+        sys.exit(-1)
+
+    class Options(object):
+        pass
+
+    exename = py.path.local(sys.executable) 
+    basedir = exename
+    while not basedir.join('include').exists():
+        _basedir = basedir.dirpath()
+        if _basedir == basedir:
+            raise ValueError('interpreter %s not inside pypy repo', 
+                                 str(exename))
+        basedir = _basedir
+    options = Options()
+    print >> sys.stderr, "There should be no failures here"
+    failures = create_cffi_import_libraries(exename, options, basedir)
+    if len(failures) > 0:
+        print 'failed to build', [f[1] for f in failures]
+        assert False
+
+    # monkey patch a failure, just to test
+    print >> sys.stderr, 'This line should be followed by a traceback'
+    for k in cffi_build_scripts:
+        setattr(options, 'no_' + k, True)
+    must_fail = '_missing_build_script.py'
+    assert not os.path.exists(str(basedir.join('lib_pypy').join(must_fail)))
+    cffi_build_scripts['should_fail'] = must_fail
+    failures = create_cffi_import_libraries(exename, options, basedir)
+    assert len(failures) == 1
diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py
--- a/pypy/tool/release/package.py
+++ b/pypy/tool/release/package.py
@@ -26,6 +26,9 @@
 
 STDLIB_VER = "2.7"
 
+from pypy.tool.build_cffi_imports import (create_cffi_import_libraries, 
+        MissingDependenciesError, cffi_build_scripts)
+
 def ignore_patterns(*patterns):
     """Function that can be used as copytree() ignore parameter.
 
@@ -41,48 +44,12 @@
 class PyPyCNotFound(Exception):
     pass
 
-class MissingDependenciesError(Exception):
-    pass
-
 def fix_permissions(dirname):
     if sys.platform != 'win32':
         os.system("chmod -R a+rX %s" % dirname)
         os.system("chmod -R g-w %s" % dirname)
 
 
-cffi_build_scripts = {
-    "sqlite3": "_sqlite3_build.py",
-    "audioop": "_audioop_build.py",
-    "tk": "_tkinter/tklib_build.py",
-    "curses": "_curses_build.py" if sys.platform != "win32" else None,
-    "syslog": "_syslog_build.py" if sys.platform != "win32" else None,
-    "gdbm": "_gdbm_build.py"  if sys.platform != "win32" else None,
-    "pwdgrp": "_pwdgrp_build.py" if sys.platform != "win32" else None,
-    "xx": None,    # for testing: 'None' should be completely ignored
-    }
-
-def create_cffi_import_libraries(pypy_c, options, basedir):
-    shutil.rmtree(str(basedir.join('lib_pypy', '__pycache__')),
-                  ignore_errors=True)
-    for key, module in sorted(cffi_build_scripts.items()):
-        if module is None or getattr(options, 'no_' + key):
-            continue
-        if module.endswith('.py'):
-            args = [str(pypy_c), module]
-            cwd = str(basedir.join('lib_pypy'))
-        else:
-            args = [str(pypy_c), '-c', 'import ' + module]
-            cwd = None
-        print >> sys.stderr, '*', ' '.join(args)
-        try:
-            subprocess.check_call(args, cwd=cwd)
-        except subprocess.CalledProcessError:
-            print >>sys.stderr, """!!!!!!!!!!\nBuilding {0} bindings failed.
-You can either install development headers package,
-add the --without-{0} option to skip packaging this
-binary CFFI extension, or say --without-cffi.""".format(key)
-            raise MissingDependenciesError(module)
-
 def pypy_runs(pypy_c, quiet=False):
     kwds = {}
     if quiet:
@@ -114,9 +81,13 @@
     if not _fake and not pypy_runs(pypy_c):
         raise OSError("Running %r failed!" % (str(pypy_c),))
     if not options.no_cffi:
-        try:
-            create_cffi_import_libraries(pypy_c, options, basedir)
-        except MissingDependenciesError:
+        failures = create_cffi_import_libraries(pypy_c, options, basedir)
+        for key, module in failures:
+            print >>sys.stderr, """!!!!!!!!!!\nBuilding {0} bindings failed.
+                You can either install development headers package,
+                add the --without-{0} option to skip packaging this
+                binary CFFI extension, or say --without-cffi.""".format(key)
+        if len(failures) > 0:
             return 1, None
 
     if sys.platform == 'win32' and not rename_pypy_c.lower().endswith('.exe'):

From noreply at buildbot.pypy.org  Wed Jun 24 00:01:02 2015
From: noreply at buildbot.pypy.org (amauryfa)
Date: Wed, 24 Jun 2015 00:01:02 +0200 (CEST)
Subject: [pypy-commit] pypy default: Attempt to fix test_tcl on 32bit
	platforms
Message-ID: <20150623220102.0E8AA1C0354@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: 
Changeset: r78278:9fa66d2aa78b
Date: 2015-06-24 00:01 +0200
http://bitbucket.org/pypy/pypy/changeset/9fa66d2aa78b/

Log:	Attempt to fix test_tcl on 32bit platforms

	(there is no WideInt at all on 64bit)

diff --git a/lib_pypy/_tkinter/tclobj.py b/lib_pypy/_tkinter/tclobj.py
--- a/lib_pypy/_tkinter/tclobj.py
+++ b/lib_pypy/_tkinter/tclobj.py
@@ -108,6 +108,8 @@
         return value.internalRep.doubleValue
     if value.typePtr == typeCache.IntType:
         return value.internalRep.longValue
+    if value.typePtr == typeCache.WideIntType:
+        return FromWideIntObj(app, value)
     if value.typePtr == typeCache.BigNumType and tklib.HAVE_LIBTOMMATH:
         return FromBignumObj(app, value)
     if value.typePtr == typeCache.ListType:

From noreply at buildbot.pypy.org  Wed Jun 24 08:57:20 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Wed, 24 Jun 2015 08:57:20 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: setting to safe default
Message-ID: <20150624065720.D540D1C0478@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r78279:1bd1ae3b3117
Date: 2015-06-24 08:57 +0200
http://bitbucket.org/pypy/pypy/changeset/1bd1ae3b3117/

Log:	setting to safe default

diff --git a/rpython/jit/metainterp/warmspot.py b/rpython/jit/metainterp/warmspot.py
--- a/rpython/jit/metainterp/warmspot.py
+++ b/rpython/jit/metainterp/warmspot.py
@@ -33,25 +33,25 @@
 
 # XXX XXX XXX
 class XXXBench(object):
-    def __init__(self, name, id, vec):
+    def __init__(self, name, uid, vec):
         self.t = []
-        self.name = name
-        self.unique_id = hex(id)
+        if name is None:
+            name = "<unkown>"
+        if uid is None:
+            uid = 0
+        self.name = str(name)
+        self.unique_id = hex(uid)
         self.vec = vec
 
     def xxx_clock_start(self):
-        if not self.vec:
-            return
         now = time.clock()
         self.t.append(now)
         debug_start("xxx-clock-start")
-        debug_print("name: %s id(jdsd): %s now: %dns" % \
+        debug_print("name: %s id: %s now: %dns" % \
                 (self.name, self.unique_id, int(now)*10**9) )
         debug_stop("xxx-clock-start")
 
     def xxx_clock_stop(self, fail=False):
-        if not self.vec:
-            return
         end = time.clock()
         if len(self.t) == 0:
             return
@@ -60,7 +60,7 @@
             del self.t[-1]
         ns = (end - start) * 10**9
         debug_start("xxx-clock-stop")
-        debug_print("name: %s id(jdsd): %s now: %dns exe time: %dns fail? %d vec? %d" % \
+        debug_print("name: %s id: %s now: %dns exe time: %dns fail? %d vec? %d" % \
                     (self.name, self.unique_id, int(end)*10**9, int(ns), int(fail), int(self.vec)))
         debug_stop("xxx-clock-stop")
 

From noreply at buildbot.pypy.org  Wed Jun 24 09:57:33 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 24 Jun 2015 09:57:33 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c8: Forgot that arrays can also be
 'stm_dont_track_raw_accesses'
Message-ID: <20150624075733.417D01C02BB@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c8
Changeset: r78280:8d2fbff1bf9e
Date: 2015-06-24 09:41 +0200
http://bitbucket.org/pypy/pypy/changeset/8d2fbff1bf9e/

Log:	Forgot that arrays can also be 'stm_dont_track_raw_accesses'

diff --git a/pypy/module/pypystm/unsafe_op.py b/pypy/module/pypystm/unsafe_op.py
--- a/pypy/module/pypystm/unsafe_op.py
+++ b/pypy/module/pypystm/unsafe_op.py
@@ -7,9 +7,9 @@
 
 @specialize.memo()
 def get_unsafe_type_ptr(TP):
-    UNSAFE = lltype.Struct('UNSAFE', ('x', TP),
-                           hints = {'stm_dont_track_raw_accesses': True})
-    return rffi.CArrayPtr(UNSAFE)
+    UNSAFE = lltype.Array(TP, hints={'nolength': True,
+                                     'stm_dont_track_raw_accesses': True})
+    return lltype.Ptr(UNSAFE)
 
 
 def unsafe_write_raw_signed_data(w_cdata, index, source, size):
@@ -17,7 +17,7 @@
         for TP, _ in misc._prim_signed_types:
             if size == rffi.sizeof(TP):
                 TPP = get_unsafe_type_ptr(TP)
-                rffi.cast(TPP, target)[index].x = rffi.cast(TP, source)
+                rffi.cast(TPP, target)[index] = rffi.cast(TP, source)
                 return
     raise NotImplementedError("bad integer size")
 
@@ -26,7 +26,7 @@
         for TP, _ in misc._prim_unsigned_types:
             if size == rffi.sizeof(TP):
                 TPP = get_unsafe_type_ptr(TP)
-                rffi.cast(TPP, target)[index].x = rffi.cast(TP, source)
+                rffi.cast(TPP, target)[index] = rffi.cast(TP, source)
                 return
     raise NotImplementedError("bad integer size")
 
@@ -35,7 +35,7 @@
         for TP, _ in misc._prim_float_types:
             if size == rffi.sizeof(TP):
                 TPP = get_unsafe_type_ptr(TP)
-                rffi.cast(TPP, target)[index].x = rffi.cast(TP, source)
+                rffi.cast(TPP, target)[index] = rffi.cast(TP, source)
                 return
     raise NotImplementedError("bad float size")
 

From noreply at buildbot.pypy.org  Wed Jun 24 09:57:34 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 24 Jun 2015 09:57:34 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c8: Add unsafe_read(),
 for a different use case: multiple threads all
Message-ID: <20150624075734.78D9A1C0478@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c8
Changeset: r78281:4da124c51415
Date: 2015-06-24 09:53 +0200
http://bitbucket.org/pypy/pypy/changeset/4da124c51415/

Log:	Add unsafe_read(), for a different use case: multiple threads all
	reading some big shared immutable raw data

diff --git a/pypy/module/pypystm/__init__.py b/pypy/module/pypystm/__init__.py
--- a/pypy/module/pypystm/__init__.py
+++ b/pypy/module/pypystm/__init__.py
@@ -31,4 +31,5 @@
         'Empty': 'space.fromcache(queue.Cache).w_Empty',
 
         'unsafe_write': 'unsafe_op.unsafe_write',
+        'unsafe_read':  'unsafe_op.unsafe_read',
     }
diff --git a/pypy/module/pypystm/test/test_unsafe_op.py b/pypy/module/pypystm/test/test_unsafe_op.py
--- a/pypy/module/pypystm/test/test_unsafe_op.py
+++ b/pypy/module/pypystm/test/test_unsafe_op.py
@@ -13,6 +13,7 @@
         pypystm.unsafe_write(x, 1, '\xAA')
         assert x[0] == 'A'
         assert x[1] == '\xAA'
+        assert pypystm.unsafe_read(x, 1) == '\xAA'
 
     def test_unsafe_write_int32(self):
         import pypystm, _cffi_backend
@@ -23,6 +24,7 @@
         pypystm.unsafe_write(x, 1, -0x05060708)
         assert x[0] == -0x01020304
         assert x[1] == -0x05060708
+        assert pypystm.unsafe_read(x, 1) == -0x05060708
 
     def test_unsafe_write_uint64(self):
         import pypystm, _cffi_backend
@@ -33,6 +35,7 @@
         pypystm.unsafe_write(x, 1, 0xF506070855667788)
         assert x[0] == 0x0102030411223344
         assert x[1] == 0xF506070855667788
+        assert pypystm.unsafe_read(x, 1) == 0xF506070855667788
 
     def test_unsafe_write_unsupported_case(self):
         import pypystm, _cffi_backend
@@ -40,6 +43,7 @@
         BUniCharP = _cffi_backend.new_pointer_type(BUniChar)
         x = _cffi_backend.newp(_cffi_backend.new_array_type(BUniCharP, 2))
         raises(TypeError, pypystm.unsafe_write, x, 0, u'X')
+        raises(TypeError, pypystm.unsafe_read, x, 1)
 
     def test_unsafe_write_float(self):
         import pypystm, _cffi_backend
@@ -50,6 +54,7 @@
         pypystm.unsafe_write(x, 1, -42.0)
         assert x[0] == 12.25
         assert x[1] == -42.0
+        assert pypystm.unsafe_read(x, 1) == -42.0
 
     def test_unsafe_write_double(self):
         import pypystm, _cffi_backend
@@ -60,3 +65,4 @@
         pypystm.unsafe_write(x, 1, -42.0)
         assert x[0] == 12.25
         assert x[1] == -42.0
+        assert pypystm.unsafe_read(x, 1) == -42.0
diff --git a/pypy/module/pypystm/unsafe_op.py b/pypy/module/pypystm/unsafe_op.py
--- a/pypy/module/pypystm/unsafe_op.py
+++ b/pypy/module/pypystm/unsafe_op.py
@@ -3,6 +3,7 @@
 from pypy.module._cffi_backend import cdataobj, ctypeptr, ctypeprim, misc
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rlib.objectmodel import specialize
+from rpython.rlib.rarithmetic import intmask
 
 
 @specialize.memo()
@@ -73,3 +74,65 @@
 
     raise oefmt(space.w_TypeError, "unsupported type in unsafe_write(): '%s'",
                 ctitem.name)
+
+# ____________________________________________________________
+
+
+def unsafe_read_raw_signed_data(w_cdata, index, size):
+    with w_cdata as target:
+        for TP, _ in misc._prim_signed_types:
+            if size == rffi.sizeof(TP):
+                TPP = get_unsafe_type_ptr(TP)
+                value = rffi.cast(TPP, target)[index]
+                return rffi.cast(lltype.Signed, value)
+    raise NotImplementedError("bad integer size")
+
+def unsafe_read_raw_unsigned_data(w_cdata, index, size):
+    with w_cdata as target:
+        for TP, _ in misc._prim_unsigned_types:
+            if size == rffi.sizeof(TP):
+                TPP = get_unsafe_type_ptr(TP)
+                value = rffi.cast(TPP, target)[index]
+                return rffi.cast(lltype.Unsigned, value)
+    raise NotImplementedError("bad integer size")
+
+def unsafe_read_raw_float_data(w_cdata, index, size):
+    with w_cdata as target:
+        for TP, _ in misc._prim_float_types:
+            if size == rffi.sizeof(TP):
+                TPP = get_unsafe_type_ptr(TP)
+                value = rffi.cast(TPP, target)[index]
+                return rffi.cast(lltype.Float, value)
+    raise NotImplementedError("bad integer size")
+
+
+ at unwrap_spec(w_cdata=cdataobj.W_CData, index=int)
+def unsafe_read(space, w_cdata, index):
+    ctype = w_cdata.ctype
+    if not isinstance(ctype, ctypeptr.W_CTypePtrOrArray):
+        raise oefmt(space.w_TypeError,
+                    "expected a cdata of type pointer or array")
+    ctitem = ctype.ctitem
+
+    if isinstance(ctitem, ctypeprim.W_CTypePrimitiveChar):
+        uintvalue = unsafe_read_raw_unsigned_data(w_cdata, index, size=1)
+        return space.wrap(chr(intmask(uintvalue)))
+
+    if isinstance(ctitem, ctypeprim.W_CTypePrimitiveSigned):
+        if ctitem.value_fits_long:
+            intvalue = unsafe_read_raw_signed_data(w_cdata, index, ctitem.size)
+            return space.wrap(intvalue)
+
+    if isinstance(ctitem, ctypeprim.W_CTypePrimitiveUnsigned):
+        if ctitem.value_fits_ulong:
+            uintvalue = unsafe_read_raw_unsigned_data(w_cdata, index,
+                                                      ctitem.size)
+            return space.wrap(uintvalue)
+
+    if isinstance(ctitem, ctypeprim.W_CTypePrimitiveFloat):
+        if not isinstance(ctitem, ctypeprim.W_CTypePrimitiveLongDouble):
+            floatvalue = unsafe_read_raw_float_data(w_cdata, index, ctitem.size)
+            return space.wrap(floatvalue)
+
+    raise oefmt(space.w_TypeError, "unsupported type in unsafe_read(): '%s'",
+                ctitem.name)

From noreply at buildbot.pypy.org  Wed Jun 24 10:36:25 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 24 Jun 2015 10:36:25 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: more passing tests
Message-ID: <20150624083625.81B721C148B@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult-unroll
Changeset: r78282:3e5b0b9e8a58
Date: 2015-06-18 13:58 +0200
http://bitbucket.org/pypy/pypy/changeset/3e5b0b9e8a58/

Log:	more passing tests

diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -77,9 +77,9 @@
             opinfo.mark_last_guard(self.optimizer)
         return opinfo
 
-    def getptrinfo(self, op, create=False, is_object=False):
+    def getptrinfo(self, op, is_object=False):
         if op.type == 'i':
-            return self.getrawptrinfo(op, create)
+            return self.getrawptrinfo(op)
         elif op.type == 'f':
             return None
         assert op.type == 'r'
@@ -288,7 +288,12 @@
             zzz
 
     def setinfo_from_preamble(self, op, old_info):
-        pass # deal with later
+        if isinstance(old_info, info.PtrInfo):
+            if op.is_constant():
+                return # nothing we can learn
+            known_class = old_info.get_known_class(self.cpu)
+            if known_class:
+                self.make_constant_class(op, known_class, False)
 
     def get_box_replacement(self, op):
         if op is None:
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -225,14 +225,14 @@
     def test_remove_guard_class_2(self):
         ops = """
         [i0]
-        p0 = new_with_vtable(ConstClass(node_vtable))
+        p0 = new_with_vtable(descr=nodesize)
         escape_n(p0)
         guard_class(p0, ConstClass(node_vtable)) []
         jump(i0)
         """
         expected = """
         [i0]
-        p0 = new_with_vtable(ConstClass(node_vtable))
+        p0 = new_with_vtable(descr=nodesize)
         escape_n(p0)
         jump(i0)
         """
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -38,8 +38,6 @@
             imp.import_value(value)
 
     def emit_operation(self, op):
-        if op.returns_bool_result():
-            self.bool_boxes[self.getvalue(op)] = None
         if self.emitting_dissabled:
             return
         if op.is_guard():
diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py
@@ -282,8 +282,10 @@
     lenbound = None
     intbound = None
     
-    def __init__(self, box, is_opaque=False):
+    def __init__(self, cpu, ptrinfo, is_opaque=False):
         self.level = LEVEL_UNKNOWN
+        if ptrinfo is not None:
+            self.known_class = ptrinfo.get_known_class(cpu)
         return
         xxx
         self.is_opaque = is_opaque
@@ -599,7 +601,11 @@
     
     def visit_not_virtual(self, box):
         is_opaque = box in self.optimizer.opaque_pointers
-        return NotVirtualStateInfo(box, is_opaque)
+        if box.type == 'r':
+            ptrinfo = self.optimizer.getptrinfo(box)
+        else:
+            return self.visit_not_ptr(box, self.optimizer.getintbound(box))
+        return NotVirtualStateInfo(self.optimizer.cpu, ptrinfo, is_opaque)
 
     def visit_virtual(self, known_class, fielddescrs):
         return VirtualStateInfo(known_class, fielddescrs)

From noreply at buildbot.pypy.org  Wed Jun 24 10:36:26 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 24 Jun 2015 10:36:26 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: kill box usage
Message-ID: <20150624083626.A56B71C148B@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r78283:3d9cfcc02bba
Date: 2015-06-24 10:36 +0200
http://bitbucket.org/pypy/pypy/changeset/3d9cfcc02bba/

Log:	kill box usage

diff --git a/rpython/jit/metainterp/graphpage.py b/rpython/jit/metainterp/graphpage.py
--- a/rpython/jit/metainterp/graphpage.py
+++ b/rpython/jit/metainterp/graphpage.py
@@ -1,7 +1,6 @@
 
 from rpython.translator.tool.graphpage import GraphPage
 from rpython.translator.tool.make_dot import DotGen
-from rpython.jit.metainterp.history import Box
 from rpython.jit.metainterp.resoperation import rop
 
 class SubGraph:
@@ -102,13 +101,8 @@
         self.dotgen = DotGen('resop')
         self.dotgen.emit('clusterrank="local"')
         self.generrmsg()
-        _prev = Box._extended_display
-        try:
-            Box._extended_display = False
-            for i, graph in enumerate(self.graphs):
-                self.gengraph(graph, i)
-        finally:
-            Box._extended_display = _prev
+        for i, graph in enumerate(self.graphs):
+            self.gengraph(graph, i)
         # we generate the edges at the end of the file; otherwise, and edge
         # could mention a node before it's declared, and this can cause the
         # node declaration to occur too early -- in the wrong subgraph.

From noreply at buildbot.pypy.org  Wed Jun 24 10:36:27 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 24 Jun 2015 10:36:27 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: fix wrong use of check isinstance(x,
	AbstractResOp)
Message-ID: <20150624083627.C8B6E1C148B@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r78284:0ec1faa7d196
Date: 2015-06-24 10:36 +0200
http://bitbucket.org/pypy/pypy/changeset/0ec1faa7d196/

Log:	fix wrong use of check isinstance(x, AbstractResOp)

diff --git a/rpython/jit/metainterp/optimizeopt/intbounds.py b/rpython/jit/metainterp/optimizeopt/intbounds.py
--- a/rpython/jit/metainterp/optimizeopt/intbounds.py
+++ b/rpython/jit/metainterp/optimizeopt/intbounds.py
@@ -134,7 +134,7 @@
             v1, v2 = v2, v1
         # if both are constant, the pure optimization will deal with it
         if v2.is_constant() and not v1.is_constant():
-            if isinstance(arg1, AbstractResOp):
+            if not self.optimizer.is_inputarg(arg1):
                 if arg1.getopnum() == rop.INT_ADD:
                     prod_arg1 = arg1.getarg(0)
                     prod_arg2 = arg1.getarg(1)
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -328,6 +328,9 @@
     def ensure_imported(self, value):
         pass
 
+    def is_inputarg(self, op):
+        return op in self.inparg_dict
+
     def get_constant_box(self, box):
         box = self.get_box_replacement(box)
         if isinstance(box, Const):
@@ -456,6 +459,9 @@
     def propagate_all_forward(self, clear=True):
         if clear:
             self.clear_newoperations()
+        self.inparg_dict = {}
+        for op in self.loop.inputargs:
+            self.inparg_dict[op] = None
         for op in self.loop.operations:
             self._really_emitted_operation = None
             self.first_optimization.propagate_forward(op)

From noreply at buildbot.pypy.org  Wed Jun 24 10:38:01 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 24 Jun 2015 10:38:01 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: merge optresult
Message-ID: <20150624083801.2C80B1C148B@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult-unroll
Changeset: r78285:c52efd0d1ac5
Date: 2015-06-24 10:37 +0200
http://bitbucket.org/pypy/pypy/changeset/c52efd0d1ac5/

Log:	merge optresult

diff too long, truncating to 2000 out of 73423 lines

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -3,11 +3,15 @@
 d8ac7d23d3ec5f9a0fa1264972f74a010dbfd07f release-1.6
 ff4af8f318821f7f5ca998613a60fca09aa137da release-1.7
 07e08e9c885ca67d89bcc304e45a32346daea2fa release-2.0-beta-1
-9b623bc48b5950cf07184462a0e48f2c4df0d720 pypy-2.1-beta1-arm
-9b623bc48b5950cf07184462a0e48f2c4df0d720 pypy-2.1-beta1-arm
 ab0dd631c22015ed88e583d9fdd4c43eebf0be21 pypy-2.1-beta1-arm
 20e51c4389ed4469b66bb9d6289ce0ecfc82c4b9 release-2.3.0
 394146e9bb673514c61f0150ab2013ccf78e8de7 release-2.3
 32f35069a16d819b58c1b6efb17c44e3e53397b2 release-2.2=3.1
 32f35069a16d819b58c1b6efb17c44e3e53397b2 release-2.3.1
 10f1b29a2bd21f837090286174a9ca030b8680b2 release-2.5.0
+9c4588d731b7fe0b08669bd732c2b676cb0a8233 release-2.5.1
+fcdb941565156385cbac04cfb891f8f4c7a92ef6 release-2.6.0
+fcdb941565156385cbac04cfb891f8f4c7a92ef6 release-2.6.0
+e03971291f3a0729ecd3ee7fae7ddb0bb82d476c release-2.6.0
+e03971291f3a0729ecd3ee7fae7ddb0bb82d476c release-2.6.0
+295ee98b69288471b0fcf2e0ede82ce5209eb90b release-2.6.0
diff --git a/.tddium.requirements.txt b/.tddium.requirements.txt
deleted file mode 100644
--- a/.tddium.requirements.txt
+++ /dev/null
@@ -1,1 +0,0 @@
-pytest
diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -38,8 +38,8 @@
   Armin Rigo
   Maciej Fijalkowski
   Carl Friedrich Bolz
+  Amaury Forgeot d'Arc
   Antonio Cuni
-  Amaury Forgeot d'Arc
   Samuele Pedroni
   Alex Gaynor
   Brian Kearns
@@ -50,9 +50,9 @@
   Holger Krekel
   Christian Tismer
   Hakan Ardo
-  Benjamin Peterson
   Manuel Jacob
   Ronan Lamy
+  Benjamin Peterson
   Anders Chrigstrom
   Eric van Riet Paap
   Wim Lavrijsen
@@ -63,8 +63,8 @@
   Sven Hager
   Anders Lehmann
   Aurelien Campeas
+  Remi Meier
   Niklaus Haldimann
-  Remi Meier
   Camillo Bruni
   Laura Creighton
   Toon Verwaest
@@ -76,10 +76,10 @@
   David Edelsohn
   Anders Hammarquist
   Jakub Gustak
+  Gregor Wegberg
   Guido Wesdorp
   Lawrence Oluyede
   Bartosz Skowron
-  Gregor Wegberg
   Daniel Roberts
   Niko Matsakis
   Adrien Di Mascio
@@ -87,10 +87,11 @@
   Ludovic Aubry
   Jacob Hallen
   Jason Creighton
+  Richard Plangger
   Alex Martelli
   Michal Bendowski
+  stian
   Jan de Mooij
-  stian
   Tyler Wade
   Michael Foord
   Stephan Diehl
@@ -133,15 +134,15 @@
   Georg Brandl
   Bert Freudenberg
   Stian Andreassen
+  Edd Barrett
   Wanja Saatkamp
   Gerald Klix
   Mike Blume
+  Tobias Pape
   Oscar Nierstrasz
   Stefan H. Muller
-  Edd Barrett
   Jeremy Thurgood
   Rami Chowdhury
-  Tobias Pape
   Eugene Oden
   Henry Mason
   Vasily Kuznetsov
@@ -167,11 +168,13 @@
   Michael Twomey
   Lucian Branescu Mihaila
   Yichao Yu
+  Anton Gulenko
   Gabriel Lavoie
   Olivier Dormond
   Jared Grubb
   Karl Bartel
   Wouter van Heyst
+  Sebastian Pawlu&#347;
   Brian Dorsey
   Victor Stinner
   Andrews Medina
@@ -188,6 +191,7 @@
   Neil Shepperd
   Stanislaw Halik
   Mikael Sch&#246;nenberg
+  Berkin Ilbeyi
   Elmo M?ntynen
   Jonathan David Riehl
   Anders Qvist
@@ -211,11 +215,11 @@
   Carl Meyer
   Karl Ramm
   Pieter Zieschang
-  Sebastian Pawlu&#347;
   Gabriel
   Lukas Vacek
   Andrew Dalke
   Sylvain Thenault
+  Jakub Stasiak
   Nathan Taylor
   Vladimir Kryachko
   Jacek Generowicz
@@ -242,6 +246,7 @@
   Tomo Cocoa
   Toni Mattis
   Lucas Stadler
+  Julian Berman
   roberto at goyle
   Yury V. Zaytsev
   Anna Katrina Dominguez
@@ -253,6 +258,8 @@
   Ben Darnell
   Roberto De Ioris
   Juan Francisco Cantero Hurtado
+  Ruochen Huang
+  Jeong YunWon
   Godefroid Chappelle
   Joshua Gilbert
   Dan Colish
@@ -271,6 +278,7 @@
   Christian Muirhead
   Berker Peksag
   James Lan
+  Volodymyr Vladymyrov
   shoma hosaka
   Daniel Neuh&#228;user
   Ben Mather
@@ -316,6 +324,7 @@
   yasirs
   Michael Chermside
   Anna Ravencroft
+  Andrey Churin
   Dan Crosta
   Julien Phalip
   Roman Podoliaka
@@ -420,3 +429,10 @@
 the terms of the GPL license version 2 or any later version.  Thus the
 gdbm module, provided in the file lib_pypy/gdbm.py, is redistributed
 under the terms of the GPL license as well.
+
+License for 'pypy/module/_vmprof/src'
+--------------------------------------
+
+The code is based on gperftools. You may see a copy of the License for it at
+
+    https://code.google.com/p/gperftools/source/browse/COPYING
diff --git a/lib-python/2.7/Cookie.py b/lib-python/2.7/Cookie.py
--- a/lib-python/2.7/Cookie.py
+++ b/lib-python/2.7/Cookie.py
@@ -528,12 +528,13 @@
 # result, the parsing rules here are less strict.
 #
 
-_LegalCharsPatt  = r"[\w\d!#%&'~_`><@,:/\$\*\+\-\.\^\|\)\(\?\}\{\=]"
+_LegalKeyChars  = r"\w\d!#%&'~_`><@,:/\$\*\+\-\.\^\|\)\(\?\}\{\="
+_LegalValueChars = _LegalKeyChars + r"\[\]"
 _CookiePattern = re.compile(
     r"(?x)"                       # This is a Verbose pattern
     r"\s*"                        # Optional whitespace at start of cookie
     r"(?P<key>"                   # Start of group 'key'
-    ""+ _LegalCharsPatt +"+?"     # Any word of at least one letter, nongreedy
+    "["+ _LegalKeyChars +"]+?"     # Any word of at least one letter, nongreedy
     r")"                          # End of group 'key'
     r"("                          # Optional group: there may not be a value.
     r"\s*=\s*"                    # Equal Sign
@@ -542,7 +543,7 @@
     r"|"                            # or
     r"\w{3},\s[\s\w\d-]{9,11}\s[\d:]{8}\sGMT" # Special case for "expires" attr
     r"|"                            # or
-    ""+ _LegalCharsPatt +"*"        # Any word or empty string
+    "["+ _LegalValueChars +"]*"        # Any word or empty string
     r")"                          # End of group 'val'
     r")?"                         # End of optional value group
     r"\s*"                        # Any number of spaces.
diff --git a/lib-python/2.7/SimpleHTTPServer.py b/lib-python/2.7/SimpleHTTPServer.py
--- a/lib-python/2.7/SimpleHTTPServer.py
+++ b/lib-python/2.7/SimpleHTTPServer.py
@@ -14,6 +14,7 @@
 import posixpath
 import BaseHTTPServer
 import urllib
+import urlparse
 import cgi
 import sys
 import shutil
@@ -68,10 +69,14 @@
         path = self.translate_path(self.path)
         f = None
         if os.path.isdir(path):
-            if not self.path.endswith('/'):
+            parts = urlparse.urlsplit(self.path)
+            if not parts.path.endswith('/'):
                 # redirect browser - doing basically what apache does
                 self.send_response(301)
-                self.send_header("Location", self.path + "/")
+                new_parts = (parts[0], parts[1], parts[2] + '/',
+                             parts[3], parts[4])
+                new_url = urlparse.urlunsplit(new_parts)
+                self.send_header("Location", new_url)
                 self.end_headers()
                 return None
             for index in "index.html", "index.htm":
diff --git a/lib-python/2.7/_LWPCookieJar.py b/lib-python/2.7/_LWPCookieJar.py
--- a/lib-python/2.7/_LWPCookieJar.py
+++ b/lib-python/2.7/_LWPCookieJar.py
@@ -18,7 +18,7 @@
                        iso2time, time2isoz)
 
 def lwp_cookie_str(cookie):
-    """Return string representation of Cookie in an the LWP cookie file format.
+    """Return string representation of Cookie in the LWP cookie file format.
 
     Actually, the format is extended a bit -- see module docstring.
 
diff --git a/lib-python/2.7/_abcoll.py b/lib-python/2.7/_abcoll.py
--- a/lib-python/2.7/_abcoll.py
+++ b/lib-python/2.7/_abcoll.py
@@ -548,23 +548,25 @@
             If E present and lacks .keys() method, does:     for (k, v) in E: D[k] = v
             In either case, this is followed by: for k, v in F.items(): D[k] = v
         '''
-        if len(args) > 2:
-            raise TypeError("update() takes at most 2 positional "
-                            "arguments ({} given)".format(len(args)))
-        elif not args:
-            raise TypeError("update() takes at least 1 argument (0 given)")
+        if not args:
+            raise TypeError("descriptor 'update' of 'MutableMapping' object "
+                            "needs an argument")
         self = args[0]
-        other = args[1] if len(args) >= 2 else ()
-
-        if isinstance(other, Mapping):
-            for key in other:
-                self[key] = other[key]
-        elif hasattr(other, "keys"):
-            for key in other.keys():
-                self[key] = other[key]
-        else:
-            for key, value in other:
-                self[key] = value
+        args = args[1:]
+        if len(args) > 1:
+            raise TypeError('update expected at most 1 arguments, got %d' %
+                            len(args))
+        if args:
+            other = args[0]
+            if isinstance(other, Mapping):
+                for key in other:
+                    self[key] = other[key]
+            elif hasattr(other, "keys"):
+                for key in other.keys():
+                    self[key] = other[key]
+            else:
+                for key, value in other:
+                    self[key] = value
         for key, value in kwds.items():
             self[key] = value
 
diff --git a/lib-python/2.7/_pyio.py b/lib-python/2.7/_pyio.py
--- a/lib-python/2.7/_pyio.py
+++ b/lib-python/2.7/_pyio.py
@@ -25,8 +25,8 @@
 DEFAULT_BUFFER_SIZE = 8 * 1024  # bytes
 
 # NOTE: Base classes defined here are registered with the "official" ABCs
-# defined in io.py. We don't use real inheritance though, because we don't
-# want to inherit the C implementations.
+# defined in io.py. We don't use real inheritance though, because we don't want
+# to inherit the C implementations.
 
 
 class BlockingIOError(IOError):
@@ -775,7 +775,7 @@
         clsname = self.__class__.__name__
         try:
             name = self.name
-        except AttributeError:
+        except Exception:
             return "<_pyio.{0}>".format(clsname)
         else:
             return "<_pyio.{0} name={1!r}>".format(clsname, name)
@@ -1216,8 +1216,10 @@
         return self.writer.flush()
 
     def close(self):
-        self.writer.close()
-        self.reader.close()
+        try:
+            self.writer.close()
+        finally:
+            self.reader.close()
 
     def isatty(self):
         return self.reader.isatty() or self.writer.isatty()
@@ -1538,7 +1540,7 @@
     def __repr__(self):
         try:
             name = self.name
-        except AttributeError:
+        except Exception:
             return "<_pyio.TextIOWrapper encoding='{0}'>".format(self.encoding)
         else:
             return "<_pyio.TextIOWrapper name={0!r} encoding='{1}'>".format(
diff --git a/lib-python/2.7/_strptime.py b/lib-python/2.7/_strptime.py
--- a/lib-python/2.7/_strptime.py
+++ b/lib-python/2.7/_strptime.py
@@ -335,9 +335,9 @@
     # though
     week_of_year = -1
     week_of_year_start = -1
-    # weekday and julian defaulted to -1 so as to signal need to calculate
+    # weekday and julian defaulted to None so as to signal need to calculate
     # values
-    weekday = julian = -1
+    weekday = julian = None
     found_dict = found.groupdict()
     for group_key in found_dict.iterkeys():
         # Directives not explicitly handled below:
@@ -434,14 +434,14 @@
         year = 1900
     # If we know the week of the year and what day of that week, we can figure
     # out the Julian day of the year.
-    if julian == -1 and week_of_year != -1 and weekday != -1:
+    if julian is None and week_of_year != -1 and weekday is not None:
         week_starts_Mon = True if week_of_year_start == 0 else False
         julian = _calc_julian_from_U_or_W(year, week_of_year, weekday,
                                             week_starts_Mon)
     # Cannot pre-calculate datetime_date() since can change in Julian
     # calculation and thus could have different value for the day of the week
     # calculation.
-    if julian == -1:
+    if julian is None:
         # Need to add 1 to result since first day of the year is 1, not 0.
         julian = datetime_date(year, month, day).toordinal() - \
                   datetime_date(year, 1, 1).toordinal() + 1
@@ -451,7 +451,7 @@
         year = datetime_result.year
         month = datetime_result.month
         day = datetime_result.day
-    if weekday == -1:
+    if weekday is None:
         weekday = datetime_date(year, month, day).weekday()
     if leap_year_fix:
         # the caller didn't supply a year but asked for Feb 29th. We couldn't
diff --git a/lib-python/2.7/aifc.py b/lib-python/2.7/aifc.py
--- a/lib-python/2.7/aifc.py
+++ b/lib-python/2.7/aifc.py
@@ -357,10 +357,13 @@
         self._soundpos = 0
 
     def close(self):
-        if self._decomp:
-            self._decomp.CloseDecompressor()
-            self._decomp = None
-        self._file.close()
+        decomp = self._decomp
+        try:
+            if decomp:
+                self._decomp = None
+                decomp.CloseDecompressor()
+        finally:
+            self._file.close()
 
     def tell(self):
         return self._soundpos
diff --git a/lib-python/2.7/binhex.py b/lib-python/2.7/binhex.py
--- a/lib-python/2.7/binhex.py
+++ b/lib-python/2.7/binhex.py
@@ -32,7 +32,8 @@
     pass
 
 # States (what have we written)
-[_DID_HEADER, _DID_DATA, _DID_RSRC] = range(3)
+_DID_HEADER = 0
+_DID_DATA = 1
 
 # Various constants
 REASONABLY_LARGE=32768  # Minimal amount we pass the rle-coder
@@ -235,17 +236,22 @@
         self._write(data)
 
     def close(self):
-        if self.state < _DID_DATA:
-            self.close_data()
-        if self.state != _DID_DATA:
-            raise Error, 'Close at the wrong time'
-        if self.rlen != 0:
-            raise Error, \
-                  "Incorrect resource-datasize, diff=%r" % (self.rlen,)
-        self._writecrc()
-        self.ofp.close()
-        self.state = None
-        del self.ofp
+        if self.state is None:
+            return
+        try:
+            if self.state < _DID_DATA:
+                self.close_data()
+            if self.state != _DID_DATA:
+                raise Error, 'Close at the wrong time'
+            if self.rlen != 0:
+                raise Error, \
+                    "Incorrect resource-datasize, diff=%r" % (self.rlen,)
+            self._writecrc()
+        finally:
+            self.state = None
+            ofp = self.ofp
+            del self.ofp
+            ofp.close()
 
 def binhex(inp, out):
     """(infilename, outfilename) - Create binhex-encoded copy of a file"""
@@ -463,11 +469,15 @@
         return self._read(n)
 
     def close(self):
-        if self.rlen:
-            dummy = self.read_rsrc(self.rlen)
-        self._checkcrc()
-        self.state = _DID_RSRC
-        self.ifp.close()
+        if self.state is None:
+            return
+        try:
+            if self.rlen:
+                dummy = self.read_rsrc(self.rlen)
+            self._checkcrc()
+        finally:
+            self.state = None
+            self.ifp.close()
 
 def hexbin(inp, out):
     """(infilename, outfilename) - Decode binhexed file"""
diff --git a/lib-python/2.7/bsddb/test/test_all.py b/lib-python/2.7/bsddb/test/test_all.py
--- a/lib-python/2.7/bsddb/test/test_all.py
+++ b/lib-python/2.7/bsddb/test/test_all.py
@@ -412,9 +412,6 @@
         def get_dbp(self) :
             return self._db
 
-    import string
-    string.letters=[chr(i) for i in xrange(65,91)]
-
     bsddb._db.DBEnv_orig = bsddb._db.DBEnv
     bsddb._db.DB_orig = bsddb._db.DB
     if bsddb.db.version() <= (4, 3) :
diff --git a/lib-python/2.7/bsddb/test/test_basics.py b/lib-python/2.7/bsddb/test/test_basics.py
--- a/lib-python/2.7/bsddb/test/test_basics.py
+++ b/lib-python/2.7/bsddb/test/test_basics.py
@@ -999,7 +999,7 @@
         for x in "The quick brown fox jumped over the lazy dog".split():
             d2.put(x, self.makeData(x))
 
-        for x in string.letters:
+        for x in string.ascii_letters:
             d3.put(x, x*70)
 
         d1.sync()
@@ -1047,7 +1047,7 @@
             if verbose:
                 print rec
             rec = c3.next()
-        self.assertEqual(count, len(string.letters))
+        self.assertEqual(count, len(string.ascii_letters))
 
 
         c1.close()
diff --git a/lib-python/2.7/bsddb/test/test_dbshelve.py b/lib-python/2.7/bsddb/test/test_dbshelve.py
--- a/lib-python/2.7/bsddb/test/test_dbshelve.py
+++ b/lib-python/2.7/bsddb/test/test_dbshelve.py
@@ -59,7 +59,7 @@
             return bytes(key, "iso8859-1")  # 8 bits
 
     def populateDB(self, d):
-        for x in string.letters:
+        for x in string.ascii_letters:
             d[self.mk('S' + x)] = 10 * x           # add a string
             d[self.mk('I' + x)] = ord(x)           # add an integer
             d[self.mk('L' + x)] = [x] * 10         # add a list
diff --git a/lib-python/2.7/bsddb/test/test_get_none.py b/lib-python/2.7/bsddb/test/test_get_none.py
--- a/lib-python/2.7/bsddb/test/test_get_none.py
+++ b/lib-python/2.7/bsddb/test/test_get_none.py
@@ -26,14 +26,14 @@
         d.open(self.filename, db.DB_BTREE, db.DB_CREATE)
         d.set_get_returns_none(1)
 
-        for x in string.letters:
+        for x in string.ascii_letters:
             d.put(x, x * 40)
 
         data = d.get('bad key')
         self.assertEqual(data, None)
 
-        data = d.get(string.letters[0])
-        self.assertEqual(data, string.letters[0]*40)
+        data = d.get(string.ascii_letters[0])
+        self.assertEqual(data, string.ascii_letters[0]*40)
 
         count = 0
         c = d.cursor()
@@ -43,7 +43,7 @@
             rec = c.next()
 
         self.assertEqual(rec, None)
-        self.assertEqual(count, len(string.letters))
+        self.assertEqual(count, len(string.ascii_letters))
 
         c.close()
         d.close()
@@ -54,14 +54,14 @@
         d.open(self.filename, db.DB_BTREE, db.DB_CREATE)
         d.set_get_returns_none(0)
 
-        for x in string.letters:
+        for x in string.ascii_letters:
             d.put(x, x * 40)
 
         self.assertRaises(db.DBNotFoundError, d.get, 'bad key')
         self.assertRaises(KeyError, d.get, 'bad key')
 
-        data = d.get(string.letters[0])
-        self.assertEqual(data, string.letters[0]*40)
+        data = d.get(string.ascii_letters[0])
+        self.assertEqual(data, string.ascii_letters[0]*40)
 
         count = 0
         exceptionHappened = 0
@@ -77,7 +77,7 @@
 
         self.assertNotEqual(rec, None)
         self.assertTrue(exceptionHappened)
-        self.assertEqual(count, len(string.letters))
+        self.assertEqual(count, len(string.ascii_letters))
 
         c.close()
         d.close()
diff --git a/lib-python/2.7/bsddb/test/test_queue.py b/lib-python/2.7/bsddb/test/test_queue.py
--- a/lib-python/2.7/bsddb/test/test_queue.py
+++ b/lib-python/2.7/bsddb/test/test_queue.py
@@ -10,7 +10,6 @@
 
 #----------------------------------------------------------------------
 
- at unittest.skip("fails on Windows; see issue 22943")
 class SimpleQueueTestCase(unittest.TestCase):
     def setUp(self):
         self.filename = get_new_database_path()
@@ -37,17 +36,17 @@
             print "before appends" + '-' * 30
             pprint(d.stat())
 
-        for x in string.letters:
+        for x in string.ascii_letters:
             d.append(x * 40)
 
-        self.assertEqual(len(d), len(string.letters))
+        self.assertEqual(len(d), len(string.ascii_letters))
 
         d.put(100, "some more data")
         d.put(101, "and some more ")
         d.put(75,  "out of order")
         d.put(1,   "replacement data")
 
-        self.assertEqual(len(d), len(string.letters)+3)
+        self.assertEqual(len(d), len(string.ascii_letters)+3)
 
         if verbose:
             print "before close" + '-' * 30
@@ -108,17 +107,17 @@
             print "before appends" + '-' * 30
             pprint(d.stat())
 
-        for x in string.letters:
+        for x in string.ascii_letters:
             d.append(x * 40)
 
-        self.assertEqual(len(d), len(string.letters))
+        self.assertEqual(len(d), len(string.ascii_letters))
 
         d.put(100, "some more data")
         d.put(101, "and some more ")
         d.put(75,  "out of order")
         d.put(1,   "replacement data")
 
-        self.assertEqual(len(d), len(string.letters)+3)
+        self.assertEqual(len(d), len(string.ascii_letters)+3)
 
         if verbose:
             print "before close" + '-' * 30
diff --git a/lib-python/2.7/bsddb/test/test_recno.py b/lib-python/2.7/bsddb/test/test_recno.py
--- a/lib-python/2.7/bsddb/test/test_recno.py
+++ b/lib-python/2.7/bsddb/test/test_recno.py
@@ -4,12 +4,11 @@
 import os, sys
 import errno
 from pprint import pprint
+import string
 import unittest
 
 from test_all import db, test_support, verbose, get_new_environment_path, get_new_database_path
 
-letters = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
-
 
 #----------------------------------------------------------------------
 
@@ -39,7 +38,7 @@
 
         d.open(self.filename, db.DB_RECNO, db.DB_CREATE)
 
-        for x in letters:
+        for x in string.ascii_letters:
             recno = d.append(x * 60)
             self.assertIsInstance(recno, int)
             self.assertGreaterEqual(recno, 1)
@@ -270,7 +269,7 @@
         d.set_re_pad(45)  # ...test both int and char
         d.open(self.filename, db.DB_RECNO, db.DB_CREATE)
 
-        for x in letters:
+        for x in string.ascii_letters:
             d.append(x * 35)    # These will be padded
 
         d.append('.' * 40)      # this one will be exact
diff --git a/lib-python/2.7/chunk.py b/lib-python/2.7/chunk.py
--- a/lib-python/2.7/chunk.py
+++ b/lib-python/2.7/chunk.py
@@ -85,8 +85,10 @@
 
     def close(self):
         if not self.closed:
-            self.skip()
-            self.closed = True
+            try:
+                self.skip()
+            finally:
+                self.closed = True
 
     def isatty(self):
         if self.closed:
diff --git a/lib-python/2.7/codecs.py b/lib-python/2.7/codecs.py
--- a/lib-python/2.7/codecs.py
+++ b/lib-python/2.7/codecs.py
@@ -20,8 +20,14 @@
            "BOM_LE", "BOM32_BE", "BOM32_LE", "BOM64_BE", "BOM64_LE",
            "BOM_UTF8", "BOM_UTF16", "BOM_UTF16_LE", "BOM_UTF16_BE",
            "BOM_UTF32", "BOM_UTF32_LE", "BOM_UTF32_BE",
+           "CodecInfo", "Codec", "IncrementalEncoder", "IncrementalDecoder",
+           "StreamReader", "StreamWriter",
+           "StreamReaderWriter", "StreamRecoder",
+           "getencoder", "getdecoder", "getincrementalencoder",
+           "getincrementaldecoder", "getreader", "getwriter",
+           "encode", "decode", "iterencode", "iterdecode",
            "strict_errors", "ignore_errors", "replace_errors",
-           "xmlcharrefreplace_errors",
+           "xmlcharrefreplace_errors", "backslashreplace_errors",
            "register_error", "lookup_error"]
 
 ### Constants
@@ -1051,7 +1057,7 @@
         during translation.
 
         One example where this happens is cp875.py which decodes
-        multiple character to \u001a.
+        multiple character to \\u001a.
 
     """
     m = {}
diff --git a/lib-python/2.7/collections.py b/lib-python/2.7/collections.py
--- a/lib-python/2.7/collections.py
+++ b/lib-python/2.7/collections.py
@@ -330,7 +330,7 @@
     #   http://code.activestate.com/recipes/259174/
     #   Knuth, TAOCP Vol. II section 4.6.3
 
-    def __init__(self, iterable=None, **kwds):
+    def __init__(*args, **kwds):
         '''Create a new, empty Counter object.  And if given, count elements
         from an input iterable.  Or, initialize the count from another mapping
         of elements to their counts.
@@ -341,8 +341,15 @@
         >>> c = Counter(a=4, b=2)                   # a new counter from keyword args
 
         '''
+        if not args:
+            raise TypeError("descriptor '__init__' of 'Counter' object "
+                            "needs an argument")
+        self = args[0]
+        args = args[1:]
+        if len(args) > 1:
+            raise TypeError('expected at most 1 arguments, got %d' % len(args))
         super(Counter, self).__init__()
-        self.update(iterable, **kwds)
+        self.update(*args, **kwds)
 
     def __missing__(self, key):
         'The count of elements not in the Counter is zero.'
@@ -393,7 +400,7 @@
         raise NotImplementedError(
             'Counter.fromkeys() is undefined.  Use Counter(iterable) instead.')
 
-    def update(self, iterable=None, **kwds):
+    def update(*args, **kwds):
         '''Like dict.update() but add counts instead of replacing them.
 
         Source can be an iterable, a dictionary, or another Counter instance.
@@ -413,6 +420,14 @@
         # contexts.  Instead, we implement straight-addition.  Both the inputs
         # and outputs are allowed to contain zero and negative counts.
 
+        if not args:
+            raise TypeError("descriptor 'update' of 'Counter' object "
+                            "needs an argument")
+        self = args[0]
+        args = args[1:]
+        if len(args) > 1:
+            raise TypeError('expected at most 1 arguments, got %d' % len(args))
+        iterable = args[0] if args else None
         if iterable is not None:
             if isinstance(iterable, Mapping):
                 if self:
@@ -428,7 +443,7 @@
         if kwds:
             self.update(kwds)
 
-    def subtract(self, iterable=None, **kwds):
+    def subtract(*args, **kwds):
         '''Like dict.update() but subtracts counts instead of replacing them.
         Counts can be reduced below zero.  Both the inputs and outputs are
         allowed to contain zero and negative counts.
@@ -444,6 +459,14 @@
         -1
 
         '''
+        if not args:
+            raise TypeError("descriptor 'subtract' of 'Counter' object "
+                            "needs an argument")
+        self = args[0]
+        args = args[1:]
+        if len(args) > 1:
+            raise TypeError('expected at most 1 arguments, got %d' % len(args))
+        iterable = args[0] if args else None
         if iterable is not None:
             self_get = self.get
             if isinstance(iterable, Mapping):
diff --git a/lib-python/2.7/cookielib.py b/lib-python/2.7/cookielib.py
--- a/lib-python/2.7/cookielib.py
+++ b/lib-python/2.7/cookielib.py
@@ -464,26 +464,42 @@
     for ns_header in ns_headers:
         pairs = []
         version_set = False
-        for ii, param in enumerate(re.split(r";\s*", ns_header)):
-            param = param.rstrip()
-            if param == "": continue
-            if "=" not in param:
-                k, v = param, None
-            else:
-                k, v = re.split(r"\s*=\s*", param, 1)
-                k = k.lstrip()
+
+        # XXX: The following does not strictly adhere to RFCs in that empty
+        # names and values are legal (the former will only appear once and will
+        # be overwritten if multiple occurrences are present). This is
+        # mostly to deal with backwards compatibility.
+        for ii, param in enumerate(ns_header.split(';')):
+            param = param.strip()
+
+            key, sep, val = param.partition('=')
+            key = key.strip()
+
+            if not key:
+                if ii == 0:
+                    break
+                else:
+                    continue
+
+            # allow for a distinction between present and empty and missing
+            # altogether
+            val = val.strip() if sep else None
+
             if ii != 0:
-                lc = k.lower()
+                lc = key.lower()
                 if lc in known_attrs:
-                    k = lc
-                if k == "version":
+                    key = lc
+
+                if key == "version":
                     # This is an RFC 2109 cookie.
-                    v = _strip_quotes(v)
+                    if val is not None:
+                        val = _strip_quotes(val)
                     version_set = True
-                if k == "expires":
+                elif key == "expires":
                     # convert expires date to seconds since epoch
-                    v = http2time(_strip_quotes(v))  # None if invalid
-            pairs.append((k, v))
+                    if val is not None:
+                        val = http2time(_strip_quotes(val))  # None if invalid
+            pairs.append((key, val))
 
         if pairs:
             if not version_set:
diff --git a/lib-python/2.7/ctypes/macholib/fetch_macholib.bat b/lib-python/2.7/ctypes/macholib/fetch_macholib.bat
--- a/lib-python/2.7/ctypes/macholib/fetch_macholib.bat
+++ b/lib-python/2.7/ctypes/macholib/fetch_macholib.bat
@@ -1,1 +1,1 @@
-svn export --force http://svn.red-bean.com/bob/macholib/trunk/macholib/ .
+svn export --force http://svn.red-bean.com/bob/macholib/trunk/macholib/ .
diff --git a/lib-python/2.7/ctypes/test/test_find.py b/lib-python/2.7/ctypes/test/test_find.py
--- a/lib-python/2.7/ctypes/test/test_find.py
+++ b/lib-python/2.7/ctypes/test/test_find.py
@@ -32,15 +32,24 @@
     def setUp(self):
         self.gl = self.glu = self.gle = None
         if lib_gl:
-            self.gl = CDLL(lib_gl, mode=RTLD_GLOBAL)
+            try:
+                self.gl = CDLL(lib_gl, mode=RTLD_GLOBAL)
+            except OSError:
+                pass
         if lib_glu:
-            self.glu = CDLL(lib_glu, RTLD_GLOBAL)
+            try:
+                self.glu = CDLL(lib_glu, RTLD_GLOBAL)
+            except OSError:
+                pass
         if lib_gle:
             try:
                 self.gle = CDLL(lib_gle)
             except OSError:
                 pass
 
+    def tearDown(self):
+        self.gl = self.glu = self.gle = None
+
     @unittest.skipUnless(lib_gl, 'lib_gl not available')
     def test_gl(self):
         if self.gl:
diff --git a/lib-python/2.7/ctypes/test/test_pickling.py b/lib-python/2.7/ctypes/test/test_pickling.py
--- a/lib-python/2.7/ctypes/test/test_pickling.py
+++ b/lib-python/2.7/ctypes/test/test_pickling.py
@@ -15,9 +15,9 @@
 class Y(X):
     _fields_ = [("str", c_char_p)]
 
-class PickleTest(unittest.TestCase):
+class PickleTest:
     def dumps(self, item):
-        return pickle.dumps(item)
+        return pickle.dumps(item, self.proto)
 
     def loads(self, item):
         return pickle.loads(item)
@@ -72,17 +72,15 @@
 
     @xfail
     def test_wchar(self):
-        pickle.dumps(c_char("x"))
+        self.dumps(c_char(b"x"))
         # Issue 5049
-        pickle.dumps(c_wchar(u"x"))
+        self.dumps(c_wchar(u"x"))
 
-class PickleTest_1(PickleTest):
-    def dumps(self, item):
-        return pickle.dumps(item, 1)
-
-class PickleTest_2(PickleTest):
-    def dumps(self, item):
-        return pickle.dumps(item, 2)
+for proto in range(pickle.HIGHEST_PROTOCOL + 1):
+    name = 'PickleTest_%s' % proto
+    globals()[name] = type(name,
+                           (PickleTest, unittest.TestCase),
+                           {'proto': proto})
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/lib-python/2.7/ctypes/test/test_pointers.py b/lib-python/2.7/ctypes/test/test_pointers.py
--- a/lib-python/2.7/ctypes/test/test_pointers.py
+++ b/lib-python/2.7/ctypes/test/test_pointers.py
@@ -7,8 +7,6 @@
                  c_long, c_ulong, c_longlong, c_ulonglong, c_double, c_float]
 python_types = [int, int, int, int, int, long,
                 int, long, long, long, float, float]
-LargeNamedType = type('T' * 2 ** 25, (Structure,), {})
-large_string = 'T' * 2 ** 25
 
 class PointersTestCase(unittest.TestCase):
 
@@ -191,9 +189,11 @@
             self.assertEqual(bool(mth), True)
 
     def test_pointer_type_name(self):
+        LargeNamedType = type('T' * 2 ** 25, (Structure,), {})
         self.assertTrue(POINTER(LargeNamedType))
 
     def test_pointer_type_str_name(self):
+        large_string = 'T' * 2 ** 25
         self.assertTrue(POINTER(large_string))
 
 if __name__ == '__main__':
diff --git a/lib-python/2.7/ctypes/util.py b/lib-python/2.7/ctypes/util.py
--- a/lib-python/2.7/ctypes/util.py
+++ b/lib-python/2.7/ctypes/util.py
@@ -178,7 +178,7 @@
             res = re.findall(expr, data)
             if not res:
                 return _get_soname(_findLib_gcc(name))
-            res.sort(cmp= lambda x,y: cmp(_num_version(x), _num_version(y)))
+            res.sort(key=_num_version)
             return res[-1]
 
     elif sys.platform == "sunos5":
diff --git a/lib-python/2.7/distutils/__init__.py b/lib-python/2.7/distutils/__init__.py
--- a/lib-python/2.7/distutils/__init__.py
+++ b/lib-python/2.7/distutils/__init__.py
@@ -15,5 +15,5 @@
 # Updated automatically by the Python release process.
 #
 #--start constants--
-__version__ = "2.7.9"
+__version__ = "2.7.10"
 #--end constants--
diff --git a/lib-python/2.7/distutils/command/check.py b/lib-python/2.7/distutils/command/check.py
--- a/lib-python/2.7/distutils/command/check.py
+++ b/lib-python/2.7/distutils/command/check.py
@@ -126,7 +126,7 @@
         """Returns warnings when the provided data doesn't compile."""
         source_path = StringIO()
         parser = Parser()
-        settings = frontend.OptionParser().get_default_values()
+        settings = frontend.OptionParser(components=(Parser,)).get_default_values()
         settings.tab_width = 4
         settings.pep_references = None
         settings.rfc_references = None
@@ -142,8 +142,8 @@
         document.note_source(source_path, -1)
         try:
             parser.parse(data, document)
-        except AttributeError:
-            reporter.messages.append((-1, 'Could not finish the parsing.',
-                                      '', {}))
+        except AttributeError as e:
+            reporter.messages.append(
+                (-1, 'Could not finish the parsing: %s.' % e, '', {}))
 
         return reporter.messages
diff --git a/lib-python/2.7/distutils/dir_util.py b/lib-python/2.7/distutils/dir_util.py
--- a/lib-python/2.7/distutils/dir_util.py
+++ b/lib-python/2.7/distutils/dir_util.py
@@ -83,7 +83,7 @@
     """Create all the empty directories under 'base_dir' needed to put 'files'
     there.
 
-    'base_dir' is just the a name of a directory which doesn't necessarily
+    'base_dir' is just the name of a directory which doesn't necessarily
     exist yet; 'files' is a list of filenames to be interpreted relative to
     'base_dir'.  'base_dir' + the directory portion of every file in 'files'
     will be created if it doesn't already exist.  'mode', 'verbose' and
diff --git a/lib-python/2.7/distutils/tests/test_check.py b/lib-python/2.7/distutils/tests/test_check.py
--- a/lib-python/2.7/distutils/tests/test_check.py
+++ b/lib-python/2.7/distutils/tests/test_check.py
@@ -1,5 +1,6 @@
 # -*- encoding: utf8 -*-
 """Tests for distutils.command.check."""
+import textwrap
 import unittest
 from test.test_support import run_unittest
 
@@ -93,6 +94,36 @@
         cmd = self._run(metadata, strict=1, restructuredtext=1)
         self.assertEqual(cmd._warnings, 0)
 
+    @unittest.skipUnless(HAS_DOCUTILS, "won't test without docutils")
+    def test_check_restructuredtext_with_syntax_highlight(self):
+        # Don't fail if there is a `code` or `code-block` directive
+
+        example_rst_docs = []
+        example_rst_docs.append(textwrap.dedent("""\
+            Here's some code:
+
+            .. code:: python
+
+                def foo():
+                    pass
+            """))
+        example_rst_docs.append(textwrap.dedent("""\
+            Here's some code:
+
+            .. code-block:: python
+
+                def foo():
+                    pass
+            """))
+
+        for rest_with_code in example_rst_docs:
+            pkg_info, dist = self.create_dist(long_description=rest_with_code)
+            cmd = check(dist)
+            cmd.check_restructuredtext()
+            self.assertEqual(cmd._warnings, 0)
+            msgs = cmd._check_rst_data(rest_with_code)
+            self.assertEqual(len(msgs), 0)
+
     def test_check_all(self):
 
         metadata = {'url': 'xxx', 'author': 'xxx'}
diff --git a/lib-python/2.7/distutils/text_file.py b/lib-python/2.7/distutils/text_file.py
--- a/lib-python/2.7/distutils/text_file.py
+++ b/lib-python/2.7/distutils/text_file.py
@@ -124,11 +124,11 @@
     def close (self):
         """Close the current file and forget everything we know about it
            (filename, current line number)."""
-
-        self.file.close ()
+        file = self.file
         self.file = None
         self.filename = None
         self.current_line = None
+        file.close()
 
 
     def gen_error (self, msg, line=None):
diff --git a/lib-python/2.7/dumbdbm.py b/lib-python/2.7/dumbdbm.py
--- a/lib-python/2.7/dumbdbm.py
+++ b/lib-python/2.7/dumbdbm.py
@@ -21,6 +21,7 @@
 
 """
 
+import ast as _ast
 import os as _os
 import __builtin__
 import UserDict
@@ -85,7 +86,7 @@
             with f:
                 for line in f:
                     line = line.rstrip()
-                    key, pos_and_siz_pair = eval(line)
+                    key, pos_and_siz_pair = _ast.literal_eval(line)
                     self._index[key] = pos_and_siz_pair
 
     # Write the index dict to the directory file.  The original directory
@@ -208,8 +209,10 @@
         return len(self._index)
 
     def close(self):
-        self._commit()
-        self._index = self._datfile = self._dirfile = self._bakfile = None
+        try:
+            self._commit()
+        finally:
+            self._index = self._datfile = self._dirfile = self._bakfile = None
 
     __del__ = close
 
diff --git a/lib-python/2.7/encodings/uu_codec.py b/lib-python/2.7/encodings/uu_codec.py
--- a/lib-python/2.7/encodings/uu_codec.py
+++ b/lib-python/2.7/encodings/uu_codec.py
@@ -84,7 +84,7 @@
             data = a2b_uu(s)
         except binascii.Error, v:
             # Workaround for broken uuencoders by /Fredrik Lundh
-            nbytes = (((ord(s[0])-32) & 63) * 4 + 5) / 3
+            nbytes = (((ord(s[0])-32) & 63) * 4 + 5) // 3
             data = a2b_uu(s[:nbytes])
             #sys.stderr.write("Warning: %s\n" % str(v))
         write(data)
diff --git a/lib-python/2.7/ensurepip/__init__.py b/lib-python/2.7/ensurepip/__init__.py
--- a/lib-python/2.7/ensurepip/__init__.py
+++ b/lib-python/2.7/ensurepip/__init__.py
@@ -12,9 +12,9 @@
 __all__ = ["version", "bootstrap"]
 
 
-_SETUPTOOLS_VERSION = "7.0"
+_SETUPTOOLS_VERSION = "15.2"
 
-_PIP_VERSION = "1.5.6"
+_PIP_VERSION = "6.1.1"
 
 # pip currently requires ssl support, so we try to provide a nicer
 # error message when that is missing (http://bugs.python.org/issue19744)
diff --git a/lib-python/2.7/ensurepip/_bundled/pip-1.5.6-py2.py3-none-any.whl b/lib-python/2.7/ensurepip/_bundled/pip-1.5.6-py2.py3-none-any.whl
deleted file mode 100644
Binary file lib-python/2.7/ensurepip/_bundled/pip-1.5.6-py2.py3-none-any.whl has changed
diff --git a/lib-python/2.7/ensurepip/_bundled/pip-6.1.1-py2.py3-none-any.whl b/lib-python/2.7/ensurepip/_bundled/pip-6.1.1-py2.py3-none-any.whl
new file mode 100644
index 0000000000000000000000000000000000000000..e59694a019051d58b9a378a1adfc9461b8cec9c3
GIT binary patch

[cut]

diff --git a/lib-python/2.7/ensurepip/_bundled/setuptools-15.2-py2.py3-none-any.whl b/lib-python/2.7/ensurepip/_bundled/setuptools-15.2-py2.py3-none-any.whl
new file mode 100644
index 0000000000000000000000000000000000000000..f153ed376684275e08fcfebdb2de8352fb074171
GIT binary patch

[cut]

diff --git a/lib-python/2.7/ensurepip/_bundled/setuptools-7.0-py2.py3-none-any.whl b/lib-python/2.7/ensurepip/_bundled/setuptools-7.0-py2.py3-none-any.whl
deleted file mode 100644
Binary file lib-python/2.7/ensurepip/_bundled/setuptools-7.0-py2.py3-none-any.whl has changed
diff --git a/lib-python/2.7/fileinput.py b/lib-python/2.7/fileinput.py
--- a/lib-python/2.7/fileinput.py
+++ b/lib-python/2.7/fileinput.py
@@ -233,8 +233,10 @@
         self.close()
 
     def close(self):
-        self.nextfile()
-        self._files = ()
+        try:
+            self.nextfile()
+        finally:
+            self._files = ()
 
     def __iter__(self):
         return self
@@ -270,23 +272,25 @@
 
         output = self._output
         self._output = 0
-        if output:
-            output.close()
+        try:
+            if output:
+                output.close()
+        finally:
+            file = self._file
+            self._file = 0
+            try:
+                if file and not self._isstdin:
+                    file.close()
+            finally:
+                backupfilename = self._backupfilename
+                self._backupfilename = 0
+                if backupfilename and not self._backup:
+                    try: os.unlink(backupfilename)
+                    except OSError: pass
 
-        file = self._file
-        self._file = 0
-        if file and not self._isstdin:
-            file.close()
-
-        backupfilename = self._backupfilename
-        self._backupfilename = 0
-        if backupfilename and not self._backup:
-            try: os.unlink(backupfilename)
-            except OSError: pass
-
-        self._isstdin = False
-        self._buffer = []
-        self._bufindex = 0
+                self._isstdin = False
+                self._buffer = []
+                self._bufindex = 0
 
     def readline(self):
         try:
diff --git a/lib-python/2.7/fnmatch.py b/lib-python/2.7/fnmatch.py
--- a/lib-python/2.7/fnmatch.py
+++ b/lib-python/2.7/fnmatch.py
@@ -47,12 +47,14 @@
     import os,posixpath
     result=[]
     pat=os.path.normcase(pat)
-    if not pat in _cache:
+    try:
+        re_pat = _cache[pat]
+    except KeyError:
         res = translate(pat)
         if len(_cache) >= _MAXCACHE:
             _cache.clear()
-        _cache[pat] = re.compile(res)
-    match=_cache[pat].match
+        _cache[pat] = re_pat = re.compile(res)
+    match = re_pat.match
     if os.path is posixpath:
         # normcase on posix is NOP. Optimize it away from the loop.
         for name in names:
@@ -71,12 +73,14 @@
     its arguments.
     """
 
-    if not pat in _cache:
+    try:
+        re_pat = _cache[pat]
+    except KeyError:
         res = translate(pat)
         if len(_cache) >= _MAXCACHE:
             _cache.clear()
-        _cache[pat] = re.compile(res)
-    return _cache[pat].match(name) is not None
+        _cache[pat] = re_pat = re.compile(res)
+    return re_pat.match(name) is not None
 
 def translate(pat):
     """Translate a shell PATTERN to a regular expression.
diff --git a/lib-python/2.7/ftplib.py b/lib-python/2.7/ftplib.py
--- a/lib-python/2.7/ftplib.py
+++ b/lib-python/2.7/ftplib.py
@@ -594,11 +594,16 @@
 
     def close(self):
         '''Close the connection without assuming anything about it.'''
-        if self.file is not None:
-            self.file.close()
-        if self.sock is not None:
-            self.sock.close()
-        self.file = self.sock = None
+        try:
+            file = self.file
+            self.file = None
+            if file is not None:
+                file.close()
+        finally:
+            sock = self.sock
+            self.sock = None
+            if sock is not None:
+                sock.close()
 
 try:
     import ssl
@@ -638,12 +643,24 @@
         '221 Goodbye.'
         >>>
         '''
-        ssl_version = ssl.PROTOCOL_TLSv1
+        ssl_version = ssl.PROTOCOL_SSLv23
 
         def __init__(self, host='', user='', passwd='', acct='', keyfile=None,
-                     certfile=None, timeout=_GLOBAL_DEFAULT_TIMEOUT):
+                     certfile=None, context=None,
+                     timeout=_GLOBAL_DEFAULT_TIMEOUT, source_address=None):
+            if context is not None and keyfile is not None:
+                raise ValueError("context and keyfile arguments are mutually "
+                                 "exclusive")
+            if context is not None and certfile is not None:
+                raise ValueError("context and certfile arguments are mutually "
+                                 "exclusive")
             self.keyfile = keyfile
             self.certfile = certfile
+            if context is None:
+                context = ssl._create_stdlib_context(self.ssl_version,
+                                                     certfile=certfile,
+                                                     keyfile=keyfile)
+            self.context = context
             self._prot_p = False
             FTP.__init__(self, host, user, passwd, acct, timeout)
 
@@ -656,12 +673,12 @@
             '''Set up secure control connection by using TLS/SSL.'''
             if isinstance(self.sock, ssl.SSLSocket):
                 raise ValueError("Already using TLS")
-            if self.ssl_version == ssl.PROTOCOL_TLSv1:
+            if self.ssl_version >= ssl.PROTOCOL_SSLv23:
                 resp = self.voidcmd('AUTH TLS')
             else:
                 resp = self.voidcmd('AUTH SSL')
-            self.sock = ssl.wrap_socket(self.sock, self.keyfile, self.certfile,
-                                        ssl_version=self.ssl_version)
+            self.sock = self.context.wrap_socket(self.sock,
+                                                 server_hostname=self.host)
             self.file = self.sock.makefile(mode='rb')
             return resp
 
@@ -692,8 +709,8 @@
         def ntransfercmd(self, cmd, rest=None):
             conn, size = FTP.ntransfercmd(self, cmd, rest)
             if self._prot_p:
-                conn = ssl.wrap_socket(conn, self.keyfile, self.certfile,
-                                       ssl_version=self.ssl_version)
+                conn = self.context.wrap_socket(conn,
+                                                server_hostname=self.host)
             return conn, size
 
         def retrbinary(self, cmd, callback, blocksize=8192, rest=None):
diff --git a/lib-python/2.7/genericpath.py b/lib-python/2.7/genericpath.py
--- a/lib-python/2.7/genericpath.py
+++ b/lib-python/2.7/genericpath.py
@@ -10,6 +10,14 @@
            'getsize', 'isdir', 'isfile']
 
 
+try:
+    _unicode = unicode
+except NameError:
+    # If Python is built without Unicode support, the unicode type
+    # will not exist. Fake one.
+    class _unicode(object):
+        pass
+
 # Does a path exist?
 # This is false for dangling symbolic links on systems that support them.
 def exists(path):
diff --git a/lib-python/2.7/gettext.py b/lib-python/2.7/gettext.py
--- a/lib-python/2.7/gettext.py
+++ b/lib-python/2.7/gettext.py
@@ -52,7 +52,9 @@
 
 __all__ = ['NullTranslations', 'GNUTranslations', 'Catalog',
            'find', 'translation', 'install', 'textdomain', 'bindtextdomain',
-           'dgettext', 'dngettext', 'gettext', 'ngettext',
+           'bind_textdomain_codeset',
+           'dgettext', 'dngettext', 'gettext', 'lgettext', 'ldgettext',
+           'ldngettext', 'lngettext', 'ngettext',
            ]
 
 _default_localedir = os.path.join(sys.prefix, 'share', 'locale')
@@ -294,11 +296,12 @@
             # See if we're looking at GNU .mo conventions for metadata
             if mlen == 0:
                 # Catalog description
-                lastk = k = None
+                lastk = None
                 for item in tmsg.splitlines():
                     item = item.strip()
                     if not item:
                         continue
+                    k = v = None
                     if ':' in item:
                         k, v = item.split(':', 1)
                         k = k.strip().lower()
diff --git a/lib-python/2.7/gzip.py b/lib-python/2.7/gzip.py
--- a/lib-python/2.7/gzip.py
+++ b/lib-python/2.7/gzip.py
@@ -238,9 +238,9 @@
             data = data.tobytes()
 
         if len(data) > 0:
-            self.size = self.size + len(data)
+            self.fileobj.write(self.compress.compress(data))
+            self.size += len(data)
             self.crc = zlib.crc32(data, self.crc) & 0xffffffffL
-            self.fileobj.write( self.compress.compress(data) )
             self.offset += len(data)
 
         return len(data)
@@ -369,19 +369,21 @@
         return self.fileobj is None
 
     def close(self):
-        if self.fileobj is None:
+        fileobj = self.fileobj
+        if fileobj is None:
             return
-        if self.mode == WRITE:
-            self.fileobj.write(self.compress.flush())
-            write32u(self.fileobj, self.crc)
-            # self.size may exceed 2GB, or even 4GB
-            write32u(self.fileobj, self.size & 0xffffffffL)
-            self.fileobj = None
-        elif self.mode == READ:
-            self.fileobj = None
-        if self.myfileobj:
-            self.myfileobj.close()
-            self.myfileobj = None
+        self.fileobj = None
+        try:
+            if self.mode == WRITE:
+                fileobj.write(self.compress.flush())
+                write32u(fileobj, self.crc)
+                # self.size may exceed 2GB, or even 4GB
+                write32u(fileobj, self.size & 0xffffffffL)
+        finally:
+            myfileobj = self.myfileobj
+            if myfileobj:
+                self.myfileobj = None
+                myfileobj.close()
 
     def flush(self,zlib_mode=zlib.Z_SYNC_FLUSH):
         self._check_closed()
diff --git a/lib-python/2.7/hashlib.py b/lib-python/2.7/hashlib.py
--- a/lib-python/2.7/hashlib.py
+++ b/lib-python/2.7/hashlib.py
@@ -187,7 +187,7 @@
 
         def prf(msg, inner=inner, outer=outer):
             # PBKDF2_HMAC uses the password as key. We can re-use the same
-            # digest objects and and just update copies to skip initialization.
+            # digest objects and just update copies to skip initialization.
             icpy = inner.copy()
             ocpy = outer.copy()
             icpy.update(msg)
diff --git a/lib-python/2.7/htmlentitydefs.py b/lib-python/2.7/htmlentitydefs.py
--- a/lib-python/2.7/htmlentitydefs.py
+++ b/lib-python/2.7/htmlentitydefs.py
@@ -1,6 +1,6 @@
 """HTML character entity references."""
 
-# maps the HTML entity name to the Unicode codepoint
+# maps the HTML entity name to the Unicode code point
 name2codepoint = {
     'AElig':    0x00c6, # latin capital letter AE = latin capital ligature AE, U+00C6 ISOlat1
     'Aacute':   0x00c1, # latin capital letter A with acute, U+00C1 ISOlat1
@@ -256,7 +256,7 @@
     'zwnj':     0x200c, # zero width non-joiner, U+200C NEW RFC 2070
 }
 
-# maps the Unicode codepoint to the HTML entity name
+# maps the Unicode code point to the HTML entity name
 codepoint2name = {}
 
 # maps the HTML entity name to the character
diff --git a/lib-python/2.7/httplib.py b/lib-python/2.7/httplib.py
--- a/lib-python/2.7/httplib.py
+++ b/lib-python/2.7/httplib.py
@@ -68,6 +68,7 @@
 
 from array import array
 import os
+import re
 import socket
 from sys import py3kwarning
 from urlparse import urlsplit
@@ -218,6 +219,38 @@
 # maximum amount of headers accepted
 _MAXHEADERS = 100
 
+# Header name/value ABNF (http://tools.ietf.org/html/rfc7230#section-3.2)
+#
+# VCHAR          = %x21-7E
+# obs-text       = %x80-FF
+# header-field   = field-name ":" OWS field-value OWS
+# field-name     = token
+# field-value    = *( field-content / obs-fold )
+# field-content  = field-vchar [ 1*( SP / HTAB ) field-vchar ]
+# field-vchar    = VCHAR / obs-text
+#
+# obs-fold       = CRLF 1*( SP / HTAB )
+#                ; obsolete line folding
+#                ; see Section 3.2.4
+
+# token          = 1*tchar
+#
+# tchar          = "!" / "#" / "$" / "%" / "&" / "'" / "*"
+#                / "+" / "-" / "." / "^" / "_" / "`" / "|" / "~"
+#                / DIGIT / ALPHA
+#                ; any VCHAR, except delimiters
+#
+# VCHAR defined in http://tools.ietf.org/html/rfc5234#appendix-B.1
+
+# the patterns for both name and value are more leniant than RFC
+# definitions to allow for backwards compatibility
+_is_legal_header_name = re.compile(r'\A[^:\s][^:\r\n]*\Z').match
+_is_illegal_header_value = re.compile(r'\n(?![ \t])|\r(?![ \t\n])').search
+
+# We always set the Content-Length header for these methods because some
+# servers will otherwise respond with a 411
+_METHODS_EXPECTING_BODY = {'PATCH', 'POST', 'PUT'}
+
 
 class HTTPMessage(mimetools.Message):
 
@@ -313,6 +346,11 @@
                 hlist.append(line)
                 self.addheader(headerseen, line[len(headerseen)+1:].strip())
                 continue
+            elif headerseen is not None:
+                # An empty header name. These aren't allowed in HTTP, but it's
+                # probably a benign mistake. Don't add the header, just keep
+                # going.
+                continue
             else:
                 # It's not a header line; throw it back and stop here.
                 if not self.dict:
@@ -522,9 +560,10 @@
         return True
 
     def close(self):
-        if self.fp:
-            self.fp.close()
+        fp = self.fp
+        if fp:
             self.fp = None
+            fp.close()
 
     def isclosed(self):
         # NOTE: it is possible that we will not ever call self.close(). This
@@ -723,7 +762,7 @@
         endpoint passed to set_tunnel. This is done by sending a HTTP CONNECT
         request to the proxy server when the connection is established.
 
-        This method must be called before the HTML connection has been
+        This method must be called before the HTTP connection has been
         established.
 
         The headers argument should be a mapping of extra HTTP headers
@@ -797,13 +836,17 @@
 
     def close(self):
         """Close the connection to the HTTP server."""
-        if self.sock:
-            self.sock.close()   # close it manually... there may be other refs
-            self.sock = None
-        if self.__response:
-            self.__response.close()
-            self.__response = None
         self.__state = _CS_IDLE
+        try:
+            sock = self.sock
+            if sock:
+                self.sock = None
+                sock.close()   # close it manually... there may be other refs
+        finally:
+            response = self.__response
+            if response:
+                self.__response = None
+                response.close()
 
     def send(self, data):
         """Send `data' to the server."""
@@ -978,7 +1021,16 @@
         if self.__state != _CS_REQ_STARTED:
             raise CannotSendHeader()
 
-        hdr = '%s: %s' % (header, '\r\n\t'.join([str(v) for v in values]))
+        header = '%s' % header
+        if not _is_legal_header_name(header):
+            raise ValueError('Invalid header name %r' % (header,))
+
+        values = [str(v) for v in values]
+        for one_value in values:
+            if _is_illegal_header_value(one_value):
+                raise ValueError('Invalid header value %r' % (one_value,))
+
+        hdr = '%s: %s' % (header, '\r\n\t'.join(values))
         self._output(hdr)
 
     def endheaders(self, message_body=None):
@@ -1000,19 +1052,25 @@
         """Send a complete request to the server."""
         self._send_request(method, url, body, headers)
 
-    def _set_content_length(self, body):
-        # Set the content-length based on the body.
+    def _set_content_length(self, body, method):
+        # Set the content-length based on the body. If the body is "empty", we
+        # set Content-Length: 0 for methods that expect a body (RFC 7230,
+        # Section 3.3.2). If the body is set for other methods, we set the
+        # header provided we can figure out what the length is.
         thelen = None
-        try:
-            thelen = str(len(body))
-        except TypeError, te:
-            # If this is a file-like object, try to
-            # fstat its file descriptor
+        if body is None and method.upper() in _METHODS_EXPECTING_BODY:
+            thelen = '0'
+        elif body is not None:
             try:
-                thelen = str(os.fstat(body.fileno()).st_size)
-            except (AttributeError, OSError):
-                # Don't send a length if this failed
-                if self.debuglevel > 0: print "Cannot stat!!"
+                thelen = str(len(body))
+            except TypeError:
+                # If this is a file-like object, try to
+                # fstat its file descriptor
+                try:
+                    thelen = str(os.fstat(body.fileno()).st_size)
+                except (AttributeError, OSError):
+                    # Don't send a length if this failed
+                    if self.debuglevel > 0: print "Cannot stat!!"
 
         if thelen is not None:
             self.putheader('Content-Length', thelen)
@@ -1028,8 +1086,8 @@
 
         self.putrequest(method, url, **skips)
 
-        if body is not None and 'content-length' not in header_names:
-            self._set_content_length(body)
+        if 'content-length' not in header_names:
+            self._set_content_length(body, method)
         for hdr, value in headers.iteritems():
             self.putheader(hdr, value)
         self.endheaders(body)
@@ -1072,20 +1130,20 @@
 
         try:
             response.begin()
+            assert response.will_close != _UNKNOWN
+            self.__state = _CS_IDLE
+
+            if response.will_close:
+                # this effectively passes the connection to the response
+                self.close()
+            else:
+                # remember this, so we can tell when it is complete
+                self.__response = response
+
+            return response
         except:
             response.close()
             raise
-        assert response.will_close != _UNKNOWN
-        self.__state = _CS_IDLE
-
-        if response.will_close:
-            # this effectively passes the connection to the response
-            self.close()
-        else:
-            # remember this, so we can tell when it is complete
-            self.__response = response
-
-        return response
 
 
 class HTTP:
@@ -1129,7 +1187,7 @@
         "Accept arguments to set the host/port, since the superclass doesn't."
 
         if host is not None:
-            self._conn._set_hostport(host, port)
+            (self._conn.host, self._conn.port) = self._conn._get_hostport(host, port)
         self._conn.connect()
 
     def getfile(self):
diff --git a/lib-python/2.7/idlelib/CodeContext.py b/lib-python/2.7/idlelib/CodeContext.py
--- a/lib-python/2.7/idlelib/CodeContext.py
+++ b/lib-python/2.7/idlelib/CodeContext.py
@@ -15,8 +15,8 @@
 from sys import maxint as INFINITY
 from idlelib.configHandler import idleConf
 
-BLOCKOPENERS = set(["class", "def", "elif", "else", "except", "finally", "for",
-                    "if", "try", "while", "with"])
+BLOCKOPENERS = {"class", "def", "elif", "else", "except", "finally", "for",
+                    "if", "try", "while", "with"}
 UPDATEINTERVAL = 100 # millisec
 FONTUPDATEINTERVAL = 1000 # millisec
 
diff --git a/lib-python/2.7/idlelib/EditorWindow.py b/lib-python/2.7/idlelib/EditorWindow.py
--- a/lib-python/2.7/idlelib/EditorWindow.py
+++ b/lib-python/2.7/idlelib/EditorWindow.py
@@ -469,13 +469,10 @@
         ("format", "F_ormat"),
         ("run", "_Run"),
         ("options", "_Options"),
-        ("windows", "_Windows"),
+        ("windows", "_Window"),
         ("help", "_Help"),
     ]
 
-    if sys.platform == "darwin":
-        menu_specs[-2] = ("windows", "_Window")
-
 
     def createmenubar(self):
         mbar = self.menubar
diff --git a/lib-python/2.7/idlelib/FormatParagraph.py b/lib-python/2.7/idlelib/FormatParagraph.py
--- a/lib-python/2.7/idlelib/FormatParagraph.py
+++ b/lib-python/2.7/idlelib/FormatParagraph.py
@@ -44,9 +44,11 @@
 
         The length limit parameter is for testing with a known value.
         """
-        if limit == None:
+        if limit is None:
+            # The default length limit is that defined by pep8
             limit = idleConf.GetOption(
-                    'main', 'FormatParagraph', 'paragraph', type='int')
+                'extensions', 'FormatParagraph', 'max-width',
+                type='int', default=72)
         text = self.editwin.text
         first, last = self.editwin.get_selection_indices()
         if first and last:
diff --git a/lib-python/2.7/idlelib/PyShell.py b/lib-python/2.7/idlelib/PyShell.py
--- a/lib-python/2.7/idlelib/PyShell.py
+++ b/lib-python/2.7/idlelib/PyShell.py
@@ -871,13 +871,10 @@
         ("edit", "_Edit"),
         ("debug", "_Debug"),
         ("options", "_Options"),
-        ("windows", "_Windows"),
+        ("windows", "_Window"),
         ("help", "_Help"),
     ]
 
-    if sys.platform == "darwin":
-        menu_specs[-2] = ("windows", "_Window")
-
 
     # New classes
     from idlelib.IdleHistory import History
@@ -1350,7 +1347,7 @@
         if type(s) not in (unicode, str, bytearray):
             # See issue #19481
             if isinstance(s, unicode):
-                s = unicode.__getslice__(s, None, None)
+                s = unicode.__getitem__(s, slice(None))
             elif isinstance(s, str):
                 s = str.__str__(s)
             elif isinstance(s, bytearray):
diff --git a/lib-python/2.7/idlelib/SearchEngine.py b/lib-python/2.7/idlelib/SearchEngine.py
--- a/lib-python/2.7/idlelib/SearchEngine.py
+++ b/lib-python/2.7/idlelib/SearchEngine.py
@@ -191,7 +191,7 @@
 
     This is done by searching forwards until there is no match.
     Prog: compiled re object with a search method returning a match.
-    Chars: line of text, without \n.
+    Chars: line of text, without \\n.
     Col: stop index for the search; the limit for match.end().
     '''
     m = prog.search(chars)
diff --git a/lib-python/2.7/idlelib/config-extensions.def b/lib-python/2.7/idlelib/config-extensions.def
--- a/lib-python/2.7/idlelib/config-extensions.def
+++ b/lib-python/2.7/idlelib/config-extensions.def
@@ -66,6 +66,7 @@
 
 [FormatParagraph]
 enable=True
+max-width=72
 [FormatParagraph_cfgBindings]
 format-paragraph=<Alt-Key-q>
 
diff --git a/lib-python/2.7/idlelib/config-main.def b/lib-python/2.7/idlelib/config-main.def
--- a/lib-python/2.7/idlelib/config-main.def
+++ b/lib-python/2.7/idlelib/config-main.def
@@ -58,9 +58,6 @@
 font-bold= 0
 encoding= none
 
-[FormatParagraph]
-paragraph=72
-
 [Indent]
 use-spaces= 1
 num-spaces= 4
diff --git a/lib-python/2.7/idlelib/configDialog.py b/lib-python/2.7/idlelib/configDialog.py
--- a/lib-python/2.7/idlelib/configDialog.py
+++ b/lib-python/2.7/idlelib/configDialog.py
@@ -371,7 +371,6 @@
         parent = self.parent
         self.winWidth = StringVar(parent)
         self.winHeight = StringVar(parent)
-        self.paraWidth = StringVar(parent)
         self.startupEdit = IntVar(parent)
         self.autoSave = IntVar(parent)
         self.encoding = StringVar(parent)
@@ -387,7 +386,6 @@
         frameSave = LabelFrame(frame, borderwidth=2, relief=GROOVE,
                                text=' Autosave Preferences ')
         frameWinSize = Frame(frame, borderwidth=2, relief=GROOVE)
-        frameParaSize = Frame(frame, borderwidth=2, relief=GROOVE)
         frameEncoding = Frame(frame, borderwidth=2, relief=GROOVE)
         frameHelp = LabelFrame(frame, borderwidth=2, relief=GROOVE,
                                text=' Additional Help Sources ')
@@ -416,11 +414,6 @@
         labelWinHeightTitle = Label(frameWinSize, text='Height')
         entryWinHeight = Entry(
                 frameWinSize, textvariable=self.winHeight, width=3)
-        #paragraphFormatWidth
-        labelParaWidthTitle = Label(
-                frameParaSize, text='Paragraph reformat width (in characters)')
-        entryParaWidth = Entry(
-                frameParaSize, textvariable=self.paraWidth, width=3)
         #frameEncoding
         labelEncodingTitle = Label(
                 frameEncoding, text="Default Source Encoding")
@@ -458,7 +451,6 @@
         frameRun.pack(side=TOP, padx=5, pady=5, fill=X)
         frameSave.pack(side=TOP, padx=5, pady=5, fill=X)
         frameWinSize.pack(side=TOP, padx=5, pady=5, fill=X)
-        frameParaSize.pack(side=TOP, padx=5, pady=5, fill=X)
         frameEncoding.pack(side=TOP, padx=5, pady=5, fill=X)
         frameHelp.pack(side=TOP, padx=5, pady=5, expand=TRUE, fill=BOTH)
         #frameRun
@@ -475,9 +467,6 @@
         labelWinHeightTitle.pack(side=RIGHT, anchor=E, pady=5)
         entryWinWidth.pack(side=RIGHT, anchor=E, padx=10, pady=5)
         labelWinWidthTitle.pack(side=RIGHT, anchor=E, pady=5)
-        #paragraphFormatWidth
-        labelParaWidthTitle.pack(side=LEFT, anchor=W, padx=5, pady=5)
-        entryParaWidth.pack(side=RIGHT, anchor=E, padx=10, pady=5)
         #frameEncoding
         labelEncodingTitle.pack(side=LEFT, anchor=W, padx=5, pady=5)
         radioEncNone.pack(side=RIGHT, anchor=E, pady=5)
@@ -509,7 +498,6 @@
         self.keysAreBuiltin.trace_variable('w', self.VarChanged_keysAreBuiltin)
         self.winWidth.trace_variable('w', self.VarChanged_winWidth)
         self.winHeight.trace_variable('w', self.VarChanged_winHeight)
-        self.paraWidth.trace_variable('w', self.VarChanged_paraWidth)
         self.startupEdit.trace_variable('w', self.VarChanged_startupEdit)
         self.autoSave.trace_variable('w', self.VarChanged_autoSave)
         self.encoding.trace_variable('w', self.VarChanged_encoding)
@@ -594,10 +582,6 @@
         value = self.winHeight.get()
         self.AddChangedItem('main', 'EditorWindow', 'height', value)
 
-    def VarChanged_paraWidth(self, *params):
-        value = self.paraWidth.get()
-        self.AddChangedItem('main', 'FormatParagraph', 'paragraph', value)
-
     def VarChanged_startupEdit(self, *params):
         value = self.startupEdit.get()
         self.AddChangedItem('main', 'General', 'editor-on-startup', value)
@@ -1094,9 +1078,6 @@
                 'main', 'EditorWindow', 'width', type='int'))
         self.winHeight.set(idleConf.GetOption(
                 'main', 'EditorWindow', 'height', type='int'))
-        #initial paragraph reformat size
-        self.paraWidth.set(idleConf.GetOption(
-                'main', 'FormatParagraph', 'paragraph', type='int'))
         # default source encoding
         self.encoding.set(idleConf.GetOption(
                 'main', 'EditorWindow', 'encoding', default='none'))
diff --git a/lib-python/2.7/idlelib/help.txt b/lib-python/2.7/idlelib/help.txt
--- a/lib-python/2.7/idlelib/help.txt
+++ b/lib-python/2.7/idlelib/help.txt
@@ -100,7 +100,7 @@
 			  which is scrolling off the top or the window.
 			  (Not present in Shell window.)
 
-Windows Menu:
+Window Menu:
 
 	Zoom Height -- toggles the window between configured size
 	and maximum height.
diff --git a/lib-python/2.7/idlelib/idle.bat b/lib-python/2.7/idlelib/idle.bat
--- a/lib-python/2.7/idlelib/idle.bat
+++ b/lib-python/2.7/idlelib/idle.bat
@@ -1,4 +1,4 @@
- at echo off
-rem Start IDLE using the appropriate Python interpreter
-set CURRDIR=%~dp0
-start "IDLE" "%CURRDIR%..\..\pythonw.exe" "%CURRDIR%idle.pyw" %1 %2 %3 %4 %5 %6 %7 %8 %9
+ at echo off
+rem Start IDLE using the appropriate Python interpreter
+set CURRDIR=%~dp0
+start "IDLE" "%CURRDIR%..\..\pythonw.exe" "%CURRDIR%idle.pyw" %1 %2 %3 %4 %5 %6 %7 %8 %9
diff --git a/lib-python/2.7/idlelib/idle_test/test_calltips.py b/lib-python/2.7/idlelib/idle_test/test_calltips.py
--- a/lib-python/2.7/idlelib/idle_test/test_calltips.py
+++ b/lib-python/2.7/idlelib/idle_test/test_calltips.py
@@ -55,7 +55,8 @@
         def gtest(obj, out):
             self.assertEqual(signature(obj), out)
 
-        gtest(List, '()\n' + List.__doc__)
+        if List.__doc__ is not None:
+            gtest(List, '()\n' + List.__doc__)
         gtest(list.__new__,
                'T.__new__(S, ...) -> a new object with type S, a subtype of T')
         gtest(list.__init__,
@@ -70,7 +71,8 @@
 
     def test_signature_wrap(self):
         # This is also a test of an old-style class
-        self.assertEqual(signature(textwrap.TextWrapper), '''\
+        if textwrap.TextWrapper.__doc__ is not None:
+            self.assertEqual(signature(textwrap.TextWrapper), '''\
 (width=70, initial_indent='', subsequent_indent='', expand_tabs=True,
     replace_whitespace=True, fix_sentence_endings=False, break_long_words=True,
     drop_whitespace=True, break_on_hyphens=True)''')
@@ -106,20 +108,23 @@
         def t5(a, b=None, *args, **kwds): 'doc'
         t5.tip = "(a, b=None, *args, **kwargs)"
 
+        doc = '\ndoc' if t1.__doc__ is not None else ''
         for func in (t1, t2, t3, t4, t5, TC):
-            self.assertEqual(signature(func), func.tip + '\ndoc')
+            self.assertEqual(signature(func), func.tip + doc)
 
     def test_methods(self):
+        doc = '\ndoc' if TC.__doc__ is not None else ''
         for meth in (TC.t1, TC.t2, TC.t3, TC.t4, TC.t5, TC.t6, TC.__call__):
-            self.assertEqual(signature(meth), meth.tip + "\ndoc")
-        self.assertEqual(signature(TC.cm), "(a)\ndoc")
-        self.assertEqual(signature(TC.sm), "(b)\ndoc")
+            self.assertEqual(signature(meth), meth.tip + doc)
+        self.assertEqual(signature(TC.cm), "(a)" + doc)
+        self.assertEqual(signature(TC.sm), "(b)" + doc)
 
     def test_bound_methods(self):
         # test that first parameter is correctly removed from argspec
+        doc = '\ndoc' if TC.__doc__ is not None else ''
         for meth, mtip  in ((tc.t1, "()"), (tc.t4, "(*args)"), (tc.t6, "(self)"),
                             (tc.__call__, '(ci)'), (tc, '(ci)'), (TC.cm, "(a)"),):
-            self.assertEqual(signature(meth), mtip + "\ndoc")
+            self.assertEqual(signature(meth), mtip + doc)
 
     def test_starred_parameter(self):
         # test that starred first parameter is *not* removed from argspec
diff --git a/lib-python/2.7/idlelib/idle_test/test_io.py b/lib-python/2.7/idlelib/idle_test/test_io.py
new file mode 100644
--- /dev/null
+++ b/lib-python/2.7/idlelib/idle_test/test_io.py
@@ -0,0 +1,267 @@
+import unittest
+import io
+from idlelib.PyShell import PseudoInputFile, PseudoOutputFile
+from test import test_support as support
+
+
+class Base(object):
+    def __str__(self):
+        return '%s:str' % type(self).__name__
+    def __unicode__(self):
+        return '%s:unicode' % type(self).__name__
+    def __len__(self):
+        return 3
+    def __iter__(self):
+        return iter('abc')
+    def __getitem__(self, *args):
+        return '%s:item' % type(self).__name__
+    def __getslice__(self, *args):
+        return '%s:slice' % type(self).__name__
+
+class S(Base, str):
+    pass
+
+class U(Base, unicode):
+    pass
+
+class BA(Base, bytearray):
+    pass
+
+class MockShell:
+    def __init__(self):
+        self.reset()
+
+    def write(self, *args):
+        self.written.append(args)
+
+    def readline(self):
+        return self.lines.pop()
+
+    def close(self):
+        pass
+
+    def reset(self):
+        self.written = []
+
+    def push(self, lines):
+        self.lines = list(lines)[::-1]
+
+
+class PseudeOutputFilesTest(unittest.TestCase):
+    def test_misc(self):
+        shell = MockShell()
+        f = PseudoOutputFile(shell, 'stdout', 'utf-8')
+        self.assertIsInstance(f, io.TextIOBase)
+        self.assertEqual(f.encoding, 'utf-8')
+        self.assertIsNone(f.errors)
+        self.assertIsNone(f.newlines)
+        self.assertEqual(f.name, '<stdout>')
+        self.assertFalse(f.closed)
+        self.assertTrue(f.isatty())
+        self.assertFalse(f.readable())
+        self.assertTrue(f.writable())
+        self.assertFalse(f.seekable())
+
+    def test_unsupported(self):
+        shell = MockShell()
+        f = PseudoOutputFile(shell, 'stdout', 'utf-8')

From noreply at buildbot.pypy.org  Wed Jun 24 12:00:35 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Wed, 24 Jun 2015 12:00:35 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: added a new resop class for casting
 operations, added a test to ensure they are created correctly
Message-ID: <20150624100035.DE69D1C034E@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r78286:e812d5febce9
Date: 2015-06-24 10:36 +0200
http://bitbucket.org/pypy/pypy/changeset/e812d5febce9/

Log:	added a new resop class for casting operations, added a test to
	ensure they are created correctly

diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -2683,14 +2683,7 @@
         tosize = tosizeloc.value
         if size == tosize:
             return # already the right size
-        if size == 4 and tosize == 2:
-            scratch = X86_64_SCRATCH_REG
-            self.mc.PSHUFLW_xxi(resloc.value, srcloc.value, 0b11111000)
-            self.mc.PEXTRW_rxi(scratch.value, srcloc.value, 4)
-            self.mc.PINSRW_xri(resloc.value, scratch.value, 2)
-            self.mc.PEXTRW_rxi(scratch.value, srcloc.value, 6)
-            self.mc.PINSRW_xri(resloc.value, scratch.value, 3)
-        elif size == 4 and tosize == 8:
+        if size == 4 and tosize == 8:
             scratch = X86_64_SCRATCH_REG.value
             self.mc.PEXTRD_rxi(scratch, srcloc.value, 1)
             self.mc.PINSRQ_xri(resloc.value, scratch, 1)
@@ -2704,7 +2697,13 @@
             self.mc.PEXTRQ_rxi(scratch, srcloc.value, 1)
             self.mc.PINSRD_xri(resloc.value, scratch, 1)
         else:
-            raise NotImplementedError("sign ext missing: " + str(size) + " -> " + str(tosize))
+            # note that all other conversions are not implemented
+            # on purpose. it needs many x86 op codes to implement
+            # the missing combinations. even if they are implemented
+            # the speedup might only be modest...
+            # the optimization does not emit such code!
+            msg = "vec int signext (%d->%d)" % (size, tosize)
+            raise NotImplementedError(msg)
 
     def genop_vec_float_expand(self, op, arglocs, resloc):
         srcloc, sizeloc = arglocs
@@ -2716,6 +2715,8 @@
             self.mc.SHUFPS_xxi(resloc.value, srcloc.value, 0)
         elif size == 8:
             self.mc.MOVDDUP(resloc, srcloc)
+        else:
+            raise AssertionError("float of size %d not supported" % (size,))
 
     def genop_vec_int_expand(self, op, arglocs, resloc):
         srcloc, sizeloc = arglocs
@@ -2737,7 +2738,7 @@
             self.mc.PINSRQ_xri(resloc.value, srcloc.value, 0)
             self.mc.PINSRQ_xri(resloc.value, srcloc.value, 1)
         else:
-            raise NotImplementedError("missing size %d for int expand" % (size,))
+            raise AssertionError("cannot handle size %d (int expand)" % (size,))
 
     def genop_vec_int_pack(self, op, arglocs, resloc):
         resultloc, sourceloc, residxloc, srcidxloc, countloc, sizeloc = arglocs
@@ -2748,7 +2749,9 @@
         residx = residxloc.value
         count = countloc.value
         # for small data type conversion this can be quite costy
-        # j = pack(i,4,4)
+        # NOTE there might be some combinations that can be handled
+        # more efficiently! e.g.
+        # v2 = pack(v0,v1,4,4)
         si = srcidx
         ri = residx
         k = count
diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/vectorize.py
@@ -642,15 +642,15 @@
 
     def profitable_pack(self, lnode, rnode, origin_pack):
         lpacknode = origin_pack.left
-        if self.prohibit_packing(lpacknode.getoperation(), lnode.getoperation()):
+        if self.prohibit_packing(origin_pack, lpacknode.getoperation(), lnode.getoperation()):
             return False
         rpacknode = origin_pack.right
-        if self.prohibit_packing(rpacknode.getoperation(), rnode.getoperation()):
+        if self.prohibit_packing(origin_pack, rpacknode.getoperation(), rnode.getoperation()):
             return False
 
         return True
 
-    def prohibit_packing(self, packed, inquestion):
+    def prohibit_packing(self, pack, packed, inquestion):
         """ Blocks the packing of some operations """
         if inquestion.vector == -1:
             return True
@@ -658,10 +658,15 @@
             if packed.getarg(1) == inquestion.result:
                 return True
         if inquestion.casts_box():
-            #input_type = packed.output_type
-            #if not input_type:
-            #    return True
-            pass
+            # prohibit the packing of signext calls that
+            # cast to int16/int8.
+            input_type = pack.output_type
+            if input_type:
+                py.test.set_trace()
+                insize = input_type.getsize()
+                outtype,outsize = inquestion.cast_to()
+                if outsize < 4 and insize != outsize:
+                    return True
         return False
 
     def combine(self, i, j):
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -1,6 +1,5 @@
 from rpython.rlib.objectmodel import we_are_translated
 
-
 def ResOperation(opnum, args, result, descr=None):
     cls = opclasses[opnum]
     op = cls(result)
@@ -26,6 +25,7 @@
     boolreflex = -1
     boolinverse = -1
     vector = -1
+    casts = ('\x00', -1, '\x00', -1)
 
     _attrs_ = ('result',)
 
@@ -190,12 +190,13 @@
         return self._cls_has_bool_result
 
     def casts_box(self):
-        opnum = self.getopnum()
-        return opnum == rop.INT_SIGNEXT or \
-               rop.CAST_FLOAT_TO_INT <= opnum <= rop.CAST_SINGLEFLOAT_TO_FLOAT or \
-               rop._VEC_CAST_FIRST <= opnum <= rop._VEC_CAST_LAST or \
-               rop.CAST_PTR_TO_INT == opnum or \
-               rop.CAST_INT_TO_PTR == opnum
+        return False
+
+    def cast_to(self):
+        return ('\x00',-1)
+
+    def cast_from(self):
+        return ('\x00',-1)
 
 # ===================
 # Top of the hierachy
@@ -204,6 +205,23 @@
 class PlainResOp(AbstractResOp):
     pass
 
+class CastResOp(AbstractResOp):
+    def casts_box(self):
+        return True
+
+    def cast_to(self):
+        _, _, to_type, size = self.casts
+        if self.casts[3] == 0:
+            if self.getopnum() == rop.INT_SIGNEXT:
+                arg = self.getarg(1)
+                assert isinstance(arg, ConstInt)
+                return (to_type,arg.value)
+            else:
+                raise NotImplementedError
+        return (to_type,size)
+
+    def cast_from(self):
+        return ('\x00',-1)
 
 class ResOpWithDescr(AbstractResOp):
 
@@ -629,6 +647,20 @@
     '_LAST',     # for the backend to add more internal operations
 ]
 
+FLOAT = 'f'
+INT = 'i'
+_cast_ops = {
+    'INT_SIGNEXT': (INT, 0, INT, 0),
+    'CAST_FLOAT_TO_INT': (FLOAT, 8, INT, 4),
+    'CAST_INT_TO_FLOAT': (INT, 4, FLOAT, 8),
+    'CAST_FLOAT_TO_SINGLEFLOAT': (FLOAT, 8, FLOAT, 4),
+    'CAST_SINGLEFLOAT_TO_FLOAT': (FLOAT, 4, FLOAT, 8),
+    'CAST_PTR_TO_INT': (INT, 0, INT, 4),
+    'CAST_INT_TO_PTR': (INT, 4, INT, 0),
+}
+del FLOAT
+del INT
+
 # ____________________________________________________________
 
 class rop(object):
@@ -639,7 +671,6 @@
 oparity = []     # mapping numbers to the arity of the operation or -1
 opwithdescr = [] # mapping numbers to a flag "takes a descr"
 
-
 def setup(debug_print=False):
     for i, name in enumerate(_oplist):
         if debug_print:
@@ -691,6 +722,8 @@
     if is_guard:
         assert withdescr
         baseclass = GuardResOp
+    elif name in _cast_ops:
+        baseclass = CastResOp
     elif withdescr:
         baseclass = ResOpWithDescr
     else:
@@ -780,21 +813,26 @@
     rop.CAST_FLOAT_TO_INT: rop.VEC_CAST_FLOAT_TO_INT,
 }
 
+
 def setup2():
     for cls in opclasses:
         if cls is None:
             continue
         opnum = cls.opnum
+        name = opname[opnum]
         if opnum in _opboolreflex:
             cls.boolreflex = _opboolreflex[opnum]
         if opnum in _opboolinverse:
             cls.boolinverse = _opboolinverse[opnum]
         if opnum in _opvector:
             cls.vector = _opvector[opnum]
+        if name in _cast_ops:
+            cls.casts = _cast_ops[name]
 setup2()
 del _opboolinverse
 del _opboolreflex
 del _opvector
+del _cast_ops
 
 def get_deep_immutable_oplist(operations):
     """
diff --git a/rpython/jit/metainterp/test/test_resoperation.py b/rpython/jit/metainterp/test/test_resoperation.py
--- a/rpython/jit/metainterp/test/test_resoperation.py
+++ b/rpython/jit/metainterp/test/test_resoperation.py
@@ -83,3 +83,12 @@
     py.test.raises(TypeError, "newops[0] = 'foobar'")
     py.test.raises(AssertionError, "newops[0].setarg(0, 'd')")
     py.test.raises(AssertionError, "newops[0].setdescr('foobar')")
+
+def test_cast_ops():
+    op = rop.ResOperation(rop.rop.INT_SIGNEXT, ['a', 1], 'c')
+    assert op.casts_box()
+    assert isinstance(op, rop.CastResOp)
+    assert op.cast_to == ('i',1)
+    op = rop.ResOperation(rop.rop.CAST_FLOAT_TO_INT, ['a'], 'c')
+    assert op.casts_box()
+    assert isinstance(op, rop.CastResOp)

From noreply at buildbot.pypy.org  Wed Jun 24 12:00:37 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Wed, 24 Jun 2015 12:00:37 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: preventing int signext from >32 -> <32
Message-ID: <20150624100037.1C30A1C034E@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r78287:3b569b13ba22
Date: 2015-06-24 12:00 +0200
http://bitbucket.org/pypy/pypy/changeset/3b569b13ba22/

Log:	preventing int signext from >32 -> <32 preventing packed int mul for
	64 bit cannot be done with an sse opcode (see assembler comment)
	interestingly SSE seems to quite well support float/double, but not
	int (other than add,sub,logicals)

diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -2598,10 +2598,12 @@
             self.mc.PMULLW(loc0, loc1)
         elif itemsize == 4:
             self.mc.PMULLD(loc0, loc1)
-        elif itemsize == 8:
-            self.mc.PMULDQ(loc0, loc1) # TODO
         else:
-            raise NotImplementedError("did not implement integer mul")
+            # NOTE see http://stackoverflow.com/questions/8866973/can-long-integer-routines-benefit-from-sse/8867025#8867025
+            # There is no 64x64 bit packed mul and I did not find one
+            # for 8 bit either. It is questionable if it gives any benefit
+            # for 8 bit.
+            raise NotImplementedError("")
 
     def genop_vec_int_add(self, op, arglocs, resloc):
         loc0, loc1, size_loc = arglocs
diff --git a/rpython/jit/metainterp/jitexc.py b/rpython/jit/metainterp/jitexc.py
--- a/rpython/jit/metainterp/jitexc.py
+++ b/rpython/jit/metainterp/jitexc.py
@@ -61,6 +61,14 @@
             self.green_int, self.green_ref, self.green_float,
             self.red_int, self.red_ref, self.red_float)
 
+class NotAVectorizeableLoop(JitException):
+    def __str__(self):
+        return 'NotAVectorizeableLoop()'
+
+class NotAProfitableLoop(JitException):
+    def __str__(self):
+        return 'NotAProfitableLoop()'
+
 
 def _get_standard_error(rtyper, Class):
     exdata = rtyper.exceptiondata
diff --git a/rpython/jit/metainterp/optimizeopt/schedule.py b/rpython/jit/metainterp/optimizeopt/schedule.py
--- a/rpython/jit/metainterp/optimizeopt/schedule.py
+++ b/rpython/jit/metainterp/optimizeopt/schedule.py
@@ -6,6 +6,7 @@
         MemoryRef, Node, IndexVar)
 from rpython.jit.metainterp.optimizeopt.util import Renamer
 from rpython.rlib.objectmodel import we_are_translated
+from rpython.jit.metainterp.jitexc import NotAProfitableLoop
 
 
 class SchedulerData(object):
@@ -238,12 +239,31 @@
         self.input_type = self.determine_input_type(op0)
         self.output_type = self.determine_output_type(op0)
 
+    def check_if_pack_supported(self, pack):
+        op0 = pack.operations[0].getoperation()
+        insize = self.input_type.getsize()
+        if op0.casts_box():
+            # prohibit the packing of signext calls that
+            # cast to int16/int8.
+            _, outsize = op0.cast_to()
+            self._prevent_signext(outsize, insize)
+        if op0.getopnum() == rop.INT_ADD:
+            if insize == 8 or insize == 1:
+                # see assembler for comment why
+                raise NotAProfitableLoop
+
+    def _prevent_signext(self, outsize, insize):
+        if outsize < 4 and insize != outsize:
+            raise NotAProfitableLoop
+
     def as_vector_operation(self, pack, sched_data, oplist):
         self.sched_data = sched_data
         self.preamble_ops = oplist
         self.costmodel = sched_data.costmodel
         self.update_input_output(pack)
         #
+        self.check_if_pack_supported(pack)
+        #
         off = 0
         stride = self.split_pack(pack, self.sched_data.vec_reg_size)
         left = len(pack.operations)
@@ -370,6 +390,7 @@
 
     def extend_int(self, vbox, newtype):
         vbox_cloned = newtype.new_vector_box(vbox.item_count)
+        self._prevent_signext(newtype.getsize(), vbox.getsize())
         op = ResOperation(rop.VEC_INT_SIGNEXT, 
                           [vbox, ConstInt(newtype.getsize())],
                           vbox_cloned)
diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/vectorize.py
@@ -9,7 +9,7 @@
 import time
 
 from rpython.jit.metainterp.resume import Snapshot
-from rpython.jit.metainterp.jitexc import JitException
+from rpython.jit.metainterp.jitexc import NotAVectorizeableLoop, NotAProfitableLoop
 from rpython.jit.metainterp.optimizeopt.unroll import optimize_unroll
 from rpython.jit.metainterp.compile import ResumeAtLoopHeaderDescr, invent_fail_descr_for_op
 from rpython.jit.metainterp.history import (ConstInt, VECTOR, FLOAT, INT,
@@ -44,14 +44,6 @@
             else:
                 print ""
 
-class NotAVectorizeableLoop(JitException):
-    def __str__(self):
-        return 'NotAVectorizeableLoop()'
-
-class NotAProfitableLoop(JitException):
-    def __str__(self):
-        return 'NotAProfitableLoop()'
-
 def optimize_vector(metainterp_sd, jitdriver_sd, loop, optimizations,
                     inline_short_preamble, start_state, cost_threshold):
     optimize_unroll(metainterp_sd, jitdriver_sd, loop, optimizations,
@@ -623,7 +615,7 @@
                     else:
                         # store only has an input
                         return Pair(lnode, rnode, ptype, None)
-                if self.profitable_pack(lnode, rnode, origin_pack):
+                if self.profitable_pack(lnode, rnode, origin_pack, forward):
                     input_type = origin_pack.output_type
                     output_type = determine_output_type(lnode, input_type)
                     return Pair(lnode, rnode, input_type, output_type)
@@ -640,33 +632,29 @@
                 return True
         return False
 
-    def profitable_pack(self, lnode, rnode, origin_pack):
+    def profitable_pack(self, lnode, rnode, origin_pack, forward):
         lpacknode = origin_pack.left
-        if self.prohibit_packing(origin_pack, lpacknode.getoperation(), lnode.getoperation()):
+        if self.prohibit_packing(origin_pack,
+                                 lpacknode.getoperation(),
+                                 lnode.getoperation(),
+                                 forward):
             return False
         rpacknode = origin_pack.right
-        if self.prohibit_packing(origin_pack, rpacknode.getoperation(), rnode.getoperation()):
+        if self.prohibit_packing(origin_pack,
+                                 rpacknode.getoperation(),
+                                 rnode.getoperation(),
+                                 forward):
             return False
 
         return True
 
-    def prohibit_packing(self, pack, packed, inquestion):
+    def prohibit_packing(self, pack, packed, inquestion, forward):
         """ Blocks the packing of some operations """
         if inquestion.vector == -1:
             return True
         if packed.is_raw_array_access():
             if packed.getarg(1) == inquestion.result:
                 return True
-        if inquestion.casts_box():
-            # prohibit the packing of signext calls that
-            # cast to int16/int8.
-            input_type = pack.output_type
-            if input_type:
-                py.test.set_trace()
-                insize = input_type.getsize()
-                outtype,outsize = inquestion.cast_to()
-                if outsize < 4 and insize != outsize:
-                    return True
         return False
 
     def combine(self, i, j):
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -214,7 +214,6 @@
         if self.casts[3] == 0:
             if self.getopnum() == rop.INT_SIGNEXT:
                 arg = self.getarg(1)
-                assert isinstance(arg, ConstInt)
                 return (to_type,arg.value)
             else:
                 raise NotImplementedError

From noreply at buildbot.pypy.org  Wed Jun 24 12:17:43 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Wed, 24 Jun 2015 12:17:43 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: added comment in doc
Message-ID: <20150624101743.0AA1F1C02A3@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r78288:16da7f0d90bd
Date: 2015-06-24 12:04 +0200
http://bitbucket.org/pypy/pypy/changeset/16da7f0d90bd/

Log:	added comment in doc

diff --git a/rpython/doc/jit/vectorization.rst b/rpython/doc/jit/vectorization.rst
--- a/rpython/doc/jit/vectorization.rst
+++ b/rpython/doc/jit/vectorization.rst
@@ -7,7 +7,7 @@
 Features
 --------
 
-Currently the following operations can be vectorized if the trace contains parallelism:
+Currently the following operations can be vectorized if the trace contains parallel operations:
 
 * float32/float64: add, substract, multiply, divide, negate, absolute
 * int8/int16/int32/int64 arithmetic: add, substract, multiply, negate, absolute
@@ -38,8 +38,11 @@
 ---------------------------
 
 * The only SIMD instruction architecture currently supported is SSE4.1
+* Packed mul for int8,int64 (see PMUL_)
 * Loop that convert types from int(8|16|32|64) to int(8|16) are not supported in
   the current SSE4.1 assembler implementation.
   The opcode needed spans over multiple instructions. In terms of performance
   there might only be little to non advantage to use SIMD instructions for this
   conversions.
+
+.. _PMUL: http://stackoverflow.com/questions/8866973/can-long-integer-routines-benefit-from-sse/8867025#8867025

From noreply at buildbot.pypy.org  Wed Jun 24 12:17:44 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Wed, 24 Jun 2015 12:17:44 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: reformated the log output to time the
	vecopt traces
Message-ID: <20150624101744.25F941C02A3@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r78289:c061d25dca84
Date: 2015-06-24 12:08 +0200
http://bitbucket.org/pypy/pypy/changeset/c061d25dca84/

Log:	reformated the log output to time the vecopt traces

diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/vectorize.py
@@ -49,6 +49,8 @@
     optimize_unroll(metainterp_sd, jitdriver_sd, loop, optimizations,
                     inline_short_preamble, start_state, False)
     orig_ops = loop.operations
+    start = -1
+    end = -1
     try:
         debug_start("vec-opt-loop")
         metainterp_sd.logger_noopt.log_loop(loop.inputargs, loop.operations, -2, None, None, "pre vectorize")
@@ -61,18 +63,6 @@
         end = time.clock()
         metainterp_sd.profiler.count(Counters.OPT_VECTORIZED)
         metainterp_sd.logger_noopt.log_loop(loop.inputargs, loop.operations, -2, None, None, "post vectorize")
-        #
-        # XXX
-        ns = int((end-start)*10.0**9)
-        debug_start("vec-opt-clock")
-        debug_print("unroll: %d gso count: %d opcount: (%d -> %d) took %dns" % \
-                      (opt.unroll_count+1,
-                       gso.strength_reduced,
-                       len(orig_ops),
-                       len(loop.operations),
-                       ns))
-        debug_stop("vec-opt-clock")
-
     except NotAVectorizeableLoop:
         # vectorization is not possible
         loop.operations = orig_ops
@@ -90,6 +80,19 @@
             raise
     finally:
         debug_stop("vec-opt-loop")
+        #
+        # XXX
+        if end != -1:
+            ns = int((end-start)*10.0**9)
+            debug_start("xxx-clock")
+            debug_print("vecopt unroll: %d gso count: %d opcount: (%d -> %d) took %dns" % \
+                          (opt.unroll_count+1,
+                           gso.strength_reduced,
+                           len(orig_ops),
+                           len(loop.operations),
+                           ns))
+            debug_stop("xxx-clock")
+
 
 class VectorizingOptimizer(Optimizer):
     """ Try to unroll the loop and find instructions to group """
diff --git a/rpython/jit/metainterp/warmspot.py b/rpython/jit/metainterp/warmspot.py
--- a/rpython/jit/metainterp/warmspot.py
+++ b/rpython/jit/metainterp/warmspot.py
@@ -46,10 +46,10 @@
     def xxx_clock_start(self):
         now = time.clock()
         self.t.append(now)
-        debug_start("xxx-clock-start")
-        debug_print("name: %s id: %s now: %dns" % \
-                (self.name, self.unique_id, int(now)*10**9) )
-        debug_stop("xxx-clock-start")
+        debug_start("xxx-clock")
+        debug_print("start name: %s id: %s clock: %f" % \
+                (self.name, self.unique_id, now) )
+        debug_stop("xxx-clock")
 
     def xxx_clock_stop(self, fail=False):
         end = time.clock()
@@ -59,10 +59,10 @@
         if not fail:
             del self.t[-1]
         ns = (end - start) * 10**9
-        debug_start("xxx-clock-stop")
-        debug_print("name: %s id: %s now: %dns exe time: %dns fail? %d vec? %d" % \
-                    (self.name, self.unique_id, int(end)*10**9, int(ns), int(fail), int(self.vec)))
-        debug_stop("xxx-clock-stop")
+        debug_start("xxx-clock")
+        debug_print("stop name: %s id: %s clock: %f exe time: %dns fail? %d vec? %d" % \
+                    (self.name, self.unique_id, end, int(ns), int(fail), int(self.vec)))
+        debug_stop("xxx-clock")
 
 
 

From noreply at buildbot.pypy.org  Wed Jun 24 12:17:45 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Wed, 24 Jun 2015 12:17:45 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: well, should be int_mul not int_add...
Message-ID: <20150624101745.4544F1C02A3@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r78290:2d5ff3f421e1
Date: 2015-06-24 12:17 +0200
http://bitbucket.org/pypy/pypy/changeset/2d5ff3f421e1/

Log:	well, should be int_mul not int_add...

diff --git a/rpython/jit/metainterp/optimizeopt/schedule.py b/rpython/jit/metainterp/optimizeopt/schedule.py
--- a/rpython/jit/metainterp/optimizeopt/schedule.py
+++ b/rpython/jit/metainterp/optimizeopt/schedule.py
@@ -241,13 +241,17 @@
 
     def check_if_pack_supported(self, pack):
         op0 = pack.operations[0].getoperation()
+        if self.input_type is None:
+            # must be a load operation
+            assert op0.is_raw_load()
+            return
         insize = self.input_type.getsize()
         if op0.casts_box():
             # prohibit the packing of signext calls that
             # cast to int16/int8.
             _, outsize = op0.cast_to()
             self._prevent_signext(outsize, insize)
-        if op0.getopnum() == rop.INT_ADD:
+        if op0.getopnum() == rop.INT_MUL:
             if insize == 8 or insize == 1:
                 # see assembler for comment why
                 raise NotAProfitableLoop

From noreply at buildbot.pypy.org  Wed Jun 24 13:58:24 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Wed, 24 Jun 2015 13:58:24 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: modified tests for missing packed int64
 mul (was incorrect before) and prevented int64->int16 conversion
Message-ID: <20150624115824.6D0FE1C02A3@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r78291:e8681afe010b
Date: 2015-06-24 13:39 +0200
http://bitbucket.org/pypy/pypy/changeset/e8681afe010b/

Log:	modified tests for missing packed int64 mul (was incorrect before)
	and prevented int64->int16 conversion

diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -262,7 +262,7 @@
         i = 8
         assert int(result) == i*16 + sum(range(7,7+i))
         # currently is is not possible to accum for types with < 8 bytes
-        self.check_vectorized(3, 2)
+        self.check_vectorized(3, 1)
 
     def define_int8_expand():
         return """
@@ -769,8 +769,8 @@
 
     def define_int_mul_array():
         return """
-        a = astype(|30|, int)
-        b = astype(|30|, int)
+        a = astype(|30|, int32)
+        b = astype(|30|, int32)
         c = a * b
         x1 = c -> 7
         x2 = c -> 8
@@ -779,6 +779,8 @@
         x1 + x2 + x3 + x4
         """
     def test_int_mul_array(self):
+        # note that int64 mul has not packed machine instr
+        # for SSE4 thus int32
         result = self.run("int_mul_array")
         assert int(result) == 7*7+8*8+11*11+12*12
         self.check_vectorized(2, 2)

From noreply at buildbot.pypy.org  Wed Jun 24 13:58:25 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Wed, 24 Jun 2015 13:58:25 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: doc additions,
 reenabled the int8 expand test (passes now)
Message-ID: <20150624115825.8D0A21C02A3@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r78292:e70ae41089d7
Date: 2015-06-24 13:58 +0200
http://bitbucket.org/pypy/pypy/changeset/e70ae41089d7/

Log:	doc additions, reenabled the int8 expand test (passes now)

diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -274,10 +274,9 @@
         sum(d)
         """
     def test_int8_expand(self):
-        py.test.skip("TODO implement assembler")
         result = self.run("int8_expand")
-        assert int(result) == 8*8 + sum(range(0,17))
-        self.check_vectorized(3, 2) # TODO sum at the end
+        assert int(result) == 17*8 + sum(range(0,17))
+        self.check_vectorized(3, 1) # TODO sum at the end
 
     def define_int32_add_const():
         return """
diff --git a/rpython/doc/jit/vectorization.rst b/rpython/doc/jit/vectorization.rst
--- a/rpython/doc/jit/vectorization.rst
+++ b/rpython/doc/jit/vectorization.rst
@@ -2,7 +2,10 @@
 Vectorization
 =============
 
-TBA
+To find parallel instructions the tracer must provide enough information about
+memory load/store operations. They must be adjacent in memory. The requirement for
+that is that they use the same index variable and offset can be expressed as a
+a linear or affine combination.
 
 Features
 --------
@@ -13,6 +16,9 @@
 * int8/int16/int32/int64 arithmetic: add, substract, multiply, negate, absolute
 * int8/int16/int32/int64 logical: and, or, xor
 
+Reduction
+---------
+
 Reduction is implemented:
 
 * sum
@@ -21,10 +27,13 @@
 
 * all, any, prod, min, max
 
-To find parallel instructions the tracer must provide enough information about
-memory load/store operations. They must be adjacent in memory. The requirement for
-that is that they use the same index variable and offset can be expressed as a
-a linear or affine combination.
+Constant & Variable Expansion
+-----------------------------
+
+Packed arithmetic operations expand scalar variables or contants into vector registers.
+
+Guard Strengthening
+-------------------
 
 Unrolled guards are strengthend on a arithmetical level (See GuardStrengthenOpt).
 The resulting vector trace will only have one guard that checks the index.
diff --git a/rpython/jit/backend/x86/rx86.py b/rpython/jit/backend/x86/rx86.py
--- a/rpython/jit/backend/x86/rx86.py
+++ b/rpython/jit/backend/x86/rx86.py
@@ -746,6 +746,7 @@
     PSHUFLW_xxi = xmminsn('\xF2', rex_nw, '\x0F\x70', register(1,8), register(2), '\xC0', immediate(3, 'b'))
     PSHUFB_xx = xmminsn('\x66', rex_nw, '\x0F\x38\x00', register(1,8), register(2), '\xC0')
     PSHUFB_xm = xmminsn('\x66', rex_nw, '\x0F\x38\x00', register(1,8), mem_reg_plus_const(2))
+    PSHUFB_xj = xmminsn('\x66', rex_nw, '\x0F\x38\x00', register(1,8), abs_(2))
 
     # SSE3
     HADDPD_xx = xmminsn('\x66', rex_nw, '\x0F\x7C', register(1,8), register(2), '\xC0')

From noreply at buildbot.pypy.org  Wed Jun 24 14:22:56 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 24 Jun 2015 14:22:56 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c8: Fix unexpected becomes-inevitable from
 some math functions
Message-ID: <20150624122256.D4DED1C02FD@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c8
Changeset: r78293:fbdf4df6f63e
Date: 2015-06-24 14:22 +0200
http://bitbucket.org/pypy/pypy/changeset/fbdf4df6f63e/

Log:	Fix unexpected becomes-inevitable from some math functions

diff --git a/TODO b/TODO
--- a/TODO
+++ b/TODO
@@ -1,8 +1,3 @@
-------------------------------------------------------------
-
-ll_math_modf(), say, causes stm_become_inevitable()
-because of the raw array read "intpart_p[0]"
-
 ------------------------------------------------------------
 
 fuse the two 32bit setfield_gc for stmflags & tid in the jit
diff --git a/rpython/rtyper/lltypesystem/module/ll_math.py b/rpython/rtyper/lltypesystem/module/ll_math.py
--- a/rpython/rtyper/lltypesystem/module/ll_math.py
+++ b/rpython/rtyper/lltypesystem/module/ll_math.py
@@ -58,8 +58,10 @@
                            [rffi.DOUBLE, rffi.DOUBLE], rffi.DOUBLE,
                            elidable_function=True)
 math_atan2 = llexternal('atan2', [rffi.DOUBLE, rffi.DOUBLE], rffi.DOUBLE)
-math_frexp = llexternal('frexp', [rffi.DOUBLE, rffi.INTP], rffi.DOUBLE)
-math_modf  = llexternal('modf',  [rffi.DOUBLE, rffi.DOUBLEP], rffi.DOUBLE)
+math_frexp = llexternal('frexp', [rffi.DOUBLE, rffi.INTP_STM_NOTRACK],
+                        rffi.DOUBLE)
+math_modf  = llexternal('modf',  [rffi.DOUBLE, rffi.DOUBLEP_STM_NOTRACK],
+                        rffi.DOUBLE)
 math_ldexp = llexternal('ldexp', [rffi.DOUBLE, rffi.INT], rffi.DOUBLE,
                         save_err=rffi.RFFI_FULL_ERRNO_ZERO)
 math_pow   = llexternal('pow', [rffi.DOUBLE, rffi.DOUBLE], rffi.DOUBLE,
@@ -188,7 +190,7 @@
         mantissa = x
         exponent = 0
     else:
-        exp_p = lltype.malloc(rffi.INTP.TO, 1, flavor='raw')
+        exp_p = lltype.malloc(rffi.INTP_STM_NOTRACK.TO, 1, flavor='raw')
         try:
             mantissa = math_frexp(x, exp_p)
             exponent = rffi.cast(lltype.Signed, exp_p[0])
@@ -229,7 +231,7 @@
             return (x, x)
         else:   # isinf(x)
             return (math_copysign(0.0, x), x)
-    intpart_p = lltype.malloc(rffi.DOUBLEP.TO, 1, flavor='raw')
+    intpart_p = lltype.malloc(rffi.DOUBLEP_STM_NOTRACK.TO, 1, flavor='raw')
     try:
         fracpart = math_modf(x, intpart_p)
         intpart = intpart_p[0]
diff --git a/rpython/rtyper/lltypesystem/rffi.py b/rpython/rtyper/lltypesystem/rffi.py
--- a/rpython/rtyper/lltypesystem/rffi.py
+++ b/rpython/rtyper/lltypesystem/rffi.py
@@ -584,6 +584,9 @@
         globals()[name.upper()] = tp
         tpp = lltype.Ptr(lltype.Array(tp, hints={'nolength': True}))
         globals()[name.upper()+'P'] = tpp
+        tppstm = lltype.Ptr(lltype.Array(tp, hints={'nolength': True,
+                                 'stm_dont_track_raw_accesses': True}))
+        globals()[name.upper()+'P_STM_NOTRACK'] = tppstm
         result.append(tp)
     return result
 
@@ -772,16 +775,25 @@
 
 # double *
 DOUBLEP = lltype.Ptr(lltype.Array(DOUBLE, hints={'nolength': True}))
+DOUBLEP_STM_NOTRACK = lltype.Ptr(lltype.Array(DOUBLE, hints={'nolength': True,
+                                        'stm_dont_track_raw_accesses': True}))
 
 # float *
 FLOATP = lltype.Ptr(lltype.Array(FLOAT, hints={'nolength': True}))
+FLOATP_STM_NOTRACK = lltype.Ptr(lltype.Array(FLOAT, hints={'nolength': True,
+                                        'stm_dont_track_raw_accesses': True}))
 
 # long double *
 LONGDOUBLEP = lltype.Ptr(lltype.Array(LONGDOUBLE, hints={'nolength': True}))
+LONGDOUBLEP_STM_NOTRACK = lltype.Ptr(lltype.Array(LONGDOUBLE, hints={
+                                        'nolength': True,
+                                        'stm_dont_track_raw_accesses': True}))
 
 # Signed, Signed *
 SIGNED = lltype.Signed
 SIGNEDP = lltype.Ptr(lltype.Array(SIGNED, hints={'nolength': True}))
+SIGNEDP_STM_NOTRACK = lltype.Ptr(lltype.Array(SIGNED, hints={'nolength': True,
+                                        'stm_dont_track_raw_accesses': True}))
 
 # various type mapping
 

From noreply at buildbot.pypy.org  Wed Jun 24 15:06:57 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Wed, 24 Jun 2015 15:06:57 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: logging condition changed for timing in
	optimize_trace
Message-ID: <20150624130657.3919E1C02A3@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r78294:26d736866fa2
Date: 2015-06-24 15:07 +0200
http://bitbucket.org/pypy/pypy/changeset/26d736866fa2/

Log:	logging condition changed for timing in optimize_trace

diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/vectorize.py
@@ -82,7 +82,7 @@
         debug_stop("vec-opt-loop")
         #
         # XXX
-        if end != -1:
+        if start > 0 and end > 0:
             ns = int((end-start)*10.0**9)
             debug_start("xxx-clock")
             debug_print("vecopt unroll: %d gso count: %d opcount: (%d -> %d) took %dns" % \

From noreply at buildbot.pypy.org  Wed Jun 24 15:19:37 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Wed, 24 Jun 2015 15:19:37 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: rpython translation issues
Message-ID: <20150624131937.2C0271C02A3@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r78295:5008c5321939
Date: 2015-06-24 15:19 +0200
http://bitbucket.org/pypy/pypy/changeset/5008c5321939/

Log:	rpython translation issues

diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/vectorize.py
@@ -63,13 +63,27 @@
         end = time.clock()
         metainterp_sd.profiler.count(Counters.OPT_VECTORIZED)
         metainterp_sd.logger_noopt.log_loop(loop.inputargs, loop.operations, -2, None, None, "post vectorize")
+        debug_stop("vec-opt-loop")
+        #
+        # XXX
+        ns = int((end-start)*10.0**9)
+        debug_start("xxx-clock")
+        debug_print("vecopt unroll: %d gso count: %d opcount: (%d -> %d) took %dns" % \
+                      (opt.unroll_count+1,
+                       gso.strength_reduced,
+                       len(orig_ops),
+                       len(loop.operations),
+                       ns))
+        debug_stop("xxx-clock")
     except NotAVectorizeableLoop:
         # vectorization is not possible
         loop.operations = orig_ops
+        debug_stop("vec-opt-loop")
     except NotAProfitableLoop:
         # cost model says to skip this loop
         loop.operations = orig_ops
     except Exception as e:
+        debug_stop("vec-opt-loop")
         loop.operations = orig_ops
         debug_print("failed to vectorize loop. THIS IS A FATAL ERROR!")
         if we_are_translated():
@@ -78,20 +92,6 @@
             llop.debug_print_traceback(lltype.Void)
         else:
             raise
-    finally:
-        debug_stop("vec-opt-loop")
-        #
-        # XXX
-        if start > 0 and end > 0:
-            ns = int((end-start)*10.0**9)
-            debug_start("xxx-clock")
-            debug_print("vecopt unroll: %d gso count: %d opcount: (%d -> %d) took %dns" % \
-                          (opt.unroll_count+1,
-                           gso.strength_reduced,
-                           len(orig_ops),
-                           len(loop.operations),
-                           ns))
-            debug_stop("xxx-clock")
 
 
 class VectorizingOptimizer(Optimizer):
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -213,7 +213,9 @@
         _, _, to_type, size = self.casts
         if self.casts[3] == 0:
             if self.getopnum() == rop.INT_SIGNEXT:
+                from rpython.jit.metainterp.history import ConstInt
                 arg = self.getarg(1)
+                assert isinstance(arg, ConstInt)
                 return (to_type,arg.value)
             else:
                 raise NotImplementedError

From noreply at buildbot.pypy.org  Wed Jun 24 15:52:54 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 24 Jun 2015 15:52:54 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Try to systematically emit "wait"
	events when we wait
Message-ID: <20150624135254.715A81C1545@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r1882:c2e8b3841c76
Date: 2015-06-24 15:53 +0200
http://bitbucket.org/pypy/stmgc/changeset/c2e8b3841c76/

Log:	Try to systematically emit "wait" events when we wait

diff --git a/c8/stm/core.c b/c8/stm/core.c
--- a/c8/stm/core.c
+++ b/c8/stm/core.c
@@ -511,10 +511,12 @@
            try to detach an inevitable transaction regularly */
         detached = fetch_detached_transaction();
         if (detached == 0) {
+            EMIT_WAIT(STM_WAIT_OTHER_INEVITABLE);
             if (!cond_wait_timeout(C_SEGMENT_FREE_OR_SAFE_POINT_REQ, 0.00001))
                 goto wait_some_more;
         }
     }
+    EMIT_WAIT_DONE();
     s_mutex_unlock();
 
     if (detached != 0)
@@ -1130,6 +1132,7 @@
 static void _do_start_transaction(stm_thread_local_t *tl)
 {
     assert(!_stm_in_transaction(tl));
+    tl->wait_event_emitted = 0;
 
     acquire_thread_segment(tl);
     /* GS invalid before this point! */
@@ -1318,6 +1321,7 @@
     }
     assert(STM_SEGMENT->running_thread->self_or_0_if_atomic ==
            (intptr_t)(STM_SEGMENT->running_thread));
+    assert(STM_SEGMENT->running_thread->wait_event_emitted == 0);
 
     dprintf(("> stm_commit_transaction(external=%d)\n", (int)external));
     minor_collection(/*commit=*/ true, external);
@@ -1582,9 +1586,8 @@
 
         if (any_soon_finished_or_inevitable_thread_segment() &&
                 num_waits <= NB_SEGMENTS) {
-#if STM_TESTS
-            timing_become_inevitable(); /* for tests: another transaction */
-            stm_abort_transaction();    /* is already inevitable, abort   */
+#if STM_TESTS                           /* for tests: another transaction */
+            stm_abort_transaction();    /*   is already inevitable, abort */
 #endif
 
             bool timed_out = false;
@@ -1594,6 +1597,7 @@
                     !safe_point_requested()) {
 
                 /* wait until C_SEGMENT_FREE_OR_SAFE_POINT_REQ is signalled */
+                EMIT_WAIT(STM_WAIT_OTHER_INEVITABLE);
                 if (!cond_wait_timeout(C_SEGMENT_FREE_OR_SAFE_POINT_REQ,
                                        0.000054321))
                     timed_out = true;
@@ -1607,14 +1611,17 @@
                    not too common.  We don't want two threads constantly
                    detaching each other. */
                 intptr_t detached = fetch_detached_transaction();
-                if (detached != 0)
+                if (detached != 0) {
+                    EMIT_WAIT_DONE();
                     commit_fetched_detached_transaction(detached);
+                }
             }
             else {
                 num_waits++;
             }
             goto retry_from_start;
         }
+        EMIT_WAIT_DONE();
         if (!_validate_and_turn_inevitable())
             goto retry_from_start;
     }
diff --git a/c8/stm/marker.h b/c8/stm/marker.h
--- a/c8/stm/marker.h
+++ b/c8/stm/marker.h
@@ -15,3 +15,28 @@
 
 #define timing_become_inevitable()                                      \
     (timing_enabled() ? _timing_become_inevitable() : (void)0)
+
+
+static inline void emit_wait(stm_thread_local_t *tl, enum stm_event_e event)
+{
+    if (!timing_enabled())
+        return;
+    if (tl->wait_event_emitted != 0) {
+        if (tl->wait_event_emitted == event)
+            return;
+        stmcb_timing_event(tl, STM_WAIT_DONE, NULL);
+    }
+    tl->wait_event_emitted = event;
+    stmcb_timing_event(tl, event, NULL);
+}
+
+static inline void emit_wait_done(stm_thread_local_t *tl)
+{
+    if (tl->wait_event_emitted != 0) {
+        tl->wait_event_emitted = 0;
+        stmcb_timing_event(tl, STM_WAIT_DONE, NULL);
+    }
+}
+
+#define EMIT_WAIT(event)  emit_wait(STM_SEGMENT->running_thread, event)
+#define EMIT_WAIT_DONE()  emit_wait_done(STM_SEGMENT->running_thread)
diff --git a/c8/stm/sync.c b/c8/stm/sync.c
--- a/c8/stm/sync.c
+++ b/c8/stm/sync.c
@@ -243,13 +243,13 @@
     /* No segment available.  Wait until release_thread_segment()
        signals that one segment has been freed.  Note that we prefer
        waiting rather than detaching an inevitable transaction, here. */
-    timing_event(tl, STM_WAIT_FREE_SEGMENT);
+    emit_wait(tl, STM_WAIT_FREE_SEGMENT);
     cond_wait(C_SEGMENT_FREE);
-    timing_event(tl, STM_WAIT_DONE);
 
     goto retry_from_start;
 
  got_num:
+    emit_wait_done(tl);
     OPT_ASSERT(num >= 0 && num < NB_SEGMENTS-1);
     sync_ctl.in_use1[num+1] = 1;
     assert(STM_SEGMENT->segment_num == num+1);
@@ -425,15 +425,15 @@
 #ifdef STM_TESTS
         abort_with_mutex();
 #endif
-        timing_event(STM_SEGMENT->running_thread, STM_WAIT_SYNC_PAUSE);
+        EMIT_WAIT(STM_WAIT_SYNC_PAUSE);
         cond_signal(C_AT_SAFE_POINT);
         STM_PSEGMENT->safe_point = SP_WAIT_FOR_C_REQUEST_REMOVED;
         cond_wait(C_REQUEST_REMOVED);
         STM_PSEGMENT->safe_point = SP_RUNNING;
-        timing_event(STM_SEGMENT->running_thread, STM_WAIT_DONE);
         assert(!STM_SEGMENT->no_safe_point_here);
         dprintf(("left safe point\n"));
     }
+    EMIT_WAIT_DONE();
 }
 
 static void synchronize_all_threads(enum sync_type_e sync_type)
@@ -461,12 +461,14 @@
 
         intptr_t detached = fetch_detached_transaction();
         if (detached != 0) {
+            EMIT_WAIT_DONE();
             remove_requests_for_safe_point();    /* => C_REQUEST_REMOVED */
             s_mutex_unlock();
             commit_fetched_detached_transaction(detached);
             s_mutex_lock();
             goto restart;
         }
+        EMIT_WAIT(STM_WAIT_SYNCING);
         STM_PSEGMENT->safe_point = SP_WAIT_FOR_C_AT_SAFE_POINT;
         cond_wait_timeout(C_AT_SAFE_POINT, 0.00001);
         /* every 10 microsec, try again fetch_detached_transaction() */
@@ -477,6 +479,7 @@
             abort_with_mutex();
         }
     }
+    EMIT_WAIT_DONE();
 
     if (UNLIKELY(sync_type == STOP_OTHERS_AND_BECOME_GLOBALLY_UNIQUE)) {
         globally_unique_transaction = true;
diff --git a/c8/stmgc.h b/c8/stmgc.h
--- a/c8/stmgc.h
+++ b/c8/stmgc.h
@@ -71,6 +71,7 @@
     /* the next fields are handled internally by the library */
     int last_associated_segment_num;   /* always a valid seg num */
     int thread_local_counter;
+    int wait_event_emitted;
     struct stm_thread_local_s *prev, *next;
     intptr_t self_or_0_if_atomic;
     void *creating_pthread[2];
@@ -580,10 +581,6 @@
     STM_TRANSACTION_COMMIT,
     STM_TRANSACTION_ABORT,
 
-    /* write-read contention: a "marker" is included in the PYPYSTM file
-       saying where the write was done.  Followed by STM_TRANSACTION_ABORT. */
-    STM_CONTENTION_WRITE_READ,
-
     /* inevitable contention: all threads that try to become inevitable
        have a STM_BECOME_INEVITABLE event with a position marker.  Then,
        if it waits it gets a STM_WAIT_OTHER_INEVITABLE.  It is possible
@@ -591,8 +588,14 @@
        STM_TRANSACTION_ABORT if it fails to become inevitable. */
     STM_BECOME_INEVITABLE,
 
-    /* always one STM_WAIT_xxx followed later by STM_WAIT_DONE */
+    /* write-read contention: a "marker" is included in the PYPYSTM file
+       saying where the write was done.  Followed by STM_TRANSACTION_ABORT. */
+    STM_CONTENTION_WRITE_READ,
+
+    /* always one STM_WAIT_xxx followed later by STM_WAIT_DONE or
+       possibly STM_TRANSACTION_ABORT */
     STM_WAIT_FREE_SEGMENT,
+    STM_WAIT_SYNCING,
     STM_WAIT_SYNC_PAUSE,
     STM_WAIT_OTHER_INEVITABLE,
     STM_WAIT_DONE,
diff --git a/c8/test/support.py b/c8/test/support.py
--- a/c8/test/support.py
+++ b/c8/test/support.py
@@ -132,10 +132,6 @@
     STM_TRANSACTION_COMMIT,
     STM_TRANSACTION_ABORT,
 
-    /* write-read contention: a "marker" is included in the PYPYSTM file
-       saying where the write was done.  Followed by STM_TRANSACTION_ABORT. */
-    STM_CONTENTION_WRITE_READ,
-
     /* inevitable contention: all threads that try to become inevitable
        have a STM_BECOME_INEVITABLE event with a position marker.  Then,
        if it waits it gets a STM_WAIT_OTHER_INEVITABLE.  It is possible
@@ -143,8 +139,14 @@
        STM_TRANSACTION_ABORT if it fails to become inevitable. */
     STM_BECOME_INEVITABLE,
 
-    /* always one STM_WAIT_xxx followed later by STM_WAIT_DONE */
+    /* write-read contention: a "marker" is included in the PYPYSTM file
+       saying where the write was done.  Followed by STM_TRANSACTION_ABORT. */
+    STM_CONTENTION_WRITE_READ,
+
+    /* always one STM_WAIT_xxx followed later by STM_WAIT_DONE or
+       possibly STM_TRANSACTION_ABORT */
     STM_WAIT_FREE_SEGMENT,
+    STM_WAIT_SYNCING,
     STM_WAIT_SYNC_PAUSE,
     STM_WAIT_OTHER_INEVITABLE,
     STM_WAIT_DONE,

From noreply at buildbot.pypy.org  Wed Jun 24 15:55:41 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 24 Jun 2015 15:55:41 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: progress
Message-ID: <20150624135541.0A0621C1545@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult-unroll
Changeset: r78296:3078ef9c2df9
Date: 2015-06-24 15:55 +0200
http://bitbucket.org/pypy/pypy/changeset/3078ef9c2df9/

Log:	progress

diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -203,11 +203,20 @@
 
         short_boxes = ShortBoxes(self.optimizer, inputargs)
 
-        self.optimizer.clear_newoperations()
-        # for i in range(len(original_jump_args)):
-        #     srcbox = jump_args[i]
-        #     if srcbox is not original_jump_args[i]:
-        #         xxx
+        inputarg_setup_ops = []
+        for i in range(len(original_jump_args)):
+            srcbox = jump_args[i]
+            if srcbox is not original_jump_args[i]:
+                if srcbox.type == 'r':
+                    info = self.optimizer.getptrinfo(srcbox)
+                    if info and info.is_virtual():
+                        xxx
+            if original_jump_args[i] is not srcbox and srcbox.is_constant():
+                inputarg_setup_ops.append((original_jump_args[i], srcbox))
+                #opnum = OpHelpers.same_as_for_type(original_jump_args[i].type)
+                #op = ResOperation(opnum, [srcbox])
+                #self.optimizer.emit_operation(op)
+            
         #     if srcbox.type != 'r':
         #         continue
         #     info = self.optimizer.getptrinfo(srcbox)
@@ -220,7 +229,8 @@
         #                                             opnum, [srcbox],
         #                                             descr=DONT_CHANGE)
         #         self.optimizer.emit_operation(op)
-        inputarg_setup_ops = self.optimizer.get_newoperations()
+        #inputarg_setup_ops = original_jump_args
+        #inputarg_setup_ops = self.optimizer.get_newoperations()
 
         target_token = targetop.getdescr()
         assert isinstance(target_token, TargetToken)
@@ -266,8 +276,8 @@
         # Setup the state of the new optimizer by emiting the
         # short operations and discarding the result
         self.optimizer.emitting_dissabled = True
-        for op in exported_state.inputarg_setup_ops:
-            self.optimizer.send_extra_operation(op)
+        for source, target in exported_state.inputarg_setup_ops:
+            source.set_forwarded(target)
 
         seen = {}
         for op in self.short_boxes.operations():
@@ -476,7 +486,7 @@
             optimizer.send_extra_operation(guard)
 
     def is_call_pure_with_exception(self, op):
-        if op.getopnum() == rop.CALL_PURE:
+        if op.is_call_pure():
             effectinfo = op.getdescr().get_extra_info()
             # Assert that only EF_ELIDABLE_CANNOT_RAISE or
             # EF_ELIDABLE_OR_MEMORYERROR end up here, not
diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py
@@ -502,6 +502,7 @@
         if optimizer.optearlyforce:
             optimizer = optimizer.optearlyforce
         assert len(inputargs) == len(self.state)
+        return [x for x in inputargs if not isinstance(x, Const)]
         return inputargs
         inputargs = [None] * self.numnotvirtuals
 

From noreply at buildbot.pypy.org  Wed Jun 24 16:35:33 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 24 Jun 2015 16:35:33 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Try to be a bit more careful in prof.c
Message-ID: <20150624143533.8F5A91C02BB@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r1883:c8ccc22dbf16
Date: 2015-06-24 16:31 +0200
http://bitbucket.org/pypy/stmgc/changeset/c8ccc22dbf16/

Log:	Try to be a bit more careful in prof.c

diff --git a/c8/stm/prof.c b/c8/stm/prof.c
--- a/c8/stm/prof.c
+++ b/c8/stm/prof.c
@@ -2,7 +2,7 @@
 #include <time.h>
 
 
-static FILE *profiling_file;
+static FILE *volatile profiling_file;
 static char *profiling_basefn = NULL;
 static stm_expand_marker_fn profiling_expand_marker;
 
@@ -26,9 +26,6 @@
 
     struct buf_s buf;
     struct timespec t;
-    clock_gettime(CLOCK_MONOTONIC, &t);
-    buf.tv_sec = t.tv_sec;
-    buf.tv_nsec = t.tv_nsec;
     buf.thread_num = tl->thread_local_counter;
     buf.event = event;
     buf.marker_length = 0;
@@ -39,10 +36,29 @@
                                                     buf.extra, MARKER_LEN_MAX);
     }
 
-    if (fwrite(&buf, offsetof(struct buf_s, extra) + buf.marker_length,
-               1, profiling_file) != 1) {
+    size_t result, outsize = offsetof(struct buf_s, extra) + buf.marker_length;
+    FILE *f = profiling_file;
+    if (f == NULL)
+        return;
+    flockfile(f);
+
+    /* We expect the following CLOCK_MONOTONIC to be really monotonic:
+       it should guarantee that the file will be perfectly ordered by time.
+       That's why we do it inside flockfile()/funlockfile(). */
+    clock_gettime(CLOCK_MONOTONIC, &t);
+    buf.tv_sec = t.tv_sec;
+    buf.tv_nsec = t.tv_nsec;
+
+    result = fwrite_unlocked(&buf, outsize, 1, f);
+    funlockfile(f);
+
+    if (result != 1) {
         fprintf(stderr, "stmgc: profiling log file closed unexpectedly: %m\n");
-        close_timing_log();
+
+        /* xxx the FILE leaks here, but it is better than random crashes if
+           we try to close it while other threads are still writing to it
+        */
+        profiling_file = NULL;
     }
 }
 
@@ -54,11 +70,12 @@
 
 static bool open_timing_log(const char *filename)
 {
-    profiling_file = fopen(filename, "w");
-    if (profiling_file == NULL)
+    FILE *f = fopen(filename, "w");
+    profiling_file = f;
+    if (f == NULL)
         return false;
 
-    fwrite("STMGC-C8-PROF01\n", 16, 1, profiling_file);
+    fwrite("STMGC-C8-PROF01\n", 16, 1, f);
     stmcb_timing_event = _stm_profiling_event;
     return true;
 }
@@ -66,9 +83,11 @@
 static bool close_timing_log(void)
 {
     if (stmcb_timing_event == &_stm_profiling_event) {
+        FILE *f = profiling_file;
         stmcb_timing_event = NULL;
-        fclose(profiling_file);
         profiling_file = NULL;
+        if (f != NULL)
+            fclose(f);
         return true;
     }
     return false;
@@ -76,8 +95,9 @@
 
 static void prof_forksupport_prepare(void)
 {
-    if (profiling_file != NULL)
-        fflush(profiling_file);
+    FILE *f = profiling_file;
+    if (f != NULL)
+        fflush(f);
 }
 
 static void prof_forksupport_child(void)

From noreply at buildbot.pypy.org  Wed Jun 24 16:35:42 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 24 Jun 2015 16:35:42 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c8: import stmgc/c8ccc22dbf16
Message-ID: <20150624143542.7AA0F1C02BB@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c8
Changeset: r78297:eeea8ac4da6b
Date: 2015-06-24 16:35 +0200
http://bitbucket.org/pypy/pypy/changeset/eeea8ac4da6b/

Log:	import stmgc/c8ccc22dbf16

diff --git a/rpython/translator/stm/src_stm/revision b/rpython/translator/stm/src_stm/revision
--- a/rpython/translator/stm/src_stm/revision
+++ b/rpython/translator/stm/src_stm/revision
@@ -1,1 +1,1 @@
-9ffba4fe03df
+c8ccc22dbf16
diff --git a/rpython/translator/stm/src_stm/stm/core.c b/rpython/translator/stm/src_stm/stm/core.c
--- a/rpython/translator/stm/src_stm/stm/core.c
+++ b/rpython/translator/stm/src_stm/stm/core.c
@@ -511,10 +511,12 @@
            try to detach an inevitable transaction regularly */
         detached = fetch_detached_transaction();
         if (detached == 0) {
+            EMIT_WAIT(STM_WAIT_OTHER_INEVITABLE);
             if (!cond_wait_timeout(C_SEGMENT_FREE_OR_SAFE_POINT_REQ, 0.00001))
                 goto wait_some_more;
         }
     }
+    EMIT_WAIT_DONE();
     s_mutex_unlock();
 
     if (detached != 0)
@@ -1130,6 +1132,7 @@
 static void _do_start_transaction(stm_thread_local_t *tl)
 {
     assert(!_stm_in_transaction(tl));
+    tl->wait_event_emitted = 0;
 
     acquire_thread_segment(tl);
     /* GS invalid before this point! */
@@ -1318,6 +1321,7 @@
     }
     assert(STM_SEGMENT->running_thread->self_or_0_if_atomic ==
            (intptr_t)(STM_SEGMENT->running_thread));
+    assert(STM_SEGMENT->running_thread->wait_event_emitted == 0);
 
     dprintf(("> stm_commit_transaction(external=%d)\n", (int)external));
     minor_collection(/*commit=*/ true, external);
@@ -1582,9 +1586,8 @@
 
         if (any_soon_finished_or_inevitable_thread_segment() &&
                 num_waits <= NB_SEGMENTS) {
-#if STM_TESTS
-            timing_become_inevitable(); /* for tests: another transaction */
-            stm_abort_transaction();    /* is already inevitable, abort   */
+#if STM_TESTS                           /* for tests: another transaction */
+            stm_abort_transaction();    /*   is already inevitable, abort */
 #endif
 
             bool timed_out = false;
@@ -1594,6 +1597,7 @@
                     !safe_point_requested()) {
 
                 /* wait until C_SEGMENT_FREE_OR_SAFE_POINT_REQ is signalled */
+                EMIT_WAIT(STM_WAIT_OTHER_INEVITABLE);
                 if (!cond_wait_timeout(C_SEGMENT_FREE_OR_SAFE_POINT_REQ,
                                        0.000054321))
                     timed_out = true;
@@ -1607,14 +1611,17 @@
                    not too common.  We don't want two threads constantly
                    detaching each other. */
                 intptr_t detached = fetch_detached_transaction();
-                if (detached != 0)
+                if (detached != 0) {
+                    EMIT_WAIT_DONE();
                     commit_fetched_detached_transaction(detached);
+                }
             }
             else {
                 num_waits++;
             }
             goto retry_from_start;
         }
+        EMIT_WAIT_DONE();
         if (!_validate_and_turn_inevitable())
             goto retry_from_start;
     }
diff --git a/rpython/translator/stm/src_stm/stm/marker.h b/rpython/translator/stm/src_stm/stm/marker.h
--- a/rpython/translator/stm/src_stm/stm/marker.h
+++ b/rpython/translator/stm/src_stm/stm/marker.h
@@ -15,3 +15,28 @@
 
 #define timing_become_inevitable()                                      \
     (timing_enabled() ? _timing_become_inevitable() : (void)0)
+
+
+static inline void emit_wait(stm_thread_local_t *tl, enum stm_event_e event)
+{
+    if (!timing_enabled())
+        return;
+    if (tl->wait_event_emitted != 0) {
+        if (tl->wait_event_emitted == event)
+            return;
+        stmcb_timing_event(tl, STM_WAIT_DONE, NULL);
+    }
+    tl->wait_event_emitted = event;
+    stmcb_timing_event(tl, event, NULL);
+}
+
+static inline void emit_wait_done(stm_thread_local_t *tl)
+{
+    if (tl->wait_event_emitted != 0) {
+        tl->wait_event_emitted = 0;
+        stmcb_timing_event(tl, STM_WAIT_DONE, NULL);
+    }
+}
+
+#define EMIT_WAIT(event)  emit_wait(STM_SEGMENT->running_thread, event)
+#define EMIT_WAIT_DONE()  emit_wait_done(STM_SEGMENT->running_thread)
diff --git a/rpython/translator/stm/src_stm/stm/prof.c b/rpython/translator/stm/src_stm/stm/prof.c
--- a/rpython/translator/stm/src_stm/stm/prof.c
+++ b/rpython/translator/stm/src_stm/stm/prof.c
@@ -2,7 +2,7 @@
 #include <stdio.h>
 #include <time.h>
 
-static FILE *profiling_file;
+static FILE *volatile profiling_file;
 static char *profiling_basefn = NULL;
 static stm_expand_marker_fn profiling_expand_marker;
 
@@ -26,9 +26,6 @@
 
     struct buf_s buf;
     struct timespec t;
-    clock_gettime(CLOCK_MONOTONIC, &t);
-    buf.tv_sec = t.tv_sec;
-    buf.tv_nsec = t.tv_nsec;
     buf.thread_num = tl->thread_local_counter;
     buf.event = event;
     buf.marker_length = 0;
@@ -39,10 +36,29 @@
                                                     buf.extra, MARKER_LEN_MAX);
     }
 
-    if (fwrite(&buf, offsetof(struct buf_s, extra) + buf.marker_length,
-               1, profiling_file) != 1) {
+    size_t result, outsize = offsetof(struct buf_s, extra) + buf.marker_length;
+    FILE *f = profiling_file;
+    if (f == NULL)
+        return;
+    flockfile(f);
+
+    /* We expect the following CLOCK_MONOTONIC to be really monotonic:
+       it should guarantee that the file will be perfectly ordered by time.
+       That's why we do it inside flockfile()/funlockfile(). */
+    clock_gettime(CLOCK_MONOTONIC, &t);
+    buf.tv_sec = t.tv_sec;
+    buf.tv_nsec = t.tv_nsec;
+
+    result = fwrite_unlocked(&buf, outsize, 1, f);
+    funlockfile(f);
+
+    if (result != 1) {
         fprintf(stderr, "stmgc: profiling log file closed unexpectedly: %m\n");
-        close_timing_log();
+
+        /* xxx the FILE leaks here, but it is better than random crashes if
+           we try to close it while other threads are still writing to it
+        */
+        profiling_file = NULL;
     }
 }
 
@@ -54,11 +70,12 @@
 
 static bool open_timing_log(const char *filename)
 {
-    profiling_file = fopen(filename, "w");
-    if (profiling_file == NULL)
+    FILE *f = fopen(filename, "w");
+    profiling_file = f;
+    if (f == NULL)
         return false;
 
-    fwrite("STMGC-C8-PROF01\n", 16, 1, profiling_file);
+    fwrite("STMGC-C8-PROF01\n", 16, 1, f);
     stmcb_timing_event = _stm_profiling_event;
     return true;
 }
@@ -66,9 +83,11 @@
 static bool close_timing_log(void)
 {
     if (stmcb_timing_event == &_stm_profiling_event) {
+        FILE *f = profiling_file;
         stmcb_timing_event = NULL;
-        fclose(profiling_file);
         profiling_file = NULL;
+        if (f != NULL)
+            fclose(f);
         return true;
     }
     return false;
@@ -76,8 +95,9 @@
 
 static void prof_forksupport_prepare(void)
 {
-    if (profiling_file != NULL)
-        fflush(profiling_file);
+    FILE *f = profiling_file;
+    if (f != NULL)
+        fflush(f);
 }
 
 static void prof_forksupport_child(void)
diff --git a/rpython/translator/stm/src_stm/stm/sync.c b/rpython/translator/stm/src_stm/stm/sync.c
--- a/rpython/translator/stm/src_stm/stm/sync.c
+++ b/rpython/translator/stm/src_stm/stm/sync.c
@@ -243,13 +243,13 @@
     /* No segment available.  Wait until release_thread_segment()
        signals that one segment has been freed.  Note that we prefer
        waiting rather than detaching an inevitable transaction, here. */
-    timing_event(tl, STM_WAIT_FREE_SEGMENT);
+    emit_wait(tl, STM_WAIT_FREE_SEGMENT);
     cond_wait(C_SEGMENT_FREE);
-    timing_event(tl, STM_WAIT_DONE);
 
     goto retry_from_start;
 
  got_num:
+    emit_wait_done(tl);
     OPT_ASSERT(num >= 0 && num < NB_SEGMENTS-1);
     sync_ctl.in_use1[num+1] = 1;
     assert(STM_SEGMENT->segment_num == num+1);
@@ -425,15 +425,15 @@
 #ifdef STM_TESTS
         abort_with_mutex();
 #endif
-        timing_event(STM_SEGMENT->running_thread, STM_WAIT_SYNC_PAUSE);
+        EMIT_WAIT(STM_WAIT_SYNC_PAUSE);
         cond_signal(C_AT_SAFE_POINT);
         STM_PSEGMENT->safe_point = SP_WAIT_FOR_C_REQUEST_REMOVED;
         cond_wait(C_REQUEST_REMOVED);
         STM_PSEGMENT->safe_point = SP_RUNNING;
-        timing_event(STM_SEGMENT->running_thread, STM_WAIT_DONE);
         assert(!STM_SEGMENT->no_safe_point_here);
         dprintf(("left safe point\n"));
     }
+    EMIT_WAIT_DONE();
 }
 
 static void synchronize_all_threads(enum sync_type_e sync_type)
@@ -461,12 +461,14 @@
 
         intptr_t detached = fetch_detached_transaction();
         if (detached != 0) {
+            EMIT_WAIT_DONE();
             remove_requests_for_safe_point();    /* => C_REQUEST_REMOVED */
             s_mutex_unlock();
             commit_fetched_detached_transaction(detached);
             s_mutex_lock();
             goto restart;
         }
+        EMIT_WAIT(STM_WAIT_SYNCING);
         STM_PSEGMENT->safe_point = SP_WAIT_FOR_C_AT_SAFE_POINT;
         cond_wait_timeout(C_AT_SAFE_POINT, 0.00001);
         /* every 10 microsec, try again fetch_detached_transaction() */
@@ -477,6 +479,7 @@
             abort_with_mutex();
         }
     }
+    EMIT_WAIT_DONE();
 
     if (UNLIKELY(sync_type == STOP_OTHERS_AND_BECOME_GLOBALLY_UNIQUE)) {
         globally_unique_transaction = true;
diff --git a/rpython/translator/stm/src_stm/stmgc.h b/rpython/translator/stm/src_stm/stmgc.h
--- a/rpython/translator/stm/src_stm/stmgc.h
+++ b/rpython/translator/stm/src_stm/stmgc.h
@@ -71,6 +71,7 @@
     /* the next fields are handled internally by the library */
     int last_associated_segment_num;   /* always a valid seg num */
     int thread_local_counter;
+    int wait_event_emitted;
     struct stm_thread_local_s *prev, *next;
     intptr_t self_or_0_if_atomic;
     void *creating_pthread[2];
@@ -580,10 +581,6 @@
     STM_TRANSACTION_COMMIT,
     STM_TRANSACTION_ABORT,
 
-    /* write-read contention: a "marker" is included in the PYPYSTM file
-       saying where the write was done.  Followed by STM_TRANSACTION_ABORT. */
-    STM_CONTENTION_WRITE_READ,
-
     /* inevitable contention: all threads that try to become inevitable
        have a STM_BECOME_INEVITABLE event with a position marker.  Then,
        if it waits it gets a STM_WAIT_OTHER_INEVITABLE.  It is possible
@@ -591,8 +588,14 @@
        STM_TRANSACTION_ABORT if it fails to become inevitable. */
     STM_BECOME_INEVITABLE,
 
-    /* always one STM_WAIT_xxx followed later by STM_WAIT_DONE */
+    /* write-read contention: a "marker" is included in the PYPYSTM file
+       saying where the write was done.  Followed by STM_TRANSACTION_ABORT. */
+    STM_CONTENTION_WRITE_READ,
+
+    /* always one STM_WAIT_xxx followed later by STM_WAIT_DONE or
+       possibly STM_TRANSACTION_ABORT */
     STM_WAIT_FREE_SEGMENT,
+    STM_WAIT_SYNCING,
     STM_WAIT_SYNC_PAUSE,
     STM_WAIT_OTHER_INEVITABLE,
     STM_WAIT_DONE,

From noreply at buildbot.pypy.org  Wed Jun 24 16:39:21 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 24 Jun 2015 16:39:21 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c8: Update print_stm_log
Message-ID: <20150624143921.E234D1C02BB@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c8
Changeset: r78298:3b9f4b42ac92
Date: 2015-06-24 16:39 +0200
http://bitbucket.org/pypy/pypy/changeset/3b9f4b42ac92/

Log:	Update print_stm_log

diff --git a/pypy/stm/print_stm_log.py b/pypy/stm/print_stm_log.py
--- a/pypy/stm/print_stm_log.py
+++ b/pypy/stm/print_stm_log.py
@@ -8,30 +8,32 @@
 STM_TRANSACTION_COMMIT  = 1
 STM_TRANSACTION_ABORT   = 2
 
-# write-read contention: a "marker" is included in the PYPYSTM file
-# saying where the write was done.  Followed by STM_TRANSACTION_ABORT.
-STM_CONTENTION_WRITE_READ  = 3
-
 # inevitable contention: all threads that try to become inevitable
 # have a STM_BECOME_INEVITABLE event with a position marker.  Then,
 # if it waits it gets a STM_WAIT_OTHER_INEVITABLE.  It is possible
 # that a thread gets STM_BECOME_INEVITABLE followed by
 # STM_TRANSACTION_ABORT if it fails to become inevitable.
-STM_BECOME_INEVITABLE      = 4
+STM_BECOME_INEVITABLE      = 3
 
-# always one STM_WAIT_xxx followed later by STM_WAIT_DONE
+# write-read contention: a "marker" is included in the PYPYSTM file
+# saying where the write was done.  Followed by STM_TRANSACTION_ABORT.
+STM_CONTENTION_WRITE_READ  = 4
+
+# always one STM_WAIT_xxx followed later by STM_WAIT_DONE or
+# possibly STM_TRANSACTION_ABORT
 STM_WAIT_FREE_SEGMENT      = 5
-STM_WAIT_SYNC_PAUSE        = 6
-STM_WAIT_OTHER_INEVITABLE  = 7
-STM_WAIT_DONE              = 8
+STM_WAIT_SYNCING           = 6
+STM_WAIT_SYNC_PAUSE        = 7
+STM_WAIT_OTHER_INEVITABLE  = 8
+STM_WAIT_DONE              = 9
 
 # start and end of GC cycles
-STM_GC_MINOR_START  = 9
-STM_GC_MINOR_DONE   = 10
-STM_GC_MAJOR_START  = 11
-STM_GC_MAJOR_DONE   = 12
+STM_GC_MINOR_START  = 10
+STM_GC_MINOR_DONE   = 11
+STM_GC_MAJOR_START  = 12
+STM_GC_MAJOR_DONE   = 13
 
-_STM_EVENT_N  = 13
+_STM_EVENT_N  = 14
 
 PAUSE_AFTER_ABORT   = 0.000001      # usleep(1) after every abort
 
@@ -70,6 +72,7 @@
         frac = 1.0 / f.tell()
         f.seek(16, 0)
         result = []
+        prev_time = -1.0
         while True:
             packet = f.read(14)
             if len(packet) < 14: break
@@ -77,9 +80,13 @@
                   struct.unpack("IIIBB", packet)
             if event >= _STM_EVENT_N:
                 raise ValueError("the file %r appears corrupted" % (filename,))
+            timestamp = sec + 0.000000001 * nsec
+            if timestamp < prev_time:
+                raise ValueError("decreasing timestamps: %.9f -> %.9f" % (
+                    prev_time, timestamp))
+            prev_time = timestamp
             marker = f.read(markerlen)
-            yield LogEntry(sec + 0.000000001 * nsec,
-                           threadnum, event, marker,
+            yield LogEntry(timestamp, threadnum, event, marker,
                            f.tell() * frac)
     finally:
         f.close()
@@ -111,7 +118,6 @@
     def progress(self, now, new_state):
         prev_time, prev_state = self._prev
         add_time = now - prev_time
-        add_time = abs(add_time) #XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
         assert add_time >= 0.0
         if prev_state == "run":
             self._transaction_cpu_time += add_time
@@ -271,6 +277,7 @@
         elif entry.event == STM_CONTENTION_WRITE_READ:
             t.contention_write_read(entry, conflicts)
         elif entry.event in (STM_WAIT_FREE_SEGMENT,
+                             STM_WAIT_SYNCING,
                              STM_WAIT_SYNC_PAUSE,
                              STM_WAIT_OTHER_INEVITABLE):
             t.transaction_pause(entry)

From noreply at buildbot.pypy.org  Wed Jun 24 17:35:54 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Wed, 24 Jun 2015 17:35:54 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: adding guards as vector instructions.
 i'm not yet sure how this will work out,
 but could help to generate better loops for reductions
Message-ID: <20150624153554.4757F1C034E@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r78299:269e30fb6042
Date: 2015-06-24 17:31 +0200
http://bitbucket.org/pypy/pypy/changeset/269e30fb6042/

Log:	adding guards as vector instructions. i'm not yet sure how this will
	work out, but could help to generate better loops for reductions

diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -553,7 +553,7 @@
     def test_all(self):
         result = self.run("all")
         assert result == 1
-        self.check_vectorized(1, 0) # success?
+        self.check_vectorized(1, 1)
 
     def define_logical_xor_reduce():
         return """
diff --git a/rpython/jit/metainterp/optimizeopt/schedule.py b/rpython/jit/metainterp/optimizeopt/schedule.py
--- a/rpython/jit/metainterp/optimizeopt/schedule.py
+++ b/rpython/jit/metainterp/optimizeopt/schedule.py
@@ -603,6 +603,18 @@
     def determine_output_type(self, op):
         return None
 
+class PassThroughOp(OpToVectorOp):
+    """ This pass through is only applicable if the target
+    operation is capable of handling vector operations.
+    Guard true/false is such an example.
+    """
+    def __init__(self, args):
+        OpToVectorOp.__init__(self, args, None)
+
+    def determine_output_type(self, op):
+        return None
+
+GUARD_TF = PassThroughOp((PT_INT_GENERIC,))
 INT_OP_TO_VOP = OpToVectorOp((PT_INT_GENERIC, PT_INT_GENERIC), INT_RES)
 FLOAT_OP_TO_VOP = OpToVectorOp((PT_FLOAT_GENERIC, PT_FLOAT_GENERIC), FLOAT_RES)
 FLOAT_SINGLE_ARG_OP_TO_VOP = OpToVectorOp((PT_FLOAT_GENERIC,), FLOAT_RES)
@@ -637,6 +649,9 @@
     rop.VEC_CAST_SINGLEFLOAT_TO_FLOAT: OpToVectorOpConv(PT_FLOAT_2, PT_DOUBLE_2),
     rop.VEC_CAST_FLOAT_TO_INT: OpToVectorOpConv(PT_DOUBLE_2, PT_INT32_2),
     rop.VEC_CAST_INT_TO_FLOAT: OpToVectorOpConv(PT_INT32_2, PT_DOUBLE_2),
+
+    rop.GUARD_TRUE: GUARD_TF,
+    rop.GUARD_FALSE: GUARD_TF,
 }
 
 def determine_output_type(node, input_type):
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_schedule.py b/rpython/jit/metainterp/optimizeopt/test/test_schedule.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_schedule.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_schedule.py
@@ -16,7 +16,12 @@
 
 class SchedulerBaseTest(DependencyBaseTest):
 
-    def parse(self, source, inc_label_jump=True):
+    def parse(self, source, inc_label_jump=True,
+              pargs=2,
+              iargs=10,
+              fargs=6,
+              additional_args=None,
+              replace_args=None):
         ns = {
             'double': self.floatarraydescr,
             'float': self.singlefloatarraydescr,
@@ -25,10 +30,24 @@
             'short': self.int16arraydescr,
             'char': self.chararraydescr,
         }
-        loop = opparse("        [p0,p1,p2,p3,p4,p5,i0,i1,i2,i3,i4,i5,i6,i7,i8,i9,f0,f1,f2,f3,f4,f5,v103204[i32|4]]\n" + source + \
-                       "\n        jump(p0,p1,p2,p3,p4,p5,i0,i1,i2,i3,i4,i5,i6,i7,i8,i9,f0,f1,f2,f3,f4,f5,v103204[i32|4])",
-                       cpu=self.cpu,
-                       namespace=ns)
+        args = []
+        for prefix, rang in [('p',range(pargs)), ('i',range(iargs)), ('f',range(fargs))]:
+            for i in rang:
+                args.append(prefix + str(i))
+
+        assert additional_args is None or isinstance(additional_args,list)
+        for arg in additional_args or []:
+            args.append(arg)
+        for k,v in (replace_args or {}).items():
+            for i,_ in enumerate(args):
+                if k == args[i]:
+                    args[i] = v
+                    break
+        indent = "        "
+        joinedargs = ','.join(args)
+        fmt = (indent, joinedargs, source, indent, joinedargs)
+        src = "%s[%s]\n%s\n%sjump(%s)" % fmt
+        loop = opparse(src, cpu=self.cpu, namespace=ns)
         if inc_label_jump:
             token = JitCellToken()
             loop.operations = \
@@ -163,21 +182,19 @@
                 return arg
         raise Exception("could not find %s in args %s" % (name, loop.inputargs))
 
-    def test_signext_int16(self):
+    def test_signext_int32(self):
         loop1 = self.parse("""
-        i10 = int_signext(i1, 2)
-        i11 = int_signext(i1, 2)
-        i12 = int_signext(i1, 2)
-        i13 = int_signext(i1, 2)
-        """)
-        pack1 = self.pack(loop1, 0, 4)
-        v103204 = self.find_input_arg('v103204', loop1)
-        def i1inv103204(var):
-            return 0, v103204
+        i10 = int_signext(i1, 4)
+        i11 = int_signext(i1, 4)
+        """, additional_args=['v10[i64|2]'])
+        pack1 = self.pack(loop1, 0, 2)
+        var = self.find_input_arg('v10', loop1)
+        def i1inv103204(v):
+            return 0, var
         loop2 = self.schedule(loop1, [pack1], prepend_invariant=True, getvboxfunc=i1inv103204)
         loop3 = self.parse("""
-        v11[i16|4] = vec_int_signext(v103204[i32|4], 2)
-        """, False)
+        v11[i32|2] = vec_int_signext(v10[i64|2], 4)
+        """, False, additional_args=['v10[i64|2]'])
         self.assert_equal(loop2, loop3)
 
     def test_cast_float_to_int(self):
@@ -275,13 +292,12 @@
         self.assert_equal(loop2, loop3)
 
     def test_all(self):
-        py.test.skip("this could be an improvement")
         loop1 = self.parse("""
         i10 = raw_load(p0, i1, descr=long)
         i11 = raw_load(p0, i2, descr=long)
         #
-        i12 = int_and(i10, i6)
-        i13 = int_and(i11, i12)
+        i12 = int_and(i10, 255)
+        i13 = int_and(i11, 255)
         #
         guard_true(i12) []
         guard_true(i13) []
@@ -289,9 +305,10 @@
         pack1 = self.pack(loop1, 0, 2)
         pack2 = self.pack(loop1, 2, 4)
         pack3 = self.pack(loop1, 4, 6)
-        loop2 = self.schedule(loop1, [pack1,pack2,pack3])
+        loop2 = self.schedule(loop1, [pack1,pack2,pack3], prepend_invariant=True)
         loop3 = self.parse("""
-        v10[i64|2] = vec_raw_load(p0, i1, 2, descr=long) 
+        v9[i64|2] = vec_int_expand(255)
+        v10[i64|2] = vec_raw_load(p0, i1, 2, descr=long)
         v11[i64|2] = vec_int_and(v10[i64|2], v9[i64|2])
         guard_true(v11[i64|2]) []
         """, False)
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -812,6 +812,10 @@
     rop.CAST_SINGLEFLOAT_TO_FLOAT: rop.VEC_CAST_SINGLEFLOAT_TO_FLOAT,
     rop.CAST_INT_TO_FLOAT: rop.VEC_CAST_INT_TO_FLOAT,
     rop.CAST_FLOAT_TO_INT: rop.VEC_CAST_FLOAT_TO_INT,
+
+    # guard
+    rop.GUARD_TRUE: rop.GUARD_TRUE,
+    rop.GUARD_FALSE: rop.GUARD_FALSE,
 }
 
 

From noreply at buildbot.pypy.org  Wed Jun 24 17:35:55 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Wed, 24 Jun 2015 17:35:55 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: docu comment
Message-ID: <20150624153555.7FFB31C034E@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r78300:01fbd63c766c
Date: 2015-06-24 17:35 +0200
http://bitbucket.org/pypy/pypy/changeset/01fbd63c766c/

Log:	docu comment

diff --git a/rpython/doc/jit/vectorization.rst b/rpython/doc/jit/vectorization.rst
--- a/rpython/doc/jit/vectorization.rst
+++ b/rpython/doc/jit/vectorization.rst
@@ -47,7 +47,8 @@
 ---------------------------
 
 * The only SIMD instruction architecture currently supported is SSE4.1
-* Packed mul for int8,int64 (see PMUL_)
+* Packed mul for int8,int64 (see PMUL_). It would be possible to use PCLMULQDQ. Only supported
+  by some CPUs and must be checked in the cpuid.
 * Loop that convert types from int(8|16|32|64) to int(8|16) are not supported in
   the current SSE4.1 assembler implementation.
   The opcode needed spans over multiple instructions. In terms of performance

From noreply at buildbot.pypy.org  Wed Jun 24 18:02:25 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 24 Jun 2015 18:02:25 +0200 (CEST)
Subject: [pypy-commit] cffi default: Clarify the weakdict example
Message-ID: <20150624160225.A47DB1C02BB@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2193:41f9e10dfdee
Date: 2015-06-24 18:03 +0200
http://bitbucket.org/cffi/cffi/changeset/41f9e10dfdee/

Log:	Clarify the weakdict example

diff --git a/doc/source/using.rst b/doc/source/using.rst
--- a/doc/source/using.rst
+++ b/doc/source/using.rst
@@ -63,16 +63,18 @@
 
     global_weakkeydict = weakref.WeakKeyDictionary()
 
-    s1   = ffi.new("struct foo *")
-    fld1 = ffi.new("struct bar *")
-    fld2 = ffi.new("struct bar *")
-    s1.thefield1 = fld1
-    s1.thefield2 = fld2
-    # here the 'fld1' and 'fld2' object must not go away,
-    # otherwise 's1.thefield1/2' will point to garbage!
-    global_weakkeydict[s1] = (fld1, fld2)
-    # now 's1' keeps alive 'fld1' and 'fld2'.  When 's1' goes
-    # away, then the weak dictionary entry will be removed.
+    def make_foo():
+        s1   = ffi.new("struct foo *")
+        fld1 = ffi.new("struct bar *")
+        fld2 = ffi.new("struct bar *")
+        s1.thefield1 = fld1
+        s1.thefield2 = fld2
+        # here the 'fld1' and 'fld2' object must not go away,
+        # otherwise 's1.thefield1/2' will point to garbage!
+        global_weakkeydict[s1] = (fld1, fld2)
+        # now 's1' keeps alive 'fld1' and 'fld2'.  When 's1' goes
+        # away, then the weak dictionary entry will be removed.
+        return s1
 
 The cdata objects support mostly the same operations as in C: you can
 read or write from pointers, arrays and structures.  Dereferencing a

From noreply at buildbot.pypy.org  Wed Jun 24 18:46:09 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 24 Jun 2015 18:46:09 +0200 (CEST)
Subject: [pypy-commit] stmgc default: - use CLOCK_MONOTONIC_RAW instead of
 CLOCK_MONOTONIC, because the latter is not
Message-ID: <20150624164609.E5FED1C02BB@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r1884:9f966d34d3be
Date: 2015-06-24 18:45 +0200
http://bitbucket.org/pypy/stmgc/changeset/9f966d34d3be/

Log:	- use CLOCK_MONOTONIC_RAW instead of CLOCK_MONOTONIC, because the
	latter is not monotonic... (thanks cfbolz)

	- don't fclose the log file after a fork(): it still seems to create
	some corruption

diff --git a/c8/stm/prof.c b/c8/stm/prof.c
--- a/c8/stm/prof.c
+++ b/c8/stm/prof.c
@@ -42,10 +42,10 @@
         return;
     flockfile(f);
 
-    /* We expect the following CLOCK_MONOTONIC to be really monotonic:
+    /* We expect the following CLOCK_MONOTONIC_RAW to be really monotonic:
        it should guarantee that the file will be perfectly ordered by time.
        That's why we do it inside flockfile()/funlockfile(). */
-    clock_gettime(CLOCK_MONOTONIC, &t);
+    clock_gettime(CLOCK_MONOTONIC_RAW, &t);
     buf.tv_sec = t.tv_sec;
     buf.tv_nsec = t.tv_nsec;
 
@@ -102,7 +102,13 @@
 
 static void prof_forksupport_child(void)
 {
-    if (close_timing_log() && profiling_basefn != NULL) {
+    /* XXX leaks the file descriptor.  I'm getting problems of
+       corrupted files if I fclose() it in the child, even though
+       we're supposed to have fflush()ed the file before the fork.
+       Why??? */
+    profiling_file = NULL;
+    stmcb_timing_event = NULL;
+    if (profiling_basefn != NULL) {
         char filename[1024];
         snprintf(filename, sizeof(filename),
                  "%s.fork%ld", profiling_basefn, (long)getpid());

From noreply at buildbot.pypy.org  Wed Jun 24 18:49:12 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 24 Jun 2015 18:49:12 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c8: more tests
Message-ID: <20150624164912.353601C02BB@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c8
Changeset: r78301:d61acebbb968
Date: 2015-06-24 18:49 +0200
http://bitbucket.org/pypy/pypy/changeset/d61acebbb968/

Log:	more tests

diff --git a/rpython/translator/stm/test/test_inevitable.py b/rpython/translator/stm/test/test_inevitable.py
--- a/rpython/translator/stm/test/test_inevitable.py
+++ b/rpython/translator/stm/test/test_inevitable.py
@@ -137,6 +137,42 @@
         res = self.interpret_inevitable(f1, [43])
         assert res == ['setfield']
 
+    def test_raw_setfield_with_hint(self):
+        X = lltype.Struct('X', ('foo', lltype.Signed),
+                          hints={'stm_dont_track_raw_accesses': True})
+        x1 = lltype.malloc(X, immortal=True)
+
+        def f1():
+            x1.foo = 42
+
+        res = self.interpret_inevitable(f1, [])
+        assert res == []
+
+    def test_raw_getarrayitem_with_hint(self):
+        X = lltype.Array(lltype.Signed,
+                         hints={'nolength': True,
+                                'stm_dont_track_raw_accesses': True})
+        x1 = lltype.malloc(X, 1, immortal=True)
+        x1[0] = 42
+
+        def f1():
+            return x1[0]
+
+        res = self.interpret_inevitable(f1, [])
+        assert res == []
+
+    def test_raw_setarrayitem_with_hint(self):
+        X = lltype.Array(lltype.Signed,
+                         hints={'nolength': True,
+                                'stm_dont_track_raw_accesses': True})
+        x1 = lltype.malloc(X, 1, immortal=True)
+
+        def f1():
+            x1[0] = 42
+
+        res = self.interpret_inevitable(f1, [])
+        assert res == []
+
     def test_malloc_no_inevitable(self):
         X = lltype.GcStruct('X', ('foo', lltype.Signed))
 

From noreply at buildbot.pypy.org  Wed Jun 24 19:10:45 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 24 Jun 2015 19:10:45 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c8: In the JIT,
 copy the stm_dont_track_raw_accesses logic for getfield_raw
Message-ID: <20150624171045.349DD1C02BB@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c8
Changeset: r78302:972b5349364b
Date: 2015-06-24 19:10 +0200
http://bitbucket.org/pypy/pypy/changeset/972b5349364b/

Log:	In the JIT, copy the stm_dont_track_raw_accesses logic for
	getfield_raw to also apply to getarrayitem_raw

diff --git a/rpython/jit/backend/llsupport/descr.py b/rpython/jit/backend/llsupport/descr.py
--- a/rpython/jit/backend/llsupport/descr.py
+++ b/rpython/jit/backend/llsupport/descr.py
@@ -84,16 +84,20 @@
 
 class ArrayOrFieldDescr(AbstractDescr):
     vinfo = None
+    stm_dont_track_raw_accesses = False
 
     def get_vinfo(self):
         return self.vinfo
 
+    def stm_should_track_raw_accesses(self):
+        return not self.stm_dont_track_raw_accesses
+
+
 class FieldDescr(ArrayOrFieldDescr):
     name = ''
     offset = 0      # help translation
     field_size = 0
     flag = '\x00'
-    stm_dont_track_raw_accesses = False
     _immutable = False
 
     def __init__(self, name, offset, field_size, flag,
@@ -147,9 +151,6 @@
     def repr_of_descr(self):
         return '<Field%s %s %s>' % (self.flag, self.name, self.offset)
 
-    def stm_should_track_raw_accesses(self):
-        return not self.stm_dont_track_raw_accesses
-
 
 def get_field_descr(gccache, STRUCT, fieldname):
     cache = gccache._cache_field
@@ -219,11 +220,13 @@
     _immutable = False
 
     def __init__(self, basesize, itemsize, lendescr, flag,
+                 stm_dont_track_raw_accesses=False,
                  immutable=False):
         self.basesize = basesize
         self.itemsize = itemsize
         self.lendescr = lendescr    # or None, if no length
         self.flag = flag
+        self.stm_dont_track_raw_accesses = stm_dont_track_raw_accesses
         self._immutable = immutable
 
     def is_immutable(self):
@@ -282,9 +285,12 @@
             lendescr = None
         else:
             lendescr = get_field_arraylen_descr(gccache, ARRAY_OR_STRUCT)
+        stm_dont_track_raw_accesses = ARRAY_INSIDE._hints.get(
+            'stm_dont_track_raw_accesses', False)
         flag = get_type_flag(ARRAY_INSIDE.OF)
         immutable = bool(ARRAY_INSIDE._immutable_field())
         arraydescr = ArrayDescr(basesize, itemsize, lendescr, flag,
+                                stm_dont_track_raw_accesses,
                                 immutable)
         if ARRAY_OR_STRUCT._gckind == 'gc':
             gccache.init_array_descr(ARRAY_OR_STRUCT, arraydescr)
diff --git a/rpython/jit/backend/llsupport/stmrewrite.py b/rpython/jit/backend/llsupport/stmrewrite.py
--- a/rpython/jit/backend/llsupport/stmrewrite.py
+++ b/rpython/jit/backend/llsupport/stmrewrite.py
@@ -1,5 +1,5 @@
 from rpython.jit.backend.llsupport.rewrite import GcRewriterAssembler
-from rpython.jit.backend.llsupport.descr import CallDescr, FieldDescr
+from rpython.jit.backend.llsupport.descr import CallDescr, ArrayOrFieldDescr
 from rpython.jit.metainterp.resoperation import ResOperation, rop
 from rpython.jit.metainterp.history import BoxPtr, ConstInt
 from rpython.rlib.objectmodel import specialize
@@ -70,8 +70,9 @@
         if opnum in (rop.COPYSTRCONTENT, rop.COPYUNICODECONTENT):
             self.handle_setters_for_pure_fields(op, 1)
             return
-        # ----------  raw getfields and setfields  ----------
-        if opnum in (rop.GETFIELD_RAW, rop.SETFIELD_RAW):
+        # ----------  raw getfields and setfields and arrays  ----------
+        if opnum in (rop.GETFIELD_RAW, rop.SETFIELD_RAW,
+                     rop.GETARRAYITEM_RAW, rop.SETARRAYITEM_RAW):
             if self.maybe_handle_raw_accesses(op):
                 return
         # ----------  labels  ----------
@@ -168,7 +169,7 @@
 
     def maybe_handle_raw_accesses(self, op):
         descr = op.getdescr()
-        assert isinstance(descr, FieldDescr)
+        assert isinstance(descr, ArrayOrFieldDescr)
         if descr.stm_dont_track_raw_accesses:
             self.newop(op)
             return True
diff --git a/rpython/jit/backend/llsupport/test/test_stmrewrite.py b/rpython/jit/backend/llsupport/test/test_stmrewrite.py
--- a/rpython/jit/backend/llsupport/test/test_stmrewrite.py
+++ b/rpython/jit/backend/llsupport/test/test_stmrewrite.py
@@ -44,8 +44,6 @@
     words.append('ZERO_ARRAY')
     words.append('ZERO_PTR_FIELD')
     # these always turn inevitable
-    words.append('GETARRAYITEM_RAW')
-    words.append('SETARRAYITEM_RAW')
     words.append('SETINTERIORFIELD_RAW')
     words.append('RAW_LOAD')
     words.append('RAW_STORE')
@@ -96,6 +94,7 @@
         to_operations  = to_operations .replace('$DUMMYALLOC', dummyalloc)
         for name, value in self.gc_ll_descr.__dict__.items():
             if name.endswith('descr') and name[1] == '2' and len(name) == 8:
+                assert name not in namespace
                 namespace[name] = value     # "X2Ydescr"
         self.gc_ll_descr.malloc_zero_filled = False
         RewriteTests.check_rewrite(self, frm_operations, to_operations,
@@ -525,6 +524,55 @@
             jump(i2)
         """, fdescr=fdescr)
 
+    def test_setfield_raw_stm_dont_track_raw_accesses(self):
+        c1 = GcCache(True)
+        F = lltype.Struct('F', ('x', lltype.Signed),
+                          hints={'stm_dont_track_raw_accesses': True})
+        fdescr = get_field_descr(c1, F, 'x')
+        self.check_rewrite("""
+            [i1]
+            setfield_raw(i1, 42, descr=fdescr)
+            jump(i1)
+        """, """
+            [i1]
+            setfield_raw(i1, 42, descr=fdescr)
+            $DUMMYALLOC
+            jump(i1)
+        """, fdescr=fdescr)
+
+    def test_getarrayitem_raw_stm_dont_track_raw_accesses(self):
+        c1 = GcCache(True)
+        A = lltype.Array(lltype.Signed, hints={'nolength': True,
+                            'stm_dont_track_raw_accesses': True})
+        aadescr = get_array_descr(c1, A)
+        assert not aadescr.stm_should_track_raw_accesses()
+        self.check_rewrite("""
+            [i1]
+            i2 = getarrayitem_raw(i1, 5, descr=aadescr)
+            jump(i2)
+        """, """
+            [i1]
+            i2 = getarrayitem_raw(i1, 5, descr=aadescr)
+            $DUMMYALLOC
+            jump(i2)
+        """, aadescr=aadescr)
+
+    def test_setarrayitem_raw_stm_dont_track_raw_accesses(self):
+        c1 = GcCache(True)
+        A = lltype.Array(lltype.Signed, hints={'nolength': True,
+                            'stm_dont_track_raw_accesses': True})
+        aadescr = get_array_descr(c1, A)
+        self.check_rewrite("""
+            [i1]
+            setarrayitem_raw(i1, 5, 42, descr=aadescr)
+            jump(i1)
+        """, """
+            [i1]
+            setarrayitem_raw(i1, 5, 42, descr=aadescr)
+            $DUMMYALLOC
+            jump(i1)
+        """, aadescr=aadescr)
+
     def test_getfield_raw_over_label(self):
         self.check_rewrite("""
             [i1, i2]
diff --git a/rpython/jit/metainterp/executor.py b/rpython/jit/metainterp/executor.py
--- a/rpython/jit/metainterp/executor.py
+++ b/rpython/jit/metainterp/executor.py
@@ -112,7 +112,8 @@
                                                 pure))
 
 def do_getarrayitem_raw(cpu, _, arraybox, indexbox, arraydescr):
-    return _do_getarrayitem_raw(cpu, False, arraybox, indexbox, arraydescr)
+    pure = not arraydescr.stm_should_track_raw_accesses()
+    return _do_getarrayitem_raw(cpu, pure, arraybox, indexbox, arraydescr)
 
 def do_getarrayitem_raw_pure(cpu, _, arraybox, indexbox, arraydescr):
     return _do_getarrayitem_raw(cpu, True, arraybox, indexbox, arraydescr)

From noreply at buildbot.pypy.org  Wed Jun 24 19:11:59 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 24 Jun 2015 19:11:59 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c8: update to stmgc/9f966d34d3be
Message-ID: <20150624171159.4D1A71C02BB@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c8
Changeset: r78303:a55f341bfd57
Date: 2015-06-24 19:11 +0200
http://bitbucket.org/pypy/pypy/changeset/a55f341bfd57/

Log:	update to stmgc/9f966d34d3be

diff --git a/rpython/translator/stm/src_stm/revision b/rpython/translator/stm/src_stm/revision
--- a/rpython/translator/stm/src_stm/revision
+++ b/rpython/translator/stm/src_stm/revision
@@ -1,1 +1,1 @@
-c8ccc22dbf16
+9f966d34d3be
diff --git a/rpython/translator/stm/src_stm/stm/prof.c b/rpython/translator/stm/src_stm/stm/prof.c
--- a/rpython/translator/stm/src_stm/stm/prof.c
+++ b/rpython/translator/stm/src_stm/stm/prof.c
@@ -42,10 +42,10 @@
         return;
     flockfile(f);
 
-    /* We expect the following CLOCK_MONOTONIC to be really monotonic:
+    /* We expect the following CLOCK_MONOTONIC_RAW to be really monotonic:
        it should guarantee that the file will be perfectly ordered by time.
        That's why we do it inside flockfile()/funlockfile(). */
-    clock_gettime(CLOCK_MONOTONIC, &t);
+    clock_gettime(CLOCK_MONOTONIC_RAW, &t);
     buf.tv_sec = t.tv_sec;
     buf.tv_nsec = t.tv_nsec;
 
@@ -102,7 +102,13 @@
 
 static void prof_forksupport_child(void)
 {
-    if (close_timing_log() && profiling_basefn != NULL) {
+    /* XXX leaks the file descriptor.  I'm getting problems of
+       corrupted files if I fclose() it in the child, even though
+       we're supposed to have fflush()ed the file before the fork.
+       Why??? */
+    profiling_file = NULL;
+    stmcb_timing_event = NULL;
+    if (profiling_basefn != NULL) {
         char filename[1024];
         snprintf(filename, sizeof(filename),
                  "%s.fork%ld", profiling_basefn, (long)getpid());

From noreply at buildbot.pypy.org  Wed Jun 24 19:25:26 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Wed, 24 Jun 2015 19:25:26 +0200 (CEST)
Subject: [pypy-commit] benchmarks default: Backed out 45bdb6aca4bb,
 re-enable cffi-dependent benchmakr
Message-ID: <20150624172526.EB18E1C034E@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: 
Changeset: r332:34f06618ef7f
Date: 2015-06-24 20:20 +0300
http://bitbucket.org/pypy/benchmarks/changeset/34f06618ef7f/

Log:	Backed out 45bdb6aca4bb, re-enable cffi-dependent benchmakr

diff --git a/benchmarks.py b/benchmarks.py
--- a/benchmarks.py
+++ b/benchmarks.py
@@ -83,8 +83,7 @@
              'raytrace-simple', 'crypto_pyaes', 'bm_mako', 'bm_chameleon',
              'json_bench', 'pidigits', 'hexiom2', 'eparse', 'deltablue',
              'bm_dulwich_log', 'bm_krakatau', 'bm_mdp', 'pypy_interp',
-             #'sqlitesynth',
-             ]:
+             'sqlitesynth']:
     _register_new_bm(name, name, globals(), **opts.get(name, {}))
 
 for name in ['names', 'iteration', 'tcp', 'pb', ]:#'web']:#, 'accepts']:

From noreply at buildbot.pypy.org  Wed Jun 24 21:33:13 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Wed, 24 Jun 2015 21:33:13 +0200 (CEST)
Subject: [pypy-commit] pypy default: remove prefix from MSVC exe name,
 since prefix was not handled in driver.compute_exe_name
Message-ID: <20150624193313.39B321C02BB@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: 
Changeset: r78304:81819d4b3f58
Date: 2015-06-24 22:33 +0300
http://bitbucket.org/pypy/pypy/changeset/81819d4b3f58/

Log:	remove prefix from MSVC exe name, since prefix was not handled in
	driver.compute_exe_name

diff --git a/rpython/translator/c/genc.py b/rpython/translator/c/genc.py
--- a/rpython/translator/c/genc.py
+++ b/rpython/translator/c/genc.py
@@ -485,7 +485,7 @@
                 else:
                     mk.definition('DEBUGFLAGS', '-O1 -g')
         if self.translator.platform.name == 'msvc':
-            mk.rule('debug_target', 'debugmode_$(DEFAULT_TARGET)', 'rem')
+            mk.rule('debug_target', '$(DEFAULT_TARGET)', 'rem')
         else:
             mk.rule('debug_target', '$(DEFAULT_TARGET)', '#')
         mk.write()

From noreply at buildbot.pypy.org  Thu Jun 25 00:06:35 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Thu, 25 Jun 2015 00:06:35 +0200 (CEST)
Subject: [pypy-commit] pypy default: be more consistent with win32
	identification macro
Message-ID: <20150624220635.7A9F61C1310@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: 
Changeset: r78305:1ebb74733012
Date: 2015-06-25 01:06 +0300
http://bitbucket.org/pypy/pypy/changeset/1ebb74733012/

Log:	be more consistent with win32 identification macro

diff --git a/pypy/module/_cffi_backend/src/parse_c_type.c b/pypy/module/_cffi_backend/src/parse_c_type.c
--- a/pypy/module/_cffi_backend/src/parse_c_type.c
+++ b/pypy/module/_cffi_backend/src/parse_c_type.c
@@ -362,7 +362,7 @@
 
             case TOK_INTEGER:
                 errno = 0;
-#ifndef MS_WIN32
+#ifndef _MSC_VER
                 if (sizeof(length) > sizeof(unsigned long))
                     length = strtoull(tok->p, &endptr, 0);
                 else

From noreply at buildbot.pypy.org  Thu Jun 25 01:03:22 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Thu, 25 Jun 2015 01:03:22 +0200 (CEST)
Subject: [pypy-commit] pypy pypy3-release-2.6.x: small win32 fixes
Message-ID: <20150624230322.A38071C034E@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: pypy3-release-2.6.x
Changeset: r78306:f3ccad00bee4
Date: 2015-06-25 01:55 +0300
http://bitbucket.org/pypy/pypy/changeset/f3ccad00bee4/

Log:	small win32 fixes

diff --git a/pypy/module/_codecs/locale_codec.c b/pypy/module/_codecs/locale_codec.c
--- a/pypy/module/_codecs/locale_codec.c
+++ b/pypy/module/_codecs/locale_codec.c
@@ -17,11 +17,12 @@
 #define PyMem_Free free
 /* C99 but recent Windows has it */
 #define HAVE_MBRTOWC 1
-/* Hopefully? */
-#define HAVE_LANGINFO_H
 
 #ifdef MS_WINDOWS
 #  include <windows.h>
+#else
+/* Hopefully? */
+#define HAVE_LANGINFO_H
 #endif
 
 #ifdef HAVE_LANGINFO_H
diff --git a/pypy/module/posix/interp_nt.py b/pypy/module/posix/interp_nt.py
--- a/pypy/module/posix/interp_nt.py
+++ b/pypy/module/posix/interp_nt.py
@@ -79,7 +79,7 @@
         with lltype.scoped_alloc(
             win32traits.BY_HANDLE_FILE_INFORMATION) as info:
             if win32traits.GetFileInformationByHandle(hFile, info) == 0:
-                raise rwin32.lastWindowsError("_getfileinformation")
+                raise rwin32.lastSavedWindowsError("_getfileinformation")
             return (rffi.cast(lltype.Signed, info.c_dwVolumeSerialNumber),
                     rffi.cast(lltype.Signed, info.c_nFileIndexHigh),
                     rffi.cast(lltype.Signed, info.c_nFileIndexLow))
@@ -101,7 +101,7 @@
                                        win32traits.FILE_FLAG_BACKUP_SEMANTICS,
                                        rwin32.NULL_HANDLE)
         if hFile == rwin32.INVALID_HANDLE_VALUE:
-            raise rwin32.lastWindowsError("CreateFile")
+            raise rwin32.lastSavedWindowsError("CreateFile")
 
         VOLUME_NAME_DOS = rffi.cast(rwin32.DWORD, win32traits.VOLUME_NAME_DOS)
         try:
@@ -111,7 +111,7 @@
                 rffi.cast(rwin32.DWORD, 0),
                 VOLUME_NAME_DOS)
             if usize == 0:
-                raise rwin32.lastWindowsError("GetFinalPathNameByHandle")
+                raise rwin32.lastSavedWindowsError("GetFinalPathNameByHandle")
 
             size = rffi.cast(lltype.Signed, usize)
             with rffi.scoped_alloc_unicodebuffer(size + 1) as buf:
@@ -121,7 +121,7 @@
                     usize,
                     VOLUME_NAME_DOS)
                 if result == 0:
-                    raise rwin32.lastWindowsError("GetFinalPathNameByHandle")
+                    raise rwin32.lastSavedWindowsError("GetFinalPathNameByHandle")
                 return buf.str(rffi.cast(lltype.Signed, result))
         finally:
             rwin32.CloseHandle(hFile)
diff --git a/pypy/module/time/interp_time.py b/pypy/module/time/interp_time.py
--- a/pypy/module/time/interp_time.py
+++ b/pypy/module/time/interp_time.py
@@ -186,7 +186,7 @@
                              "RPY_EXTERN "
                              "char** pypy_get_tzname();\n"
                              "RPY_EXTERN "
-                             "void* pypy__tzset();"],
+                             "void pypy__tzset();"],
         separate_module_sources = ["""
         long pypy_get_timezone() { return timezone; }
         int pypy_get_daylight() { return daylight; }

From noreply at buildbot.pypy.org  Thu Jun 25 12:10:39 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Thu, 25 Jun 2015 12:10:39 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: moved timing functions into the execute
 assembler, commented out the guard as vec opt for this translation test
Message-ID: <20150625101039.369731C0EFC@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r78307:f5d62531d777
Date: 2015-06-25 12:10 +0200
http://bitbucket.org/pypy/pypy/changeset/f5d62531d777/

Log:	moved timing functions into the execute assembler, commented out the
	guard as vec opt for this translation test

diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -530,9 +530,6 @@
             self.status = hash & self.ST_SHIFT_MASK
 
     def handle_fail(self, deadframe, metainterp_sd, jitdriver_sd):
-        # XXX debug purpose only
-        jitdriver_sd.xxxbench.xxx_clock_stop(fail=True)
-        # XXX debug purpose only end
         if self.must_compile(deadframe, metainterp_sd, jitdriver_sd):
             self.start_compiling()
             try:
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -814,8 +814,8 @@
     rop.CAST_FLOAT_TO_INT: rop.VEC_CAST_FLOAT_TO_INT,
 
     # guard
-    rop.GUARD_TRUE: rop.GUARD_TRUE,
-    rop.GUARD_FALSE: rop.GUARD_FALSE,
+    #rop.GUARD_TRUE: rop.GUARD_TRUE,
+    #rop.GUARD_FALSE: rop.GUARD_FALSE,
 }
 
 
diff --git a/rpython/jit/metainterp/warmspot.py b/rpython/jit/metainterp/warmspot.py
--- a/rpython/jit/metainterp/warmspot.py
+++ b/rpython/jit/metainterp/warmspot.py
@@ -53,11 +53,9 @@
 
     def xxx_clock_stop(self, fail=False):
         end = time.clock()
-        if len(self.t) == 0:
-            return
+        assert len(self.t) > 0
         start = self.t[-1]
-        if not fail:
-            del self.t[-1]
+        del self.t[-1]
         ns = (end - start) * 10**9
         debug_start("xxx-clock")
         debug_print("stop name: %s id: %s clock: %f exe time: %dns fail? %d vec? %d" % \
@@ -869,10 +867,6 @@
                     else:
                         value = cast_base_ptr_to_instance(Exception, value)
                         raise Exception, value
-                finally:
-                    # XXX debug purpose only
-                    jd.xxxbench.xxx_clock_stop(fail=False)
-                    # XXX debug purpose only end
 
 
         def handle_jitexception(e):
diff --git a/rpython/jit/metainterp/warmstate.py b/rpython/jit/metainterp/warmstate.py
--- a/rpython/jit/metainterp/warmstate.py
+++ b/rpython/jit/metainterp/warmstate.py
@@ -368,6 +368,7 @@
             if vinfo is not None:
                 virtualizable = args[index_of_virtualizable]
                 vinfo.clear_vable_token(virtualizable)
+
             # XXX debug purpose only
             jitdriver_sd.xxxbench.xxx_clock_start()
             # XXX debug purpose only end
@@ -377,6 +378,9 @@
             # Record in the memmgr that we just ran this loop,
             # so that it will keep it alive for a longer time
             warmrunnerdesc.memory_manager.keep_loop_alive(loop_token)
+            # XXX debug purpose only
+            jitdriver_sd.xxxbench.xxx_clock_stop()
+            # XXX debug purpose only end
             #
             # Handle the failure
             fail_descr = cpu.get_latest_descr(deadframe)

From noreply at buildbot.pypy.org  Thu Jun 25 20:24:31 2015
From: noreply at buildbot.pypy.org (amauryfa)
Date: Thu, 25 Jun 2015 20:24:31 +0200 (CEST)
Subject: [pypy-commit] pypy more-rposix: jit.dont_look_inside(forkpty)
Message-ID: <20150625182431.3F86E1C0683@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: more-rposix
Changeset: r78308:f4d30da9a715
Date: 2015-06-12 13:22 +0200
http://bitbucket.org/pypy/pypy/changeset/f4d30da9a715/

Log:	jit.dont_look_inside(forkpty)

diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -804,6 +804,7 @@
         lltype.free(slave_p, flavor='raw')
 
 @replace_os_function('forkpty')
+ at jit.dont_look_inside
 def forkpty():
     master_p = lltype.malloc(rffi.INTP.TO, 1, flavor='raw')
     master_p[0] = rffi.cast(rffi.INT, -1)

From noreply at buildbot.pypy.org  Thu Jun 25 22:47:29 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 25 Jun 2015 22:47:29 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c8: Progress, but still something TO DO
Message-ID: <20150625204729.D31A71C0460@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c8
Changeset: r78309:163959b2883e
Date: 2015-06-25 10:16 +0200
http://bitbucket.org/pypy/pypy/changeset/163959b2883e/

Log:	Progress, but still something TO DO

diff --git a/TODO b/TODO
--- a/TODO
+++ b/TODO
@@ -7,13 +7,25 @@
 
 ------------------------------------------------------------
 
-better heuristic of when to break transactions? e.g., we should
-rarely break if there are not threads running in parallel.
-But we need to break sometimes in order to run finalizers...
+check what occurs with finalizers: in single-threaded programs
+they are fine, but in this example they seem not called often
+enough:
 
-IMPROVED, but we should check if we break often enough to run
-finaliers from time to time, or if we really make infinite
-transactions
+    import gc, thread, time
+    class Foo(object):
+        count = 0
+        def __init__(self):
+            Foo.count += 1
+        def __del__(self):
+            Foo.count -= 1
+    def f():
+        while True:
+            [Foo() for j in range(100000)]
+            gc.collect()
+            print Foo.count
+    thread.start_new_thread(f, ())
+    thread.start_new_thread(f, ())
+    time.sleep(99)
 
 ------------------------------------------------------------
 

From noreply at buildbot.pypy.org  Thu Jun 25 22:47:31 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 25 Jun 2015 22:47:31 +0200 (CEST)
Subject: [pypy-commit] pypy default: Plug the logic from
 find_initializing_stores() to also detect and remove
Message-ID: <20150625204731.05DA41C0460@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78310:f24607693daa
Date: 2015-06-25 11:11 +0200
http://bitbucket.org/pypy/pypy/changeset/f24607693daa/

Log:	Plug the logic from find_initializing_stores() to also detect and
	remove duplicate write barrier calls done without any malloc()

diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py
--- a/rpython/memory/gctransform/framework.py
+++ b/rpython/memory/gctransform/framework.py
@@ -52,21 +52,19 @@
             return (op.opname in LL_OPERATIONS and
                     LL_OPERATIONS[op.opname].canmallocgc)
 
-def find_initializing_stores(collect_analyzer, graph):
-    from rpython.flowspace.model import mkentrymap
-    entrymap = mkentrymap(graph)
-    # a bit of a hackish analysis: if a block contains a malloc and check that
-    # the result is not zero, then the block following the True link will
-    # usually initialize the newly allocated object
-    result = set()
-    def find_in_block(block, mallocvars):
+def propagate_no_write_barrier_needed(result, block, mallocvars,
+                                      collect_analyzer, entrymap):
+    # We definitely know that no write barrier is needed in the 'block'
+    # for any of the variables in 'mallocvars'.  Propagate this information
+    # forward.  Note that "definitely know" implies that we just did either
+    # a fixed-size malloc (variable-size might require card marking), or
+    # that we just did a full write barrier (not just for card marking).
+    if 1:       # keep indentation
         for i, op in enumerate(block.operations):
             if op.opname in ("cast_pointer", "same_as"):
                 if op.args[0] in mallocvars:
                     mallocvars[op.result] = True
             elif op.opname in ("setfield", "setarrayitem", "setinteriorfield"):
-                # note that 'mallocvars' only tracks fixed-size mallocs,
-                # so no risk that they use card marking
                 TYPE = op.args[-1].concretetype
                 if (op.args[0] in mallocvars and
                     isinstance(TYPE, lltype.Ptr) and
@@ -83,7 +81,15 @@
                 if var in mallocvars:
                     newmallocvars[exit.target.inputargs[i]] = True
             if newmallocvars:
-                find_in_block(exit.target, newmallocvars)
+                propagate_no_write_barrier_needed(result, exit.target,
+                                                  newmallocvars,
+                                                  collect_analyzer, entrymap)
+
+def find_initializing_stores(collect_analyzer, graph, entrymap):
+    # a bit of a hackish analysis: if a block contains a malloc and check that
+    # the result is not zero, then the block following the True link will
+    # usually initialize the newly allocated object
+    result = set()
     mallocnum = 0
     blockset = set(graph.iterblocks())
     while blockset:
@@ -113,7 +119,8 @@
         target = exit.target
         mallocvars = {target.inputargs[index]: True}
         mallocnum += 1
-        find_in_block(target, mallocvars)
+        propagate_no_write_barrier_needed(result, target, mallocvars,
+                                          collect_analyzer, entrymap)
     #if result:
     #    print "found %s initializing stores in %s" % (len(result), graph.name)
     return result
@@ -698,8 +705,11 @@
                                 " %s" % func)
 
         if self.write_barrier_ptr:
+            from rpython.flowspace.model import mkentrymap
+            self._entrymap = mkentrymap(graph)
             self.clean_sets = (
-                find_initializing_stores(self.collect_analyzer, graph))
+                find_initializing_stores(self.collect_analyzer, graph,
+                                         self._entrymap))
             if self.gcdata.gc.can_optimize_clean_setarrayitems():
                 self.clean_sets = self.clean_sets.union(
                     find_clean_setarrayitems(self.collect_analyzer, graph))
@@ -1269,6 +1279,15 @@
                 hop.genop("direct_call", [self.write_barrier_ptr,
                                           self.c_const_gc,
                                           v_structaddr])
+                # we just did a full write barrier here, so we can use
+                # this helper to propagate this knowledge forward and
+                # avoid to repeat the write barrier.
+                if self.curr_block is not None:   # for tests
+                    propagate_no_write_barrier_needed(self.clean_sets,
+                                                      self.curr_block,
+                                                      {v_struct: True},
+                                                      self.collect_analyzer,
+                                                      self._entrymap)
         hop.rename('bare_' + opname)
 
     def transform_getfield_typeptr(self, hop):
diff --git a/rpython/memory/gctransform/test/test_framework.py b/rpython/memory/gctransform/test/test_framework.py
--- a/rpython/memory/gctransform/test/test_framework.py
+++ b/rpython/memory/gctransform/test/test_framework.py
@@ -1,6 +1,6 @@
 from rpython.annotator.listdef import s_list_of_strings
 from rpython.annotator.model import SomeInteger
-from rpython.flowspace.model import Constant, SpaceOperation
+from rpython.flowspace.model import Constant, SpaceOperation, mkentrymap
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rtyper.lltypesystem.lloperation import llop
 from rpython.memory.gc.semispace import SemiSpaceGC
@@ -231,6 +231,33 @@
          Constant('b', lltype.Void), varoftype(PTR_TYPE2)],
         varoftype(lltype.Void)))
 
+def test_remove_duplicate_write_barrier():
+    from rpython.translator.c.genc import CStandaloneBuilder
+    from rpython.flowspace.model import summary
+
+    class A(object):
+        pass
+    glob_a_1 = A()
+    glob_a_2 = A()
+
+    def f(a, cond):
+        a.x = a
+        a.z = a
+        if cond:
+            a.y = a
+    def g():
+        f(glob_a_1, 5)
+        f(glob_a_2, 0)
+    t = rtype(g, [])
+    t.config.translation.gc = "minimark"
+    cbuild = CStandaloneBuilder(t, g, t.config,
+                                gcpolicy=FrameworkGcPolicy2)
+    db = cbuild.generate_graphs_for_llinterp()
+
+    ff = graphof(t, f)
+    #ff.show()
+    assert summary(ff)['direct_call'] == 1    # only one remember_young_pointer
+
 def test_find_initializing_stores():
 
     class A(object):
@@ -246,7 +273,8 @@
     etrafo = ExceptionTransformer(t)
     graphs = etrafo.transform_completely()
     collect_analyzer = CollectAnalyzer(t)
-    init_stores = find_initializing_stores(collect_analyzer, t.graphs[0])
+    init_stores = find_initializing_stores(collect_analyzer, t.graphs[0],
+                                           mkentrymap(t.graphs[0]))
     assert len(init_stores) == 1
 
 def test_find_initializing_stores_across_blocks():
@@ -271,7 +299,8 @@
     etrafo = ExceptionTransformer(t)
     graphs = etrafo.transform_completely()
     collect_analyzer = CollectAnalyzer(t)
-    init_stores = find_initializing_stores(collect_analyzer, t.graphs[0])
+    init_stores = find_initializing_stores(collect_analyzer, t.graphs[0],
+                                           mkentrymap(t.graphs[0]))
     assert len(init_stores) == 5
 
 def test_find_clean_setarrayitems():
diff --git a/rpython/memory/gctransform/transform.py b/rpython/memory/gctransform/transform.py
--- a/rpython/memory/gctransform/transform.py
+++ b/rpython/memory/gctransform/transform.py
@@ -83,6 +83,7 @@
 
 class BaseGCTransformer(object):
     finished_helpers = False
+    curr_block = None
 
     def __init__(self, translator, inline=False):
         self.translator = translator
@@ -159,7 +160,7 @@
 
     def transform_block(self, block, is_borrowed):
         llops = LowLevelOpList()
-        #self.curr_block = block
+        self.curr_block = block
         self.livevars = [var for var in block.inputargs
                     if var_needsgc(var) and not is_borrowed(var)]
         allvars = [var for var in block.getvariables() if var_needsgc(var)]
@@ -205,6 +206,7 @@
             block.operations[:] = llops
         self.livevars = None
         self.var_last_needed_in = None
+        self.curr_block = None
 
     def transform_graph(self, graph):
         if graph in self.minimal_transform:

From noreply at buildbot.pypy.org  Thu Jun 25 22:47:32 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 25 Jun 2015 22:47:32 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c8: Plug the logic from
 find_initializing_stores() to also detect and remove
Message-ID: <20150625204732.2B7CF1C0460@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c8
Changeset: r78311:8043ab320b22
Date: 2015-06-25 11:11 +0200
http://bitbucket.org/pypy/pypy/changeset/8043ab320b22/

Log:	Plug the logic from find_initializing_stores() to also detect and
	remove duplicate write barrier calls done without any malloc()

diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py
--- a/rpython/memory/gctransform/framework.py
+++ b/rpython/memory/gctransform/framework.py
@@ -52,23 +52,19 @@
             return (op.opname in LL_OPERATIONS and
                     LL_OPERATIONS[op.opname].canmallocgc)
 
-
-
-def find_initializing_stores(collect_analyzer, graph):
-    from rpython.flowspace.model import mkentrymap
-    entrymap = mkentrymap(graph)
-    # a bit of a hackish analysis: if a block contains a malloc and check that
-    # the result is not zero, then the block following the True link will
-    # usually initialize the newly allocated object
-    result = set()
-    def find_in_block(block, mallocvars):
+def propagate_no_write_barrier_needed(result, block, mallocvars,
+                                      collect_analyzer, entrymap):
+    # We definitely know that no write barrier is needed in the 'block'
+    # for any of the variables in 'mallocvars'.  Propagate this information
+    # forward.  Note that "definitely know" implies that we just did either
+    # a fixed-size malloc (variable-size might require card marking), or
+    # that we just did a full write barrier (not just for card marking).
+    if 1:       # keep indentation
         for i, op in enumerate(block.operations):
             if op.opname in ("cast_pointer", "same_as"):
                 if op.args[0] in mallocvars:
                     mallocvars[op.result] = True
             elif op.opname in ("setfield", "setarrayitem", "setinteriorfield"):
-                # note that 'mallocvars' only tracks fixed-size mallocs,
-                # so no risk that they use card marking
                 if op.args[0] in mallocvars:
                     # Collect all assignments, even if they are not storing
                     # a pointer.  This is needed for stmframework and doesn't
@@ -85,7 +81,15 @@
                 if var in mallocvars:
                     newmallocvars[exit.target.inputargs[i]] = True
             if newmallocvars:
-                find_in_block(exit.target, newmallocvars)
+                propagate_no_write_barrier_needed(result, exit.target,
+                                                  newmallocvars,
+                                                  collect_analyzer, entrymap)
+
+def find_initializing_stores(collect_analyzer, graph, entrymap):
+    # a bit of a hackish analysis: if a block contains a malloc and check that
+    # the result is not zero, then the block following the True link will
+    # usually initialize the newly allocated object
+    result = set()
     mallocnum = 0
     blockset = set(graph.iterblocks())
     while blockset:
@@ -115,7 +119,8 @@
         target = exit.target
         mallocvars = {target.inputargs[index]: True}
         mallocnum += 1
-        find_in_block(target, mallocvars)
+        propagate_no_write_barrier_needed(result, target, mallocvars,
+                                          collect_analyzer, entrymap)
     #if result:
     #    print "found %s initializing stores in %s" % (len(result), graph.name)
     return result
@@ -713,8 +718,11 @@
                                 " %s\n%s" % (func, err.getvalue()))
 
         if self.write_barrier_ptr:
+            from rpython.flowspace.model import mkentrymap
+            self._entrymap = mkentrymap(graph)
             self.clean_sets = (
-                find_initializing_stores(self.collect_analyzer, graph))
+                find_initializing_stores(self.collect_analyzer, graph,
+                                         self._entrymap))
             if self.gcdata.gc.can_optimize_clean_setarrayitems():
                 self.clean_sets = self.clean_sets.union(
                     find_clean_setarrayitems(self.collect_analyzer, graph))
@@ -1328,6 +1336,15 @@
                 hop.genop("direct_call", [self.write_barrier_ptr,
                                           self.c_const_gc,
                                           v_structaddr])
+                # we just did a full write barrier here, so we can use
+                # this helper to propagate this knowledge forward and
+                # avoid to repeat the write barrier.
+                if self.curr_block is not None:   # for tests
+                    propagate_no_write_barrier_needed(self.clean_sets,
+                                                      self.curr_block,
+                                                      {v_struct: True},
+                                                      self.collect_analyzer,
+                                                      self._entrymap)
         hop.rename('bare_' + opname)
 
     def transform_getfield_typeptr(self, hop):
diff --git a/rpython/memory/gctransform/test/test_framework.py b/rpython/memory/gctransform/test/test_framework.py
--- a/rpython/memory/gctransform/test/test_framework.py
+++ b/rpython/memory/gctransform/test/test_framework.py
@@ -1,6 +1,6 @@
 from rpython.annotator.listdef import s_list_of_strings
 from rpython.annotator.model import SomeInteger
-from rpython.flowspace.model import Constant, SpaceOperation
+from rpython.flowspace.model import Constant, SpaceOperation, mkentrymap
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rtyper.lltypesystem.lloperation import llop
 from rpython.memory.gc.semispace import SemiSpaceGC
@@ -244,6 +244,33 @@
          Constant('b', lltype.Void), varoftype(PTR_TYPE2)],
         varoftype(lltype.Void)))
 
+def test_remove_duplicate_write_barrier():
+    from rpython.translator.c.genc import CStandaloneBuilder
+    from rpython.flowspace.model import summary
+
+    class A(object):
+        pass
+    glob_a_1 = A()
+    glob_a_2 = A()
+
+    def f(a, cond):
+        a.x = a
+        a.z = a
+        if cond:
+            a.y = a
+    def g():
+        f(glob_a_1, 5)
+        f(glob_a_2, 0)
+    t = rtype(g, [])
+    t.config.translation.gc = "minimark"
+    cbuild = CStandaloneBuilder(t, g, t.config,
+                                gcpolicy=FrameworkGcPolicy2)
+    db = cbuild.generate_graphs_for_llinterp()
+
+    ff = graphof(t, f)
+    #ff.show()
+    assert summary(ff)['direct_call'] == 1    # only one remember_young_pointer
+
 def test_find_initializing_stores():
 
     class A(object):
@@ -259,7 +286,8 @@
     etrafo = ExceptionTransformer(t)
     graphs = etrafo.transform_completely()
     collect_analyzer = CollectAnalyzer(t)
-    init_stores = find_initializing_stores(collect_analyzer, t.graphs[0])
+    init_stores = find_initializing_stores(collect_analyzer, t.graphs[0],
+                                           mkentrymap(t.graphs[0]))
     assert len(init_stores) == 1
 
 def test_find_initializing_stores_across_blocks():
@@ -284,7 +312,8 @@
     etrafo = ExceptionTransformer(t)
     graphs = etrafo.transform_completely()
     collect_analyzer = CollectAnalyzer(t)
-    init_stores = find_initializing_stores(collect_analyzer, t.graphs[0])
+    init_stores = find_initializing_stores(collect_analyzer, t.graphs[0],
+                                           mkentrymap(t.graphs[0]))
     assert len(init_stores) == 5
 
 def test_find_clean_setarrayitems():
diff --git a/rpython/memory/gctransform/transform.py b/rpython/memory/gctransform/transform.py
--- a/rpython/memory/gctransform/transform.py
+++ b/rpython/memory/gctransform/transform.py
@@ -83,6 +83,7 @@
 
 class BaseGCTransformer(object):
     finished_helpers = False
+    curr_block = None
 
     def __init__(self, translator, inline=False):
         self.translator = translator
@@ -159,7 +160,7 @@
 
     def transform_block(self, block, is_borrowed):
         llops = LowLevelOpList()
-        #self.curr_block = block
+        self.curr_block = block
         self.livevars = [var for var in block.inputargs
                     if var_needsgc(var) and not is_borrowed(var)]
         allvars = [var for var in block.getvariables() if var_needsgc(var)]
@@ -205,6 +206,7 @@
             block.operations[:] = llops
         self.livevars = None
         self.var_last_needed_in = None
+        self.curr_block = None
 
     def transform_graph(self, graph):
         if graph in self.minimal_transform:

From noreply at buildbot.pypy.org  Thu Jun 25 22:47:33 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 25 Jun 2015 22:47:33 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c8: Copy this logic here
Message-ID: <20150625204733.4B09F1C0460@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c8
Changeset: r78312:59c5a1d7f46b
Date: 2015-06-25 11:41 +0200
http://bitbucket.org/pypy/pypy/changeset/59c5a1d7f46b/

Log:	Copy this logic here

diff --git a/rpython/memory/gctransform/stmframework.py b/rpython/memory/gctransform/stmframework.py
--- a/rpython/memory/gctransform/stmframework.py
+++ b/rpython/memory/gctransform/stmframework.py
@@ -3,7 +3,8 @@
 from rpython.rtyper.lltypesystem.lloperation import llop
 from rpython.memory.gctransform.support import get_rtti
 from rpython.memory.gctransform.framework import (TYPE_ID,
-     BaseFrameworkGCTransformer, BaseRootWalker, sizeofaddr)
+     BaseFrameworkGCTransformer, BaseRootWalker, sizeofaddr,
+     propagate_no_write_barrier_needed)
 from rpython.memory.gctypelayout import WEAKREF, WEAKREFPTR
 from rpython.memory.gc.stmgc import StmGC
 from rpython.rlib.debug import ll_assert
@@ -168,6 +169,15 @@
             else:
                 self.write_barrier_calls += 1
                 hop.genop("stm_write", [v_struct])
+                # we just did a full write barrier here, so we can use
+                # this helper to propagate this knowledge forward and
+                # avoid to repeat the write barrier.
+                if self.curr_block is not None:   # for tests
+                    propagate_no_write_barrier_needed(self.clean_sets,
+                                                      self.curr_block,
+                                                      {v_struct: True},
+                                                      self.collect_analyzer,
+                                                      self._entrymap)
         hop.rename('bare_' + opname)
 
     def gct_gc_writebarrier(self, hop):

From noreply at buildbot.pypy.org  Thu Jun 25 22:47:34 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 25 Jun 2015 22:47:34 +0200 (CEST)
Subject: [pypy-commit] pypy default: Theoretical fix.  In practice,
 I think it cannot really occur that a
Message-ID: <20150625204734.67E7F1C0460@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78313:8b07e976dab8
Date: 2015-06-25 22:47 +0200
http://bitbucket.org/pypy/pypy/changeset/8b07e976dab8/

Log:	Theoretical fix. In practice, I think it cannot really occur that a
	block contains a canmallocgc operation anywhere else than near the
	end, because it should be followed by the exception check.

diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py
--- a/rpython/memory/gctransform/framework.py
+++ b/rpython/memory/gctransform/framework.py
@@ -53,7 +53,8 @@
                     LL_OPERATIONS[op.opname].canmallocgc)
 
 def propagate_no_write_barrier_needed(result, block, mallocvars,
-                                      collect_analyzer, entrymap):
+                                      collect_analyzer, entrymap,
+                                      startindex=0):
     # We definitely know that no write barrier is needed in the 'block'
     # for any of the variables in 'mallocvars'.  Propagate this information
     # forward.  Note that "definitely know" implies that we just did either
@@ -61,6 +62,8 @@
     # that we just did a full write barrier (not just for card marking).
     if 1:       # keep indentation
         for i, op in enumerate(block.operations):
+            if i < startindex:
+                continue
             if op.opname in ("cast_pointer", "same_as"):
                 if op.args[0] in mallocvars:
                     mallocvars[op.result] = True
@@ -1283,11 +1286,13 @@
                 # this helper to propagate this knowledge forward and
                 # avoid to repeat the write barrier.
                 if self.curr_block is not None:   # for tests
+                    assert self.curr_block.operations[hop.index] is hop.spaceop
                     propagate_no_write_barrier_needed(self.clean_sets,
                                                       self.curr_block,
                                                       {v_struct: True},
                                                       self.collect_analyzer,
-                                                      self._entrymap)
+                                                      self._entrymap,
+                                                      hop.index + 1)
         hop.rename('bare_' + opname)
 
     def transform_getfield_typeptr(self, hop):

From noreply at buildbot.pypy.org  Thu Jun 25 22:47:35 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 25 Jun 2015 22:47:35 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge heads
Message-ID: <20150625204735.ABA771C0460@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78314:6d7ee832797e
Date: 2015-06-25 22:47 +0200
http://bitbucket.org/pypy/pypy/changeset/6d7ee832797e/

Log:	merge heads

diff --git a/pypy/module/_cffi_backend/src/parse_c_type.c b/pypy/module/_cffi_backend/src/parse_c_type.c
--- a/pypy/module/_cffi_backend/src/parse_c_type.c
+++ b/pypy/module/_cffi_backend/src/parse_c_type.c
@@ -362,7 +362,7 @@
 
             case TOK_INTEGER:
                 errno = 0;
-#ifndef MS_WIN32
+#ifndef _MSC_VER
                 if (sizeof(length) > sizeof(unsigned long))
                     length = strtoull(tok->p, &endptr, 0);
                 else
diff --git a/rpython/translator/c/genc.py b/rpython/translator/c/genc.py
--- a/rpython/translator/c/genc.py
+++ b/rpython/translator/c/genc.py
@@ -485,7 +485,7 @@
                 else:
                     mk.definition('DEBUGFLAGS', '-O1 -g')
         if self.translator.platform.name == 'msvc':
-            mk.rule('debug_target', 'debugmode_$(DEFAULT_TARGET)', 'rem')
+            mk.rule('debug_target', '$(DEFAULT_TARGET)', 'rem')
         else:
             mk.rule('debug_target', '$(DEFAULT_TARGET)', '#')
         mk.write()

From noreply at buildbot.pypy.org  Thu Jun 25 22:58:09 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 25 Jun 2015 22:58:09 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c8: Theoretical fix.  In practice,
 I think it cannot really occur that a
Message-ID: <20150625205809.487761C0460@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c8
Changeset: r78315:60f39963e5a8
Date: 2015-06-25 22:47 +0200
http://bitbucket.org/pypy/pypy/changeset/60f39963e5a8/

Log:	Theoretical fix. In practice, I think it cannot really occur that a
	block contains a canmallocgc operation anywhere else than near the
	end, because it should be followed by the exception check.

diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py
--- a/rpython/memory/gctransform/framework.py
+++ b/rpython/memory/gctransform/framework.py
@@ -53,7 +53,8 @@
                     LL_OPERATIONS[op.opname].canmallocgc)
 
 def propagate_no_write_barrier_needed(result, block, mallocvars,
-                                      collect_analyzer, entrymap):
+                                      collect_analyzer, entrymap,
+                                      startindex=0):
     # We definitely know that no write barrier is needed in the 'block'
     # for any of the variables in 'mallocvars'.  Propagate this information
     # forward.  Note that "definitely know" implies that we just did either
@@ -61,6 +62,8 @@
     # that we just did a full write barrier (not just for card marking).
     if 1:       # keep indentation
         for i, op in enumerate(block.operations):
+            if i < startindex:
+                continue
             if op.opname in ("cast_pointer", "same_as"):
                 if op.args[0] in mallocvars:
                     mallocvars[op.result] = True
@@ -1340,11 +1343,13 @@
                 # this helper to propagate this knowledge forward and
                 # avoid to repeat the write barrier.
                 if self.curr_block is not None:   # for tests
+                    assert self.curr_block.operations[hop.index] is hop.spaceop
                     propagate_no_write_barrier_needed(self.clean_sets,
                                                       self.curr_block,
                                                       {v_struct: True},
                                                       self.collect_analyzer,
-                                                      self._entrymap)
+                                                      self._entrymap,
+                                                      hop.index + 1)
         hop.rename('bare_' + opname)
 
     def transform_getfield_typeptr(self, hop):

From noreply at buildbot.pypy.org  Thu Jun 25 23:10:11 2015
From: noreply at buildbot.pypy.org (amauryfa)
Date: Thu, 25 Jun 2015 23:10:11 +0200 (CEST)
Subject: [pypy-commit] pypy more-rposix: Another function to hide from the
	jit.
Message-ID: <20150625211011.1406D1C0589@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: more-rposix
Changeset: r78316:2151aaead9c9
Date: 2015-06-25 23:09 +0200
http://bitbucket.org/pypy/pypy/changeset/2151aaead9c9/

Log:	Another function to hide from the jit. I don't know why they are
	needed now. Maybe because llimpl functions were also never jitted?

diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -1349,6 +1349,7 @@
     return handle_posix_error('killpg', c_killpg(pgrp, sig))
 
 @replace_os_function('_exit')
+ at jit.dont_look_inside
 def exit(status):
     debug.debug_flush()
     c_exit(status)

From noreply at buildbot.pypy.org  Fri Jun 26 09:56:34 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Fri, 26 Jun 2015 09:56:34 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: passing all scheduling tests again
Message-ID: <20150626075634.296031C02BB@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r78317:369284ff1423
Date: 2015-06-25 12:21 +0200
http://bitbucket.org/pypy/pypy/changeset/369284ff1423/

Log:	passing all scheduling tests again

diff --git a/rpython/jit/metainterp/optimizeopt/schedule.py b/rpython/jit/metainterp/optimizeopt/schedule.py
--- a/rpython/jit/metainterp/optimizeopt/schedule.py
+++ b/rpython/jit/metainterp/optimizeopt/schedule.py
@@ -250,16 +250,12 @@
             # prohibit the packing of signext calls that
             # cast to int16/int8.
             _, outsize = op0.cast_to()
-            self._prevent_signext(outsize, insize)
+            self.sched_data._prevent_signext(outsize, insize)
         if op0.getopnum() == rop.INT_MUL:
             if insize == 8 or insize == 1:
                 # see assembler for comment why
                 raise NotAProfitableLoop
 
-    def _prevent_signext(self, outsize, insize):
-        if outsize < 4 and insize != outsize:
-            raise NotAProfitableLoop
-
     def as_vector_operation(self, pack, sched_data, oplist):
         self.sched_data = sched_data
         self.preamble_ops = oplist
@@ -394,7 +390,7 @@
 
     def extend_int(self, vbox, newtype):
         vbox_cloned = newtype.new_vector_box(vbox.item_count)
-        self._prevent_signext(newtype.getsize(), vbox.getsize())
+        self.sched_data._prevent_signext(newtype.getsize(), vbox.getsize())
         op = ResOperation(rop.VEC_INT_SIGNEXT, 
                           [vbox, ConstInt(newtype.getsize())],
                           vbox_cloned)
@@ -676,6 +672,10 @@
         self.expanded_map = {}
         self.costmodel = costmodel
 
+    def _prevent_signext(self, outsize, insize):
+        if outsize < 4 and insize != outsize:
+            raise NotAProfitableLoop
+
     def as_vector_operation(self, pack, preproc_renamer):
         assert pack.opcount() > 1
         # properties that hold for the pack are:
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_schedule.py b/rpython/jit/metainterp/optimizeopt/test/test_schedule.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_schedule.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_schedule.py
@@ -61,7 +61,7 @@
     def pack(self, loop, l, r):
         return Pack([Node(op,1+l+i) for i,op in enumerate(loop.operations[1+l:1+r])], None, None)
 
-    def schedule(self, loop_orig, packs, vec_reg_size=16, prepend_invariant=False, getvboxfunc=None):
+    def schedule(self, loop_orig, packs, vec_reg_size=16, prepend_invariant=False, overwrite_funcs=None):
         loop = get_model(False).ExtendedTreeLoop("loop")
         loop.original_jitcell_token = loop_orig.original_jitcell_token
         loop.inputargs = loop_orig.inputargs
@@ -69,8 +69,8 @@
         ops = []
         cm = X86_CostModel(0, vec_reg_size)
         vsd = VecScheduleData(vec_reg_size, cm)
-        if getvboxfunc is not None:
-            vsd.getvector_of_box = getvboxfunc
+        for name, overwrite in (overwrite_funcs or {}).items():
+            setattr(vsd, name, overwrite)
         renamer = Renamer()
         for pack in packs:
             if pack.opcount() == 1:
@@ -191,7 +191,10 @@
         var = self.find_input_arg('v10', loop1)
         def i1inv103204(v):
             return 0, var
-        loop2 = self.schedule(loop1, [pack1], prepend_invariant=True, getvboxfunc=i1inv103204)
+        loop2 = self.schedule(loop1, [pack1], prepend_invariant=True,
+                              overwrite_funcs = {
+                                'getvector_of_box': i1inv103204,
+                              })
         loop3 = self.parse("""
         v11[i32|2] = vec_int_signext(v10[i64|2], 4)
         """, False, additional_args=['v10[i64|2]'])
@@ -239,7 +242,12 @@
         pack2 = self.pack(loop1, 8, 16)
         pack3 = self.pack(loop1, 16, 24)
         pack4 = self.pack(loop1, 24, 32)
-        loop2 = self.schedule(loop1, [pack1,pack2,pack3,pack4])
+        def void(b,c):
+            pass
+        loop2 = self.schedule(loop1, [pack1,pack2,pack3,pack4],
+                              overwrite_funcs={
+                                  '_prevent_signext': void
+                              })
         loop3 = self.parse("""
         v10[f64|2] = vec_raw_load(p0, i1, 2, descr=double)
         v11[f64|2] = vec_raw_load(p0, i3, 2, descr=double)
diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/vectorize.py
@@ -76,10 +76,11 @@
                        ns))
         debug_stop("xxx-clock")
     except NotAVectorizeableLoop:
+        debug_stop("vec-opt-loop")
         # vectorization is not possible
         loop.operations = orig_ops
+    except NotAProfitableLoop:
         debug_stop("vec-opt-loop")
-    except NotAProfitableLoop:
         # cost model says to skip this loop
         loop.operations = orig_ops
     except Exception as e:
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -814,8 +814,8 @@
     rop.CAST_FLOAT_TO_INT: rop.VEC_CAST_FLOAT_TO_INT,
 
     # guard
-    #rop.GUARD_TRUE: rop.GUARD_TRUE,
-    #rop.GUARD_FALSE: rop.GUARD_FALSE,
+    rop.GUARD_TRUE: rop.GUARD_TRUE,
+    rop.GUARD_FALSE: rop.GUARD_FALSE,
 }
 
 

From noreply at buildbot.pypy.org  Fri Jun 26 09:56:35 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Fri, 26 Jun 2015 09:56:35 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: don't follow wrong dependency chains,
	excluded fail args
Message-ID: <20150626075635.93F0E1C02BB@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r78318:8afb499c0584
Date: 2015-06-26 09:56 +0200
http://bitbucket.org/pypy/pypy/changeset/8afb499c0584/

Log:	don't follow wrong dependency chains, excluded fail args only store
	is not allowed to compute operations if the vector is not fully
	packed

diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -2722,7 +2722,9 @@
 
     def genop_vec_int_expand(self, op, arglocs, resloc):
         srcloc, sizeloc = arglocs
-        assert isinstance(srcloc, RegLoc)
+        if not isinstance(srcloc, RegLoc):
+            self.mov(X86_64_SCRATCH_REG, srcloc)
+            srcloc = X86_64_SCRATCH_REG
         assert not srcloc.is_xmm
         size = sizeloc.value
         if size == 1:
diff --git a/rpython/jit/metainterp/optimizeopt/schedule.py b/rpython/jit/metainterp/optimizeopt/schedule.py
--- a/rpython/jit/metainterp/optimizeopt/schedule.py
+++ b/rpython/jit/metainterp/optimizeopt/schedule.py
@@ -269,8 +269,10 @@
         left = len(pack.operations)
         assert stride > 0
         while off < len(pack.operations):
-            if left < stride:
-                self.preamble_ops.append(pack.operations[off].getoperation())
+            print left, "<", stride
+            if stride == 1:
+                op = pack.operations[off].getoperation()
+                self.preamble_ops.append(op)
                 off += 1
                 continue
             ops = pack.operations[off:off+stride]
@@ -294,9 +296,6 @@
         if bytes > vec_reg_size:
             # too many bytes. does not fit into the vector register
             return vec_reg_size // self.getscalarsize()
-        if bytes < vec_reg_size:
-            # not enough to fill the vector register
-            return 1
         return pack.opcount()
 
     def getscalarsize(self):
@@ -316,12 +315,16 @@
             if isinstance(arg, BoxVector):
                 continue
             if self.is_vector_arg(i):
-                args[i] = self.transform_argument(args[i], i, off)
+                args[i] = self.transform_argument(args[i], i, off, stride)
         #
         result = op.result
         result = self.transform_result(result, off)
         #
         vop = ResOperation(op.vector, args, result, op.getdescr())
+        if op.is_guard():
+            assert isinstance(op, GuardResOp)
+            vop.setfailargs(op.getfailargs())
+            vop.rd_snapshot = op.rd_snapshot
         self.preamble_ops.append(vop)
 
     def transform_result(self, result, off):
@@ -342,7 +345,7 @@
         signed = self.output_type.signed
         return BoxVector(type, count, size, signed)
 
-    def transform_argument(self, arg, argidx, off):
+    def transform_argument(self, arg, argidx, off, stride):
         ops = self.pack.operations
         box_pos, vbox = self.sched_data.getvector_of_box(arg)
         if not vbox:
@@ -359,7 +362,8 @@
         packed = vbox.item_count
         assert packed >= 0
         assert packable >= 0
-        if packed < packable:
+        vboxes = self.vector_boxes_for_args(argidx)
+        if len(vboxes) > 1: # packed < packable and packed < stride:
             # the argument is scattered along different vector boxes
             args = [op.getoperation().getarg(argidx) for op in ops]
             vbox = self._pack(vbox, packed, args, packable)
@@ -379,8 +383,20 @@
             vbox = self.unpack(vbox, args, off, len(ops), self.input_type)
             self.update_input_output(self.pack)
         #
+        assert vbox is not None
         return vbox
 
+    def vector_boxes_for_args(self, index):
+        args = [op.getoperation().getarg(index) for op in self.pack.operations]
+        vboxes = []
+        last_vbox = None
+        for arg in args:
+            pos, vbox = self.sched_data.getvector_of_box(arg)
+            if vbox != last_vbox and vbox is not None:
+                vboxes.append(vbox)
+        return vboxes
+
+
     def extend(self, vbox, newtype):
         assert vbox.gettype() == newtype.gettype()
         if vbox.gettype() == INT:
@@ -443,6 +459,7 @@
                 self.sched_data.setvector_of_box(arg, j, new_box)
             tgt_box = new_box
         _, vbox = self.sched_data.getvector_of_box(args[0])
+        assert vbox is not None
         return vbox
 
     def _check_vec_pack(self, op):
@@ -589,6 +606,11 @@
         return BoxVector(type, count, size, signed)
 
 class StoreToVectorStore(OpToVectorOp):
+    """
+    Storing operations are special because they are not allowed
+    to store to memory if the vector is not fully filled.
+    Thus a modified split_pack function
+    """
     def __init__(self):
         OpToVectorOp.__init__(self, (None, None, PT_GENERIC), None)
         self.has_descr = True
@@ -599,6 +621,20 @@
     def determine_output_type(self, op):
         return None
 
+    def split_pack(self, pack, vec_reg_size):
+        """ Returns how many items of the pack should be
+            emitted as vector operation. """
+        bytes = pack.opcount() * self.getscalarsize()
+        if bytes > vec_reg_size:
+            # too many bytes. does not fit into the vector register
+            return vec_reg_size // self.getscalarsize()
+        if bytes < vec_reg_size:
+            # special case for store, even though load is allowed
+            # to load more, store is not!
+            # not enough to fill the vector register
+            return 1
+        return pack.opcount()
+
 class PassThroughOp(OpToVectorOp):
     """ This pass through is only applicable if the target
     operation is capable of handling vector operations.
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_schedule.py b/rpython/jit/metainterp/optimizeopt/test/test_schedule.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_schedule.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_schedule.py
@@ -79,6 +79,16 @@
                 for op in vsd.as_vector_operation(pack, renamer):
                     ops.append(op)
         loop.operations = ops
+        metainterp_sd = FakeMetaInterpStaticData(self.cpu)
+        jitdriver_sd = FakeJitDriverStaticData()
+        opt = VectorizingOptimizer(metainterp_sd, jitdriver_sd, loop, 0)
+        opt.clear_newoperations()
+        for op in ops:
+            opt.unpack_from_vector(op, vsd, renamer)
+            opt.emit_operation(op)
+        ops = opt._newoperations
+        loop.operations = ops
+
         if prepend_invariant:
             loop.operations = vsd.invariant_oplist + ops
         return loop
@@ -100,8 +110,7 @@
         loop2 = self.schedule(loop1, [pack1])
         loop3 = self.parse("""
         v10[i32|4] = vec_raw_load(p0, i0, 4, descr=float)
-        i14 = raw_load(p0, i4, descr=float)
-        i15 = raw_load(p0, i5, descr=float)
+        v11[i32|2] = vec_raw_load(p0, i4, 2, descr=float)
         """, False)
         self.assert_equal(loop2, loop3)
 
@@ -109,12 +118,15 @@
         loop1 = self.parse("""
         i10 = raw_load(p0, i0, descr=long)
         i11 = raw_load(p0, i1, descr=long)
-        f10 = cast_int_to_float(i10)
-        f11 = cast_int_to_float(i11)
+        i12 = int_signext(i10, 4)
+        i13 = int_signext(i11, 4)
+        f10 = cast_int_to_float(i12)
+        f11 = cast_int_to_float(i13)
         """)
         pack1 = self.pack(loop1, 0, 2)
         pack2 = self.pack(loop1, 2, 4)
-        loop2 = self.schedule(loop1, [pack1, pack2])
+        pack3 = self.pack(loop1, 4, 6)
+        loop2 = self.schedule(loop1, [pack1, pack2, pack3])
         loop3 = self.parse("""
         v10[i64|2] = vec_raw_load(p0, i0, 2, descr=long)
         v20[i32|2] = vec_int_signext(v10[i64|2], 4)
@@ -321,3 +333,54 @@
         guard_true(v11[i64|2]) []
         """, False)
         self.assert_equal(loop2, loop3)
+
+
+    def test_split_load_store(self):
+        loop1 = self.parse("""
+        i10 = raw_load(p0, i1, descr=float)
+        i11 = raw_load(p0, i2, descr=float)
+        raw_store(p0, i3, i10, descr=float)
+        raw_store(p0, i4, i11, descr=float)
+        """)
+        pack1 = self.pack(loop1, 0, 2)
+        pack2 = self.pack(loop1, 2, 4)
+        loop2 = self.schedule(loop1, [pack1,pack2], prepend_invariant=True)
+        loop3 = self.parse("""
+        v1[ui32|2] = vec_raw_load(p0, i1, 2, descr=float)
+        i10 = vec_int_unpack(v1[ui32|2], 0, 1)
+        raw_store(p0, i3, i10, descr=float)
+        i11 = vec_int_unpack(v1[ui32|2], 1, 1)
+        raw_store(p0, i4, i11, descr=float)
+        """, False)
+        # unfortunate ui32 is the type for float32... the unsigned u is for
+        # the tests
+        self.assert_equal(loop2, loop3)
+
+    def test_split_arith(self):
+        loop1 = self.parse("""
+        i10 = int_and(255, i1)
+        i11 = int_and(255, i1)
+        """)
+        pack1 = self.pack(loop1, 0, 2)
+        loop2 = self.schedule(loop1, [pack1], prepend_invariant=True)
+        loop3 = self.parse("""
+        v1[i64|2] = vec_int_expand(255)
+        v2[i64|2] = vec_int_expand(i1)
+        v3[i64|2] = vec_int_and(v1[i64|2], v2[i64|2])
+        """, False)
+        self.assert_equal(loop2, loop3)
+
+    def test_split_arith(self):
+        loop1 = self.parse("""
+        i10 = int_and(255, i1)
+        i11 = int_and(255, i1)
+        """)
+        pack1 = self.pack(loop1, 0, 2)
+        loop2 = self.schedule(loop1, [pack1], prepend_invariant=True)
+        loop3 = self.parse("""
+        v1[i64|2] = vec_int_expand(255)
+        v2[i64|2] = vec_int_expand(i1)
+        v3[i64|2] = vec_int_and(v1[i64|2], v2[i64|2])
+        """, False)
+        self.assert_equal(loop2, loop3)
+
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py b/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
@@ -1397,21 +1397,18 @@
         jump(p0, p1, p5, p6, p7, p17, p19, i68, i39, i44, i49, i51)
         """
         trace="""
-        [p3, i4, p1, i5, i6, i7]
-        guard_early_exit(descr=<ResumeAtLoopHeaderDescr object at 0x7f3afe4fb830>) [p1, i5, i4, p3]
-        i8 = raw_load(i6, i5, descr=intarraydescr)
-        guard_not_invalidated(descr=<ResumeGuardNotInvalidated object at 0x7f3afe4fb888>) [p1, i8, i5, i4, p3]
-        i10 = int_and(i8, 255)
-        guard_false(i10, descr=<ResumeGuardFalseDescr object at 0x7f3afe4fb8e0>) [p1, i5, i4, p3]
-        i13 = getarrayitem_raw(139891327308826, 2, descr=chararraydescr)
-        guard_value(i13, 1, descr=<ResumeGuardValueDescr object at 0x7f3afe4fb938>) [i13, p1, i5, i4, p3]
-        i17 = getarrayitem_raw(139891327308824, 1, descr=chararraydescr)
-        i19 = int_add(i4, 1)
-        i21 = int_add(i5, 8)
-        i22 = int_ge(i19, i7)
-        guard_false(i22, descr=<ResumeGuardFalseDescr object at 0x7f3afe4fb990>) [i17, p1, i21, i19, None, None, p3]
-        guard_value(i17, 2, descr=<ResumeGuardValueDescr object at 0x7f3afe4fb9e8>) [i17, p1, i21, i19, None, None, p3]
-        jump(p3, i19, p1, i21, i6, i7)
+        [p0, p3, i4, i5, i6, i7]
+        guard_early_exit(descr=<rpython.jit.metainterp.compile.ResumeAtLoopHeaderDescr object at 0x7f492da84250>) [p0, p3, i4, i5]
+        f8 = raw_load(i6, i5, descr=floatarraydescr)
+        guard_not_invalidated(descr=<rpython.jit.metainterp.compile.ResumeGuardNotInvalidated object at 0x7f492da846d0>) [p0, f8, p3, i4, i5]
+        i9 = cast_float_to_int(f8)
+        i11 = int_and(i9, 255)
+        guard_true(i11, descr=<rpython.jit.metainterp.compile.ResumeGuardTrueDescr object at 0x7f492da8b790>) [p0, p3, i4, i5]
+        i13 = int_add(i4, 1)
+        i15 = int_add(i5, 8)
+        i16 = int_ge(i13, i7)
+        guard_false(i16, descr=<rpython.jit.metainterp.compile.ResumeGuardFalseDescr object at 0x7f492da93610>) [p0, i13, i15, p3, None, None]
+        jump(p0, p3, i13, i15, i6, i7)
         """
         opt = self.vectorize(self.parse_loop(trace))
         self.debug_print_operations(opt.loop)
diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/vectorize.py
@@ -339,6 +339,11 @@
             for rdep in pack.right.depends():
                 lnode = ldep.to
                 rnode = rdep.to
+                # only valid if the result of the left is in args of pack left
+                result = lnode.getoperation().result
+                args = pack.left.getoperation().getarglist()
+                if result is None or result not in args:
+                    continue
                 isomorph = isomorphic(lnode.getoperation(), rnode.getoperation())
                 if isomorph and lnode.is_before(rnode):
                     pair = self.packset.can_be_packed(lnode, rnode, pack, False)
@@ -351,6 +356,10 @@
             for rdep in pack.right.provides():
                 lnode = ldep.to
                 rnode = rdep.to
+                result = pack.left.getoperation().result
+                args = lnode.getoperation().getarglist()
+                if result is None or result not in args:
+                    continue
                 isomorph = isomorphic(lnode.getoperation(), rnode.getoperation())
                 if isomorph and lnode.is_before(rnode):
                     pair = self.packset.can_be_packed(lnode, rnode, pack, True)
diff --git a/rpython/jit/metainterp/warmstate.py b/rpython/jit/metainterp/warmstate.py
--- a/rpython/jit/metainterp/warmstate.py
+++ b/rpython/jit/metainterp/warmstate.py
@@ -379,7 +379,7 @@
             # so that it will keep it alive for a longer time
             warmrunnerdesc.memory_manager.keep_loop_alive(loop_token)
             # XXX debug purpose only
-            jitdriver_sd.xxxbench.xxx_clock_stop()
+            jitdriver_sd.xxxbench.xxx_clock_stop(fail=True)
             # XXX debug purpose only end
             #
             # Handle the failure

From noreply at buildbot.pypy.org  Fri Jun 26 10:39:32 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Fri, 26 Jun 2015 10:39:32 +0200 (CEST)
Subject: [pypy-commit] pypy pypy3-release-2.6.x: fix annotation on narrow
	builds (win32)
Message-ID: <20150626083932.864561C02BB@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: pypy3-release-2.6.x
Changeset: r78319:17eb1202a1c0
Date: 2015-06-26 11:39 +0300
http://bitbucket.org/pypy/pypy/changeset/17eb1202a1c0/

Log:	fix annotation on narrow builds (win32)

diff --git a/pypy/module/_codecs/locale.py b/pypy/module/_codecs/locale.py
--- a/pypy/module/_codecs/locale.py
+++ b/pypy/module/_codecs/locale.py
@@ -10,6 +10,7 @@
 from rpython.rlib.runicode import (code_to_unichr,
     default_unicode_error_decode, default_unicode_error_encode)
 from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.rlib.rarithmetic import widen
 from rpython.translator import cdir
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
 
@@ -144,8 +145,11 @@
 def rawwcharp2unicoden(wcp, maxlen):
     b = UnicodeBuilder(maxlen)
     i = 0
-    while i < maxlen and rffi.cast(lltype.Signed, wcp[i]) != 0:
-        b.append(code_to_unichr(wcp[i]))
+    while i < maxlen:
+        wcp_i = widen(wcp[i])
+        if wcp_i == 0:
+            break
+        b.append(code_to_unichr(wcp_i))
         i += 1
     return assert_str0(b.build())
 rawwcharp2unicoden._annenforceargs_ = [None, int]

From noreply at buildbot.pypy.org  Fri Jun 26 12:29:05 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Fri, 26 Jun 2015 12:29:05 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: splitting must be done differently
Message-ID: <20150626102905.D773B1C0460@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r78320:835955fe1216
Date: 2015-06-26 10:22 +0200
http://bitbucket.org/pypy/pypy/changeset/835955fe1216/

Log:	splitting must be done differently

diff --git a/rpython/jit/metainterp/optimizeopt/schedule.py b/rpython/jit/metainterp/optimizeopt/schedule.py
--- a/rpython/jit/metainterp/optimizeopt/schedule.py
+++ b/rpython/jit/metainterp/optimizeopt/schedule.py
@@ -369,7 +369,7 @@
             vbox = self._pack(vbox, packed, args, packable)
             self.update_input_output(self.pack)
             box_pos = 0
-        elif packed > packable and box_pos != 0:
+        elif packed > packable:
             # box_pos == 0 then it is already at the right place
             # the argument has more items than the operation is able to process!
             args = [op.getoperation().getarg(argidx) for op in ops]
@@ -377,6 +377,7 @@
             self.update_input_output(self.pack)
             box_pos = 0
         elif off != 0 and box_pos != 0:
+            import py; py.test.set_trace()
             # The original box is at a position != 0 but it
             # is required to be at position 0. Unpack it!
             args = [op.getoperation().getarg(argidx) for op in ops]
@@ -542,11 +543,11 @@
         return self.result_ptype
 
     def split_pack(self, pack, vec_reg_size):
-        op0 = pack.operations[0].getoperation()
-        _, vbox = self.sched_data.getvector_of_box(op0.getarg(0))
-        if vbox.getcount() * self.to_size > vec_reg_size:
-            return vec_reg_size // self.to_size
-        return vbox.getcount()
+        count = self.arg_ptypes[0].getcount()
+        bytes = pack.opcount() * self.getscalarsize()
+        if bytes > count * self.from_size:
+            return bytes // (count * self.from_size)
+        return pack.opcount()
 
     def new_result_vector_box(self):
         type = self.output_type.gettype()
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py b/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
@@ -1397,18 +1397,21 @@
         jump(p0, p1, p5, p6, p7, p17, p19, i68, i39, i44, i49, i51)
         """
         trace="""
-        [p0, p3, i4, i5, i6, i7]
-        guard_early_exit(descr=<rpython.jit.metainterp.compile.ResumeAtLoopHeaderDescr object at 0x7f492da84250>) [p0, p3, i4, i5]
-        f8 = raw_load(i6, i5, descr=floatarraydescr)
-        guard_not_invalidated(descr=<rpython.jit.metainterp.compile.ResumeGuardNotInvalidated object at 0x7f492da846d0>) [p0, f8, p3, i4, i5]
-        i9 = cast_float_to_int(f8)
-        i11 = int_and(i9, 255)
-        guard_true(i11, descr=<rpython.jit.metainterp.compile.ResumeGuardTrueDescr object at 0x7f492da8b790>) [p0, p3, i4, i5]
-        i13 = int_add(i4, 1)
-        i15 = int_add(i5, 8)
-        i16 = int_ge(i13, i7)
-        guard_false(i16, descr=<rpython.jit.metainterp.compile.ResumeGuardFalseDescr object at 0x7f492da93610>) [p0, i13, i15, p3, None, None]
-        jump(p0, p3, i13, i15, i6, i7)
+        [p0, p1, p9, i10, p4, i11, p3, p6, p12, i13, i14, i15, f16, i17, i18]
+        guard_early_exit(descr=<rpython.jit.metainterp.compile.ResumeAtLoopHeaderDescr object at 0x7f2327d4b390>) [p6, p4, p3, p1, p0, i14, i10, i13, i11, p9, p12]
+        i19 = raw_load(i15, i11, descr=singlefloatarraydescr)
+        guard_not_invalidated(descr=<rpython.jit.metainterp.compile.ResumeGuardNotInvalidated object at 0x7f23284786d0>) [p6, p4, p3, p1, p0, i19, i14, i10, i13, i11, p9, p12]
+        i21 = int_add(i11, 4)
+        f22 = cast_singlefloat_to_float(i19)
+        f23 = float_add(f22, f16)
+        i24 = cast_float_to_singlefloat(f23)
+        raw_store(i17, i14, i24, descr=singlefloatarraydescr)
+        i26 = int_add(i13, 1)
+        i28 = int_add(i14, 4)
+        i29 = int_ge(i26, i18)
+        guard_false(i29, descr=<rpython.jit.metainterp.compile.ResumeGuardFalseDescr object at 0x7f2327d53910>) [p6, p4, p3, p1, p0, i28, i21, i26, None, i10, None, None, p9, p12]
+        debug_merge_point(0, 0, '(numpy_call2: no get_printable_location)')
+        jump(p0, p1, p9, i10, p4, i21, p3, p6, p12, i26, i28, i15, f16, i17, i18)
         """
         opt = self.vectorize(self.parse_loop(trace))
         self.debug_print_operations(opt.loop)

From noreply at buildbot.pypy.org  Fri Jun 26 12:29:07 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Fri, 26 Jun 2015 12:29:07 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: rewritten splitting of packs,
 added asserts to ensure the impl assumptions are correct. some
 tests broke (it is not yet finished)
Message-ID: <20150626102907.0A1731C0683@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r78321:2e935c4aa59d
Date: 2015-06-26 12:29 +0200
http://bitbucket.org/pypy/pypy/changeset/2e935c4aa59d/

Log:	rewritten splitting of packs, added asserts to ensure the impl
	assumptions are correct. some tests broke (it is not yet finished)

diff --git a/rpython/jit/metainterp/optimizeopt/schedule.py b/rpython/jit/metainterp/optimizeopt/schedule.py
--- a/rpython/jit/metainterp/optimizeopt/schedule.py
+++ b/rpython/jit/metainterp/optimizeopt/schedule.py
@@ -264,23 +264,8 @@
         #
         self.check_if_pack_supported(pack)
         #
-        off = 0
-        stride = self.split_pack(pack, self.sched_data.vec_reg_size)
-        left = len(pack.operations)
-        assert stride > 0
-        while off < len(pack.operations):
-            print left, "<", stride
-            if stride == 1:
-                op = pack.operations[off].getoperation()
-                self.preamble_ops.append(op)
-                off += 1
-                continue
-            ops = pack.operations[off:off+stride]
-            self.pack = Pack(ops, pack.input_type, pack.output_type)
-            self.costmodel.record_pack_savings(self.pack)
-            self.transform_pack(ops, off, stride)
-            off += stride
-            left -= stride
+        self.pack = pack
+        self.transform_pack()
 
         self.pack = None
         self.costmodel = None
@@ -305,35 +290,52 @@
     def before_argument_transform(self, args):
         pass
 
-    def transform_pack(self, ops, off, stride):
-        op = self.pack.operations[0].getoperation()
-        args = op.getarglist()
-        #
-        self.before_argument_transform(args)
-        #
-        for i,arg in enumerate(args):
-            if isinstance(arg, BoxVector):
-                continue
-            if self.is_vector_arg(i):
-                args[i] = self.transform_argument(args[i], i, off, stride)
-        #
-        result = op.result
-        result = self.transform_result(result, off)
-        #
-        vop = ResOperation(op.vector, args, result, op.getdescr())
-        if op.is_guard():
-            assert isinstance(op, GuardResOp)
-            vop.setfailargs(op.getfailargs())
-            vop.rd_snapshot = op.rd_snapshot
-        self.preamble_ops.append(vop)
+    def transform_pack(self):
+        self.off = 0
+        while self.off < self.pack.opcount():
+            op = self.pack.operations[self.off].getoperation()
+            args = op.getarglist()
+            #
+            self.before_argument_transform(args)
+            #
+            argument_infos = []
+            self.transform_arguments(args, argument_infos)
+            #
+            result = op.result
+            result = self.transform_result(result)
+            #
+            vop = ResOperation(op.vector, args, result, op.getdescr())
+            if op.is_guard():
+                assert isinstance(op, GuardResOp)
+                vop.setfailargs(op.getfailargs())
+                vop.rd_snapshot = op.rd_snapshot
+            self.preamble_ops.append(vop)
+            stride = self.consumed_operations(argument_infos, result)
+            self.costmodel.record_pack_savings(self.pack, stride)
+            assert stride != 0
+            self.off += stride
 
-    def transform_result(self, result, off):
+    def consumed_operations(self, argument_infos, result):
+        ops = self.getoperations()
+        if len(argument_infos) == 0:
+            return result.getcount()
+        if len(argument_infos) == 1:
+            return argument_infos[0]
+        if not we_are_translated():
+            first = argument_infos[0]
+            for ai in argument_infos:
+                assert first == ai
+        return argument_infos[0]
+
+    def transform_result(self, result):
         if result is None:
             return None
         vbox = self.new_result_vector_box()
         #
         # mark the position and the vbox in the hash
-        for i, node in enumerate(self.pack.operations):
+        for i, node in enumerate(self.getoperations()):
+            if i >= vbox.item_count:
+                break
             op = node.getoperation()
             self.sched_data.setvector_of_box(op.result, i, vbox)
         return vbox
@@ -345,56 +347,99 @@
         signed = self.output_type.signed
         return BoxVector(type, count, size, signed)
 
-    def transform_argument(self, arg, argidx, off, stride):
-        ops = self.pack.operations
-        box_pos, vbox = self.sched_data.getvector_of_box(arg)
-        if not vbox:
-            # constant/variable expand this box
-            vbox = self.expand(ops, arg, argidx)
-            box_pos = 0
-        # convert size i64 -> i32, i32 -> i64, ...
-        if self.input_type.getsize() > 0 and \
-           self.input_type.getsize() != vbox.getsize():
-            vbox = self.extend(vbox, self.input_type)
+    def getoperations(self):
+        return self.pack.operations[self.off:]
 
-        # use the input as an indicator for the pack type
-        packable = self.input_type.getcount()
-        packed = vbox.item_count
-        assert packed >= 0
-        assert packable >= 0
-        vboxes = self.vector_boxes_for_args(argidx)
-        if len(vboxes) > 1: # packed < packable and packed < stride:
-            # the argument is scattered along different vector boxes
-            args = [op.getoperation().getarg(argidx) for op in ops]
-            vbox = self._pack(vbox, packed, args, packable)
-            self.update_input_output(self.pack)
-            box_pos = 0
-        elif packed > packable:
-            # box_pos == 0 then it is already at the right place
-            # the argument has more items than the operation is able to process!
-            args = [op.getoperation().getarg(argidx) for op in ops]
-            vbox = self.unpack(vbox, args, off, packable, self.input_type)
-            self.update_input_output(self.pack)
-            box_pos = 0
-        elif off != 0 and box_pos != 0:
-            import py; py.test.set_trace()
-            # The original box is at a position != 0 but it
-            # is required to be at position 0. Unpack it!
-            args = [op.getoperation().getarg(argidx) for op in ops]
-            vbox = self.unpack(vbox, args, off, len(ops), self.input_type)
-            self.update_input_output(self.pack)
-        #
-        assert vbox is not None
-        return vbox
+    def transform_arguments(self, args, argument_info):
+        for i,arg in enumerate(args):
+            if isinstance(arg, BoxVector):
+                continue
+            if not self.is_vector_arg(i):
+                continue
+            box_pos, vbox = self.sched_data.getvector_of_box(arg)
+            if not vbox:
+                # constant/variable expand this box
+                vbox = self.expand(arg, i)
+                self.sched_data.setvector_of_box(arg, 0, vbox)
+                box_pos = 0
+            # convert size i64 -> i32, i32 -> i64, ...
+            if self.input_type.getsize() > 0 and \
+               self.input_type.getsize() != vbox.getsize():
+                vbox = self.extend(vbox, self.input_type)
+
+            # use the input as an indicator for the pack type
+            packable = self.input_type.getcount()
+            packed = vbox.item_count
+            assert packed >= 0
+            assert packable >= 0
+            if packed > packable:
+                # the argument has more items than the operation is able to process!
+                # box_pos == 0 then it is already at the right place
+                argument_info.append(packable)
+                if box_pos != 0:
+                    args[i] = self.unpack(vbox, self.off, packable, self.input_type)
+                    self.update_arg_in_vector_pos(i, args[i])
+                    #self.update_input_output(self.pack)
+                    continue
+                else:
+                    assert vbox is not None
+                    args[i] = vbox
+                    continue
+            vboxes = self.vector_boxes_for_args(i)
+            if packed < packable and len(vboxes) > 1:
+                # the argument is scattered along different vector boxes
+                args[i] = self.gather(vboxes, packable)
+                self.update_arg_in_vector_pos(i, args[i])
+                argument_info.append(args[i].item_count)
+                continue
+            if box_pos != 0:
+                # The vector box is at a position != 0 but it
+                # is required to be at position 0. Unpack it!
+                args[i] = self.unpack(vbox, self.off, packable, self.input_type)
+                self.update_arg_in_vector_pos(i, args[i])
+                argument_info.append(args[i].item_count)
+                continue
+                #self.update_input_output(self.pack)
+            #
+            assert vbox is not None
+            args[i] = vbox
+            argument_info.append(args[i].item_count)
+
+    def gather(self, vboxes, target_count): # packed < packable and packed < stride:
+        i = 0
+        (_, box) = vboxes[0]
+        while i < len(vboxes):
+            if i+1 >= len(vboxes):
+                break
+            (box2_pos, box2) = vboxes[i+1]
+            if box.getcount() + box2.getcount() <= target_count:
+                box = self.package(box, box.getcount(),
+                                   box2, box2_pos, box2.getcount())
+            i += 2
+        return box
+        pass
+                # OLD
+                #args = [op.getoperation().getarg(argidx) for op in ops]
+                #vbox = self._pack(vbox, packed, args, packable)
+                #self.update_input_output(self.pack)
+                #box_pos = 0
+
+    def update_arg_in_vector_pos(self, argidx, box):
+        arguments = [op.getoperation().getarg(argidx) for op in self.getoperations()]
+        for i,arg in enumerate(arguments):
+            if i >= box.item_count:
+                break
+            self.sched_data.setvector_of_box(arg, i, box)
 
     def vector_boxes_for_args(self, index):
-        args = [op.getoperation().getarg(index) for op in self.pack.operations]
+        args = [op.getoperation().getarg(index) for op in self.getoperations()]
         vboxes = []
         last_vbox = None
         for arg in args:
             pos, vbox = self.sched_data.getvector_of_box(arg)
-            if vbox != last_vbox and vbox is not None:
-                vboxes.append(vbox)
+            if vbox is not last_vbox and vbox is not None:
+                vboxes.append((pos, vbox))
+                last_vbox = vbox
         return vboxes
 
 
@@ -415,22 +460,37 @@
         self.preamble_ops.append(op)
         return vbox_cloned
 
-    def unpack(self, vbox, args, index, count, arg_ptype):
+    def unpack(self, vbox, index, count, arg_ptype):
+        assert index < vbox.item_count
+        assert index + count <= vbox.item_count
         vbox_cloned = vectorbox_clone_set(vbox, count=count)
         opnum = getunpackopnum(vbox.item_type)
         op = ResOperation(opnum, [vbox, ConstInt(index), ConstInt(count)], vbox_cloned)
         self.costmodel.record_vector_unpack(vbox, index, count)
         self.preamble_ops.append(op)
         #
-        for i,arg in enumerate(args):
-            self.sched_data.setvector_of_box(arg, i, vbox_cloned)
-        #
         return vbox_cloned
 
-    def _pack(self, tgt_box, index, args, packable):
+    def package(self, tgt, tidx, src, sidx, scount):
+        """ tgt = [1,2,3,4,_,_,_,_]
+            src = [5,6,_,_]
+            new_box = [1,2,3,4,5,6,_,_] after the operation, tidx=4, scount=2
+        """
+        assert sidx == 0 # restriction
+        count = tgt.item_count + src.item_count
+        new_box = vectorbox_clone_set(tgt, count=count)
+        opnum = getpackopnum(tgt.item_type)
+        op = ResOperation(opnum, [tgt, src, ConstInt(tidx), ConstInt(scount)], new_box)
+        self.preamble_ops.append(op)
+        self.costmodel.record_vector_pack(src, sidx, scount)
+        if not we_are_translated():
+            self._check_vec_pack(op)
+        return new_box
+
+    def package2(self, tgt_box, index, args, packable):
         """ If there are two vector boxes:
-          v1 = [<empty>,<emtpy>,X,Y]
-          v2 = [A,B,<empty>,<empty>]
+          v1 = [_,_,X,Y]
+          v2 = [A,B,_,_]
           this function creates a box pack instruction to merge them to:
           v1/2 = [A,B,X,Y]
         """
@@ -482,8 +542,9 @@
         assert index.value + count.value <= result.item_count
         assert result.item_count > arg0.item_count
 
-    def expand(self, nodes, arg, argidx):
-        vbox = self.input_type.new_vector_box(len(nodes))
+    def expand(self, arg, argidx):
+        elem_count = self.input_type.getcount()
+        vbox = self.input_type.new_vector_box(elem_count)
         box_type = arg.type
         expanded_map = self.sched_data.expanded_map
         invariant_ops = self.sched_data.invariant_oplist
@@ -496,7 +557,7 @@
         if already_expanded:
             return already_expanded
 
-        for i, node in enumerate(nodes):
+        for i, node in enumerate(self.getoperations()):
             op = node.getoperation()
             if not arg.same_box(op.getarg(argidx)):
                 break
@@ -509,10 +570,10 @@
             expanded_map[arg] = vbox
             return vbox
 
-        op = ResOperation(rop.VEC_BOX, [ConstInt(len(nodes))], vbox)
+        op = ResOperation(rop.VEC_BOX, [ConstInt(elem_count)], vbox)
         invariant_ops.append(op)
         opnum = getpackopnum(arg.type)
-        for i,node in enumerate(nodes):
+        for i,node in enumerate(self.getoperations()):
             op = node.getoperation()
             arg = op.getarg(argidx)
             new_box = vbox.clonebox()
@@ -737,6 +798,7 @@
         return self.box_to_vbox.get(arg, (-1, None))
 
     def setvector_of_box(self, box, off, vector):
+        assert off < vector.item_count
         self.box_to_vbox[box] = (off, vector)
 
     def prepend_invariant_operations(self, oplist):
diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/vectorize.py
@@ -530,7 +530,7 @@
     def record_cast_int(self, op):
         raise NotImplementedError
 
-    def record_pack_savings(self, pack):
+    def record_pack_savings(self, pack, times):
         raise NotImplementedError
 
     def record_vector_pack(self, box, index, count):
@@ -550,8 +550,7 @@
 
 class X86_CostModel(CostModel):
 
-    def record_pack_savings(self, pack):
-        times = pack.opcount()
+    def record_pack_savings(self, pack, times):
         cost, benefit_factor = (1,1)
         node = pack.operations[0]
         op = node.getoperation()

From noreply at buildbot.pypy.org  Fri Jun 26 14:48:48 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Fri, 26 Jun 2015 14:48:48 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: tyring to make things more easier,
 complexity gets hard to manage if extending the accumulation.
 trying to prevent the splitting entering the scheduling (work in progress)
Message-ID: <20150626124848.CEB4F1C0222@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r78322:7419dfb817a7
Date: 2015-06-26 14:48 +0200
http://bitbucket.org/pypy/pypy/changeset/7419dfb817a7/

Log:	tyring to make things more easier, complexity gets hard to manage if
	extending the accumulation. trying to prevent the splitting entering
	the scheduling (work in progress)

diff --git a/rpython/jit/metainterp/optimizeopt/schedule.py b/rpython/jit/metainterp/optimizeopt/schedule.py
--- a/rpython/jit/metainterp/optimizeopt/schedule.py
+++ b/rpython/jit/metainterp/optimizeopt/schedule.py
@@ -188,6 +188,9 @@
     def new_vector_box(self, count = -1):
         if count == -1:
             count = self.count
+        assert count > 1
+        assert self.type in ('i','f')
+        assert self.size > 0
         return BoxVector(self.type, count, self.size, self.signed)
 
     def __repr__(self):
@@ -291,29 +294,29 @@
         pass
 
     def transform_pack(self):
-        self.off = 0
-        while self.off < self.pack.opcount():
-            op = self.pack.operations[self.off].getoperation()
-            args = op.getarglist()
-            #
-            self.before_argument_transform(args)
-            #
-            argument_infos = []
-            self.transform_arguments(args, argument_infos)
-            #
-            result = op.result
-            result = self.transform_result(result)
-            #
-            vop = ResOperation(op.vector, args, result, op.getdescr())
-            if op.is_guard():
-                assert isinstance(op, GuardResOp)
-                vop.setfailargs(op.getfailargs())
-                vop.rd_snapshot = op.rd_snapshot
-            self.preamble_ops.append(vop)
-            stride = self.consumed_operations(argument_infos, result)
-            self.costmodel.record_pack_savings(self.pack, stride)
-            assert stride != 0
-            self.off += stride
+        #self.off = 0
+        #while self.off < self.pack.opcount():
+        op = self.pack.operations[0].getoperation()
+        args = op.getarglist()
+        #
+        self.before_argument_transform(args)
+        #
+        argument_infos = []
+        self.transform_arguments(args, argument_infos)
+        #
+        result = op.result
+        result = self.transform_result(result)
+        #
+        vop = ResOperation(op.vector, args, result, op.getdescr())
+        if op.is_guard():
+            assert isinstance(op, GuardResOp)
+            vop.setfailargs(op.getfailargs())
+            vop.rd_snapshot = op.rd_snapshot
+        self.preamble_ops.append(vop)
+        #stride = self.consumed_operations(argument_infos, result)
+        self.costmodel.record_pack_savings(self.pack, self.pack.opcount())
+        #assert stride != 0
+        #self.off += stride
 
     def consumed_operations(self, argument_infos, result):
         ops = self.getoperations()
@@ -348,7 +351,7 @@
         return BoxVector(type, count, size, signed)
 
     def getoperations(self):
-        return self.pack.operations[self.off:]
+        return self.pack.operations
 
     def transform_arguments(self, args, argument_info):
         for i,arg in enumerate(args):
@@ -406,16 +409,14 @@
             argument_info.append(args[i].item_count)
 
     def gather(self, vboxes, target_count): # packed < packable and packed < stride:
-        i = 0
         (_, box) = vboxes[0]
+        i = 1
         while i < len(vboxes):
-            if i+1 >= len(vboxes):
-                break
-            (box2_pos, box2) = vboxes[i+1]
+            (box2_pos, box2) = vboxes[i]
             if box.getcount() + box2.getcount() <= target_count:
                 box = self.package(box, box.getcount(),
                                    box2, box2_pos, box2.getcount())
-            i += 2
+            i += 1
         return box
         pass
                 # OLD
@@ -453,8 +454,10 @@
     def extend_int(self, vbox, newtype):
         vbox_cloned = newtype.new_vector_box(vbox.item_count)
         self.sched_data._prevent_signext(newtype.getsize(), vbox.getsize())
+        newsize = newtype.getsize()
+        assert newsize > 0
         op = ResOperation(rop.VEC_INT_SIGNEXT, 
-                          [vbox, ConstInt(newtype.getsize())],
+                          [vbox, ConstInt(newsize)],
                           vbox_cloned)
         self.costmodel.record_cast_int(vbox.getsize(), newtype.getsize(), vbox.getcount())
         self.preamble_ops.append(op)
@@ -618,6 +621,9 @@
         if count * size > vec_reg_size:
             count = vec_reg_size // size
         signed = self.output_type.signed
+        assert type in ('i','f')
+        assert size > 0
+        assert count > 1
         return BoxVector(type, count, size, signed)
 
 class SignExtToVectorOp(OpToVectorOp):
@@ -625,15 +631,10 @@
         OpToVectorOp.__init__(self, intype, outtype)
         self.size = -1
 
-    def split_pack(self, pack, vec_reg_size):
-        op0 = pack.operations[0].getoperation()
-        sizearg = op0.getarg(1)
+    def before_argument_transform(self, args):
+        sizearg = args[1]
         assert isinstance(sizearg, ConstInt)
         self.size = sizearg.value
-        _, vbox = self.sched_data.getvector_of_box(op0.getarg(0))
-        if vbox.getcount() * self.size > vec_reg_size:
-            return vec_reg_size // self.size
-        return vbox.getcount()
 
     def new_result_vector_box(self):
         type = self.output_type.gettype()
@@ -642,6 +643,9 @@
         if count * self.size > vec_reg_size:
             count = vec_reg_size // self.size
         signed = self.input_type.signed
+        assert type in ('i','f')
+        assert self.size > 0
+        assert count > 1
         return BoxVector(type, count, self.size, signed)
 
 class LoadToVectorLoad(OpToVectorOp):
@@ -655,18 +659,12 @@
         return PackType.by_descr(op.getdescr(), self.sched_data.vec_reg_size)
 
     def before_argument_transform(self, args):
-        args.append(ConstInt(len(self.pack.operations)))
+        count = min(self.output_type.getcount(), len(self.getoperations()))
+        args.append(ConstInt(count))
 
     def getscalarsize(self):
         return self.output_type.getsize()
 
-    def new_result_vector_box(self):
-        type = self.output_type.gettype()
-        size = self.output_type.getsize()
-        count = len(self.pack.operations)
-        signed = self.output_type.signed
-        return BoxVector(type, count, size, signed)
-
 class StoreToVectorStore(OpToVectorOp):
     """
     Storing operations are special because they are not allowed
@@ -846,6 +844,28 @@
     def opcount(self):
         return len(self.operations)
 
+    def process_count(self):
+        return len(self.operations)
+
+    def is_full(self, vec_reg_size):
+        """ if one input element times the opcount is equal
+        to the vector register size, we are full!
+        """
+        ptype = self.input_type
+        if self.input_type is None:
+            # load does not have an input type, but only an output type
+            assert self.operations[0].getoperation().is_raw_load()
+            ptype = self.output_type
+        bytes = ptype.getsize() * self.process_count()
+        assert bytes <= vec_reg_size
+        if bytes == vec_reg_size:
+            return True
+        if ptype.getcount() != -1:
+            size = ptype.getcount() * ptype.getsize()
+            assert bytes <= size
+            return bytes == size
+        return False
+
     def opnum(self):
         assert len(self.operations) > 0
         return self.operations[0].getoperation().getopnum()
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py b/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
@@ -747,8 +747,8 @@
         self.assert_packset_empty(vopt.packset, len(loop.operations),
                                   [(6,12), (5,11), (7,13)])
 
-    @pytest.mark.parametrize("descr", ['char','float','int','singlefloat'])
-    def test_packset_combine_simple(self,descr):
+    @pytest.mark.parametrize("descr,size", [('char',16),('float',2),('int',2),('singlefloat',4)])
+    def test_packset_combine_simple(self,descr,size):
         ops = """
         [p0,i0]
         i3 = getarrayitem_raw(p0, i0, descr={descr}arraydescr)
@@ -758,18 +758,7 @@
         loop = self.parse_loop(ops)
         vopt = self.combine_packset(loop,3)
         assert len(vopt.dependency_graph.memory_refs) == 4
-        assert len(vopt.packset.packs) == 1
-        self.assert_pack(vopt.packset.packs[0], (1,3,5,7))
-        ops = """
-        [p0,i0]
-        i3 = getarrayitem_raw(p0, i0, descr={descr}arraydescr)
-        i1 = int_add(i0,1)
-        jump(p0,i1)
-        """.format(descr=descr)
-        loop = self.parse_loop(ops)
-        vopt = self.combine_packset(loop,3)
-        assert len(vopt.dependency_graph.memory_refs) == 4
-        assert len(vopt.packset.packs) == 1
+        assert len(vopt.packset.packs) == 16 // size
         self.assert_pack(vopt.packset.packs[0], (1,3,5,7))
 
     @pytest.mark.parametrize("descr,stride",
@@ -786,7 +775,7 @@
         loop = self.parse_loop(ops)
         vopt = self.combine_packset(loop,3)
         assert len(vopt.dependency_graph.memory_refs) == 8
-        assert len(vopt.packset.packs) == 1
+        assert len(vopt.packset.packs) == (16//stride) * 2
         self.assert_pack(vopt.packset.packs[0], (1,3,5,7,9,11,13,15))
 
     def test_packset_combine_2_loads_one_redundant(self):
diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/vectorize.py
@@ -24,6 +24,7 @@
         getunpackopnum, PackType, determine_output_type, determine_trans)
 from rpython.jit.metainterp.optimizeopt.guard import GuardStrengthenOpt
 from rpython.jit.metainterp.resoperation import (rop, ResOperation, GuardResOp)
+from rpython.rlib import listsort
 from rpython.rlib.objectmodel import we_are_translated
 from rpython.rlib.debug import debug_print, debug_start, debug_stop
 from rpython.rlib.jit import Counters
@@ -94,6 +95,9 @@
         else:
             raise
 
+def cmp_pack_lt(a,b):
+    return a.left.getindex() < b.left.getindex()
+packsort = listsort.make_timsort_class(lt=cmp_pack_lt)
 
 class VectorizingOptimizer(Optimizer):
     """ Try to unroll the loop and find instructions to group """
@@ -327,10 +331,13 @@
         pack_count = self.packset.pack_count()
         while True:
             for pack in self.packset.packs:
-                self.follow_use_defs(pack)
                 self.follow_def_uses(pack)
             if pack_count == self.packset.pack_count():
-                break
+                pack_count = self.packset.pack_count()
+                for pack in self.packset.packs:
+                    self.follow_use_defs(pack)
+                if pack_count == self.packset.pack_count():
+                    break
             pack_count = self.packset.pack_count()
 
     def follow_use_defs(self, pack):
@@ -371,6 +378,7 @@
             raise NotAVectorizeableLoop()
         i = 0
         j = 0
+        packsort(self.packset.packs)
         end_ij = len(self.packset.packs)
         while True:
             len_before = len(self.packset.packs)
@@ -381,6 +389,8 @@
                         j += 1
                         continue
                     pack1 = self.packset.packs[i]
+                    if pack1.is_full(self.cpu.vector_register_size):
+                        break
                     pack2 = self.packset.packs[j]
                     if pack1.rightmost_match_leftmost(pack2):
                         end_ij = self.packset.combine(i,j)

From noreply at buildbot.pypy.org  Fri Jun 26 16:26:10 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Fri, 26 Jun 2015 16:26:10 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: hg merge default
Message-ID: <20150626142610.684011C02A3@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob <me at manueljacob.de>
Branch: py3k
Changeset: r78323:85bc12fb4725
Date: 2015-06-26 16:26 +0200
http://bitbucket.org/pypy/pypy/changeset/85bc12fb4725/

Log:	hg merge default

diff --git a/lib_pypy/_tkinter/tclobj.py b/lib_pypy/_tkinter/tclobj.py
--- a/lib_pypy/_tkinter/tclobj.py
+++ b/lib_pypy/_tkinter/tclobj.py
@@ -103,6 +103,8 @@
         return value.internalRep.doubleValue
     if value.typePtr == typeCache.IntType:
         return value.internalRep.longValue
+    if value.typePtr == typeCache.WideIntType:
+        return FromWideIntObj(app, value)
     if value.typePtr == typeCache.BigNumType and tklib.HAVE_LIBTOMMATH:
         return FromBignumObj(app, value)
     if value.typePtr == typeCache.ListType:
diff --git a/lib_pypy/_tkinter/tklib_build.py b/lib_pypy/_tkinter/tklib_build.py
--- a/lib_pypy/_tkinter/tklib_build.py
+++ b/lib_pypy/_tkinter/tklib_build.py
@@ -179,6 +179,7 @@
 typedef int... Tcl_WideInt;
 
 int Tcl_GetWideIntFromObj(Tcl_Interp *interp, Tcl_Obj *obj, Tcl_WideInt *value);
+Tcl_Obj *Tcl_NewWideIntObj(Tcl_WideInt value);
 """)
 
 if HAVE_LIBTOMMATH:
diff --git a/pypy/doc/embedding.rst b/pypy/doc/embedding.rst
--- a/pypy/doc/embedding.rst
+++ b/pypy/doc/embedding.rst
@@ -6,15 +6,9 @@
 C. It was developed in collaboration with Roberto De Ioris from the `uwsgi`_
 project. The `PyPy uwsgi plugin`_ is a good example of using the embedding API.
 
-**NOTE**: As of 1st of December, PyPy comes with ``--shared`` by default
-on linux, linux64 and windows. We will make it the default on all platforms
-by the time of the next release.
-
-The first thing that you need is to compile PyPy yourself with the option
-``--shared``. We plan to make ``--shared`` the default in the future. Consult
-the `how to compile PyPy`_ doc for details. This will result in ``libpypy.so``
-or ``pypy.dll`` file or something similar, depending on your platform. Consult
-your platform specification for details.
+**NOTE**: You need a PyPy compiled with the option ``--shared``, i.e.
+with a ``libpypy-c.so`` or ``pypy-c.dll`` file.  This is the default in
+recent versions of PyPy.
 
 The resulting shared library exports very few functions, however they are
 enough to accomplish everything you need, provided you follow a few principles.
@@ -75,10 +69,12 @@
 Note that this API is a lot more minimal than say CPython C API, so at first
 it's obvious to think that you can't do much. However, the trick is to do
 all the logic in Python and expose it via `cffi`_ callbacks. Let's assume
-we're on linux and pypy is installed in ``/opt/pypy`` with the
+we're on linux and pypy is installed in ``/opt/pypy`` (with
+subdirectories like ``lib-python`` and ``lib_pypy``), and with the
 library in ``/opt/pypy/bin/libpypy-c.so``.  (It doesn't need to be
-installed; you can also replace this path with your local checkout.)
-We write a little C program:
+installed; you can also replace these paths with a local extract of the
+installation tarballs, or with your local checkout of pypy.) We write a
+little C program:
 
 .. code-block:: c
 
@@ -92,7 +88,9 @@
         int res;
 
         rpython_startup_code();
-        res = pypy_setup_home("/opt/pypy/bin/libpypy-c.so", 1);
+        /* note: in the path /opt/pypy/x, the final x is ignored and
+           replaced with lib-python and lib_pypy. */
+        res = pypy_setup_home("/opt/pypy/x", 1);
         if (res) {
             printf("Error setting pypy home!\n");
             return 1;
@@ -179,7 +177,7 @@
         int res;
 
         rpython_startup_code();
-        res = pypy_setup_home("/opt/pypy/bin/libpypy-c.so", 1);
+        res = pypy_setup_home("/opt/pypy/x", 1);
         if (res) {
             fprintf(stderr, "Error setting pypy home!\n");
             return -1;
@@ -220,9 +218,15 @@
 Finding pypy_home
 -----------------
 
-Function pypy_setup_home takes one parameter - the path to libpypy. There's 
-currently no "clean" way (pkg-config comes to mind) how to find this path. You 
-can try the following (GNU-specific) hack (don't forget to link against *dl*):
+The function pypy_setup_home() takes as first parameter the path to a
+file from which it can deduce the location of the standard library.
+More precisely, it tries to remove final components until it finds
+``lib-python`` and ``lib_pypy``.  There is currently no "clean" way
+(pkg-config comes to mind) to find this path.  You can try the following
+(GNU-specific) hack (don't forget to link against *dl*), which assumes
+that the ``libpypy-c.so`` is inside the standard library directory.
+(This must more-or-less be the case anyway, otherwise the ``pypy``
+program itself would not run.)
 
 .. code-block:: c
 
@@ -236,7 +240,7 @@
 
     // caller should free returned pointer to avoid memleaks
     // returns NULL on error
-    char* guess_pypyhome() {
+    char* guess_pypyhome(void) {
         // glibc-only (dladdr is why we #define _GNU_SOURCE)
         Dl_info info;
         void *_rpython_startup_code = dlsym(0,"rpython_startup_code");
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -11,3 +11,14 @@
 .. branch: stdlib-2.7.10
 
 Update stdlib to version 2.7.10
+
+.. branch: issue2062
+
+.. branch: disable-unroll-for-short-loops
+The JIT no longer performs loop unrolling if the loop compiles to too much code.
+
+.. branch: run-create_cffi_imports
+
+Build cffi import libraries as part of translation by monkey-patching an 
+aditional task into translation
+
diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -1,6 +1,6 @@
 import py
 
-import os, sys
+import os, sys, subprocess
 
 import pypy
 from pypy.interpreter import gateway
@@ -104,13 +104,16 @@
         from pypy.module.sys.initpath import pypy_find_stdlib
         verbose = rffi.cast(lltype.Signed, verbose)
         if ll_home:
-            home = rffi.charp2str(ll_home)
+            home1 = rffi.charp2str(ll_home)
+            home = os.path.join(home1, 'x') # <- so that 'll_home' can be
+                                            # directly the root directory
         else:
-            home = pypydir
+            home = home1 = pypydir
         w_path = pypy_find_stdlib(space, home)
         if space.is_none(w_path):
             if verbose:
-                debug("Failed to find library based on pypy_find_stdlib")
+                debug("pypy_setup_home: directories 'lib-python' and 'lib_pypy'"
+                      " not found in '%s' or in any parent directory" % home1)
             return rffi.cast(rffi.INT, 1)
         space.startup()
         space.call_function(w_pathsetter, w_path)
@@ -301,6 +304,44 @@
         wrapstr = 'space.wrap(%r)' % (options)
         pypy.module.sys.Module.interpleveldefs['pypy_translation_info'] = wrapstr
 
+        # HACKHACKHACK
+        # ugly hack to modify target goal from compile_c to build_cffi_imports
+        # this should probably get cleaned up and merged with driver.create_exe
+        from rpython.translator.driver import taskdef
+        import types
+
+        class Options(object):
+            pass
+
+
+        def mkexename(name):
+            if sys.platform == 'win32':
+                name = name.new(ext='exe')
+            return name
+
+        @taskdef(['compile_c'], "Create cffi bindings for modules")
+        def task_build_cffi_imports(self):
+            from pypy.tool.build_cffi_imports import create_cffi_import_libraries
+            ''' Use cffi to compile cffi interfaces to modules'''
+            exename = mkexename(driver.compute_exe_name())
+            basedir = exename
+            while not basedir.join('include').exists():
+                _basedir = basedir.dirpath()
+                if _basedir == basedir:
+                    raise ValueError('interpreter %s not inside pypy repo', 
+                                     str(exename))
+                basedir = _basedir
+            modules = self.config.objspace.usemodules.getpaths()
+            options = Options()
+            # XXX possibly adapt options using modules
+            failures = create_cffi_import_libraries(exename, options, basedir)
+            # if failures, they were already printed
+            print  >> sys.stderr, str(exename),'successfully built, but errors while building the above modules will be ignored'
+        driver.task_build_cffi_imports = types.MethodType(task_build_cffi_imports, driver)
+        driver.tasks['build_cffi_imports'] = driver.task_build_cffi_imports, ['compile_c']
+        driver.default_goal = 'build_cffi_imports'
+        # HACKHACKHACK end
+
         return self.get_entry_point(config)
 
     def jitpolicy(self, driver):
diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py
--- a/pypy/interpreter/app_main.py
+++ b/pypy/interpreter/app_main.py
@@ -766,6 +766,7 @@
     # This is important for py3k
     sys.executable = executable
 
+ at hidden_applevel
 def entry_point(executable, argv):
     # note that before calling setup_bootstrap_path, we are limited because we
     # cannot import stdlib modules. In particular, we cannot use unicode
diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -1,6 +1,7 @@
 import sys
 from pypy.interpreter.error import OperationError, get_cleared_operation_error
 from rpython.rlib.unroll import unrolling_iterable
+from rpython.rlib.objectmodel import specialize
 from rpython.rlib import jit
 
 TICK_COUNTER_STEP = 100
diff --git a/pypy/interpreter/pytraceback.py b/pypy/interpreter/pytraceback.py
--- a/pypy/interpreter/pytraceback.py
+++ b/pypy/interpreter/pytraceback.py
@@ -61,7 +61,6 @@
 
 
 def check_traceback(space, w_tb, msg):
-    from pypy.interpreter.typedef import PyTraceback
     if w_tb is None or not space.isinstance_w(w_tb, space.gettypeobject(PyTraceback.typedef)):
         raise OperationError(space.w_TypeError, space.wrap(msg))
     return w_tb
diff --git a/pypy/module/_cffi_backend/ctypefunc.py b/pypy/module/_cffi_backend/ctypefunc.py
--- a/pypy/module/_cffi_backend/ctypefunc.py
+++ b/pypy/module/_cffi_backend/ctypefunc.py
@@ -143,7 +143,7 @@
     @jit.unroll_safe
     def _call(self, funcaddr, args_w):
         space = self.space
-        cif_descr = self.cif_descr
+        cif_descr = self.cif_descr   # 'self' should have been promoted here
         size = cif_descr.exchange_size
         mustfree_max_plus_1 = 0
         buffer = lltype.malloc(rffi.CCHARP.TO, size, flavor='raw')
diff --git a/pypy/module/_cffi_backend/ctypeprim.py b/pypy/module/_cffi_backend/ctypeprim.py
--- a/pypy/module/_cffi_backend/ctypeprim.py
+++ b/pypy/module/_cffi_backend/ctypeprim.py
@@ -134,8 +134,7 @@
 
     def convert_to_object(self, cdata):
         unichardata = rffi.cast(rffi.CWCHARP, cdata)
-        s = rffi.wcharpsize2unicode(unichardata, 1)
-        return self.space.wrap(s)
+        return self.space.wrap(unichardata[0])
 
     def string(self, cdataobj, maxlen):
         with cdataobj as ptr:
diff --git a/pypy/module/_cffi_backend/lib_obj.py b/pypy/module/_cffi_backend/lib_obj.py
--- a/pypy/module/_cffi_backend/lib_obj.py
+++ b/pypy/module/_cffi_backend/lib_obj.py
@@ -60,12 +60,12 @@
             self.ffi, self.ctx.c_types, getarg(g.c_type_op))
         assert isinstance(rawfunctype, realize_c_type.W_RawFuncType)
         #
-        w_ct, locs = rawfunctype.unwrap_as_nostruct_fnptr(self.ffi)
+        rawfunctype.prepare_nostruct_fnptr(self.ffi)
         #
         ptr = rffi.cast(rffi.CCHARP, g.c_address)
         assert ptr
-        return W_FunctionWrapper(self.space, ptr, g.c_size_or_direct_fn, w_ct,
-                                 locs, rawfunctype, fnname, self.libname)
+        return W_FunctionWrapper(self.space, ptr, g.c_size_or_direct_fn,
+                                 rawfunctype, fnname, self.libname)
 
     @jit.elidable_promote()
     def _get_attr_elidable(self, attr):
@@ -173,6 +173,8 @@
             if w_value is None:
                 if is_getattr and attr == '__all__':
                     return self.dir1(ignore_type=cffi_opcode.OP_GLOBAL_VAR)
+                if is_getattr and attr == '__dict__':
+                    return self.full_dict_copy()
                 raise oefmt(self.space.w_AttributeError,
                             "cffi library '%s' has no function, constant "
                             "or global variable named '%s'",
@@ -212,6 +214,17 @@
                 names_w.append(space.wrap(rffi.charp2str(g[i].c_name)))
         return space.newlist(names_w)
 
+    def full_dict_copy(self):
+        space = self.space
+        total = rffi.getintfield(self.ctx, 'c_num_globals')
+        g = self.ctx.c_globals
+        w_result = space.newdict()
+        for i in range(total):
+            w_attr = space.wrap(rffi.charp2str(g[i].c_name))
+            w_value = self._get_attr(w_attr)
+            space.setitem(w_result, w_attr, w_value)
+        return w_result
+
     def address_of_func_or_global_var(self, varname):
         # rebuild a string object from 'varname', to do typechecks and
         # to force a unicode back to a plain string
@@ -224,7 +237,8 @@
         if isinstance(w_value, W_FunctionWrapper):
             # '&func' returns a regular cdata pointer-to-function
             if w_value.directfnptr:
-                return W_CData(space, w_value.directfnptr, w_value.ctype)
+                ctype = w_value.typeof(self.ffi)
+                return W_CData(space, w_value.directfnptr, ctype)
             else:
                 return w_value    # backward compatibility
         #
diff --git a/pypy/module/_cffi_backend/realize_c_type.py b/pypy/module/_cffi_backend/realize_c_type.py
--- a/pypy/module/_cffi_backend/realize_c_type.py
+++ b/pypy/module/_cffi_backend/realize_c_type.py
@@ -1,4 +1,5 @@
 import sys
+from rpython.rlib import jit
 from rpython.rlib.rarithmetic import intmask
 from rpython.rlib.objectmodel import specialize
 from rpython.rtyper.lltypesystem import lltype, rffi
@@ -135,8 +136,12 @@
 
 class W_RawFuncType(W_Root):
     """Temporary: represents a C function type (not a function pointer)"""
+
+    _immutable_fields_ = ['nostruct_ctype', 'nostruct_locs', 'nostruct_nargs']
     _ctfuncptr = None
-    _nostruct_ctfuncptr = (None, None)
+    nostruct_ctype = None
+    nostruct_locs = None
+    nostruct_nargs = 0
 
     def __init__(self, opcodes, base_index):
         self.opcodes = opcodes
@@ -168,14 +173,16 @@
         assert self._ctfuncptr is not None
         return self._ctfuncptr
 
-    def unwrap_as_nostruct_fnptr(self, ffi):
-        # tweaked version: instead of returning the ctfuncptr corresponding
-        # exactly to the OP_FUNCTION ... OP_FUNCTION_END opcodes, return
-        # another one in which the struct args are replaced with ptr-to-
-        # struct, and a struct return value is replaced with a hidden first
-        # arg of type ptr-to-struct.  This is how recompiler.py produces
+    @jit.dont_look_inside
+    def prepare_nostruct_fnptr(self, ffi):
+        # tweaked version: instead of returning the ctfuncptr
+        # corresponding exactly to the OP_FUNCTION ... OP_FUNCTION_END
+        # opcodes, this builds in self.nostruct_ctype another one in
+        # which the struct args are replaced with ptr-to- struct, and
+        # a struct return value is replaced with a hidden first arg of
+        # type ptr-to-struct.  This is how recompiler.py produces
         # trampoline functions for PyPy.
-        if self._nostruct_ctfuncptr[0] is None:
+        if self.nostruct_ctype is None:
             fargs, fret, ellipsis = self._unpack(ffi)
             # 'locs' will be a string of the same length as the final fargs,
             # containing 'A' where a struct argument was detected, and 'R'
@@ -198,8 +205,10 @@
                 locs = None
             else:
                 locs = ''.join(locs)
-            self._nostruct_ctfuncptr = (ctfuncptr, locs)
-        return self._nostruct_ctfuncptr
+            self.nostruct_ctype = ctfuncptr
+            self.nostruct_locs = locs
+            self.nostruct_nargs = len(ctfuncptr.fargs) - (locs is not None and
+                                                          locs[0] == 'R')
 
     def unexpected_fn_type(self, ffi):
         fargs, fret, ellipsis = self._unpack(ffi)
diff --git a/pypy/module/_cffi_backend/src/parse_c_type.c b/pypy/module/_cffi_backend/src/parse_c_type.c
--- a/pypy/module/_cffi_backend/src/parse_c_type.c
+++ b/pypy/module/_cffi_backend/src/parse_c_type.c
@@ -362,7 +362,7 @@
 
             case TOK_INTEGER:
                 errno = 0;
-#ifndef MS_WIN32
+#ifndef _MSC_VER
                 if (sizeof(length) > sizeof(unsigned long))
                     length = strtoull(tok->p, &endptr, 0);
                 else
diff --git a/pypy/module/_cffi_backend/test/test_recompiler.py b/pypy/module/_cffi_backend/test/test_recompiler.py
--- a/pypy/module/_cffi_backend/test/test_recompiler.py
+++ b/pypy/module/_cffi_backend/test/test_recompiler.py
@@ -276,6 +276,15 @@
         """)
         lib.aa = 5
         assert dir(lib) == ['aa', 'ff', 'my_constant']
+        #
+        aaobj = lib.__dict__['aa']
+        assert not isinstance(aaobj, int)    # some internal object instead
+        assert lib.__dict__ == {
+            'ff': lib.ff,
+            'aa': aaobj,
+            'my_constant': -45}
+        lib.__dict__['ff'] = "??"
+        assert lib.ff(10) == 15
 
     def test_verify_opaque_struct(self):
         ffi, lib = self.prepare(
@@ -819,6 +828,22 @@
         assert isinstance(addr, ffi.CData)
         assert ffi.typeof(addr) == ffi.typeof("long(*)(long)")
 
+    def test_address_of_function_with_struct(self):
+        ffi, lib = self.prepare(
+            "struct foo_s { int x; }; long myfunc(struct foo_s);",
+            "test_addressof_function_with_struct", """
+                struct foo_s { int x; };
+                char myfunc(struct foo_s input) { return (char)(input.x + 42); }
+            """)
+        s = ffi.new("struct foo_s *", [5])[0]
+        assert lib.myfunc(s) == 47
+        assert not isinstance(lib.myfunc, ffi.CData)
+        assert ffi.typeof(lib.myfunc) == ffi.typeof("long(*)(struct foo_s)")
+        addr = ffi.addressof(lib, 'myfunc')
+        assert addr(s) == 47
+        assert isinstance(addr, ffi.CData)
+        assert ffi.typeof(addr) == ffi.typeof("long(*)(struct foo_s)")
+
     def test_issue198(self):
         ffi, lib = self.prepare("""
             typedef struct{...;} opaque_t;
@@ -984,5 +1009,5 @@
         assert sys.modules['_CFFI_test_import_from_lib.lib'] is lib
         from _CFFI_test_import_from_lib.lib import MYFOO
         assert MYFOO == 42
-        assert not hasattr(lib, '__dict__')
+        assert hasattr(lib, '__dict__')
         assert lib.__all__ == ['MYFOO', 'mybar']   # but not 'myvar'
diff --git a/pypy/module/_cffi_backend/wrapper.py b/pypy/module/_cffi_backend/wrapper.py
--- a/pypy/module/_cffi_backend/wrapper.py
+++ b/pypy/module/_cffi_backend/wrapper.py
@@ -19,12 +19,20 @@
     wrapper is callable, and the arguments it expects and returns
     are directly the struct/union.  Calling ffi.typeof(wrapper)
     also returns the original struct/union signature.
+
+    This class cannot be used for variadic functions.
     """
     _immutable_ = True
     common_doc_str = 'direct call to the C function of the same name'
 
-    def __init__(self, space, fnptr, directfnptr, ctype,
-                 locs, rawfunctype, fnname, modulename):
+    def __init__(self, space, fnptr, directfnptr,
+                 rawfunctype, fnname, modulename):
+        # everything related to the type of the function is accessed
+        # as immutable attributes of the 'rawfunctype' object, which
+        # is a W_RawFuncType.  This gives us an obvious thing to
+        # promote in order to do the call.
+        ctype = rawfunctype.nostruct_ctype
+        locs = rawfunctype.nostruct_locs
         assert isinstance(ctype, W_CTypeFunc)
         assert ctype.cif_descr is not None     # not for '...' functions
         assert locs is None or len(ctype.fargs) == len(locs)
@@ -32,83 +40,86 @@
         self.space = space
         self.fnptr = fnptr
         self.directfnptr = directfnptr
-        self.ctype = ctype
-        self.locs = locs
         self.rawfunctype = rawfunctype
         self.fnname = fnname
         self.modulename = modulename
-        self.nargs_expected = len(ctype.fargs) - (locs is not None and
-                                                  locs[0] == 'R')
 
     def typeof(self, ffi):
         return self.rawfunctype.unwrap_as_fnptr(ffi)
 
-    @jit.unroll_safe
-    def _prepare(self, args_w, start_index):
-        # replaces struct/union arguments with ptr-to-struct/union arguments
+    def descr_call(self, args_w):
         space = self.space
-        locs = self.locs
-        fargs = self.ctype.fargs
-        for i in range(start_index, len(locs)):
-            if locs[i] != 'A':
-                continue
-            w_arg = args_w[i]
-            farg = fargs[i]      # <ptr to struct/union>
-            assert isinstance(farg, W_CTypePtrOrArray)
-            if isinstance(w_arg, W_CData) and w_arg.ctype is farg.ctitem:
-                # fast way: we are given a W_CData "struct", so just make
-                # a new W_CData "ptr-to-struct" which points to the same
-                # raw memory.  We use unsafe_escaping_ptr(), so we have to
-                # make sure the original 'w_arg' stays alive; the easiest
-                # is to build an instance of W_CDataPtrToStructOrUnion.
-                w_arg = W_CDataPtrToStructOrUnion(
-                    space, w_arg.unsafe_escaping_ptr(), farg, w_arg)
-            else:
-                # slow way: build a new "ptr to struct" W_CData by calling
-                # the equivalent of ffi.new()
-                if space.is_w(w_arg, space.w_None):
-                    continue
-                w_arg = farg.newp(w_arg)
-            args_w[i] = w_arg
-
-    def descr_call(self, args_w):
-        if len(args_w) != self.nargs_expected:
-            space = self.space
-            if self.nargs_expected == 0:
+        rawfunctype = jit.promote(self.rawfunctype)
+        ctype = rawfunctype.nostruct_ctype
+        locs = rawfunctype.nostruct_locs
+        nargs_expected = rawfunctype.nostruct_nargs
+        #
+        if len(args_w) != nargs_expected:
+            if nargs_expected == 0:
                 raise oefmt(space.w_TypeError,
                             "%s() takes no arguments (%d given)",
                             self.fnname, len(args_w))
-            elif self.nargs_expected == 1:
+            elif nargs_expected == 1:
                 raise oefmt(space.w_TypeError,
                             "%s() takes exactly one argument (%d given)",
                             self.fnname, len(args_w))
             else:
                 raise oefmt(space.w_TypeError,
                             "%s() takes exactly %d arguments (%d given)",
-                            self.fnname, self.nargs_expected, len(args_w))
+                            self.fnname, nargs_expected, len(args_w))
         #
-        if self.locs is not None:
+        if locs is not None:
             # This case is if there are structs as arguments or return values.
             # If the result we want to present to the user is "returns struct",
             # then internally allocate the struct and pass a pointer to it as
             # a first argument.
-            if self.locs[0] == 'R':
-                w_result_cdata = self.ctype.fargs[0].newp(self.space.w_None)
+            if locs[0] == 'R':
+                w_result_cdata = ctype.fargs[0].newp(space.w_None)
                 args_w = [w_result_cdata] + args_w
-                self._prepare(args_w, 1)
-                self.ctype._call(self.fnptr, args_w)    # returns w_None
+                prepare_args(space, rawfunctype, args_w, 1)
+                #
+                ctype._call(self.fnptr, args_w)    # returns w_None
+                #
                 assert isinstance(w_result_cdata, W_CDataPtrToStructOrUnion)
                 return w_result_cdata.structobj
             else:
                 args_w = args_w[:]
-                self._prepare(args_w, 0)
+                prepare_args(space, rawfunctype, args_w, 0)
         #
-        return self.ctype._call(self.fnptr, args_w)
+        return ctype._call(self.fnptr, args_w)
 
     def descr_repr(self, space):
         return space.wrap("<FFIFunctionWrapper for %s()>" % (self.fnname,))
 
 
+ at jit.unroll_safe
+def prepare_args(space, rawfunctype, args_w, start_index):
+    # replaces struct/union arguments with ptr-to-struct/union arguments
+    locs = rawfunctype.nostruct_locs
+    fargs = rawfunctype.nostruct_ctype.fargs
+    for i in range(start_index, len(locs)):
+        if locs[i] != 'A':
+            continue
+        w_arg = args_w[i]
+        farg = fargs[i]      # <ptr to struct/union>
+        assert isinstance(farg, W_CTypePtrOrArray)
+        if isinstance(w_arg, W_CData) and w_arg.ctype is farg.ctitem:
+            # fast way: we are given a W_CData "struct", so just make
+            # a new W_CData "ptr-to-struct" which points to the same
+            # raw memory.  We use unsafe_escaping_ptr(), so we have to
+            # make sure the original 'w_arg' stays alive; the easiest
+            # is to build an instance of W_CDataPtrToStructOrUnion.
+            w_arg = W_CDataPtrToStructOrUnion(
+                space, w_arg.unsafe_escaping_ptr(), farg, w_arg)
+        else:
+            # slow way: build a new "ptr to struct" W_CData by calling
+            # the equivalent of ffi.new()
+            if space.is_w(w_arg, space.w_None):
+                continue
+            w_arg = farg.newp(w_arg)
+        args_w[i] = w_arg
+
+
 W_FunctionWrapper.typedef = TypeDef(
         'FFIFunctionWrapper',
         __repr__ = interp2app(W_FunctionWrapper.descr_repr),
diff --git a/pypy/module/_io/interp_textio.py b/pypy/module/_io/interp_textio.py
--- a/pypy/module/_io/interp_textio.py
+++ b/pypy/module/_io/interp_textio.py
@@ -625,6 +625,7 @@
 
     def read_w(self, space, w_size=None):
         self._check_attached(space)
+        self._check_closed(space)
         if not self.w_decoder:
             self._unsupportedoperation(space, "not readable")
 
@@ -666,6 +667,7 @@
 
     def readline_w(self, space, w_limit=None):
         self._check_attached(space)
+        self._check_closed(space)
         self._writeflush(space)
 
         limit = convert_size(space, w_limit)
@@ -761,7 +763,7 @@
 
     def write_w(self, space, w_text):
         self._check_attached(space)
-        # self._check_closed(space)
+        self._check_closed(space)
 
         if not self.w_encoder:
             self._unsupportedoperation(space, "not writable")
diff --git a/pypy/module/_io/test/test_io.py b/pypy/module/_io/test/test_io.py
--- a/pypy/module/_io/test/test_io.py
+++ b/pypy/module/_io/test/test_io.py
@@ -429,3 +429,55 @@
             f.seek(1, 0)
             f.read(buffer_size * 2)
             assert f.tell() == 1 + buffer_size * 2
+
+
+class AppTestIoAferClose:
+    spaceconfig = dict(usemodules=['_io'])
+
+    def setup_class(cls):
+        tmpfile = udir.join('tmpfile').ensure()
+        cls.w_tmpfile = cls.space.wrap(str(tmpfile))
+
+    def test_io_after_close(self):
+        import _io
+        for kwargs in [
+                {"mode": "w"},
+                {"mode": "wb"},
+                {"mode": "w", "buffering": 1},
+                {"mode": "w", "buffering": 2},
+                {"mode": "wb", "buffering": 0},
+                {"mode": "r"},
+                {"mode": "rb"},
+                {"mode": "r", "buffering": 1},
+                {"mode": "r", "buffering": 2},
+                {"mode": "rb", "buffering": 0},
+                {"mode": "w+"},
+                {"mode": "w+b"},
+                {"mode": "w+", "buffering": 1},
+                {"mode": "w+", "buffering": 2},
+                {"mode": "w+b", "buffering": 0},
+            ]:
+            print kwargs
+            f = _io.open(self.tmpfile, **kwargs)
+            f.close()
+            raises(ValueError, f.flush)
+            raises(ValueError, f.fileno)
+            raises(ValueError, f.isatty)
+            raises(ValueError, f.__iter__)
+            if hasattr(f, "peek"):
+                raises(ValueError, f.peek, 1)
+            raises(ValueError, f.read)
+            if hasattr(f, "read1"):
+                raises(ValueError, f.read1, 1024)
+            if hasattr(f, "readall"):
+                raises(ValueError, f.readall)
+            if hasattr(f, "readinto"):
+                raises(ValueError, f.readinto, bytearray(1024))
+            raises(ValueError, f.readline)
+            raises(ValueError, f.readlines)
+            raises(ValueError, f.seek, 0)
+            raises(ValueError, f.tell)
+            raises(ValueError, f.truncate)
+            raises(ValueError, f.write, b"" if "b" in kwargs['mode'] else u"")
+            raises(ValueError, f.writelines, [])
+            raises(ValueError, next, f)
diff --git a/pypy/module/_socket/__init__.py b/pypy/module/_socket/__init__.py
--- a/pypy/module/_socket/__init__.py
+++ b/pypy/module/_socket/__init__.py
@@ -18,6 +18,10 @@
         from rpython.rlib.rsocket import rsocket_startup
         rsocket_startup()
 
+    def shutdown(self, space):
+        from pypy.module._socket.interp_socket import close_all_sockets
+        close_all_sockets(space)
+
     def buildloaders(cls):
         from rpython.rlib import rsocket
         for name in """
diff --git a/pypy/module/_socket/interp_func.py b/pypy/module/_socket/interp_func.py
--- a/pypy/module/_socket/interp_func.py
+++ b/pypy/module/_socket/interp_func.py
@@ -2,7 +2,7 @@
 from rpython.rlib.rsocket import SocketError, INVALID_SOCKET
 from rpython.rlib.rarithmetic import intmask
 
-from pypy.interpreter.error import OperationError
+from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.gateway import unwrap_spec, WrappedDefault
 from pypy.module._socket.interp_socket import (
     converted_error, W_Socket, addr_as_object, fill_from_object, get_error,
@@ -147,6 +147,19 @@
     newfd = rsocket.dup(fd)
     return space.wrap(newfd)
 
+ at unwrap_spec(fd=int, family=int, type=int, proto=int)
+def fromfd(space, fd, family, type, proto=0):
+    """fromfd(fd, family, type[, proto]) -> socket object
+
+    Create a socket object from the given file descriptor.
+    The remaining arguments are the same as for socket().
+    """
+    try:
+        sock = rsocket.fromfd(fd, family, type, proto)
+    except SocketError, e:
+        raise converted_error(space, e)
+    return space.wrap(W_Socket(space, sock))
+
 @unwrap_spec(family=int, type=int, proto=int)
 def socketpair(space, family=rsocket.socketpair_default_family,
                       type  =rsocket.SOCK_STREAM,
@@ -163,8 +176,8 @@
     except SocketError, e:
         raise converted_error(space, e)
     return space.newtuple([
-        space.wrap(W_Socket(sock1)),
-        space.wrap(W_Socket(sock2))
+        space.wrap(W_Socket(space, sock1)),
+        space.wrap(W_Socket(space, sock2))
     ])
 
 # The following 4 functions refuse all negative numbers, like CPython 2.6.
@@ -250,9 +263,9 @@
         ip = rsocket.inet_ntop(family, packed)
     except SocketError, e:
         raise converted_error(space, e)
-    except ValueError, e:     # XXX the message is lost in RPython
-        raise OperationError(space.w_ValueError,
-                  space.wrap(str(e)))
+    except ValueError:
+        raise oefmt(space.w_ValueError,
+                    "invalid length of packed IP address string")
     return space.wrap(ip)
 
 @unwrap_spec(family=int, type=int, proto=int, flags=int)
diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py
--- a/pypy/module/_socket/interp_socket.py
+++ b/pypy/module/_socket/interp_socket.py
@@ -1,4 +1,5 @@
-from rpython.rlib import rsocket
+import sys
+from rpython.rlib import rsocket, rweaklist
 from rpython.rlib.rarithmetic import intmask
 from rpython.rlib.rsocket import (
     RSocket, AF_INET, SOCK_STREAM, SocketError, SocketErrorWithErrno,
@@ -158,12 +159,10 @@
 
 
 class W_Socket(W_Root):
-
-    # for _dealloc_warn
-    space = None
-
-    def __init__(self, sock):
+    def __init__(self, space, sock):
+        self.space = space
         self.sock = sock
+        register_socket(space, sock)
 
     def descr_new(space, w_subtype, __args__):
         sock = space.allocate_instance(W_Socket, w_subtype)
@@ -179,8 +178,7 @@
                                fd=space.c_filedescriptor_w(w_fileno))
             else:
                 sock = RSocket(family, type, proto)
-            W_Socket.__init__(self, sock)
-            self.space = space
+            W_Socket.__init__(self, space, sock)
         except SocketError, e:
             raise converted_error(space, e)
 
@@ -617,6 +615,45 @@
 
 
 # ____________________________________________________________
+# Automatic shutdown()/close()
+
+# On some systems, the C library does not guarantee that when the program
+# finishes, all data sent so far is really sent even if the socket is not
+# explicitly closed.  This behavior has been observed on Windows but not
+# on Linux, so far.
+NEED_EXPLICIT_CLOSE = (sys.platform == 'win32')
+
+class OpenRSockets(rweaklist.RWeakListMixin):
+    pass
+class OpenRSocketsState:
+    def __init__(self, space):
+        self.openrsockets = OpenRSockets()
+        self.openrsockets.initialize()
+
+def getopenrsockets(space):
+    if NEED_EXPLICIT_CLOSE and space.config.translation.rweakref:
+        return space.fromcache(OpenRSocketsState).openrsockets
+    else:
+        return None
+
+def register_socket(space, socket):
+    openrsockets = getopenrsockets(space)
+    if openrsockets is not None:
+        openrsockets.add_handle(socket)
+
+def close_all_sockets(space):
+    openrsockets = getopenrsockets(space)
+    if openrsockets is not None:
+        for sock_wref in openrsockets.get_all_handles():
+            sock = sock_wref()
+            if sock is not None:
+                try:
+                    sock.close()
+                except SocketError:
+                    pass
+
+
+# ____________________________________________________________
 # Error handling
 
 class SocketAPI:
diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py
--- a/pypy/module/_socket/test/test_sock_app.py
+++ b/pypy/module/_socket/test/test_sock_app.py
@@ -301,10 +301,16 @@
 
 
 class AppTestSocket:
+    spaceconfig = dict(usemodules=['_socket', '_weakref', 'struct'])
+
     def setup_class(cls):
         cls.space = space
         cls.w_udir = space.wrap(str(udir))
 
+    def teardown_class(cls):
+        if not cls.runappdirect:
+            cls.space.sys.getmodule('_socket').shutdown(cls.space)
+
     def test_module(self):
         import _socket
         assert _socket.socket.__name__ == 'socket'
@@ -602,6 +608,12 @@
         finally:
             os.chdir(oldcwd)
 
+    def test_automatic_shutdown(self):
+        # doesn't really test anything, but at least should not explode
+        # in close_all_sockets()
+        import _socket
+        self.foo = _socket.socket()
+
     def test_subclass(self):
         # Socket is not created in __new__, but in __init__.
         import socket
diff --git a/pypy/module/cpyext/test/test_version.py b/pypy/module/cpyext/test/test_version.py
--- a/pypy/module/cpyext/test/test_version.py
+++ b/pypy/module/cpyext/test/test_version.py
@@ -24,7 +24,7 @@
         }
         """
         module = self.import_module(name='foo', init=init)
-        assert module.py_version == sys.version[:5]
+        assert module.py_version == '%d.%d.%d' % sys.version_info[:3]
         assert module.py_major_version == sys.version_info.major
         assert module.py_minor_version == sys.version_info.minor
         assert module.py_micro_version == sys.version_info.micro
diff --git a/pypy/module/math/interp_math.py b/pypy/module/math/interp_math.py
--- a/pypy/module/math/interp_math.py
+++ b/pypy/module/math/interp_math.py
@@ -361,7 +361,7 @@
             else:
                 partials.append(v)
     if special_sum != 0.0:
-        if rfloat.isnan(special_sum):
+        if rfloat.isnan(inf_sum):
             raise OperationError(space.w_ValueError, space.wrap("-inf + inf"))
         return space.wrap(special_sum)
     hi = 0.0
diff --git a/pypy/module/math/test/test_math.py b/pypy/module/math/test/test_math.py
--- a/pypy/module/math/test/test_math.py
+++ b/pypy/module/math/test/test_math.py
@@ -1,5 +1,6 @@
 from __future__ import with_statement
 
+import py
 from pypy.interpreter.function import Function
 from pypy.interpreter.gateway import BuiltinCode
 from pypy.module.math.test import test_direct
@@ -113,6 +114,10 @@
             ([2.**n - 2.**(n+50) + 2.**(n+52) for n in range(-1074, 972, 2)] +
              [-2.**1022],
              float.fromhex('0x1.5555555555555p+970')),
+            # infinity and nans
+            ([float("inf")], float("inf")),
+            ([float("-inf")], float("-inf")),
+            ([float("nan")], float("nan")),
             ]
 
         for i, (vals, expected) in enumerate(test_values):
@@ -124,7 +129,8 @@
             except ValueError:
                 py.test.fail("test %d failed: got ValueError, expected %r "
                           "for math.fsum(%.100r)" % (i, expected, vals))
-            assert actual == expected
+            assert actual == expected or (
+                math.isnan(actual) and math.isnan(expected))
 
     def test_factorial(self):
         import math, sys
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
@@ -266,6 +266,15 @@
     """)
     lib.aa = 5
     assert dir(lib) == ['aa', 'ff', 'my_constant']
+    #
+    aaobj = lib.__dict__['aa']
+    assert not isinstance(aaobj, int)    # some internal object instead
+    assert lib.__dict__ == {
+        'ff': lib.ff,
+        'aa': aaobj,
+        'my_constant': -45}
+    lib.__dict__['ff'] = "??"
+    assert lib.ff(10) == 15
 
 def test_verify_opaque_struct():
     ffi = FFI()
@@ -1053,5 +1062,5 @@
     assert sys.modules['_CFFI_test_import_from_lib.lib'] is lib
     from _CFFI_test_import_from_lib.lib import MYFOO
     assert MYFOO == 42
-    assert not hasattr(lib, '__dict__')
+    assert hasattr(lib, '__dict__')
     assert lib.__all__ == ['MYFOO', 'mybar']   # but not 'myvar'
diff --git a/pypy/tool/build_cffi_imports.py b/pypy/tool/build_cffi_imports.py
new file mode 100644
--- /dev/null
+++ b/pypy/tool/build_cffi_imports.py
@@ -0,0 +1,75 @@
+import sys, shutil
+from rpython.tool.runsubprocess import run_subprocess
+
+class MissingDependenciesError(Exception):
+    pass
+
+
+cffi_build_scripts = {
+    "sqlite3": "_sqlite3_build.py",
+    "audioop": "_audioop_build.py",
+    "tk": "_tkinter/tklib_build.py",
+    "curses": "_curses_build.py" if sys.platform != "win32" else None,
+    "syslog": "_syslog_build.py" if sys.platform != "win32" else None,
+    "_gdbm": "_gdbm_build.py"  if sys.platform != "win32" else None,
+    "pwdgrp": "_pwdgrp_build.py" if sys.platform != "win32" else None,
+    "xx": None,    # for testing: 'None' should be completely ignored
+    }
+
+def create_cffi_import_libraries(pypy_c, options, basedir):
+    shutil.rmtree(str(basedir.join('lib_pypy', '__pycache__')),
+                  ignore_errors=True)
+    failures = []
+    for key, module in sorted(cffi_build_scripts.items()):
+        if module is None or getattr(options, 'no_' + key, False):
+            continue
+        if module.endswith('.py'):
+            args = [module]
+            cwd = str(basedir.join('lib_pypy'))
+        else:
+            args = ['-c', 'import ' + module]
+            cwd = None
+        print >> sys.stderr, '*', ' '.join(args)
+        try:
+            status, stdout, stderr = run_subprocess(str(pypy_c), args, cwd=cwd)
+            if status != 0:
+                print >> sys.stderr, stdout, stderr
+                failures.append((key, module))
+        except:
+            import traceback;traceback.print_exc()
+            failures.append((key, module))
+    return failures
+
+if __name__ == '__main__':
+    import py, os
+    if '__pypy__' not in sys.builtin_module_names:
+        print 'Call with a pypy interpreter'
+        sys.exit(-1)
+
+    class Options(object):
+        pass
+
+    exename = py.path.local(sys.executable) 
+    basedir = exename
+    while not basedir.join('include').exists():
+        _basedir = basedir.dirpath()
+        if _basedir == basedir:
+            raise ValueError('interpreter %s not inside pypy repo', 
+                                 str(exename))
+        basedir = _basedir
+    options = Options()
+    print >> sys.stderr, "There should be no failures here"
+    failures = create_cffi_import_libraries(exename, options, basedir)
+    if len(failures) > 0:
+        print 'failed to build', [f[1] for f in failures]
+        assert False
+
+    # monkey patch a failure, just to test
+    print >> sys.stderr, 'This line should be followed by a traceback'
+    for k in cffi_build_scripts:
+        setattr(options, 'no_' + k, True)
+    must_fail = '_missing_build_script.py'
+    assert not os.path.exists(str(basedir.join('lib_pypy').join(must_fail)))
+    cffi_build_scripts['should_fail'] = must_fail
+    failures = create_cffi_import_libraries(exename, options, basedir)
+    assert len(failures) == 1
diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py
--- a/pypy/tool/release/package.py
+++ b/pypy/tool/release/package.py
@@ -29,6 +29,9 @@
 # XXX: don't hardcode the version
 POSIX_EXE = 'pypy3.2'
 
+from pypy.tool.build_cffi_imports import (create_cffi_import_libraries, 
+        MissingDependenciesError, cffi_build_scripts)
+
 def ignore_patterns(*patterns):
     """Function that can be used as copytree() ignore parameter.
 
@@ -44,48 +47,12 @@
 class PyPyCNotFound(Exception):
     pass
 
-class MissingDependenciesError(Exception):
-    pass
-
 def fix_permissions(dirname):
     if sys.platform != 'win32':
         os.system("chmod -R a+rX %s" % dirname)
         os.system("chmod -R g-w %s" % dirname)
 
 
-cffi_build_scripts = {
-    "sqlite3": "_sqlite3_build.py",
-    "audioop": "_audioop_build.py",
-    "tk": "_tkinter/tklib_build.py",
-    "curses": "_curses_build.py" if sys.platform != "win32" else None,
-    "syslog": "_syslog_build.py" if sys.platform != "win32" else None,
-    "_gdbm": "_gdbm_build.py"  if sys.platform != "win32" else None,
-    "pwdgrp": "_pwdgrp_build.py" if sys.platform != "win32" else None,
-    "xx": None,    # for testing: 'None' should be completely ignored
-    }
-
-def create_cffi_import_libraries(pypy_c, options, basedir):
-    shutil.rmtree(str(basedir.join('lib_pypy', '__pycache__')),
-                  ignore_errors=True)
-    for key, module in sorted(cffi_build_scripts.items()):
-        if module is None or getattr(options, 'no_' + key):
-            continue
-        if module.endswith('.py'):
-            args = [str(pypy_c), module]
-            cwd = str(basedir.join('lib_pypy'))
-        else:
-            args = [str(pypy_c), '-c', 'import ' + module]
-            cwd = None
-        print >> sys.stderr, '*', ' '.join(args)
-        try:
-            subprocess.check_call(args, cwd=cwd)
-        except subprocess.CalledProcessError:
-            print >>sys.stderr, """!!!!!!!!!!\nBuilding {0} bindings failed.
-You can either install development headers package,
-add the --without-{0} option to skip packaging this
-binary CFFI extension, or say --without-cffi.""".format(key)
-            raise MissingDependenciesError(module)
-
 def pypy_runs(pypy_c, quiet=False):
     kwds = {}
     if quiet:
@@ -117,9 +84,13 @@
     if not _fake and not pypy_runs(pypy_c):
         raise OSError("Running %r failed!" % (str(pypy_c),))
     if not options.no_cffi:
-        try:
-            create_cffi_import_libraries(pypy_c, options, basedir)
-        except MissingDependenciesError:
+        failures = create_cffi_import_libraries(pypy_c, options, basedir)
+        for key, module in failures:
+            print >>sys.stderr, """!!!!!!!!!!\nBuilding {0} bindings failed.
+                You can either install development headers package,
+                add the --without-{0} option to skip packaging this
+                binary CFFI extension, or say --without-cffi.""".format(key)
+        if len(failures) > 0:
             return 1, None
 
     if sys.platform == 'win32' and not rename_pypy_c.lower().endswith('.exe'):
diff --git a/rpython/flowspace/flowcontext.py b/rpython/flowspace/flowcontext.py
--- a/rpython/flowspace/flowcontext.py
+++ b/rpython/flowspace/flowcontext.py
@@ -1207,7 +1207,8 @@
     def nomoreblocks(self, ctx):
         w_exc = self.w_exc
         if w_exc.w_type == const(ImportError):
-            msg = 'import statement always raises %s' % self
+            msg = 'ImportError is raised in RPython: %s' % (
+                getattr(w_exc.w_value, 'value', '<not a constant message>'),)
             raise ImportError(msg)
         link = Link([w_exc.w_type, w_exc.w_value], ctx.graph.exceptblock)
         ctx.recorder.crnt_block.closeblock(link)
diff --git a/rpython/flowspace/test/cant_import.py b/rpython/flowspace/test/cant_import.py
new file mode 100644
--- /dev/null
+++ b/rpython/flowspace/test/cant_import.py
@@ -0,0 +1,1 @@
+raise ImportError("some explanation here")
diff --git a/rpython/flowspace/test/test_objspace.py b/rpython/flowspace/test/test_objspace.py
--- a/rpython/flowspace/test/test_objspace.py
+++ b/rpython/flowspace/test/test_objspace.py
@@ -816,6 +816,12 @@
             from rpython import this_does_not_exist
         py.test.raises(ImportError, 'self.codetest(f)')
 
+    def test_importerror_3(self):
+        def f():
+            import rpython.flowspace.test.cant_import
+        e = py.test.raises(ImportError, 'self.codetest(f)')
+        assert "some explanation here" in str(e.value)
+
     def test_relative_import(self):
         def f():
             from ..objspace import build_flow
diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py
--- a/rpython/jit/backend/llsupport/rewrite.py
+++ b/rpython/jit/backend/llsupport/rewrite.py
@@ -73,8 +73,6 @@
                 self.emit_pending_zeros()
             elif op.can_malloc():
                 self.emitting_an_operation_that_can_collect()
-            elif op.getopnum() == rop.DEBUG_MERGE_POINT:
-                continue # ignore debug_merge_points
             elif op.getopnum() == rop.LABEL:
                 self.emitting_an_operation_that_can_collect()
                 self.known_lengths.clear()
diff --git a/rpython/jit/metainterp/heapcache.py b/rpython/jit/metainterp/heapcache.py
--- a/rpython/jit/metainterp/heapcache.py
+++ b/rpython/jit/metainterp/heapcache.py
@@ -60,6 +60,26 @@
             if not value.is_unescaped:
                 del d[value]
 
+
+class FieldUpdater(object):
+    def __init__(self, heapcache, value, cache, fieldvalue):
+        self.heapcache = heapcache
+        self.value = value
+        self.cache = cache
+        if fieldvalue is not None:
+            self.currfieldbox = fieldvalue.box
+        else:
+            self.currfieldbox = None
+
+    def getfield_now_known(self, fieldbox):
+        fieldvalue = self.heapcache.getvalue(fieldbox)
+        self.cache.read_now_known(self.value, fieldvalue)
+
+    def setfield(self, fieldbox):
+        fieldvalue = self.heapcache.getvalue(fieldbox)
+        self.cache.do_write_with_aliasing(self.value, fieldvalue)
+
+
 class HeapCache(object):
     def __init__(self):
         self.reset()
@@ -98,9 +118,9 @@
         self.heap_cache = {}
         self.heap_array_cache = {}
 
-    def getvalue(self, box):
+    def getvalue(self, box, create=True):
         value = self.values.get(box, None)
-        if not value:
+        if not value and create:
             value = self.values[box] = HeapCacheValue(box)
         return value
 
@@ -111,25 +131,26 @@
         self.mark_escaped(opnum, descr, argboxes)
         self.clear_caches(opnum, descr, argboxes)
 
+    def _escape_from_write(self, box, fieldbox):
+        value = self.getvalue(box, create=False)
+        fieldvalue = self.getvalue(fieldbox, create=False)
+        if (value is not None and value.is_unescaped and
+                fieldvalue is not None and fieldvalue.is_unescaped):
+            if value.dependencies is None:
+                value.dependencies = []
+            value.dependencies.append(fieldvalue)
+        elif fieldvalue is not None:
+            self._escape(fieldvalue)
+
     def mark_escaped(self, opnum, descr, argboxes):
         if opnum == rop.SETFIELD_GC:
             assert len(argboxes) == 2
-            value, fieldvalue = self.getvalues(argboxes)
-            if value.is_unescaped and fieldvalue.is_unescaped:
-                if value.dependencies is None:
-                    value.dependencies = []
-                value.dependencies.append(fieldvalue)
-            else:
-                self._escape(fieldvalue)
+            box, fieldbox = argboxes
+            self._escape_from_write(box, fieldbox)
         elif opnum == rop.SETARRAYITEM_GC:
             assert len(argboxes) == 3
-            value, indexvalue, fieldvalue = self.getvalues(argboxes)
-            if value.is_unescaped and fieldvalue.is_unescaped:
-                if value.dependencies is None:
-                    value.dependencies = []
-                value.dependencies.append(fieldvalue)
-            else:
-                self._escape(fieldvalue)
+            box, indexbox, fieldbox = argboxes
+            self._escape_from_write(box, fieldbox)
         elif (opnum == rop.CALL and
               descr.get_extra_info().oopspecindex == descr.get_extra_info().OS_ARRAYCOPY and
               isinstance(argboxes[3], ConstInt) and
@@ -153,7 +174,7 @@
                 self._escape_box(box)
 
     def _escape_box(self, box):
-        value = self.values.get(box, None)
+        value = self.getvalue(box, create=False)
         if not value:
             return
         self._escape(value)
@@ -261,7 +282,7 @@
         self.reset_keep_likely_virtuals()
 
     def is_class_known(self, box):
-        value = self.values.get(box, None)
+        value = self.getvalue(box, create=False)
         if value:
             return value.known_class
         return False
@@ -270,7 +291,7 @@
         self.getvalue(box).known_class = True
 
     def is_nonstandard_virtualizable(self, box):
-        value = self.values.get(box, None)
+        value = self.getvalue(box, create=False)
         if value:
             return value.nonstandard_virtualizable
         return False
@@ -279,13 +300,13 @@
         self.getvalue(box).nonstandard_virtualizable = True
 
     def is_unescaped(self, box):
-        value = self.values.get(box, None)
+        value = self.getvalue(box, create=False)
         if value:
             return value.is_unescaped
         return False
 
     def is_likely_virtual(self, box):
-        value = self.values.get(box, None)
+        value = self.getvalue(box, create=False)
         if value:
             return value.likely_virtual
         return False
@@ -301,7 +322,7 @@
         self.arraylen_now_known(box, lengthbox)
 
     def getfield(self, box, descr):
-        value = self.values.get(box, None)
+        value = self.getvalue(box, create=False)
         if value:
             cache = self.heap_cache.get(descr, None)
             if cache:
@@ -310,26 +331,28 @@
                     return tovalue.box
         return None
 
-    def getfield_now_known(self, box, descr, fieldbox):
+    def get_field_updater(self, box, descr):
         value = self.getvalue(box)
-        fieldvalue = self.getvalue(fieldbox)
         cache = self.heap_cache.get(descr, None)
         if cache is None:
             cache = self.heap_cache[descr] = CacheEntry()
-        cache.read_now_known(value, fieldvalue)
+            fieldvalue = None
+        else:
+            fieldvalue = cache.read(value)
+        return FieldUpdater(self, value, cache, fieldvalue)
+
+    def getfield_now_known(self, box, descr, fieldbox):
+        upd = self.get_field_updater(box, descr)
+        upd.getfield_now_known(fieldbox)
 
     def setfield(self, box, fieldbox, descr):
-        cache = self.heap_cache.get(descr, None)
-        if cache is None:
-            cache = self.heap_cache[descr] = CacheEntry()
-        value = self.getvalue(box)
-        fieldvalue = self.getvalue(fieldbox)
-        cache.do_write_with_aliasing(value, fieldvalue)
+        upd = self.get_field_updater(box, descr)
+        upd.setfield(fieldbox)
 
     def getarrayitem(self, box, indexbox, descr):
         if not isinstance(indexbox, ConstInt):
             return None
-        value = self.values.get(box, None)
+        value = self.getvalue(box, create=False)
         if value is None:
             return None
         index = indexbox.getint()
@@ -373,7 +396,7 @@
             indexcache.do_write_with_aliasing(value, fieldvalue)
 
     def arraylen(self, box):
-        value = self.values.get(box, None)
+        value = self.getvalue(box, create=False)
         if value and value.length:
             return value.length.box
         return None
@@ -383,7 +406,7 @@
         value.length = self.getvalue(lengthbox)
 
     def replace_box(self, oldbox, newbox):
-        value = self.values.get(oldbox, None)
+        value = self.getvalue(oldbox, create=False)
         if value is None:
             return
         value.box = newbox
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -649,16 +649,16 @@
 
     @specialize.arg(1)
     def _opimpl_getfield_gc_any_pureornot(self, opnum, box, fielddescr):
-        tobox = self.metainterp.heapcache.getfield(box, fielddescr)
-        if tobox is not None:
+        upd = self.metainterp.heapcache.get_field_updater(box, fielddescr)
+        if upd.currfieldbox is not None:
             # sanity check: see whether the current struct value
             # corresponds to what the cache thinks the value is
             resbox = executor.execute(self.metainterp.cpu, self.metainterp,
                                       rop.GETFIELD_GC, fielddescr, box)
-            assert resbox.constbox().same_constant(tobox.constbox())
-            return tobox
+            assert resbox.constbox().same_constant(upd.currfieldbox.constbox())
+            return upd.currfieldbox
         resbox = self.execute_with_descr(opnum, fielddescr, box)
-        self.metainterp.heapcache.getfield_now_known(box, fielddescr, resbox)
+        upd.getfield_now_known(resbox)
         return resbox
 
     @arguments("box", "descr", "orgpc")
@@ -679,10 +679,11 @@
 
     @arguments("box", "box", "descr")
     def _opimpl_setfield_gc_any(self, box, valuebox, fielddescr):
-        tobox = self.metainterp.heapcache.getfield(box, fielddescr)
-        if tobox is valuebox:
+        upd = self.metainterp.heapcache.get_field_updater(box, fielddescr)
+        if upd.currfieldbox is valuebox:
             return
-        self.metainterp.execute_setfield_gc(fielddescr, box, valuebox)
+        self.metainterp.execute_and_record(rop.SETFIELD_GC, fielddescr, box, valuebox)
+        upd.setfield(valuebox)
         # The following logic is disabled because buggy.  It is supposed
         # to be: not(we're writing null into a freshly allocated object)
         # but the bug is that is_unescaped() can be True even after the
@@ -1922,9 +1923,10 @@
         resbox = executor.execute(self.cpu, self, opnum, descr, *argboxes)
         if rop._ALWAYS_PURE_FIRST <= opnum <= rop._ALWAYS_PURE_LAST:
             return self._record_helper_pure(opnum, resbox, descr, *argboxes)
-        else:
-            return self._record_helper_nonpure_varargs(opnum, resbox, descr,
-                                                       list(argboxes))
+        if rop._OVF_FIRST <= opnum <= rop._OVF_LAST:
+            return self._record_helper_ovf(opnum, resbox, descr, *argboxes)
+        return self._record_helper_nonpure_varargs(opnum, resbox, descr,
+                                                   list(argboxes))
 
     @specialize.arg(1)
     def execute_and_record_varargs(self, opnum, argboxes, descr=None):
@@ -1951,6 +1953,12 @@
             resbox = resbox.nonconstbox()    # ensure it is a Box
             return self._record_helper_nonpure_varargs(opnum, resbox, descr, list(argboxes))
 
+    def _record_helper_ovf(self, opnum, resbox, descr, *argboxes):
+        if (self.last_exc_value_box is None and
+                self._all_constants(*argboxes)):
+            return resbox.constbox()
+        return self._record_helper_nonpure_varargs(opnum, resbox, descr, list(argboxes))
+
     def _record_helper_pure_varargs(self, opnum, resbox, descr, argboxes):
         canfold = self._all_constants_varargs(argboxes)
         if canfold:
@@ -1962,10 +1970,6 @@
 
     def _record_helper_nonpure_varargs(self, opnum, resbox, descr, argboxes):
         assert resbox is None or isinstance(resbox, Box)
-        if (rop._OVF_FIRST <= opnum <= rop._OVF_LAST and
-            self.last_exc_value_box is None and
-            self._all_constants_varargs(argboxes)):
-            return resbox.constbox()
         # record the operation
         profiler = self.staticdata.profiler
         profiler.count_ops(opnum, Counters.RECORDED_OPS)
diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py
--- a/rpython/memory/gctransform/framework.py
+++ b/rpython/memory/gctransform/framework.py
@@ -52,21 +52,22 @@
             return (op.opname in LL_OPERATIONS and
                     LL_OPERATIONS[op.opname].canmallocgc)
 
-def find_initializing_stores(collect_analyzer, graph):
-    from rpython.flowspace.model import mkentrymap
-    entrymap = mkentrymap(graph)
-    # a bit of a hackish analysis: if a block contains a malloc and check that
-    # the result is not zero, then the block following the True link will
-    # usually initialize the newly allocated object
-    result = set()
-    def find_in_block(block, mallocvars):
+def propagate_no_write_barrier_needed(result, block, mallocvars,
+                                      collect_analyzer, entrymap,
+                                      startindex=0):
+    # We definitely know that no write barrier is needed in the 'block'
+    # for any of the variables in 'mallocvars'.  Propagate this information
+    # forward.  Note that "definitely know" implies that we just did either
+    # a fixed-size malloc (variable-size might require card marking), or
+    # that we just did a full write barrier (not just for card marking).
+    if 1:       # keep indentation
         for i, op in enumerate(block.operations):
+            if i < startindex:
+                continue
             if op.opname in ("cast_pointer", "same_as"):
                 if op.args[0] in mallocvars:
                     mallocvars[op.result] = True
             elif op.opname in ("setfield", "setarrayitem", "setinteriorfield"):
-                # note that 'mallocvars' only tracks fixed-size mallocs,
-                # so no risk that they use card marking
                 TYPE = op.args[-1].concretetype
                 if (op.args[0] in mallocvars and
                     isinstance(TYPE, lltype.Ptr) and
@@ -83,7 +84,15 @@
                 if var in mallocvars:
                     newmallocvars[exit.target.inputargs[i]] = True
             if newmallocvars:
-                find_in_block(exit.target, newmallocvars)
+                propagate_no_write_barrier_needed(result, exit.target,
+                                                  newmallocvars,
+                                                  collect_analyzer, entrymap)
+
+def find_initializing_stores(collect_analyzer, graph, entrymap):
+    # a bit of a hackish analysis: if a block contains a malloc and check that
+    # the result is not zero, then the block following the True link will
+    # usually initialize the newly allocated object
+    result = set()
     mallocnum = 0
     blockset = set(graph.iterblocks())
     while blockset:
@@ -113,7 +122,8 @@
         target = exit.target
         mallocvars = {target.inputargs[index]: True}
         mallocnum += 1
-        find_in_block(target, mallocvars)
+        propagate_no_write_barrier_needed(result, target, mallocvars,
+                                          collect_analyzer, entrymap)
     #if result:
     #    print "found %s initializing stores in %s" % (len(result), graph.name)
     return result
@@ -698,8 +708,11 @@
                                 " %s" % func)
 
         if self.write_barrier_ptr:
+            from rpython.flowspace.model import mkentrymap
+            self._entrymap = mkentrymap(graph)
             self.clean_sets = (
-                find_initializing_stores(self.collect_analyzer, graph))
+                find_initializing_stores(self.collect_analyzer, graph,
+                                         self._entrymap))
             if self.gcdata.gc.can_optimize_clean_setarrayitems():
                 self.clean_sets = self.clean_sets.union(
                     find_clean_setarrayitems(self.collect_analyzer, graph))
@@ -1269,6 +1282,17 @@
                 hop.genop("direct_call", [self.write_barrier_ptr,
                                           self.c_const_gc,
                                           v_structaddr])
+                # we just did a full write barrier here, so we can use
+                # this helper to propagate this knowledge forward and
+                # avoid to repeat the write barrier.
+                if self.curr_block is not None:   # for tests
+                    assert self.curr_block.operations[hop.index] is hop.spaceop
+                    propagate_no_write_barrier_needed(self.clean_sets,
+                                                      self.curr_block,
+                                                      {v_struct: True},
+                                                      self.collect_analyzer,
+                                                      self._entrymap,
+                                                      hop.index + 1)
         hop.rename('bare_' + opname)
 
     def transform_getfield_typeptr(self, hop):
diff --git a/rpython/memory/gctransform/test/test_framework.py b/rpython/memory/gctransform/test/test_framework.py
--- a/rpython/memory/gctransform/test/test_framework.py
+++ b/rpython/memory/gctransform/test/test_framework.py
@@ -1,6 +1,6 @@
 from rpython.annotator.listdef import s_list_of_strings
 from rpython.annotator.model import SomeInteger
-from rpython.flowspace.model import Constant, SpaceOperation
+from rpython.flowspace.model import Constant, SpaceOperation, mkentrymap
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rtyper.lltypesystem.lloperation import llop
 from rpython.memory.gc.semispace import SemiSpaceGC
@@ -231,6 +231,33 @@
          Constant('b', lltype.Void), varoftype(PTR_TYPE2)],
         varoftype(lltype.Void)))
 
+def test_remove_duplicate_write_barrier():
+    from rpython.translator.c.genc import CStandaloneBuilder
+    from rpython.flowspace.model import summary
+
+    class A(object):
+        pass
+    glob_a_1 = A()
+    glob_a_2 = A()
+
+    def f(a, cond):
+        a.x = a
+        a.z = a
+        if cond:
+            a.y = a
+    def g():
+        f(glob_a_1, 5)
+        f(glob_a_2, 0)
+    t = rtype(g, [])
+    t.config.translation.gc = "minimark"
+    cbuild = CStandaloneBuilder(t, g, t.config,
+                                gcpolicy=FrameworkGcPolicy2)
+    db = cbuild.generate_graphs_for_llinterp()
+
+    ff = graphof(t, f)
+    #ff.show()
+    assert summary(ff)['direct_call'] == 1    # only one remember_young_pointer
+
 def test_find_initializing_stores():
 
     class A(object):
@@ -246,7 +273,8 @@
     etrafo = ExceptionTransformer(t)
     graphs = etrafo.transform_completely()
     collect_analyzer = CollectAnalyzer(t)
-    init_stores = find_initializing_stores(collect_analyzer, t.graphs[0])
+    init_stores = find_initializing_stores(collect_analyzer, t.graphs[0],
+                                           mkentrymap(t.graphs[0]))
     assert len(init_stores) == 1
 
 def test_find_initializing_stores_across_blocks():
@@ -271,7 +299,8 @@
     etrafo = ExceptionTransformer(t)
     graphs = etrafo.transform_completely()
     collect_analyzer = CollectAnalyzer(t)
-    init_stores = find_initializing_stores(collect_analyzer, t.graphs[0])
+    init_stores = find_initializing_stores(collect_analyzer, t.graphs[0],
+                                           mkentrymap(t.graphs[0]))
     assert len(init_stores) == 5
 
 def test_find_clean_setarrayitems():
diff --git a/rpython/memory/gctransform/transform.py b/rpython/memory/gctransform/transform.py
--- a/rpython/memory/gctransform/transform.py
+++ b/rpython/memory/gctransform/transform.py
@@ -83,6 +83,7 @@
 
 class BaseGCTransformer(object):
     finished_helpers = False
+    curr_block = None
 
     def __init__(self, translator, inline=False):
         self.translator = translator
@@ -159,7 +160,7 @@
 
     def transform_block(self, block, is_borrowed):
         llops = LowLevelOpList()
-        #self.curr_block = block
+        self.curr_block = block
         self.livevars = [var for var in block.inputargs
                     if var_needsgc(var) and not is_borrowed(var)]
         allvars = [var for var in block.getvariables() if var_needsgc(var)]
@@ -205,6 +206,7 @@
             block.operations[:] = llops
         self.livevars = None
         self.var_last_needed_in = None
+        self.curr_block = None
 
     def transform_graph(self, graph):
         if graph in self.minimal_transform:
diff --git a/rpython/rlib/jit_libffi.py b/rpython/rlib/jit_libffi.py
--- a/rpython/rlib/jit_libffi.py
+++ b/rpython/rlib/jit_libffi.py
@@ -109,6 +109,11 @@
 def jit_ffi_call(cif_description, func_addr, exchange_buffer):
     """Wrapper around ffi_call().  Must receive a CIF_DESCRIPTION_P that
     describes the layout of the 'exchange_buffer'.
+
+    Note that this cannot be optimized if 'cif_description' is not
+    a constant for the JIT, so if it is ever possible, consider promoting
+    it.  The promotion of 'cif_description' must be done earlier, before
+    the raw malloc of 'exchange_buffer'.
     """
     reskind = types.getkind(cif_description.rtype)
     if reskind == 'v':
diff --git a/rpython/translator/c/genc.py b/rpython/translator/c/genc.py
--- a/rpython/translator/c/genc.py
+++ b/rpython/translator/c/genc.py
@@ -485,7 +485,7 @@
                 else:
                     mk.definition('DEBUGFLAGS', '-O1 -g')
         if self.translator.platform.name == 'msvc':
-            mk.rule('debug_target', 'debugmode_$(DEFAULT_TARGET)', 'rem')
+            mk.rule('debug_target', '$(DEFAULT_TARGET)', 'rem')
         else:
             mk.rule('debug_target', '$(DEFAULT_TARGET)', '#')
         mk.write()

From noreply at buildbot.pypy.org  Fri Jun 26 17:46:10 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Fri, 26 Jun 2015 17:46:10 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: finishing up changes
Message-ID: <20150626154610.1513C1C0222@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r78324:313bcd2938c3
Date: 2015-06-26 16:34 +0200
http://bitbucket.org/pypy/pypy/changeset/313bcd2938c3/

Log:	finishing up changes

diff --git a/rpython/jit/metainterp/optimizeopt/schedule.py b/rpython/jit/metainterp/optimizeopt/schedule.py
--- a/rpython/jit/metainterp/optimizeopt/schedule.py
+++ b/rpython/jit/metainterp/optimizeopt/schedule.py
@@ -361,6 +361,7 @@
                 continue
             box_pos, vbox = self.sched_data.getvector_of_box(arg)
             if not vbox:
+                import pdb; pdb.set_trace()
                 # constant/variable expand this box
                 vbox = self.expand(arg, i)
                 self.sched_data.setvector_of_box(arg, 0, vbox)
@@ -875,6 +876,11 @@
             node.pack = None
             node.pack_position = -1
 
+    def update_pack_of_nodes(self):
+        for i,node in enumerate(self.operations):
+            node.pack = self
+            node.pack_position = i
+
     def rightmost_match_leftmost(self, other):
         assert isinstance(other, Pack)
         rightmost = self.operations[-1]
@@ -889,7 +895,8 @@
         return rightmost is leftmost and accum
 
     def __repr__(self):
-        return "Pack(%r)" % self.operations
+        opname = self.operations[0].getoperation().getopname()
+        return "Pack(%s,%r)" % (opname, self.operations)
 
     def is_accumulating(self):
         return self.accum is not None
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py b/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
@@ -747,8 +747,12 @@
         self.assert_packset_empty(vopt.packset, len(loop.operations),
                                   [(6,12), (5,11), (7,13)])
 
-    @pytest.mark.parametrize("descr,size", [('char',16),('float',2),('int',2),('singlefloat',4)])
-    def test_packset_combine_simple(self,descr,size):
+    @pytest.mark.parametrize("descr,packs,packidx", 
+                             [('char',1,  [(0,(1,3,5,7))]),
+                              ('float',2, [(0,(1,3)),(1,(5,7))]),
+                              ('int',2,   [(0,(1,3)),(1,(5,7))]),
+                              ('singlefloat',1,[(0,(1,3,5,7))])])
+    def test_packset_combine_simple(self,descr,packs,packidx):
         ops = """
         [p0,i0]
         i3 = getarrayitem_raw(p0, i0, descr={descr}arraydescr)
@@ -758,12 +762,13 @@
         loop = self.parse_loop(ops)
         vopt = self.combine_packset(loop,3)
         assert len(vopt.dependency_graph.memory_refs) == 4
-        assert len(vopt.packset.packs) == 16 // size
-        self.assert_pack(vopt.packset.packs[0], (1,3,5,7))
+        assert len(vopt.packset.packs) == packs
+        for i,t in packidx:
+            self.assert_pack(vopt.packset.packs[i], t)
 
-    @pytest.mark.parametrize("descr,stride",
-            [('char',1),('float',8),('int',8),('singlefloat',4)])
-    def test_packset_combine_2_loads_in_trace(self, descr, stride):
+    @pytest.mark.parametrize("descr,stride,packs",
+            [('char',1,1),('float',8,4),('int',8,4),('singlefloat',4,2)])
+    def test_packset_combine_2_loads_in_trace(self, descr, stride,packs):
         ops = """
         [p0,i0]
         i3 = raw_load(p0, i0, descr={type}arraydescr)
@@ -775,24 +780,7 @@
         loop = self.parse_loop(ops)
         vopt = self.combine_packset(loop,3)
         assert len(vopt.dependency_graph.memory_refs) == 8
-        assert len(vopt.packset.packs) == (16//stride) * 2
-        self.assert_pack(vopt.packset.packs[0], (1,3,5,7,9,11,13,15))
-
-    def test_packset_combine_2_loads_one_redundant(self):
-        py.test.skip("apply redundant load elimination?")
-        ops = """
-        [p0,i0]
-        i3 = getarrayitem_raw(p0, i0, descr=floatarraydescr)
-        i1 = int_add(i0,1)
-        i4 = getarrayitem_raw(p0, i1, descr=floatarraydescr)
-        jump(p0,i1)
-        """
-        loop = self.parse_loop(ops)
-        vopt = self.combine_packset(loop,3)
-        assert len(vopt.dependency_graph.memory_refs) == 8
-        assert len(vopt.packset.packs) == 2
-        self.assert_pack(vopt.packset.packs[0], (1,5,9))
-        self.assert_pack(vopt.packset.packs[1], (3,7,11))
+        assert len(vopt.packset.packs) == packs
 
     def test_packset_combine_no_candidates_packset_empty(self):
         ops = """
@@ -847,7 +835,10 @@
         loop = self.parse_loop(ops)
         vopt = self.combine_packset(loop,3)
         assert len(vopt.dependency_graph.memory_refs) == 12
-        assert len(vopt.packset.packs) == 4
+        if stride == 8:
+            assert len(vopt.packset.packs) == 8
+        else:
+            assert len(vopt.packset.packs) == 4
 
         for opindices in [(5,12,19,26),(6,13,20,27),
                           (7,14,21,28),(8,15,22,29)]:
@@ -859,7 +850,6 @@
              ('float_mul','float',8),
              ('int_add','int',8),
              ('int_sub','int',8),
-             ('int_mul','int',8),
             ])
     def test_schedule_vector_operation(self, op, descr, stride):
         ops = """
@@ -981,7 +971,7 @@
         [p0,i0]
         guard_early_exit() [p0,i0]
         i1 = getarrayitem_raw(p0, i0, descr=floatarraydescr)
-        i4 = int_mul(i1, 42)
+        i4 = int_sub(i1, 42)
         i3 = int_add(i0,1)
         i5 = int_lt(i3, 10)
         guard_true(i5) [p0, i0]
@@ -1000,7 +990,7 @@
         i4 = int_add(i0, 2)
         i5 = int_lt(i2, 10)
         v1 = vec_getarrayitem_raw(p0, i0, 2, descr=floatarraydescr)
-        v2 = vec_int_mul(v1, v3)
+        v2 = vec_int_sub(v1, v3)
         jump(p0,i2,v3)
         """
         vopt = self.vectorize(self.parse_loop(ops),1)
@@ -1011,7 +1001,7 @@
         [p0,i0,f3]
         guard_early_exit() [p0,i0]
         f1 = getarrayitem_raw(p0, i0, descr=floatarraydescr)
-        f4 = int_mul(f1, f3)
+        f4 = int_add(f1, f3)
         i3 = int_add(i0,1)
         i5 = int_lt(i3, 10)
         guard_true(i5) [p0, i0]
@@ -1030,7 +1020,7 @@
         i4 = int_add(i0, 2)
         i5 = int_lt(i2, 10)
         v1 = vec_getarrayitem_raw(p0, i0, 2, descr=floatarraydescr)
-        v2 = vec_int_mul(v1, v3)
+        v2 = vec_int_add(v1, v3)
         jump(p0,i2,f3,v3)
         """
         vopt = self.vectorize(self.parse_loop(ops),1)
@@ -1157,8 +1147,8 @@
         i7 = int_add(i1, 4)
         i14 = int_ge(i50, 36)
         v17 = vec_getarrayitem_raw(p0, i1, 2, descr=floatarraydescr)
+        v19 = vec_cast_float_to_singlefloat(v17)
         v18 = vec_getarrayitem_raw(p0, i5, 2, descr=floatarraydescr)
-        v19 = vec_cast_float_to_singlefloat(v17)
         v20 = vec_cast_float_to_singlefloat(v18)
         v21 = vec_float_pack(v19, v20, 2, 2)
         vec_setarrayitem_raw(p1, i1, v21, descr=singlefloatarraydescr)
diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/vectorize.py
@@ -374,12 +374,26 @@
                         self.packset.add_pack(pair)
 
     def combine_packset(self):
+        """ Combination is done iterating the packs that have
+        a sorted op index of the first operation (= left).
+        If a pack is marked as 'full', the next pack that is
+        encountered having the full_pack.right == pack.left,
+        the pack is removed. This is because the packs have
+        intersecting edges.
+        """
         if len(self.packset.packs) == 0:
             raise NotAVectorizeableLoop()
+        packsort(self.packset.packs).sort()
+        if not we_are_translated():
+            # ensure we are really sorted!
+            x = 0
+            for i,pack in enumerate(self.packset.packs):
+                assert x <= pack.left.getindex()
+                x = pack.left.getindex()
         i = 0
         j = 0
-        packsort(self.packset.packs)
         end_ij = len(self.packset.packs)
+        remove_left = {}
         while True:
             len_before = len(self.packset.packs)
             i = 0
@@ -389,14 +403,29 @@
                         j += 1
                         continue
                     pack1 = self.packset.packs[i]
+                    pack2 = self.packset.packs[j]
+                    # remove intermediate
+                    left = pack1.operations[0]
+                    if left in remove_left:
+                        remove_left[left] = pack1
+                        del self.packset.packs[i]
+                        end_ij -= 1
+                        continue
+                    # check if the pack is already full
                     if pack1.is_full(self.cpu.vector_register_size):
+                        pack1.update_pack_of_nodes()
+                        right = pack1.operations[-1]
+                        remove_left[right] = None
                         break
-                    pack2 = self.packset.packs[j]
                     if pack1.rightmost_match_leftmost(pack2):
                         end_ij = self.packset.combine(i,j)
-                    elif pack2.rightmost_match_leftmost(pack1):
-                        end_ij = self.packset.combine(j,i)
-                    j += 1
+                    else:
+                        # do not inc in rightmost_match_leftmost
+                        # this could miss some pack
+                        j += 1
+                # set for each node to which pack it belongs
+                self.packset.packs[i].update_pack_of_nodes()
+
                 j = 0
                 i += 1
             if len_before == len(self.packset.packs):
@@ -406,7 +435,15 @@
             # some test cases check the accumulation variables
             self.packset.accum_vars = {}
             print "packs:"
+            check = {}
+            fail = False
             for pack in self.packset.packs:
+                left = pack.operations[0]
+                right = pack.operations[-1]
+                if left in check or right in check:
+                    fail = True
+                check[left] = None
+                check[right] = None
                 accum = pack.accum
                 if accum:
                     self.packset.accum_vars[accum.var] = accum.pos
@@ -414,6 +451,8 @@
                 print " %dx %s (accum? %d) " % (len(pack.operations),
                                                 pack.operations[0].op.getopname(),
                                                 accum is not None)
+            if fail:
+                assert False
 
     def schedule(self, vector=False):
         self.guard_early_exit = -1
@@ -463,6 +502,8 @@
     def _unpack_from_vector(self, i, arg, sched_data, renamer):
         (j, vbox) = sched_data.box_to_vbox.get(arg, (-1, None))
         if vbox:
+            if vbox in sched_data.invariant_vector_vars:
+                return arg
             arg_cloned = arg.clonebox()
             renamer.start_renaming(arg, arg_cloned)
             cj = ConstInt(j)
@@ -684,8 +725,6 @@
         is not iterated when calling this method. """
         pack_i = self.packs[i]
         pack_j = self.packs[j]
-        pack_i.clear()
-        pack_j.clear()
         operations = pack_i.operations
         for op in pack_j.operations[1:]:
             operations.append(op)
@@ -697,16 +736,19 @@
         pack.accum = pack_i.accum
         pack_i.accum = pack_j.accum = None
 
+        del self.packs[j]
+        return len(self.packs)
+        # OLD
         # instead of deleting an item in the center of pack array,
         # the last element is assigned to position j and
         # the last slot is freed. Order of packs doesn't matter
-        last_pos = len(self.packs) - 1
-        if j == last_pos:
-            del self.packs[j]
-        else:
-            self.packs[j] = self.packs[last_pos]
-            del self.packs[last_pos]
-        return last_pos
+        #last_pos = len(self.packs) - 1
+        #if j == last_pos:
+        #    del self.packs[j]
+        #else:
+        #    self.packs[j] = self.packs[last_pos]
+        #    del self.packs[last_pos]
+        #return last_pos
 
     def accumulates_pair(self, lnode, rnode, origin_pack):
         # lnode and rnode are isomorphic and dependent

From noreply at buildbot.pypy.org  Fri Jun 26 17:46:11 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Fri, 26 Jun 2015 17:46:11 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: test_vectorize passing again
Message-ID: <20150626154611.561D51C0222@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r78325:2df6133da026
Date: 2015-06-26 16:56 +0200
http://bitbucket.org/pypy/pypy/changeset/2df6133da026/

Log:	test_vectorize passing again

diff --git a/rpython/jit/metainterp/optimizeopt/schedule.py b/rpython/jit/metainterp/optimizeopt/schedule.py
--- a/rpython/jit/metainterp/optimizeopt/schedule.py
+++ b/rpython/jit/metainterp/optimizeopt/schedule.py
@@ -294,15 +294,12 @@
         pass
 
     def transform_pack(self):
-        #self.off = 0
-        #while self.off < self.pack.opcount():
         op = self.pack.operations[0].getoperation()
         args = op.getarglist()
         #
         self.before_argument_transform(args)
         #
-        argument_infos = []
-        self.transform_arguments(args, argument_infos)
+        self.transform_arguments(args)
         #
         result = op.result
         result = self.transform_result(result)
@@ -313,22 +310,7 @@
             vop.setfailargs(op.getfailargs())
             vop.rd_snapshot = op.rd_snapshot
         self.preamble_ops.append(vop)
-        #stride = self.consumed_operations(argument_infos, result)
         self.costmodel.record_pack_savings(self.pack, self.pack.opcount())
-        #assert stride != 0
-        #self.off += stride
-
-    def consumed_operations(self, argument_infos, result):
-        ops = self.getoperations()
-        if len(argument_infos) == 0:
-            return result.getcount()
-        if len(argument_infos) == 1:
-            return argument_infos[0]
-        if not we_are_translated():
-            first = argument_infos[0]
-            for ai in argument_infos:
-                assert first == ai
-        return argument_infos[0]
 
     def transform_result(self, result):
         if result is None:
@@ -353,7 +335,7 @@
     def getoperations(self):
         return self.pack.operations
 
-    def transform_arguments(self, args, argument_info):
+    def transform_arguments(self, args):
         for i,arg in enumerate(args):
             if isinstance(arg, BoxVector):
                 continue
@@ -361,7 +343,6 @@
                 continue
             box_pos, vbox = self.sched_data.getvector_of_box(arg)
             if not vbox:
-                import pdb; pdb.set_trace()
                 # constant/variable expand this box
                 vbox = self.expand(arg, i)
                 self.sched_data.setvector_of_box(arg, 0, vbox)
@@ -379,9 +360,8 @@
             if packed > packable:
                 # the argument has more items than the operation is able to process!
                 # box_pos == 0 then it is already at the right place
-                argument_info.append(packable)
                 if box_pos != 0:
-                    args[i] = self.unpack(vbox, self.off, packable, self.input_type)
+                    args[i] = self.unpack(vbox, box_pos, packable, self.input_type)
                     self.update_arg_in_vector_pos(i, args[i])
                     #self.update_input_output(self.pack)
                     continue
@@ -394,20 +374,17 @@
                 # the argument is scattered along different vector boxes
                 args[i] = self.gather(vboxes, packable)
                 self.update_arg_in_vector_pos(i, args[i])
-                argument_info.append(args[i].item_count)
                 continue
             if box_pos != 0:
                 # The vector box is at a position != 0 but it
                 # is required to be at position 0. Unpack it!
-                args[i] = self.unpack(vbox, self.off, packable, self.input_type)
+                args[i] = self.unpack(vbox, box_pos, packable, self.input_type)
                 self.update_arg_in_vector_pos(i, args[i])
-                argument_info.append(args[i].item_count)
                 continue
                 #self.update_input_output(self.pack)
             #
             assert vbox is not None
             args[i] = vbox
-            argument_info.append(args[i].item_count)
 
     def gather(self, vboxes, target_count): # packed < packable and packed < stride:
         (_, box) = vboxes[0]
@@ -798,6 +775,7 @@
 
     def setvector_of_box(self, box, off, vector):
         assert off < vector.item_count
+        print "set" , box, "[",off,"] =", vector
         self.box_to_vbox[box] = (off, vector)
 
     def prepend_invariant_operations(self, oplist):
@@ -845,8 +823,8 @@
     def opcount(self):
         return len(self.operations)
 
-    def process_count(self):
-        return len(self.operations)
+    def leftmost(self):
+        return self.operations[0].getoperation()
 
     def is_full(self, vec_reg_size):
         """ if one input element times the opcount is equal
@@ -855,9 +833,15 @@
         ptype = self.input_type
         if self.input_type is None:
             # load does not have an input type, but only an output type
-            assert self.operations[0].getoperation().is_raw_load()
+            assert self.leftmost().is_raw_load()
             ptype = self.output_type
-        bytes = ptype.getsize() * self.process_count()
+
+        op = self.leftmost()
+        if op.casts_box():
+            assert self.output_type.getcount() <= ptype.getcount()
+            return self.output_type.getcount() <= ptype.getcount()
+
+        bytes = ptype.getsize() * len(self.operations)
         assert bytes <= vec_reg_size
         if bytes == vec_reg_size:
             return True
diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/vectorize.py
@@ -413,7 +413,7 @@
                         continue
                     # check if the pack is already full
                     if pack1.is_full(self.cpu.vector_register_size):
-                        pack1.update_pack_of_nodes()
+                        #pack1.update_pack_of_nodes()
                         right = pack1.operations[-1]
                         remove_left[right] = None
                         break
@@ -424,8 +424,8 @@
                         # this could miss some pack
                         j += 1
                 # set for each node to which pack it belongs
-                self.packset.packs[i].update_pack_of_nodes()
-
+                pack = self.packset.packs[i]
+                pack.update_pack_of_nodes()
                 j = 0
                 i += 1
             if len_before == len(self.packset.packs):

From noreply at buildbot.pypy.org  Fri Jun 26 17:46:12 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Fri, 26 Jun 2015 17:46:12 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: rewritten scheduling tests to add type
 (could not be inferred easily), cost model passing again
Message-ID: <20150626154612.7A8731C0222@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r78326:d1a942296dd8
Date: 2015-06-26 17:46 +0200
http://bitbucket.org/pypy/pypy/changeset/d1a942296dd8/

Log:	rewritten scheduling tests to add type (could not be inferred
	easily), cost model passing again

diff --git a/rpython/jit/metainterp/optimizeopt/schedule.py b/rpython/jit/metainterp/optimizeopt/schedule.py
--- a/rpython/jit/metainterp/optimizeopt/schedule.py
+++ b/rpython/jit/metainterp/optimizeopt/schedule.py
@@ -229,6 +229,7 @@
         self.output_type = None
         self.costmodel = None
 
+
     def determine_input_type(self, op):
         arg = op.getarg(0)
         _, vbox = self.sched_data.getvector_of_box(arg)
@@ -267,9 +268,13 @@
         #
         self.check_if_pack_supported(pack)
         #
-        self.pack = pack
-        self.transform_pack()
-
+        if self.must_be_full_but_is_not(pack):
+            for op in pack.operations:
+                self.preamble_ops.append(op.getoperation())
+        else:
+            self.pack = pack
+            self.transform_pack()
+        #
         self.pack = None
         self.costmodel = None
         self.preamble_ops = None
@@ -277,6 +282,9 @@
         self.input_type = None
         self.output_type = None
 
+    def must_be_full_but_is_not(self, pack):
+        return False
+
     def split_pack(self, pack, vec_reg_size):
         """ Returns how many items of the pack should be
             emitted as vector operation. """
@@ -294,11 +302,9 @@
         pass
 
     def transform_pack(self):
-        op = self.pack.operations[0].getoperation()
+        op = self.pack.leftmost()
         args = op.getarglist()
-        #
         self.before_argument_transform(args)
-        #
         self.transform_arguments(args)
         #
         result = op.result
@@ -614,6 +620,7 @@
         assert isinstance(sizearg, ConstInt)
         self.size = sizearg.value
 
+
     def new_result_vector_box(self):
         type = self.output_type.gettype()
         count = self.input_type.getcount()
@@ -656,6 +663,11 @@
     def determine_input_type(self, op):
         return PackType.by_descr(op.getdescr(), self.sched_data.vec_reg_size)
 
+    def must_be_full_but_is_not(self, pack):
+        vrs = self.sched_data.vec_reg_size
+        it = pack.input_type
+        return it.getsize() * it.getcount() < vrs
+
     def determine_output_type(self, op):
         return None
 
@@ -833,7 +845,6 @@
         ptype = self.input_type
         if self.input_type is None:
             # load does not have an input type, but only an output type
-            assert self.leftmost().is_raw_load()
             ptype = self.output_type
 
         op = self.leftmost()
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_costmodel.py b/rpython/jit/metainterp/optimizeopt/test/test_costmodel.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_costmodel.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_costmodel.py
@@ -136,7 +136,7 @@
         savings = self.savings(loop1)
         assert savings == 2
 
-    @py.test.mark.parametrize("bytes,s", [(1,-1),(2,-1),(4,0),(8,-1)])
+    @py.test.mark.parametrize("bytes,s", [(1,None),(2,None),(4,0),(8,-1)])
     def test_sum_float_to_int(self, bytes, s):
         loop1 = self.parse("""
         f10 = raw_load(p0, i0, descr=double)
@@ -150,13 +150,19 @@
         i15 = int_add(i16, i13)
         i17 = int_signext(i15, {c})
         """.format(c=bytes))
-        savings = self.savings(loop1)
-        # it does not benefit because signext has
-        # a very inefficient implementation (x86
-        # does not provide nice instr to convert
-        # integer sizes)
-        # signext -> no benefit, + 2x unpack
-        assert savings <= s
+        try:
+            savings = self.savings(loop1)
+            if s is None:
+                py.test.fail("must fail")
+            # it does not benefit because signext has
+            # a very inefficient implementation (x86
+            # does not provide nice instr to convert
+            # integer sizes)
+            # signext -> no benefit, + 2x unpack
+            assert savings <= s
+        except NotAProfitableLoop:
+            if s is not None:
+                py.test.fail("must not fail")
 
     def test_cast(self):
         loop1 = self.parse("""
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_schedule.py b/rpython/jit/metainterp/optimizeopt/test/test_schedule.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_schedule.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_schedule.py
@@ -3,7 +3,8 @@
 from rpython.jit.metainterp.history import TargetToken, JitCellToken, TreeLoop
 from rpython.jit.metainterp.optimizeopt.util import equaloplists, Renamer
 from rpython.jit.metainterp.optimizeopt.vectorize import (VecScheduleData,
-        Pack, NotAProfitableLoop, VectorizingOptimizer, X86_CostModel)
+        Pack, Pair, NotAProfitableLoop, VectorizingOptimizer, X86_CostModel,
+        PackSet)
 from rpython.jit.metainterp.optimizeopt.dependency import Node
 from rpython.jit.metainterp.optimizeopt.schedule import PackType
 from rpython.jit.metainterp.optimizeopt.test.test_util import LLtypeMixin
@@ -14,6 +15,14 @@
 from rpython.jit.tool.oparser import parse as opparse
 from rpython.jit.tool.oparser_model import get_model
 
+F64 = PackType('f',8,True,2)
+F32 = PackType('f',4,True,4)
+F32_2 =  PackType('f',4,True,2)
+I64 = PackType('i',8,True,2)
+I32 = PackType('i',4,True,4)
+I32_2 =  PackType('i',4,True,2)
+I16 = PackType('i',2,True,8)
+
 class SchedulerBaseTest(DependencyBaseTest):
 
     def parse(self, source, inc_label_jump=True,
@@ -58,8 +67,8 @@
         del loop.operations[-1]
         return loop
 
-    def pack(self, loop, l, r):
-        return Pack([Node(op,1+l+i) for i,op in enumerate(loop.operations[1+l:1+r])], None, None)
+    def pack(self, loop, l, r, input_type, output_type):
+        return Pack([Node(op,1+l+i) for i,op in enumerate(loop.operations[1+l:1+r])], input_type, output_type)
 
     def schedule(self, loop_orig, packs, vec_reg_size=16, prepend_invariant=False, overwrite_funcs=None):
         loop = get_model(False).ExtendedTreeLoop("loop")
@@ -72,16 +81,32 @@
         for name, overwrite in (overwrite_funcs or {}).items():
             setattr(vsd, name, overwrite)
         renamer = Renamer()
+        metainterp_sd = FakeMetaInterpStaticData(self.cpu)
+        jitdriver_sd = FakeJitDriverStaticData()
+        opt = VectorizingOptimizer(metainterp_sd, jitdriver_sd, loop, 0)
+        pairs = []
         for pack in packs:
+            for i in range(len(pack.operations)-1):
+                o1 = pack.operations[i]
+                o2 = pack.operations[i+1]
+                pairs.append(Pair(o1,o2,pack.input_type,pack.output_type))
+
+        class FakePackSet(PackSet):
+            def __init__(self):
+                self.packs = None
+
+        opt.packset = FakePackSet()
+        opt.packset.packs = pairs
+
+        opt.combine_packset()
+
+        for pack in opt.packset.packs:
             if pack.opcount() == 1:
                 ops.append(pack.operations[0].getoperation())
             else:
                 for op in vsd.as_vector_operation(pack, renamer):
                     ops.append(op)
         loop.operations = ops
-        metainterp_sd = FakeMetaInterpStaticData(self.cpu)
-        jitdriver_sd = FakeJitDriverStaticData()
-        opt = VectorizingOptimizer(metainterp_sd, jitdriver_sd, loop, 0)
         opt.clear_newoperations()
         for op in ops:
             opt.unpack_from_vector(op, vsd, renamer)
@@ -106,7 +131,7 @@
         i14 = raw_load(p0, i4, descr=float)
         i15 = raw_load(p0, i5, descr=float)
         """)
-        pack1 = self.pack(loop1, 0, 6)
+        pack1 = self.pack(loop1, 0, 6, None, F32)
         loop2 = self.schedule(loop1, [pack1])
         loop3 = self.parse("""
         v10[i32|4] = vec_raw_load(p0, i0, 4, descr=float)
@@ -123,9 +148,9 @@
         f10 = cast_int_to_float(i12)
         f11 = cast_int_to_float(i13)
         """)
-        pack1 = self.pack(loop1, 0, 2)
-        pack2 = self.pack(loop1, 2, 4)
-        pack3 = self.pack(loop1, 4, 6)
+        pack1 = self.pack(loop1, 0, 2, None, I64)
+        pack2 = self.pack(loop1, 2, 4, I64, I32_2)
+        pack3 = self.pack(loop1, 4, 6, I32_2, F32_2)
         loop2 = self.schedule(loop1, [pack1, pack2, pack3])
         loop3 = self.parse("""
         v10[i64|2] = vec_raw_load(p0, i0, 2, descr=long)
@@ -139,7 +164,7 @@
         i10 = int_add(i0, 73)
         i11 = int_add(i1, 73)
         """)
-        pack1 = self.pack(loop1, 0, 2)
+        pack1 = self.pack(loop1, 0, 2, I64, I64)
         loop2 = self.schedule(loop1, [pack1], prepend_invariant=True)
         loop3 = self.parse("""
         v10[i64|2] = vec_box(2)
@@ -155,7 +180,7 @@
         f10 = float_add(f0, 73.0)
         f11 = float_add(f1, 73.0)
         """)
-        pack1 = self.pack(loop1, 0, 2)
+        pack1 = self.pack(loop1, 0, 2, I64, I64)
         loop2 = self.schedule(loop1, [pack1], prepend_invariant=True)
         loop3 = self.parse("""
         v10[f64|2] = vec_box(2)
@@ -174,8 +199,8 @@
         f12 = float_add(f10, f5)
         f13 = float_add(f11, f5)
         """)
-        pack1 = self.pack(loop1, 0, 2)
-        pack2 = self.pack(loop1, 2, 4)
+        pack1 = self.pack(loop1, 0, 2, F64, F64)
+        pack2 = self.pack(loop1, 2, 4, F64, F64)
         loop2 = self.schedule(loop1, [pack1, pack2], prepend_invariant=True)
         loop3 = self.parse("""
         v10[f64|2] = vec_box(2)
@@ -199,7 +224,7 @@
         i10 = int_signext(i1, 4)
         i11 = int_signext(i1, 4)
         """, additional_args=['v10[i64|2]'])
-        pack1 = self.pack(loop1, 0, 2)
+        pack1 = self.pack(loop1, 0, 2, I64, I32_2)
         var = self.find_input_arg('v10', loop1)
         def i1inv103204(v):
             return 0, var
@@ -250,10 +275,11 @@
         raw_store(p1, i7, i24, descr=short)
         raw_store(p1, i8, i25, descr=short)
         """)
-        pack1 = self.pack(loop1, 0, 8)
-        pack2 = self.pack(loop1, 8, 16)
-        pack3 = self.pack(loop1, 16, 24)
-        pack4 = self.pack(loop1, 24, 32)
+        pack1 = self.pack(loop1, 0, 8, None, I64)
+        pack2 = self.pack(loop1, 8, 16, I64, I32_2)
+        I16_2 = PackType('i',2,True,2)
+        pack3 = self.pack(loop1, 16, 24, I32, I16_2)
+        pack4 = self.pack(loop1, 24, 32, I16, None)
         def void(b,c):
             pass
         loop2 = self.schedule(loop1, [pack1,pack2,pack3,pack4],
@@ -297,9 +323,9 @@
         raw_store(p1, i3, i12, descr=float)
         raw_store(p1, i4, i13, descr=float)
         """)
-        pack1 = self.pack(loop1, 0, 4)
-        pack2 = self.pack(loop1, 4, 8)
-        pack3 = self.pack(loop1, 8, 12)
+        pack1 = self.pack(loop1, 0, 4, None, I64)
+        pack2 = self.pack(loop1, 4, 8, I64, I32_2)
+        pack3 = self.pack(loop1, 8, 12, I32, None)
         loop2 = self.schedule(loop1, [pack1,pack2,pack3])
         loop3 = self.parse("""
         v44[f64|2] = vec_raw_load(p0, i1, 2, descr=double) 
@@ -322,9 +348,9 @@
         guard_true(i12) []
         guard_true(i13) []
         """)
-        pack1 = self.pack(loop1, 0, 2)
-        pack2 = self.pack(loop1, 2, 4)
-        pack3 = self.pack(loop1, 4, 6)
+        pack1 = self.pack(loop1, 0, 2, None, I64)
+        pack2 = self.pack(loop1, 2, 4, I64, I64)
+        pack3 = self.pack(loop1, 4, 6, None, I64)
         loop2 = self.schedule(loop1, [pack1,pack2,pack3], prepend_invariant=True)
         loop3 = self.parse("""
         v9[i64|2] = vec_int_expand(255)
@@ -342,8 +368,8 @@
         raw_store(p0, i3, i10, descr=float)
         raw_store(p0, i4, i11, descr=float)
         """)
-        pack1 = self.pack(loop1, 0, 2)
-        pack2 = self.pack(loop1, 2, 4)
+        pack1 = self.pack(loop1, 0, 2, None, I32_2)
+        pack2 = self.pack(loop1, 2, 4, I32_2, None)
         loop2 = self.schedule(loop1, [pack1,pack2], prepend_invariant=True)
         loop3 = self.parse("""
         v1[ui32|2] = vec_raw_load(p0, i1, 2, descr=float)
@@ -361,7 +387,7 @@
         i10 = int_and(255, i1)
         i11 = int_and(255, i1)
         """)
-        pack1 = self.pack(loop1, 0, 2)
+        pack1 = self.pack(loop1, 0, 2, I64, I64)
         loop2 = self.schedule(loop1, [pack1], prepend_invariant=True)
         loop3 = self.parse("""
         v1[i64|2] = vec_int_expand(255)
@@ -375,7 +401,7 @@
         i10 = int_and(255, i1)
         i11 = int_and(255, i1)
         """)
-        pack1 = self.pack(loop1, 0, 2)
+        pack1 = self.pack(loop1, 0, 2, I64, I64)
         loop2 = self.schedule(loop1, [pack1], prepend_invariant=True)
         loop3 = self.parse("""
         v1[i64|2] = vec_int_expand(255)
diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/vectorize.py
@@ -408,6 +408,7 @@
                     left = pack1.operations[0]
                     if left in remove_left:
                         remove_left[left] = pack1
+                        pack1.clear()
                         del self.packset.packs[i]
                         end_ij -= 1
                         continue

From noreply at buildbot.pypy.org  Fri Jun 26 23:46:53 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 26 Jun 2015 23:46:53 +0200 (CEST)
Subject: [pypy-commit] cffi default: Test and fix: Python 3.5 (at least the
 current beta) is unhappy
Message-ID: <20150626214653.5CEF01C0460@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2194:9d86b893ad22
Date: 2015-06-26 23:47 +0200
http://bitbucket.org/cffi/cffi/changeset/9d86b893ad22/

Log:	Test and fix: Python 3.5 (at least the current beta) is unhappy with
	module-like objects without a '__name__'

diff --git a/c/lib_obj.c b/c/lib_obj.c
--- a/c/lib_obj.c
+++ b/c/lib_obj.c
@@ -451,6 +451,11 @@
         PyErr_Clear();
         return _lib_dict(lib);
     }
+    /* this hack is for Python 3.5 */
+    if (strcmp(PyText_AsUTF8(name), "__name__") == 0) {
+        PyErr_Clear();
+        return lib_repr(lib);
+    }
     return NULL;
 }
 
diff --git a/testing/cffi1/test_recompiler.py b/testing/cffi1/test_recompiler.py
--- a/testing/cffi1/test_recompiler.py
+++ b/testing/cffi1/test_recompiler.py
@@ -1,3 +1,4 @@
+
 import sys, os, py
 from cffi import FFI, VerificationError
 from cffi import recompiler
@@ -1079,3 +1080,4 @@
     assert MYFOO == 42
     assert hasattr(lib, '__dict__')
     assert lib.__all__ == ['MYFOO', 'mybar']   # but not 'myvar'
+    assert lib.__name__ == repr(lib)

From noreply at buildbot.pypy.org  Fri Jun 26 23:51:12 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 26 Jun 2015 23:51:12 +0200 (CEST)
Subject: [pypy-commit] pypy default: cffi/9d86b893ad22: add 'lib.__name__'
Message-ID: <20150626215112.55AA61C0460@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78327:c636e65f2d90
Date: 2015-06-26 23:51 +0200
http://bitbucket.org/pypy/pypy/changeset/c636e65f2d90/

Log:	cffi/9d86b893ad22: add 'lib.__name__'

diff --git a/pypy/module/_cffi_backend/lib_obj.py b/pypy/module/_cffi_backend/lib_obj.py
--- a/pypy/module/_cffi_backend/lib_obj.py
+++ b/pypy/module/_cffi_backend/lib_obj.py
@@ -175,6 +175,8 @@
                     return self.dir1(ignore_type=cffi_opcode.OP_GLOBAL_VAR)
                 if is_getattr and attr == '__dict__':
                     return self.full_dict_copy()
+                if is_getattr and attr == '__name__':
+                    return self.descr_repr()
                 raise oefmt(self.space.w_AttributeError,
                             "cffi library '%s' has no function, constant "
                             "or global variable named '%s'",
diff --git a/pypy/module/_cffi_backend/test/test_recompiler.py b/pypy/module/_cffi_backend/test/test_recompiler.py
--- a/pypy/module/_cffi_backend/test/test_recompiler.py
+++ b/pypy/module/_cffi_backend/test/test_recompiler.py
@@ -1011,3 +1011,4 @@
         assert MYFOO == 42
         assert hasattr(lib, '__dict__')
         assert lib.__all__ == ['MYFOO', 'mybar']   # but not 'myvar'
+        assert lib.__name__ == repr(lib)

From noreply at buildbot.pypy.org  Sat Jun 27 13:33:47 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 27 Jun 2015 13:33:47 +0200 (CEST)
Subject: [pypy-commit] pypy.org extradoc: update the values
Message-ID: <20150627113347.3843A1C120C@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r620:c32b43f64350
Date: 2015-06-27 13:34 +0200
http://bitbucket.org/pypy/pypy.org/changeset/c32b43f64350/

Log:	update the values

diff --git a/don1.html b/don1.html
--- a/don1.html
+++ b/don1.html
@@ -15,7 +15,7 @@
 </script>
 
    <!-- Income:PyPy:Donations:Py3k / 0.95 -->
-   $59712 of $105000 (56.9%)
+   $59778 of $105000 (56.9%)
    <div id="progressbar">
    </div>
 
@@ -23,7 +23,7 @@
   <li>
   <span id="donate_text">This donation goes towards supporting Python 3 in PyPy.
   Current status:
-    we have $5165 left
+    we have $5226 left
   in the account.</br> <a href="py3donate.html">Read proposal</a></span>
   </li>
   <li>
diff --git a/don4.html b/don4.html
--- a/don4.html
+++ b/don4.html
@@ -17,7 +17,7 @@
    2nd call:
    <!-- Income:PyPy:Donations:Transactional Memory from April 8, 2014,
    divided by 0.9 -->
-   $29163 of $80000 (36.5%)
+   $29183 of $80000 (36.5%)
    <div id="progressbar">
    </div>
 
@@ -25,7 +25,7 @@
   <li>
   <span id="donate_text">This donation goes towards supporting the Transactional Memory in PyPy.
   Current status:
-    we have $22060 left
+    we have $22077 left
   in the account.</br> <a href="tmdonate2.html">Read proposal (2nd call)</a></span>
   </li>
   <li>

From noreply at buildbot.pypy.org  Sat Jun 27 13:50:32 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 27 Jun 2015 13:50:32 +0200 (CEST)
Subject: [pypy-commit] pypy default: Attempt to fix the test on some
 machines, where the open() fails with
Message-ID: <20150627115032.8D4A41C0271@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78328:655704b3f51b
Date: 2015-06-27 13:50 +0200
http://bitbucket.org/pypy/pypy/changeset/655704b3f51b/

Log:	Attempt to fix the test on some machines, where the open() fails
	with "IOError: could not determine default encoding"

diff --git a/pypy/module/_io/test/test_io.py b/pypy/module/_io/test/test_io.py
--- a/pypy/module/_io/test/test_io.py
+++ b/pypy/module/_io/test/test_io.py
@@ -420,6 +420,8 @@
                 {"mode": "w+b", "buffering": 0},
             ]:
             print kwargs
+            if "b" not in kwargs["mode"]:
+                kwargs["encoding"] = "ascii"
             f = _io.open(self.tmpfile, **kwargs)
             f.close()
             raises(ValueError, f.flush)

From noreply at buildbot.pypy.org  Sat Jun 27 15:31:47 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 27 Jun 2015 15:31:47 +0200 (CEST)
Subject: [pypy-commit] pypy default: Test and fix for a case of mutating
 kwargs dictionaries. Thanks
Message-ID: <20150627133147.AB9FC1C120C@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78329:f35f84a382a9
Date: 2015-06-27 15:19 +0200
http://bitbucket.org/pypy/pypy/changeset/f35f84a382a9/

Log:	Test and fix for a case of mutating kwargs dictionaries. Thanks
	Mitsuhiko

diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py
--- a/pypy/objspace/std/dictmultiobject.py
+++ b/pypy/objspace/std/dictmultiobject.py
@@ -711,7 +711,7 @@
     next_item = _new_next('item')
 
 
-def create_iterator_classes(dictimpl, override_next_item=None):
+def create_iterator_classes(dictimpl):
     if not hasattr(dictimpl, 'wrapkey'):
         wrapkey = lambda space, key: key
     else:
@@ -754,15 +754,12 @@
             self.iterator = strategy.getiteritems(impl)
             BaseIteratorImplementation.__init__(self, space, strategy, impl)
 
-        if override_next_item is not None:
-            next_item_entry = override_next_item
-        else:
-            def next_item_entry(self):
-                for key, value in self.iterator:
-                    return (wrapkey(self.space, key),
-                            wrapvalue(self.space, value))
-                else:
-                    return None, None
+        def next_item_entry(self):
+            for key, value in self.iterator:
+                return (wrapkey(self.space, key),
+                        wrapvalue(self.space, value))
+            else:
+                return None, None
 
     class IterClassReversed(BaseKeyIterator):
         def __init__(self, space, strategy, impl):
@@ -795,22 +792,7 @@
     def rev_update1_dict_dict(self, w_dict, w_updatedict):
         # the logic is to call prepare_dict_update() after the first setitem():
         # it gives the w_updatedict a chance to switch its strategy.
-        if override_next_item is not None:
-            # this is very similar to the general version, but the difference
-            # is that it is specialized to call a specific next_item()
-            iteritems = IterClassItems(self.space, self, w_dict)
-            w_key, w_value = iteritems.next_item()
-            if w_key is None:
-                return
-            w_updatedict.setitem(w_key, w_value)
-            w_updatedict.strategy.prepare_update(w_updatedict,
-                                                 w_dict.length() - 1)
-            while True:
-                w_key, w_value = iteritems.next_item()
-                if w_key is None:
-                    return
-                w_updatedict.setitem(w_key, w_value)
-        else:
+        if 1:     # (preserve indentation)
             iteritems = self.getiteritems(w_dict)
             if not same_strategy(self, w_updatedict):
                 # Different strategy.  Try to copy one item of w_dict
diff --git a/pypy/objspace/std/kwargsdict.py b/pypy/objspace/std/kwargsdict.py
--- a/pypy/objspace/std/kwargsdict.py
+++ b/pypy/objspace/std/kwargsdict.py
@@ -166,19 +166,26 @@
         return iter(self.unerase(w_dict.dstorage)[1])
 
     def getiteritems(self, w_dict):
-        keys = self.unerase(w_dict.dstorage)[0]
-        return iter(range(len(keys)))
+        return Zip(*self.unerase(w_dict.dstorage))
 
     wrapkey = _wrapkey
 
 
-def next_item(self):
-    strategy = self.strategy
-    assert isinstance(strategy, KwargsDictStrategy)
-    for i in self.iterator:
-        keys, values_w = strategy.unerase(self.dictimplementation.dstorage)
-        return _wrapkey(self.space, keys[i]), values_w[i]
-    else:
-        return None, None
+class Zip(object):
+    def __init__(self, list1, list2):
+        assert len(list1) == len(list2)
+        self.list1 = list1
+        self.list2 = list2
+        self.i = 0
 
-create_iterator_classes(KwargsDictStrategy, override_next_item=next_item)
+    def __iter__(self):
+        return self
+
+    def next(self):
+        i = self.i
+        if i >= len(self.list1):
+            raise StopIteration
+        self.i = i + 1
+        return (self.list1[i], self.list2[i])
+
+create_iterator_classes(KwargsDictStrategy)
diff --git a/pypy/objspace/std/test/test_kwargsdict.py b/pypy/objspace/std/test/test_kwargsdict.py
--- a/pypy/objspace/std/test/test_kwargsdict.py
+++ b/pypy/objspace/std/test/test_kwargsdict.py
@@ -159,3 +159,10 @@
         assert a == 3
         assert "KwargsDictStrategy" in self.get_strategy(d)
 
+    def test_iteritems_bug(self):
+        def f(**args):
+            return args
+
+        d = f(a=2, b=3, c=4)
+        for key, value in d.iteritems():
+            None in d

From noreply at buildbot.pypy.org  Sat Jun 27 18:09:51 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Sat, 27 Jun 2015 18:09:51 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: added guard_true/false for vector
 register as first argument
Message-ID: <20150627160951.6D5281C0222@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r78330:9150ce6cdf52
Date: 2015-06-27 18:10 +0200
http://bitbucket.org/pypy/pypy/changeset/9150ce6cdf52/

Log:	added guard_true/false for vector register as first argument

diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -276,7 +276,11 @@
     def test_int8_expand(self):
         result = self.run("int8_expand")
         assert int(result) == 17*8 + sum(range(0,17))
-        self.check_vectorized(3, 1) # TODO sum at the end
+        # does not pay off to cast float64 -> int8
+        # neither does sum
+        # a + c should work, but it is given as a parameter
+        # thus the accum must handle this!
+        self.check_vectorized(3, 0) # TODO
 
     def define_int32_add_const():
         return """
@@ -535,25 +539,109 @@
 
     def define_any():
         return """
+        a = astype([0,0,0,0,0,0,0,1,0,0,0],int8)
+        any(a)
+        """
+
+    def define_any_int():
+        return """
+        a = astype([0,0,0,0,256,65537,0,0,0,0,0],int16)
+        any(a)
+        """
+
+    def define_any_ret_0():
+        return """
+        a = astype([0,0,0,0,0,0,0,0,0,0,0],int64)
+        any(a)
+        """
+
+    def define_float_any():
+        return """
         a = [0,0,0,0,0,0,0,1,0,0,0]
         any(a)
         """
 
+    def define_float32_any():
+        return """
+        a = astype([0,0,0,0,0,0,0,1,0,0,0], float32)
+        any(a)
+        """
+
+    def test_float_any(self):
+        result = self.run("float_any")
+        assert int(result) == 1
+        self.check_vectorized(2, 2)
+
+    def test_float32_any(self):
+        result = self.run("float32_any")
+        assert int(result) == 1
+        self.check_vectorized(1, 1)
+
     def test_any(self):
-        result = self.run("any")
-        assert result == 1
-        self.check_vectorized(1, 0)
+        result = self.run("float_any")
+        assert int(result) == 1
+        self.check_vectorized(1, 1)
+
+    def test_any_int(self):
+        result = self.run("any_int")
+        assert int(result) == 1
+        self.check_vectorized(2, 1)
+
+    def test_any_ret_0(self):
+        result = self.run("any_ret_0")
+        assert int(result) == 0
+        self.check_vectorized(2, 2)
 
     def define_all():
         return """
+        a = astype([1,1,1,1,1,1,1,1],int32)
+        all(a)
+        """
+    def define_all_int():
+        return """
+        a = astype([1,100,255,1,3,1,1,1],int32)
+        all(a)
+        """
+    def define_all_ret_0():
+        return """
+        a = astype([1,1,1,1,1,0,1,1],int32)
+        all(a)
+        """
+    def define_float_all():
+        return """
         a = [1,1,1,1,1,1,1,1]
         all(a)
         """
+    def define_float32_all():
+        return """
+        a = astype([1,1,1,1,1,1,1,1],float32)
+        all(a)
+        """
+
+    def test_float_all(self):
+        result = self.run("float_all")
+        assert int(result) == 1
+        self.check_vectorized(2, 2)
+
+    def test_float_all(self):
+        result = self.run("float32_all")
+        assert int(result) == 1
+        self.check_vectorized(2, 2)
 
     def test_all(self):
         result = self.run("all")
-        assert result == 1
-        self.check_vectorized(1, 1)
+        assert int(result) == 1
+        self.check_vectorized(2, 2)
+
+    def test_all_int(self):
+        result = self.run("all_int")
+        assert int(result) == 1
+        self.check_vectorized(2, 2)
+
+    def test_all_ret_0(self):
+        result = self.run("all_ret_0")
+        assert int(result) == 0
+        self.check_vectorized(2, 2)
 
     def define_logical_xor_reduce():
         return """
diff --git a/rpython/doc/jit/vectorization.rst b/rpython/doc/jit/vectorization.rst
--- a/rpython/doc/jit/vectorization.rst
+++ b/rpython/doc/jit/vectorization.rst
@@ -54,5 +54,8 @@
   The opcode needed spans over multiple instructions. In terms of performance
   there might only be little to non advantage to use SIMD instructions for this
   conversions.
+* For a guard that checks true/false on a vector integer regsiter, it would be handy
+  to have 2 xmm registers (one filled with zero bits and the other with one every bit).
+  This cuts down 2 instructions for guard checking, trading for higher register pressure.
 
 .. _PMUL: http://stackoverflow.com/questions/8866973/can-long-integer-routines-benefit-from-sse/8867025#8867025
diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -1644,10 +1644,38 @@
             self.mc.MOVD32_xr(resloc.value, eax.value)
             self.mc.PUNPCKLDQ_xx(resloc.value, loc1.value)
 
+    def genop_guard_vector_arg(self, guard_op, loc):
+        arg = guard_op.getarg(0)
+        assert isinstance(arg, BoxVector)
+        size = arg.item_size
+        temp = X86_64_XMM_SCRATCH_REG
+        #
+        self.mc.PXOR(temp, temp)
+        # if the vector is not fully packed blend 1s
+        if not arg.fully_packed(self.cpu.vector_register_size):
+            self.mc.PCMPEQQ(temp, temp) # fill with ones
+            select = 0
+            bits_used = (arg.item_count * arg.item_size * 8)
+            index = bits_used // 16
+            while index < 8:
+                select |= (1 << index)
+                index += 1
+            self.mc.PBLENDW_xxi(loc, temp, select)
+            # reset to zeros
+            self.mc.PXOR(temp, temp)
+
+        self.mc.PCMPEQ(size, loc, temp)
+        self.mc.PCMPEQQ(temp, temp)
+        self.mc.PTEST(loc, temp)
+
     def genop_guard_guard_true(self, ign_1, guard_op, guard_token, locs, ign_2):
         loc = locs[0]
-        self.mc.TEST(loc, loc)
-        self.implement_guard(guard_token, 'Z')
+        if loc.is_xmm:
+            self.genop_guard_vector_arg(guard_op, loc)
+            self.implement_guard(guard_token, 'Z')
+        else:
+            self.mc.TEST(loc, loc)
+            self.implement_guard(guard_token, 'Z')
     genop_guard_guard_nonnull = genop_guard_guard_true
 
     def genop_guard_guard_no_exception(self, ign_1, guard_op, guard_token,
@@ -1724,8 +1752,12 @@
 
     def genop_guard_guard_false(self, ign_1, guard_op, guard_token, locs, ign_2):
         loc = locs[0]
-        self.mc.TEST(loc, loc)
-        self.implement_guard(guard_token, 'NZ')
+        if loc.is_xmm:
+            self.genop_guard_vector_arg(guard_op, loc)
+            self.implement_guard(guard_token, 'Z')
+        else:
+            self.mc.TEST(loc, loc)
+            self.implement_guard(guard_token, 'NZ')
     genop_guard_guard_isnull = genop_guard_guard_false
 
     def genop_guard_guard_value(self, ign_1, guard_op, guard_token, locs, ign_2):
@@ -2723,7 +2755,7 @@
     def genop_vec_int_expand(self, op, arglocs, resloc):
         srcloc, sizeloc = arglocs
         if not isinstance(srcloc, RegLoc):
-            self.mov(X86_64_SCRATCH_REG, srcloc)
+            self.mov(srcloc, X86_64_SCRATCH_REG)
             srcloc = X86_64_SCRATCH_REG
         assert not srcloc.is_xmm
         size = sizeloc.value
diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -390,12 +390,22 @@
             return self.xrm.loc(v)
         return self.rm.loc(v)
 
+    def _consider_guard_tf(self, op):
+        arg = op.getarg(0)
+        if arg.type == VECTOR:
+            assert arg.item_type == INT
+            loc = self.xrm.make_sure_var_in_reg(arg)
+        else:
+            loc = self.rm.make_sure_var_in_reg(arg)
+        self.perform_guard(op, [loc], None)
+
+    consider_guard_true = _consider_guard_tf
+    consider_guard_false = _consider_guard_tf
+
     def _consider_guard(self, op):
         loc = self.rm.make_sure_var_in_reg(op.getarg(0))
         self.perform_guard(op, [loc], None)
 
-    consider_guard_true = _consider_guard
-    consider_guard_false = _consider_guard
     consider_guard_nonnull = _consider_guard
     consider_guard_isnull = _consider_guard
 
diff --git a/rpython/jit/backend/x86/regloc.py b/rpython/jit/backend/x86/regloc.py
--- a/rpython/jit/backend/x86/regloc.py
+++ b/rpython/jit/backend/x86/regloc.py
@@ -601,6 +601,28 @@
         self._reuse_scratch_register = False
         self._scratch_register_known = False
 
+    def _vector_size_choose(name):
+        def invoke(self, suffix, val1, val2):
+            methname = name + suffix
+            _rx86_getattr(self, methname)(val1, val2)
+        invoke._annspecialcase_ = 'specialize:arg(1)'
+
+        def INSN(self, size, loc1, loc2):
+            code1 = loc1.location_code()
+            code2 = loc2.location_code()
+            val1 = getattr(loc1, "value_" + code1)()
+            val2 = getattr(loc2, "value_" + code2)()
+            suffix = 'B'
+            if size == 2:
+                suffix = 'W'
+            elif size == 4:
+                suffix = 'D'
+            else:
+                suffix = 'Q'
+            invoke(self, suffix + "_"+ code1+code2, val1, val2)
+
+        return INSN
+
     AND = _binaryop('AND')
     OR  = _binaryop('OR')
     OR8 = _binaryop('OR8')
@@ -610,6 +632,7 @@
     SHR = _binaryop('SHR')
     SAR = _binaryop('SAR')
     TEST = _binaryop('TEST')
+    PTEST = _binaryop('PTEST')
     TEST8 = _binaryop('TEST8')
     BTS = _binaryop('BTS')
 
@@ -621,6 +644,11 @@
 
     CMP = _binaryop('CMP')
     CMP16 = _binaryop('CMP16')
+    PCMPEQQ = _binaryop('PCMPEQQ')
+    PCMPEQD = _binaryop('PCMPEQD')
+    PCMPEQW = _binaryop('PCMPEQW')
+    PCMPEQB = _binaryop('PCMPEQB')
+    PCMPEQ = _vector_size_choose('PCMPEQ')
     MOV = _binaryop('MOV')
     MOV8 = _binaryop('MOV8')
     MOV16 = _binaryop('MOV16')
@@ -698,7 +726,6 @@
     PAND  = _binaryop('PAND')
     POR   = _binaryop('POR')
     PXOR  = _binaryop('PXOR')
-    PCMPEQD = _binaryop('PCMPEQD')
     PSRLDQ = _binaryop('PSRLDQ')
 
     MOVDQ = _binaryop('MOVDQ')
diff --git a/rpython/jit/backend/x86/rx86.py b/rpython/jit/backend/x86/rx86.py
--- a/rpython/jit/backend/x86/rx86.py
+++ b/rpython/jit/backend/x86/rx86.py
@@ -766,6 +766,8 @@
     PINSRB_xri = xmminsn('\x66', rex_nw, '\x0F\x3A\x20', register(1,8), register(2), '\xC0', immediate(3, 'b'))
     INSERTPS_xxi = xmminsn('\x66', rex_nw, '\x0F\x3A\x21', register(1,8), register(2), '\xC0', immediate(3, 'b'))
 
+    PTEST_xx = xmminsn('\x66', rex_nw, '\x0F\x38\x17', register(1,8), register(2), '\xC0')
+    PBLENDW_xxi = xmminsn('\x66', rex_nw, '\x0F\x3A\x0E', register(1,8), register(2), '\xC0', immediate(3, 'b'))
 
     # ------------------------------------------------------------
 
@@ -1003,7 +1005,10 @@
 define_pxmm_insn('PUNPCKHDQ_x*', '\x6A')
 define_pxmm_insn('PUNPCKLQDQ_x*', '\x6C')
 define_pxmm_insn('PUNPCKHQDQ_x*', '\x6D')
+define_pxmm_insn('PCMPEQQ_x*',   '\x38\x29')
 define_pxmm_insn('PCMPEQD_x*',   '\x76')
+define_pxmm_insn('PCMPEQW_x*',   '\x75')
+define_pxmm_insn('PCMPEQB_x*',   '\x74')
 
 # ____________________________________________________________
 
diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py
--- a/rpython/jit/metainterp/history.py
+++ b/rpython/jit/metainterp/history.py
@@ -540,6 +540,9 @@
     def getcount(self):
         return self.item_count
 
+    def fully_packed(self, vec_reg_size):
+        return self.item_size * self.item_count == vec_reg_size
+
     def forget_value(self):
         raise NotImplementedError("cannot forget value of vector")
 
diff --git a/rpython/jit/metainterp/optimizeopt/schedule.py b/rpython/jit/metainterp/optimizeopt/schedule.py
--- a/rpython/jit/metainterp/optimizeopt/schedule.py
+++ b/rpython/jit/metainterp/optimizeopt/schedule.py
@@ -367,7 +367,7 @@
                 # the argument has more items than the operation is able to process!
                 # box_pos == 0 then it is already at the right place
                 if box_pos != 0:
-                    args[i] = self.unpack(vbox, box_pos, packable, self.input_type)
+                    args[i] = self.unpack(vbox, box_pos, packed - box_pos, self.input_type)
                     self.update_arg_in_vector_pos(i, args[i])
                     #self.update_input_output(self.pack)
                     continue
@@ -384,7 +384,7 @@
             if box_pos != 0:
                 # The vector box is at a position != 0 but it
                 # is required to be at position 0. Unpack it!
-                args[i] = self.unpack(vbox, box_pos, packable, self.input_type)
+                args[i] = self.unpack(vbox, box_pos, packed - box_pos, self.input_type)
                 self.update_arg_in_vector_pos(i, args[i])
                 continue
                 #self.update_input_output(self.pack)
@@ -450,6 +450,7 @@
     def unpack(self, vbox, index, count, arg_ptype):
         assert index < vbox.item_count
         assert index + count <= vbox.item_count
+        assert count > 0
         vbox_cloned = vectorbox_clone_set(vbox, count=count)
         opnum = getunpackopnum(vbox.item_type)
         op = ResOperation(opnum, [vbox, ConstInt(index), ConstInt(count)], vbox_cloned)
@@ -787,7 +788,6 @@
 
     def setvector_of_box(self, box, off, vector):
         assert off < vector.item_count
-        print "set" , box, "[",off,"] =", vector
         self.box_to_vbox[box] = (off, vector)
 
     def prepend_invariant_operations(self, oplist):
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py b/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
@@ -1357,40 +1357,23 @@
 
     def test_abc(self):
         trace="""
-        label(p0, p1, p5, p6, p7, p17, p19, i53, i39, i44, i49, i51, descr=TargetToken(140531585719072))
-        guard_not_invalidated(descr=<Guard0x7fd00f3ebdb0>) [p1, p0, p5, p6, p7, p17, p19]
-        i63 = int_ge(i53, 2024)
-        guard_false(i63, descr=<Guard0x7fd00f3ebe08>) [p1, p0, p5, p6, p7, p17, p19, i53]
-        i64 = int_lt(i53, i39)
-        guard_true(i64, descr=<Guard0x7fd00f3ebe60>) [p1, p0, i53, p5, p6, p7, p17, p19, None]
-        f65 = getarrayitem_raw(i44, i53, descr=floatarraydescr)
-        f66 = float_add(f65, 1.000000)
-        i67 = int_lt(i53, i49)
-        guard_true(i67, descr=<Guard0x7fd00f3ebeb8>) [p1, p0, i53, p5, p6, p7, p17, p19, f66, None]
-        setarrayitem_raw(i51, i53, f66, descr=floatarraydescr)
-        i68 = int_add(i53, 1)
-        i69 = getfield_raw(140531584083072, descr=<FieldS pypysig_long_struct.c_value 0>)
-        setfield_gc(59, i68, descr=<FieldS pypy.objspace.std.typeobject.IntMutableCell.inst_intvalue 8>)
-        i70 = int_lt(i69, 0)
-        guard_false(i70, descr=<Guard0x7fd00f3ebf10>) [p1, p0, p5, p6, p7, p17, p19, None, None]
-        jump(p0, p1, p5, p6, p7, p17, p19, i68, i39, i44, i49, i51)
-        """
-        trace="""
-        [p0, p1, p9, i10, p4, i11, p3, p6, p12, i13, i14, i15, f16, i17, i18]
-        guard_early_exit(descr=<rpython.jit.metainterp.compile.ResumeAtLoopHeaderDescr object at 0x7f2327d4b390>) [p6, p4, p3, p1, p0, i14, i10, i13, i11, p9, p12]
-        i19 = raw_load(i15, i11, descr=singlefloatarraydescr)
-        guard_not_invalidated(descr=<rpython.jit.metainterp.compile.ResumeGuardNotInvalidated object at 0x7f23284786d0>) [p6, p4, p3, p1, p0, i19, i14, i10, i13, i11, p9, p12]
-        i21 = int_add(i11, 4)
-        f22 = cast_singlefloat_to_float(i19)
-        f23 = float_add(f22, f16)
-        i24 = cast_float_to_singlefloat(f23)
-        raw_store(i17, i14, i24, descr=singlefloatarraydescr)
-        i26 = int_add(i13, 1)
-        i28 = int_add(i14, 4)
-        i29 = int_ge(i26, i18)
-        guard_false(i29, descr=<rpython.jit.metainterp.compile.ResumeGuardFalseDescr object at 0x7f2327d53910>) [p6, p4, p3, p1, p0, i28, i21, i26, None, i10, None, None, p9, p12]
-        debug_merge_point(0, 0, '(numpy_call2: no get_printable_location)')
-        jump(p0, p1, p9, i10, p4, i21, p3, p6, p12, i26, i28, i15, f16, i17, i18)
+        [p0, p9, i10, p3, i11, p12, i13, p6, i14, p7, p15, i16, i17, i18, i19, i20, i21]
+        guard_early_exit(descr=<rpython.jit.metainterp.compile.ResumeAtLoopHeaderDescr object at 0x7f09b34aad50>) [p7, p6, p3, p0, i14, i17, i16, p9, p15, i11, i10, p12, i13]
+        i22 = raw_load(i18, i11, descr=singlefloatarraydescr)
+        guard_not_invalidated(descr=<rpython.jit.metainterp.compile.ResumeGuardNotInvalidated object at 0x7f09b34fd390>) [p7, p6, p3, p0, i22, i14, i17, i16, p9, p15, i11, i10, p12, i13]
+        i24 = int_add(i11, 4)
+        i25 = raw_load(i19, i17, descr=singlefloatarraydescr)
+        i27 = int_add(i17, 4)
+        f28 = cast_singlefloat_to_float(i22)
+        f29 = cast_singlefloat_to_float(i25)
+        f30 = float_add(f28, f29)
+        i31 = cast_float_to_singlefloat(f30)
+        raw_store(i20, i14, i31, descr=singlefloatarraydescr)
+        i33 = int_add(i13, 1)
+        i35 = int_add(i14, 4)
+        i36 = int_ge(i33, i21)
+        guard_false(i36, descr=<rpython.jit.metainterp.compile.ResumeGuardFalseDescr object at 0x7f09b34b7c10>) [p7, p6, p3, p0, i35, i24, i33, i27, None, None, i16, p9, p15, None, i10, p12, None]
+        jump(p0, p9, i10, p3, i24, p12, i33, p6, i35, p7, p15, i16, i27, i18, i19, i20, i21)
         """
         opt = self.vectorize(self.parse_loop(trace))
         self.debug_print_operations(opt.loop)

From noreply at buildbot.pypy.org  Sat Jun 27 18:53:50 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Sat, 27 Jun 2015 18:53:50 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: all & any (but not any casting from
 float) execute correctly
Message-ID: <20150627165350.228C91C120C@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r78331:12de4acb3b37
Date: 2015-06-27 18:54 +0200
http://bitbucket.org/pypy/pypy/changeset/12de4acb3b37/

Log:	all & any (but not any casting from float) execute correctly

diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -545,7 +545,7 @@
 
     def define_any_int():
         return """
-        a = astype([0,0,0,0,256,65537,0,0,0,0,0],int16)
+        a = astype([0,0,0,0,256,0,0,0,0,0,0],int16)
         any(a)
         """
 
@@ -567,20 +567,20 @@
         any(a)
         """
 
-    def test_float_any(self):
+    def test_any_float(self):
         result = self.run("float_any")
         assert int(result) == 1
         self.check_vectorized(2, 2)
 
-    def test_float32_any(self):
+    def test_any_float32(self):
         result = self.run("float32_any")
         assert int(result) == 1
         self.check_vectorized(1, 1)
 
     def test_any(self):
-        result = self.run("float_any")
+        result = self.run("any")
         assert int(result) == 1
-        self.check_vectorized(1, 1)
+        self.check_vectorized(2, 1)
 
     def test_any_int(self):
         result = self.run("any_int")
@@ -618,12 +618,12 @@
         all(a)
         """
 
-    def test_float_all(self):
+    def test_all_float(self):
         result = self.run("float_all")
         assert int(result) == 1
-        self.check_vectorized(2, 2)
+        self.check_vectorized(1, 1)
 
-    def test_float_all(self):
+    def test_all_float32(self):
         result = self.run("float32_all")
         assert int(result) == 1
         self.check_vectorized(2, 2)
diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -1644,7 +1644,7 @@
             self.mc.MOVD32_xr(resloc.value, eax.value)
             self.mc.PUNPCKLDQ_xx(resloc.value, loc1.value)
 
-    def genop_guard_vector_arg(self, guard_op, loc):
+    def _guard_vector_arg(self, guard_op, loc, zero=False):
         arg = guard_op.getarg(0)
         assert isinstance(arg, BoxVector)
         size = arg.item_size
@@ -1653,16 +1653,18 @@
         self.mc.PXOR(temp, temp)
         # if the vector is not fully packed blend 1s
         if not arg.fully_packed(self.cpu.vector_register_size):
-            self.mc.PCMPEQQ(temp, temp) # fill with ones
+            if not zero:
+                self.mc.PCMPEQQ(temp, temp) # fill with ones
             select = 0
             bits_used = (arg.item_count * arg.item_size * 8)
             index = bits_used // 16
             while index < 8:
                 select |= (1 << index)
                 index += 1
-            self.mc.PBLENDW_xxi(loc, temp, select)
+            self.mc.PBLENDW_xxi(loc.value, temp.value, select)
             # reset to zeros
-            self.mc.PXOR(temp, temp)
+            if not zero:
+                self.mc.PXOR(temp, temp)
 
         self.mc.PCMPEQ(size, loc, temp)
         self.mc.PCMPEQQ(temp, temp)
@@ -1671,8 +1673,8 @@
     def genop_guard_guard_true(self, ign_1, guard_op, guard_token, locs, ign_2):
         loc = locs[0]
         if loc.is_xmm:
-            self.genop_guard_vector_arg(guard_op, loc)
-            self.implement_guard(guard_token, 'Z')
+            self._guard_vector_arg(guard_op, loc, zero=False)
+            self.implement_guard(guard_token, 'NZ')
         else:
             self.mc.TEST(loc, loc)
             self.implement_guard(guard_token, 'Z')
@@ -1753,7 +1755,7 @@
     def genop_guard_guard_false(self, ign_1, guard_op, guard_token, locs, ign_2):
         loc = locs[0]
         if loc.is_xmm:
-            self.genop_guard_vector_arg(guard_op, loc)
+            self._guard_vector_arg(guard_op, loc, zero=True)
             self.implement_guard(guard_token, 'Z')
         else:
             self.mc.TEST(loc, loc)

From noreply at buildbot.pypy.org  Sat Jun 27 19:24:29 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Sat, 27 Jun 2015 19:24:29 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: fixed guard_false for packed arguments,
 the last test (set_slice) not working
Message-ID: <20150627172429.65F991C0271@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r78332:ccca4d45dd30
Date: 2015-06-27 19:24 +0200
http://bitbucket.org/pypy/pypy/changeset/ccca4d45dd30/

Log:	fixed guard_false for packed arguments, the last test (set_slice)
	not working

diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -570,12 +570,12 @@
     def test_any_float(self):
         result = self.run("float_any")
         assert int(result) == 1
-        self.check_vectorized(2, 2)
+        self.check_vectorized(1, 1)
 
     def test_any_float32(self):
         result = self.run("float32_any")
         assert int(result) == 1
-        self.check_vectorized(1, 1)
+        self.check_vectorized(2, 2)
 
     def test_any(self):
         result = self.run("any")
diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -1644,7 +1644,7 @@
             self.mc.MOVD32_xr(resloc.value, eax.value)
             self.mc.PUNPCKLDQ_xx(resloc.value, loc1.value)
 
-    def _guard_vector_arg(self, guard_op, loc, zero=False):
+    def _guard_vector_true(self, guard_op, loc, zero=False):
         arg = guard_op.getarg(0)
         assert isinstance(arg, BoxVector)
         size = arg.item_size
@@ -1653,8 +1653,7 @@
         self.mc.PXOR(temp, temp)
         # if the vector is not fully packed blend 1s
         if not arg.fully_packed(self.cpu.vector_register_size):
-            if not zero:
-                self.mc.PCMPEQQ(temp, temp) # fill with ones
+            self.mc.PCMPEQQ(temp, temp) # fill with ones
             select = 0
             bits_used = (arg.item_count * arg.item_size * 8)
             index = bits_used // 16
@@ -1663,8 +1662,7 @@
                 index += 1
             self.mc.PBLENDW_xxi(loc.value, temp.value, select)
             # reset to zeros
-            if not zero:
-                self.mc.PXOR(temp, temp)
+            self.mc.PXOR(temp, temp)
 
         self.mc.PCMPEQ(size, loc, temp)
         self.mc.PCMPEQQ(temp, temp)
@@ -1673,7 +1671,7 @@
     def genop_guard_guard_true(self, ign_1, guard_op, guard_token, locs, ign_2):
         loc = locs[0]
         if loc.is_xmm:
-            self._guard_vector_arg(guard_op, loc, zero=False)
+            self._guard_vector_true(guard_op, loc)
             self.implement_guard(guard_token, 'NZ')
         else:
             self.mc.TEST(loc, loc)
@@ -1752,11 +1750,29 @@
         self.mc.IMUL(arglocs[0], arglocs[1])
         return self._gen_guard_overflow(guard_op, guard_token)
 
+    def _guard_vector_false(self, guard_op, loc):
+        arg = guard_op.getarg(0)
+        assert isinstance(arg, BoxVector)
+        #
+        # if the vector is not fully packed blend 1s
+        if not arg.fully_packed(self.cpu.vector_register_size):
+            temp = X86_64_XMM_SCRATCH_REG
+            self.mc.PXOR(temp, temp)
+            select = 0
+            bits_used = (arg.item_count * arg.item_size * 8)
+            index = bits_used // 16
+            while index < 8:
+                select |= (1 << index)
+                index += 1
+            self.mc.PBLENDW_xxi(loc.value, temp.value, select)
+
+        self.mc.PTEST(loc, loc)
+
     def genop_guard_guard_false(self, ign_1, guard_op, guard_token, locs, ign_2):
         loc = locs[0]
         if loc.is_xmm:
-            self._guard_vector_arg(guard_op, loc, zero=True)
-            self.implement_guard(guard_token, 'Z')
+            self._guard_vector_false(guard_op, loc)
+            self.implement_guard(guard_token, 'NZ')
         else:
             self.mc.TEST(loc, loc)
             self.implement_guard(guard_token, 'NZ')
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py b/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
@@ -1375,6 +1375,7 @@
         guard_false(i36, descr=<rpython.jit.metainterp.compile.ResumeGuardFalseDescr object at 0x7f09b34b7c10>) [p7, p6, p3, p0, i35, i24, i33, i27, None, None, i16, p9, p15, None, i10, p12, None]
         jump(p0, p9, i10, p3, i24, p12, i33, p6, i35, p7, p15, i16, i27, i18, i19, i20, i21)
         """
+        # schedule 885 -> ptype is non for raw_load?
         opt = self.vectorize(self.parse_loop(trace))
         self.debug_print_operations(opt.loop)
 

From noreply at buildbot.pypy.org  Sun Jun 28 10:26:46 2015
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sun, 28 Jun 2015 10:26:46 +0200 (CEST)
Subject: [pypy-commit] pypy py3.3: hg merge more-rposix,
	directly in py3.3 branch.
Message-ID: <20150628082646.C7C661C063D@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3.3
Changeset: r78333:7d4ae17b382a
Date: 2015-06-25 08:37 +0200
http://bitbucket.org/pypy/pypy/changeset/7d4ae17b382a/

Log:	hg merge more-rposix, directly in py3.3 branch.

diff too long, truncating to 2000 out of 8516 lines

diff --git a/pypy/module/_io/interp_io.py b/pypy/module/_io/interp_io.py
--- a/pypy/module/_io/interp_io.py
+++ b/pypy/module/_io/interp_io.py
@@ -6,7 +6,9 @@
     TypeDef, interp_attrproperty, generic_new_descr)
 from pypy.module._io.interp_fileio import W_FileIO
 from pypy.module._io.interp_textio import W_TextIOWrapper
-from rpython.rtyper.module.ll_os_stat import STAT_FIELD_TYPES
+from rpython.rlib.rposix_stat import STAT_FIELD_TYPES
+
+HAS_BLKSIZE = 'st_blksize' in STAT_FIELD_TYPES
 
 
 class Cache:
@@ -102,7 +104,7 @@
     if buffering < 0:
         buffering = DEFAULT_BUFFER_SIZE
 
-        if space.config.translation.type_system == 'lltype' and 'st_blksize' in STAT_FIELD_TYPES:
+        if HAS_BLKSIZE:
             fileno = space.c_int_w(space.call_method(w_raw, "fileno"))
             try:
                 st = os.fstat(fileno)
diff --git a/pypy/module/posix/__init__.py b/pypy/module/posix/__init__.py
--- a/pypy/module/posix/__init__.py
+++ b/pypy/module/posix/__init__.py
@@ -1,5 +1,5 @@
 from pypy.interpreter.mixedmodule import MixedModule
-from rpython.rtyper.module.ll_os import RegisterOs
+from rpython.rlib import rposix
 from rpython.rlib import rdynload
 
 import os
@@ -176,7 +176,7 @@
     if hasattr(os, 'chroot'):
         interpleveldefs['chroot'] = 'interp_posix.chroot'
 
-    for name in RegisterOs.w_star:
+    for name in rposix.WAIT_MACROS:
         if hasattr(os, name):
             interpleveldefs[name] = 'interp_posix.' + name
 
diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -1,12 +1,11 @@
 import os
 import sys
 
-from rpython.rlib import rposix, objectmodel, rurandom
+from rpython.rlib import rposix, rposix_stat
+from rpython.rlib import objectmodel, rurandom
 from rpython.rlib.objectmodel import specialize
 from rpython.rlib.rarithmetic import r_longlong, intmask
 from rpython.rlib.unroll import unrolling_iterable
-from rpython.rtyper.module import ll_os_stat
-from rpython.rtyper.module.ll_os import RegisterOs
 
 from pypy.interpreter.gateway import unwrap_spec, WrappedDefault
 from pypy.interpreter.error import (OperationError, wrap_oserror,
@@ -37,6 +36,8 @@
                                  space.wrap("integer out of range"))
 
 class FileEncoder(object):
+    is_unicode = True
+
     def __init__(self, space, w_obj):
         self.space = space
         self.w_obj = w_obj
@@ -48,6 +49,8 @@
         return self.space.unicode0_w(self.w_obj)
 
 class FileDecoder(object):
+    is_unicode = False
+
     def __init__(self, space, w_obj):
         self.space = space
         self.w_obj = w_obj
@@ -222,13 +225,13 @@
 
 # ____________________________________________________________
 
-STAT_FIELDS = unrolling_iterable(enumerate(ll_os_stat.STAT_FIELDS))
+STAT_FIELDS = unrolling_iterable(enumerate(rposix_stat.STAT_FIELDS))
 
-STATVFS_FIELDS = unrolling_iterable(enumerate(ll_os_stat.STATVFS_FIELDS))
+STATVFS_FIELDS = unrolling_iterable(enumerate(rposix_stat.STATVFS_FIELDS))
 
 def build_stat_result(space, st):
     FIELDS = STAT_FIELDS    # also when not translating at all
-    lst = [None] * ll_os_stat.N_INDEXABLE_FIELDS
+    lst = [None] * rposix_stat.N_INDEXABLE_FIELDS
     w_keywords = space.newdict()
     stat_float_times = space.fromcache(StatState).stat_float_times
     for i, (name, TYPE) in FIELDS:
@@ -236,7 +239,7 @@
         if name in ('st_atime', 'st_mtime', 'st_ctime'):
             value = int(value)   # rounded to an integer for indexed access
         w_value = space.wrap(value)
-        if i < ll_os_stat.N_INDEXABLE_FIELDS:
+        if i < rposix_stat.N_INDEXABLE_FIELDS:
             lst[i] = w_value
         else:
             space.setitem(w_keywords, space.wrap(name), w_value)
@@ -264,7 +267,7 @@
 
 
 def build_statvfs_result(space, st):
-    vals_w = [None] * len(ll_os_stat.STATVFS_FIELDS)
+    vals_w = [None] * len(rposix_stat.STATVFS_FIELDS)
     for i, (name, _) in STATVFS_FIELDS:
         vals_w[i] = space.wrap(getattr(st, name))
     w_tuple = space.newtuple(vals_w)
@@ -277,7 +280,7 @@
     """Perform a stat system call on the file referenced to by an open
 file descriptor."""
     try:
-        st = os.fstat(fd)
+        st = rposix_stat.fstat(fd)
     except OSError, e:
         raise wrap_oserror(space, e)
     else:
@@ -299,7 +302,7 @@
 """
 
     try:
-        st = dispatch_filename(rposix.stat)(space, w_path)
+        st = dispatch_filename(rposix_stat.stat)(space, w_path)
     except OSError, e:
         raise wrap_oserror2(space, e, w_path)
     else:
@@ -308,7 +311,7 @@
 def lstat(space, w_path):
     "Like stat(path), but do no follow symbolic links."
     try:
-        st = dispatch_filename(rposix.lstat)(space, w_path)
+        st = dispatch_filename(rposix_stat.lstat)(space, w_path)
     except OSError, e:
         raise wrap_oserror2(space, e, w_path)
     else:
@@ -337,7 +340,7 @@
 @unwrap_spec(fd=c_int)
 def fstatvfs(space, fd):
     try:
-        st = os.fstatvfs(fd)
+        st = rposix_stat.fstatvfs(fd)
     except OSError as e:
         raise wrap_oserror(space, e)
     else:
@@ -346,7 +349,7 @@
 
 def statvfs(space, w_path):
     try:
-        st = dispatch_filename(rposix.statvfs)(space, w_path)
+        st = dispatch_filename(rposix_stat.statvfs)(space, w_path)
     except OSError as e:
         raise wrap_oserror2(space, e, w_path)
     else:
@@ -437,11 +440,11 @@
     try:
         if space.isinstance_w(w_path, space.w_unicode):
             path = FileEncoder(space, w_path)
-            fullpath = rposix._getfullpathname(path)
+            fullpath = rposix.getfullpathname(path)
             w_fullpath = space.wrap(fullpath)
         else:
             path = space.str0_w(w_path)
-            fullpath = rposix._getfullpathname(path)
+            fullpath = rposix.getfullpathname(path)
             w_fullpath = space.wrapbytes(fullpath)
     except OSError, e:
         raise wrap_oserror2(space, e, w_path)
@@ -688,7 +691,7 @@
 def kill(space, pid, sig):
     "Kill a process with a signal."
     try:
-        rposix.os_kill(pid, sig)
+        rposix.kill(pid, sig)
     except OSError, e:
         raise wrap_oserror(space, e)
 
@@ -704,7 +707,7 @@
     """Abort the interpreter immediately.  This 'dumps core' or otherwise fails
 in the hardest way possible on the hosting operating system."""
     import signal
-    rposix.os_kill(os.getpid(), signal.SIGABRT)
+    rposix.kill(os.getpid(), signal.SIGABRT)
 
 @unwrap_spec(src='fsencode', dst='fsencode')
 def link(space, src, dst):
@@ -1237,7 +1240,7 @@
         raise wrap_oserror(space, e)
 
 def declare_new_w_star(name):
-    if name in RegisterOs.w_star_returning_int:
+    if name in ('WEXITSTATUS', 'WSTOPSIG', 'WTERMSIG'):
         @unwrap_spec(status=c_int)
         def WSTAR(space, status):
             return space.wrap(getattr(os, name)(status))
@@ -1249,7 +1252,7 @@
     WSTAR.func_name = name
     return WSTAR
 
-for name in RegisterOs.w_star:
+for name in rposix.WAIT_MACROS:
     if hasattr(os, name):
         func = declare_new_w_star(name)
         globals()[name] = func
diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py
--- a/pypy/module/posix/test/test_posix2.py
+++ b/pypy/module/posix/test/test_posix2.py
@@ -6,8 +6,8 @@
 from rpython.tool.udir import udir
 from pypy.tool.pytest.objspace import gettestobjspace
 from pypy.conftest import pypydir
-from rpython.rtyper.module.ll_os import RegisterOs
 from rpython.translator.c.test.test_extfunc import need_sparse_files
+from rpython.rlib import rposix
 import os
 import py
 import sys
@@ -539,7 +539,7 @@
         raises(TypeError, "os.utime('xxx', 3)")
         raises(OSError, "os.utime('somefilewhichihopewouldneverappearhere', None)")
 
-    for name in RegisterOs.w_star:
+    for name in rposix.WAIT_MACROS:
         if hasattr(os, name):
             values = [0, 1, 127, 128, 255]
             code = py.code.Source("""
diff --git a/pypy/module/time/interp_time.py b/pypy/module/time/interp_time.py
--- a/pypy/module/time/interp_time.py
+++ b/pypy/module/time/interp_time.py
@@ -486,13 +486,6 @@
     secs = pytime.time()
     return space.wrap(secs)
 
-if _WIN:
-    class PCCache:
-        pass
-    pccache = PCCache()
-    pccache.divisor = 0.0
-    pccache.ctrStart = 0
-
 def clock(space):
     """clock() -> floating point number
 
diff --git a/rpython/annotator/bookkeeper.py b/rpython/annotator/bookkeeper.py
--- a/rpython/annotator/bookkeeper.py
+++ b/rpython/annotator/bookkeeper.py
@@ -237,10 +237,11 @@
             else:
                 result = SomeString(no_nul=no_nul)
         elif tp is unicode:
+            no_nul = not u'\x00' in x
             if len(x) == 1:
-                result = SomeUnicodeCodePoint()
+                result = SomeUnicodeCodePoint(no_nul=no_nul)
             else:
-                result = SomeUnicodeString()
+                result = SomeUnicodeString(no_nul=no_nul)
         elif tp is bytearray:
             result = SomeByteArray()
         elif tp is tuple:
diff --git a/rpython/annotator/unaryop.py b/rpython/annotator/unaryop.py
--- a/rpython/annotator/unaryop.py
+++ b/rpython/annotator/unaryop.py
@@ -569,7 +569,7 @@
         enc = s_enc.const
         if enc not in ('ascii', 'latin-1', 'utf-8'):
             raise AnnotatorError("Encoding %s not supported for unicode" % (enc,))
-        return SomeString()
+        return SomeString(no_nul=self.no_nul)
     method_encode.can_only_throw = [UnicodeEncodeError]
 
 
@@ -602,7 +602,7 @@
         enc = s_enc.const
         if enc not in ('ascii', 'latin-1', 'utf-8'):
             raise AnnotatorError("Encoding %s not supported for strings" % (enc,))
-        return SomeUnicodeString()
+        return SomeUnicodeString(no_nul=self.no_nul)
     method_decode.can_only_throw = [UnicodeDecodeError]
 
 class __extend__(SomeChar, SomeUnicodeCodePoint):
diff --git a/rpython/flowspace/objspace.py b/rpython/flowspace/objspace.py
--- a/rpython/flowspace/objspace.py
+++ b/rpython/flowspace/objspace.py
@@ -18,7 +18,7 @@
     if func.func_code.co_cellvars:
         raise ValueError(
 """RPython functions cannot create closures
-Possible casues:
+Possible causes:
     Function is inner function
     Function uses generator expressions
     Lambda expressions
diff --git a/rpython/jit/metainterp/jitprof.py b/rpython/jit/metainterp/jitprof.py
--- a/rpython/jit/metainterp/jitprof.py
+++ b/rpython/jit/metainterp/jitprof.py
@@ -51,7 +51,7 @@
 
 class Profiler(BaseProfiler):
     initialized = False
-    timer = time.time
+    timer = staticmethod(time.time)
     starttime = 0
     t1 = 0
     times = None
diff --git a/rpython/memory/gc/inspector.py b/rpython/memory/gc/inspector.py
--- a/rpython/memory/gc/inspector.py
+++ b/rpython/memory/gc/inspector.py
@@ -3,7 +3,6 @@
 """
 from rpython.rtyper.lltypesystem import lltype, llmemory, rffi, llgroup
 from rpython.rlib.objectmodel import free_non_gc_object
-from rpython.rtyper.module.ll_os import UNDERSCORE_ON_WIN32
 from rpython.rlib import rposix, rgc, jit
 
 from rpython.memory.support import AddressDict, get_address_stack
@@ -94,7 +93,7 @@
 
 # ----------
 
-raw_os_write = rffi.llexternal(UNDERSCORE_ON_WIN32 + 'write',
+raw_os_write = rffi.llexternal(rposix.UNDERSCORE_ON_WIN32 + 'write',
                                [rffi.INT, llmemory.Address, rffi.SIZE_T],
                                rffi.SIZE_T,
                                sandboxsafe=True, _nowrapper=True)
diff --git a/rpython/memory/gctransform/support.py b/rpython/memory/gctransform/support.py
--- a/rpython/memory/gctransform/support.py
+++ b/rpython/memory/gctransform/support.py
@@ -73,15 +73,19 @@
         hop.exception_cannot_occur()
         return hop.inputconst(hop.r_result.lowleveltype, hop.s_result.const)
 
+def write(fd, string):
+    from rpython.rlib.rposix import c_write
+    return c_write(fd, string, len(string))
+
 def ll_call_destructor(destrptr, destr_v, typename):
     try:
         destrptr(destr_v)
     except Exception, e:
         try:
-            os.write(2, "a destructor of type ")
-            os.write(2, typename)
-            os.write(2, " raised an exception ")
-            os.write(2, str(e))
-            os.write(2, " ignoring it\n")
+            write(2, "a destructor of type ")
+            write(2, typename)
+            write(2, " raised an exception ")
+            write(2, str(e))
+            write(2, " ignoring it\n")
         except:
             pass
diff --git a/rpython/rlib/_rposix_repr.py b/rpython/rlib/_rposix_repr.py
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/_rposix_repr.py
@@ -0,0 +1,122 @@
+"""
+RTyping support for os.stat_result objects.
+They are rtyped just like a tuple of the correct length supporting
+only indexing and the st_xxx attributes.  We need a custom StatResultRepr
+because when rtyping for LL backends we have extra platform-dependent
+items at the end of the tuple, but for OO backends we only want the
+portable items.  This allows the OO backends to assume a fixed shape for
+the tuples returned by os.stat().
+"""
+from rpython.annotator import model as annmodel
+from rpython.rtyper.llannotation import lltype_to_annotation
+from rpython.flowspace.model import Constant
+from rpython.flowspace.operation import op
+from rpython.tool.pairtype import pairtype
+from rpython.rtyper.rmodel import Repr
+from rpython.rtyper.rint import IntegerRepr
+from rpython.rtyper.error import TyperError
+from rpython.rlib import rposix_stat
+
+
+class StatResultRepr(Repr):
+
+    def __init__(self, rtyper):
+        self.rtyper = rtyper
+        self.stat_fields = rposix_stat.STAT_FIELDS
+
+        self.stat_field_indexes = {}
+        for i, (name, TYPE) in enumerate(self.stat_fields):
+            self.stat_field_indexes[name] = i
+
+        self.s_tuple = annmodel.SomeTuple([lltype_to_annotation(TYPE)
+                                           for name, TYPE in self.stat_fields])
+        self.r_tuple = rtyper.getrepr(self.s_tuple)
+        self.lowleveltype = self.r_tuple.lowleveltype
+
+    def redispatch_getfield(self, hop, index):
+        rtyper = self.rtyper
+        s_index = rtyper.annotator.bookkeeper.immutablevalue(index)
+        hop2 = hop.copy()
+        spaceop = op.getitem(hop.args_v[0], Constant(index))
+        spaceop.result = hop.spaceop.result
+        hop2.spaceop = spaceop
+        hop2.args_v = spaceop.args
+        hop2.args_s = [self.s_tuple, s_index]
+        hop2.args_r = [self.r_tuple, rtyper.getrepr(s_index)]
+        return hop2.dispatch()
+
+    def rtype_getattr(self, hop):
+        s_attr = hop.args_s[1]
+        attr = s_attr.const
+        try:
+            index = self.stat_field_indexes[attr]
+        except KeyError:
+            raise TyperError("os.stat().%s: field not available" % (attr,))
+        return self.redispatch_getfield(hop, index)
+
+
+class __extend__(pairtype(StatResultRepr, IntegerRepr)):
+
+    def rtype_getitem((r_sta, r_int), hop):
+        s_int = hop.args_s[1]
+        index = s_int.const
+        return r_sta.redispatch_getfield(hop, index)
+
+
+def specialize_make_stat_result(hop):
+    r_StatResult = hop.rtyper.getrepr(rposix_stat.s_StatResult)
+    [v_result] = hop.inputargs(r_StatResult.r_tuple)
+    # no-op conversion from r_StatResult.r_tuple to r_StatResult
+    hop.exception_cannot_occur()
+    return v_result
+
+
+class StatvfsResultRepr(Repr):
+
+    def __init__(self, rtyper):
+        self.rtyper = rtyper
+        self.statvfs_fields = rposix_stat.STATVFS_FIELDS
+
+        self.statvfs_field_indexes = {}
+        for i, (name, TYPE) in enumerate(self.statvfs_fields):
+            self.statvfs_field_indexes[name] = i
+
+        self.s_tuple = annmodel.SomeTuple([lltype_to_annotation(TYPE)
+                                           for name, TYPE in self.statvfs_fields])
+        self.r_tuple = rtyper.getrepr(self.s_tuple)
+        self.lowleveltype = self.r_tuple.lowleveltype
+
+    def redispatch_getfield(self, hop, index):
+        rtyper = self.rtyper
+        s_index = rtyper.annotator.bookkeeper.immutablevalue(index)
+        hop2 = hop.copy()
+        spaceop = op.getitem(hop.args_v[0], Constant(index))
+        spaceop.result = hop.spaceop.result
+        hop2.spaceop = spaceop
+        hop2.args_v = spaceop.args
+        hop2.args_s = [self.s_tuple, s_index]
+        hop2.args_r = [self.r_tuple, rtyper.getrepr(s_index)]
+        return hop2.dispatch()
+
+    def rtype_getattr(self, hop):
+        s_attr = hop.args_s[1]
+        attr = s_attr.const
+        try:
+            index = self.statvfs_field_indexes[attr]
+        except KeyError:
+            raise TyperError("os.statvfs().%s: field not available" % (attr,))
+        return self.redispatch_getfield(hop, index)
+
+
+class __extend__(pairtype(StatvfsResultRepr, IntegerRepr)):
+    def rtype_getitem((r_sta, r_int), hop):
+        s_int = hop.args_s[1]
+        index = s_int.const
+        return r_sta.redispatch_getfield(hop, index)
+
+
+def specialize_make_statvfs_result(hop):
+    r_StatvfsResult = hop.rtyper.getrepr(rposix_stat.s_StatvfsResult)
+    [v_result] = hop.inputargs(r_StatvfsResult.r_tuple)
+    hop.exception_cannot_occur()
+    return v_result
diff --git a/rpython/rlib/objectmodel.py b/rpython/rlib/objectmodel.py
--- a/rpython/rlib/objectmodel.py
+++ b/rpython/rlib/objectmodel.py
@@ -290,6 +290,20 @@
 def sc_we_are_translated(ctx):
     return Constant(True)
 
+def register_replacement_for(replaced_function, sandboxed_name=None):
+    def wrap(func):
+        from rpython.rtyper.extregistry import ExtRegistryEntry
+        class ExtRegistry(ExtRegistryEntry):
+            _about_ = replaced_function
+            def compute_annotation(self):
+                if sandboxed_name:
+                    config = self.bookkeeper.annotator.translator.config
+                    if config.translation.sandbox:
+                        func._sandbox_external_name = sandboxed_name
+                        func._dont_inline_ = True
+                return self.bookkeeper.immutablevalue(func)
+        return func
+    return wrap
 
 def keepalive_until_here(*values):
     pass
diff --git a/rpython/rlib/rarithmetic.py b/rpython/rlib/rarithmetic.py
--- a/rpython/rlib/rarithmetic.py
+++ b/rpython/rlib/rarithmetic.py
@@ -534,7 +534,7 @@
 else:
     r_int64 = int
 
-# needed for ll_os_stat.time_t_to_FILE_TIME in the 64 bit case
+# needed for rposix_stat.time_t_to_FILE_TIME in the 64 bit case
 r_uint32 = build_int('r_uint32', False, 32)
 
 SHRT_MIN = -2**(_get_bitsize('h') - 1)
diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py
--- a/rpython/rlib/rfile.py
+++ b/rpython/rlib/rfile.py
@@ -173,7 +173,6 @@
 
 def create_fdopen_rfile(fd, mode="r", buffering=-1):
     newmode = _sanitize_mode(mode)
-    fd = rffi.cast(rffi.INT, fd)
     rposix.validate_fd(fd)
     ll_mode = rffi.str2charp(newmode)
     try:
diff --git a/rpython/rlib/rpath.py b/rpython/rlib/rpath.py
--- a/rpython/rlib/rpath.py
+++ b/rpython/rlib/rpath.py
@@ -146,7 +146,7 @@
     try:
         if path == '':
             path = os.getcwd()
-        return rposix._getfullpathname(path)
+        return rposix.getfullpathname(path)
     except OSError:
         return path
 
diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -1,15 +1,28 @@
 import os
+import sys
+import errno
 from rpython.rtyper.lltypesystem.rffi import CConstant, CExternVariable, INT
+from rpython.rtyper.lltypesystem import lltype, ll2ctypes, rffi
+from rpython.rtyper.module.support import StringTraits, UnicodeTraits
 from rpython.rtyper.tool import rffi_platform
-from rpython.rtyper.lltypesystem import ll2ctypes, rffi
+from rpython.tool.sourcetools import func_renamer
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
-from rpython.rlib.rarithmetic import intmask
-from rpython.rlib.objectmodel import specialize
+from rpython.rlib.rarithmetic import intmask, widen
+from rpython.rlib.objectmodel import (
+    specialize, enforceargs, register_replacement_for)
 from rpython.rlib import jit
 from rpython.translator.platform import platform
+from rpython.rlib import rstring
+from rpython.rlib import debug, rthread
 
-WIN32 = os.name == "nt"
+_WIN32 = sys.platform.startswith('win')
+_CYGWIN = sys.platform == 'cygwin'
+UNDERSCORE_ON_WIN32 = '_' if _WIN32 else ''
+_MACRO_ON_POSIX = True if not _WIN32 else None
 
+if _WIN32:
+    from rpython.rlib import rwin32 
+    from rpython.rlib.rwin32file import make_win32_traits
 
 class CConfig:
     _compilation_info_ = ExternalCompilationInfo(
@@ -162,7 +175,7 @@
             _set_errno(rthread.tlfield_rpy_errno.getraw())
     elif save_err & rffi.RFFI_ZERO_ERRNO_BEFORE:
         _set_errno(rffi.cast(rffi.INT, 0))
-    if WIN32 and (save_err & rffi.RFFI_READSAVED_LASTERROR):
+    if _WIN32 and (save_err & rffi.RFFI_READSAVED_LASTERROR):
         from rpython.rlib import rthread, rwin32
         if save_err & rffi.RFFI_ALT_ERRNO:
             err = rthread.tlfield_alt_lasterror.getraw()
@@ -175,7 +188,7 @@
 
 @specialize.call_location()
 def _errno_after(save_err):
-    if WIN32:
+    if _WIN32:
         if save_err & rffi.RFFI_SAVE_LASTERROR:
             from rpython.rlib import rthread, rwin32
             err = rwin32._GetLastError()
@@ -205,6 +218,7 @@
         "_PyVerify_fd", [rffi.INT], rffi.INT,
         compilation_info=errno_eci,
         ))
+    @enforceargs(int)
     def validate_fd(fd):
         if not is_valid_fd(fd):
             from errno import EBADF
@@ -213,6 +227,7 @@
     def is_valid_fd(fd):
         return 1
 
+    @enforceargs(int)
     def validate_fd(fd):
         pass
 
@@ -225,6 +240,68 @@
         except OSError:
             pass
 
+if _WIN32:
+    includes = ['io.h', 'sys/utime.h', 'sys/types.h']
+    libraries = []
+else:
+    includes = ['unistd.h',  'sys/types.h', 'sys/wait.h',
+                'utime.h', 'sys/time.h', 'sys/times.h',
+                'grp.h', 'dirent.h']
+    libraries = ['util']
+eci = ExternalCompilationInfo(
+    includes=includes,
+    libraries=libraries,
+)
+
+class CConfig:
+    _compilation_info_ = eci
+    SEEK_SET = rffi_platform.DefinedConstantInteger('SEEK_SET')
+    SEEK_CUR = rffi_platform.DefinedConstantInteger('SEEK_CUR')
+    SEEK_END = rffi_platform.DefinedConstantInteger('SEEK_END')
+    OFF_T_SIZE = rffi_platform.SizeOf('off_t')
+
+    HAVE_UTIMES = rffi_platform.Has('utimes')
+    UTIMBUF = rffi_platform.Struct('struct %sutimbuf' % UNDERSCORE_ON_WIN32,
+                                   [('actime', rffi.INT),
+                                    ('modtime', rffi.INT)])
+    if not _WIN32:
+        CLOCK_T = rffi_platform.SimpleType('clock_t', rffi.INT)
+
+        TMS = rffi_platform.Struct(
+            'struct tms', [('tms_utime', rffi.INT),
+                           ('tms_stime', rffi.INT),
+                           ('tms_cutime', rffi.INT),
+                           ('tms_cstime', rffi.INT)])
+
+    GETPGRP_HAVE_ARG = rffi_platform.Has("getpgrp(0)")
+    SETPGRP_HAVE_ARG = rffi_platform.Has("setpgrp(0, 0)")
+
+config = rffi_platform.configure(CConfig)
+globals().update(config)
+
+def external(name, args, result, compilation_info=eci, **kwds):
+    return rffi.llexternal(name, args, result,
+                           compilation_info=compilation_info, **kwds)
+
+# For now we require off_t to be the same size as LONGLONG, which is the
+# interface required by callers of functions that thake an argument of type
+# off_t.
+if not _WIN32:
+    assert OFF_T_SIZE == rffi.sizeof(rffi.LONGLONG)
+
+c_dup = external(UNDERSCORE_ON_WIN32 + 'dup', [rffi.INT], rffi.INT,
+                 save_err=rffi.RFFI_SAVE_ERRNO)
+c_dup2 = external(UNDERSCORE_ON_WIN32 + 'dup2', [rffi.INT, rffi.INT], rffi.INT,
+                  save_err=rffi.RFFI_SAVE_ERRNO)
+c_open = external(UNDERSCORE_ON_WIN32 + 'open',
+                  [rffi.CCHARP, rffi.INT, rffi.MODE_T], rffi.INT,
+                  save_err=rffi.RFFI_SAVE_ERRNO)
+
+# Win32 Unicode functions
+c_wopen = external(UNDERSCORE_ON_WIN32 + 'wopen',
+                   [rffi.CWCHARP, rffi.INT, rffi.MODE_T], rffi.INT,
+                   save_err=rffi.RFFI_SAVE_ERRNO)
+
 #___________________________________________________________________
 # Wrappers around posix functions, that accept either strings, or
 # instances with a "as_bytes()" method.
@@ -237,26 +314,63 @@
     assert path is not None
     if isinstance(path, str):
         return path
+    elif isinstance(path, unicode):
+        # This never happens in PyPy's Python interpreter!
+        # Only in raw RPython code that uses unicode strings.
+        # We implement python2 behavior: silently convert to ascii.
+        return path.encode('ascii')
     else:
         return path.as_bytes()
 
 @specialize.argtype(0)
-def open(path, flags, mode):
-    return os.open(_as_bytes(path), flags, mode)
+def _as_bytes0(path):
+    """Crashes translation if the path contains NUL characters."""
+    res = _as_bytes(path)
+    rstring.check_str0(res)
+    return res
 
 @specialize.argtype(0)
-def stat(path):
-    return os.stat(_as_bytes(path))
+def _as_unicode(path):
+    assert path is not None
+    if isinstance(path, unicode):
+        return path
+    else:
+        return path.as_unicode()
 
 @specialize.argtype(0)
-def lstat(path):
-    return os.lstat(_as_bytes(path))
+def _as_unicode0(path):
+    """Crashes translation if the path contains NUL characters."""
+    res = _as_unicode(path)
+    rstring.check_str0(res)
+    return res
 
+# Returns True when the unicode function should be called:
+# - on Windows
+# - if the path is Unicode.
+unicode_traits = UnicodeTraits()
+string_traits = StringTraits()
+if _WIN32:
+    @specialize.argtype(0)
+    def _prefer_unicode(path):
+        if isinstance(path, str):
+            return False
+        elif isinstance(path, unicode):
+            return True
+        else:
+            return path.is_unicode
 
- at specialize.argtype(0)
-def statvfs(path):
-    return os.statvfs(_as_bytes(path))
+    @specialize.argtype(0)
+    def _preferred_traits(path):
+        if _prefer_unicode(path):
+            return unicode_traits
+        else:
+            return string_traits
+else:
+    @specialize.argtype(0)
+    def _prefer_unicode(path):
+        return False
 
+<<<<<<< local
 
 @specialize.argtype(0)
 def unlink(path):
@@ -315,10 +429,12 @@
 
 if os.name == 'nt':
     import nt
+=======
+>>>>>>> other
     @specialize.argtype(0)
-    def _getfullpathname(path):
-        return nt._getfullpathname(_as_bytes(path))
-
+    def _preferred_traits(path):
+        return string_traits
+    
 @specialize.argtype(0, 1)
 def putenv(name, value):
     os.environ[_as_bytes(name)] = _as_bytes(value)
@@ -327,8 +443,1355 @@
 def unsetenv(name):
     del os.environ[_as_bytes(name)]
 
-if os.name == 'nt':
+#___________________________________________________________________
+# Implementation of many posix functions.
+# They usually check the return value and raise an (RPython) OSError
+# with errno.
+
+def replace_os_function(name):
+    func = getattr(os, name, None)
+    if func is None:
+        return lambda f: f
+    return register_replacement_for(
+        func,
+        sandboxed_name='ll_os.ll_os_%s' % name)
+
+ at specialize.arg(0)
+def handle_posix_error(name, result):
+    result = widen(result)
+    if result < 0:
+        raise OSError(get_saved_errno(), '%s failed' % name)
+    return result
+
+ at replace_os_function('dup')
+def dup(fd):
+    validate_fd(fd)
+    return handle_posix_error('dup', c_dup(fd))
+
+ at replace_os_function('dup2')
+def dup2(fd, newfd):
+    validate_fd(fd)
+    handle_posix_error('dup2', c_dup2(fd, newfd))
+
+#___________________________________________________________________
+
+ at replace_os_function('open')
+ at specialize.argtype(0)
+def open(path, flags, mode):
+    if _prefer_unicode(path):
+        fd = c_wopen(_as_unicode0(path), flags, mode)
+    else:
+        fd = c_open(_as_bytes0(path), flags, mode)
+    return handle_posix_error('open', fd)
+
+c_read = external(UNDERSCORE_ON_WIN32 + 'read',
+                  [rffi.INT, rffi.VOIDP, rffi.SIZE_T], rffi.SSIZE_T,
+                  save_err=rffi.RFFI_SAVE_ERRNO)
+c_write = external(UNDERSCORE_ON_WIN32 + 'write',
+                   [rffi.INT, rffi.VOIDP, rffi.SIZE_T], rffi.SSIZE_T,
+                   save_err=rffi.RFFI_SAVE_ERRNO)
+c_close = external(UNDERSCORE_ON_WIN32 + 'close', [rffi.INT], rffi.INT,
+                   releasegil=False, save_err=rffi.RFFI_SAVE_ERRNO)
+
+ at replace_os_function('read')
+ at enforceargs(int, int)
+def read(fd, count):
+    if count < 0:
+        raise OSError(errno.EINVAL, None)
+    validate_fd(fd)
+    with rffi.scoped_alloc_buffer(count) as buf:
+        void_buf = rffi.cast(rffi.VOIDP, buf.raw)
+        got = handle_posix_error('read', c_read(fd, void_buf, count))
+        return buf.str(got)
+
+ at replace_os_function('write')
+ at enforceargs(int, None)
+def write(fd, data):
+    count = len(data)
+    validate_fd(fd)
+    with rffi.scoped_nonmovingbuffer(data) as buf:
+        return handle_posix_error('write', c_write(fd, buf, count))
+
+ at replace_os_function('close')
+def close(fd):
+    validate_fd(fd)
+    handle_posix_error('close', c_close(fd))
+
+c_lseek = external('_lseeki64' if _WIN32 else 'lseek',
+                   [rffi.INT, rffi.LONGLONG, rffi.INT], rffi.LONGLONG,
+                   macro=_MACRO_ON_POSIX, save_err=rffi.RFFI_SAVE_ERRNO)
+
+ at replace_os_function('lseek')
+def lseek(fd, pos, how):
+    validate_fd(fd)
+    if SEEK_SET is not None:
+        if how == 0:
+            how = SEEK_SET
+        elif how == 1: 
+            how = SEEK_CUR
+        elif how == 2:
+            how = SEEK_END
+    return handle_posix_error('lseek', c_lseek(fd, pos, how))
+
+c_ftruncate = external('ftruncate', [rffi.INT, rffi.LONGLONG], rffi.INT,
+                       macro=_MACRO_ON_POSIX, save_err=rffi.RFFI_SAVE_ERRNO)
+c_fsync = external('fsync' if not _WIN32 else '_commit', [rffi.INT], rffi.INT,
+                   save_err=rffi.RFFI_SAVE_ERRNO)
+c_fdatasync = external('fdatasync', [rffi.INT], rffi.INT,
+                       save_err=rffi.RFFI_SAVE_ERRNO)
+
+ at replace_os_function('ftruncate')
+def ftruncate(fd, length):
+    validate_fd(fd)
+    handle_posix_error('ftruncate', c_ftruncate(fd, length))
+
+ at replace_os_function('fsync')
+def fsync(fd):
+    validate_fd(fd)
+    handle_posix_error('fsync', c_fsync(fd))
+
+ at replace_os_function('fdatasync')
+def fdatasync(fd):
+    validate_fd(fd)
+    handle_posix_error('fdatasync', c_fdatasync(fd))
+
+#___________________________________________________________________
+
+c_chdir = external('chdir', [rffi.CCHARP], rffi.INT,
+                   save_err=rffi.RFFI_SAVE_ERRNO)
+c_fchdir = external('fchdir', [rffi.INT], rffi.INT,
+                    save_err=rffi.RFFI_SAVE_ERRNO)
+c_access = external(UNDERSCORE_ON_WIN32 + 'access',
+                    [rffi.CCHARP, rffi.INT], rffi.INT)
+c_waccess = external(UNDERSCORE_ON_WIN32 + 'waccess',
+                     [rffi.CWCHARP, rffi.INT], rffi.INT)
+
+ at replace_os_function('chdir')
+ at specialize.argtype(0)
+def chdir(path):
+    if not _WIN32:
+        handle_posix_error('chdir', c_chdir(_as_bytes0(path)))
+    else:
+        traits = _preferred_traits(path)
+        win32traits = make_win32_traits(traits)
+        path = traits.as_str0(path)
+
+        # This is a reimplementation of the C library's chdir
+        # function, but one that produces Win32 errors instead of DOS
+        # error codes.
+        # chdir is essentially a wrapper around SetCurrentDirectory;
+        # however, it also needs to set "magic" environment variables
+        # indicating the per-drive current directory, which are of the
+        # form =<drive>:
+        if not win32traits.SetCurrentDirectory(path):
+            raise rwin32.lastSavedWindowsError()
+        MAX_PATH = rwin32.MAX_PATH
+        assert MAX_PATH > 0
+
+        with traits.scoped_alloc_buffer(MAX_PATH) as path:
+            res = win32traits.GetCurrentDirectory(MAX_PATH + 1, path.raw)
+            if not res:
+                raise rwin32.lastSavedWindowsError()
+            res = rffi.cast(lltype.Signed, res)
+            assert res > 0
+            if res <= MAX_PATH + 1:
+                new_path = path.str(res)
+            else:
+                with traits.scoped_alloc_buffer(res) as path:
+                    res = win32traits.GetCurrentDirectory(res, path.raw)
+                    if not res:
+                        raise rwin32.lastSavedWindowsError()
+                    res = rffi.cast(lltype.Signed, res)
+                    assert res > 0
+                    new_path = path.str(res)
+        if traits.str is unicode:
+            if new_path[0] == u'\\' or new_path[0] == u'/':  # UNC path
+                return
+            magic_envvar = u'=' + new_path[0] + u':'
+        else:
+            if new_path[0] == '\\' or new_path[0] == '/':  # UNC path
+                return
+            magic_envvar = '=' + new_path[0] + ':'
+        if not win32traits.SetEnvironmentVariable(magic_envvar, new_path):
+            raise rwin32.lastSavedWindowsError()
+
+ at replace_os_function('fchdir')
+def fchdir(fd):
+    validate_fd(fd)
+    handle_posix_error('fchdir', c_fchdir(fd))
+
+ at replace_os_function('access')
+ at specialize.argtype(0)
+def access(path, mode):
+    if _WIN32:
+        # All files are executable on Windows
+        mode = mode & ~os.X_OK
+    if _prefer_unicode(path):
+        error = c_waccess(_as_unicode0(path), mode)
+    else:
+        error = c_access(_as_bytes0(path), mode)
+    return error == 0
+
+# This Win32 function is not exposed via os, but needed to get a
+# correct implementation of os.path.abspath.
+ at specialize.argtype(0)
+def getfullpathname(path):
+    length = rwin32.MAX_PATH + 1
+    traits = _preferred_traits(path)
+    win32traits = make_win32_traits(traits)
+    with traits.scoped_alloc_buffer(length) as buf:
+        res = win32traits.GetFullPathName(
+            traits.as_str0(path), rffi.cast(rwin32.DWORD, length),
+            buf.raw, lltype.nullptr(win32traits.LPSTRP.TO))
+        if res == 0:
+            raise rwin32.lastSavedWindowsError("_getfullpathname failed")
+        return buf.str(intmask(res))
+
+c_getcwd = external(UNDERSCORE_ON_WIN32 + 'getcwd',
+                    [rffi.CCHARP, rffi.SIZE_T], rffi.CCHARP,
+                    save_err=rffi.RFFI_SAVE_ERRNO)
+c_wgetcwd = external(UNDERSCORE_ON_WIN32 + 'wgetcwd',
+                     [rffi.CWCHARP, rffi.SIZE_T], rffi.CWCHARP,
+                     save_err=rffi.RFFI_SAVE_ERRNO)
+
+ at replace_os_function('getcwd')
+def getcwd():
+    bufsize = 256
+    while True:
+        buf = lltype.malloc(rffi.CCHARP.TO, bufsize, flavor='raw')
+        res = c_getcwd(buf, bufsize)
+        if res:
+            break   # ok
+        error = get_saved_errno()
+        lltype.free(buf, flavor='raw')
+        if error != errno.ERANGE:
+            raise OSError(error, "getcwd failed")
+        # else try again with a larger buffer, up to some sane limit
+        bufsize *= 4
+        if bufsize > 1024*1024:  # xxx hard-coded upper limit
+            raise OSError(error, "getcwd result too large")
+    result = rffi.charp2str(res)
+    lltype.free(buf, flavor='raw')
+    return result
+
+ at replace_os_function('getcwdu')
+def getcwdu():
+    bufsize = 256
+    while True:
+        buf = lltype.malloc(rffi.CWCHARP.TO, bufsize, flavor='raw')
+        res = c_wgetcwd(buf, bufsize)
+        if res:
+            break   # ok
+        error = get_saved_errno()
+        lltype.free(buf, flavor='raw')
+        if error != errno.ERANGE:
+            raise OSError(error, "getcwd failed")
+        # else try again with a larger buffer, up to some sane limit
+        bufsize *= 4
+        if bufsize > 1024*1024:  # xxx hard-coded upper limit
+            raise OSError(error, "getcwd result too large")
+    result = rffi.wcharp2unicode(res)
+    lltype.free(buf, flavor='raw')
+    return result
+
+if not _WIN32:
+    class CConfig:
+        _compilation_info_ = eci
+        DIRENT = rffi_platform.Struct('struct dirent',
+            [('d_name', lltype.FixedSizeArray(rffi.CHAR, 1))])
+
+    DIRP = rffi.COpaquePtr('DIR')
+    config = rffi_platform.configure(CConfig)
+    DIRENT = config['DIRENT']
+    DIRENTP = lltype.Ptr(DIRENT)
+    c_opendir = external('opendir', [rffi.CCHARP], DIRP,
+                         save_err=rffi.RFFI_SAVE_ERRNO)
+    # XXX macro=True is hack to make sure we get the correct kind of
+    # dirent struct (which depends on defines)
+    c_readdir = external('readdir', [DIRP], DIRENTP,
+                         macro=True, save_err=rffi.RFFI_FULL_ERRNO_ZERO)
+    c_closedir = external('closedir', [DIRP], rffi.INT)
+
+ at replace_os_function('listdir')
+ at specialize.argtype(0)
+def listdir(path):
+    if not _WIN32:
+        path = _as_bytes0(path)
+        dirp = c_opendir(path)
+        if not dirp:
+            raise OSError(get_saved_errno(), "opendir failed")
+        result = []
+        while True:
+            direntp = c_readdir(dirp)
+            if not direntp:
+                error = get_saved_errno()
+                break
+            namep = rffi.cast(rffi.CCHARP, direntp.c_d_name)
+            name = rffi.charp2str(namep)
+            if name != '.' and name != '..':
+                result.append(name)
+        c_closedir(dirp)
+        if error:
+            raise OSError(error, "readdir failed")
+        return result
+    else:  # _WIN32 case
+        traits = _preferred_traits(path)
+        win32traits = make_win32_traits(traits)
+        path = traits.as_str0(path)
+
+        if traits.str is unicode:
+            if path and path[-1] not in (u'/', u'\\', u':'):
+                path += u'/'
+            mask = path + u'*.*'
+        else:
+            if path and path[-1] not in ('/', '\\', ':'):
+                path += '/'
+            mask = path + '*.*'
+
+        filedata = lltype.malloc(win32traits.WIN32_FIND_DATA, flavor='raw')
+        try:
+            result = []
+            hFindFile = win32traits.FindFirstFile(mask, filedata)
+            if hFindFile == rwin32.INVALID_HANDLE_VALUE:
+                error = rwin32.GetLastError_saved()
+                if error == win32traits.ERROR_FILE_NOT_FOUND:
+                    return result
+                else:
+                    raise WindowsError(error,  "FindFirstFile failed")
+            while True:
+                name = traits.charp2str(rffi.cast(traits.CCHARP,
+                                                  filedata.c_cFileName))
+                if traits.str is unicode:
+                    if not (name == u"." or name == u".."):
+                        result.append(name)
+                else:
+                    if not (name == "." or name == ".."):
+                        result.append(name)
+                if not win32traits.FindNextFile(hFindFile, filedata):
+                    break
+            # FindNextFile sets error to ERROR_NO_MORE_FILES if
+            # it got to the end of the directory
+            error = rwin32.GetLastError_saved()
+            win32traits.FindClose(hFindFile)
+            if error == win32traits.ERROR_NO_MORE_FILES:
+                return result
+            else:
+                raise WindowsError(error,  "FindNextFile failed")
+        finally:
+            lltype.free(filedata, flavor='raw')
+
+#___________________________________________________________________
+
+c_execv = external('execv', [rffi.CCHARP, rffi.CCHARPP], rffi.INT,
+                   save_err=rffi.RFFI_SAVE_ERRNO)
+c_execve = external('execve',
+                    [rffi.CCHARP, rffi.CCHARPP, rffi.CCHARPP], rffi.INT,
+                    save_err=rffi.RFFI_SAVE_ERRNO)
+c_spawnv = external('spawnv',
+                    [rffi.INT, rffi.CCHARP, rffi.CCHARPP], rffi.INT,
+                    save_err=rffi.RFFI_SAVE_ERRNO)
+c_spawnve = external('spawnve',
+                    [rffi.INT, rffi.CCHARP, rffi.CCHARPP, rffi.CCHARPP],
+                     rffi.INT,
+                     save_err=rffi.RFFI_SAVE_ERRNO)
+
+ at replace_os_function('execv')
+def execv(path, args):
+    rstring.check_str0(path)
+    # This list conversion already takes care of NUL bytes.
+    l_args = rffi.ll_liststr2charpp(args)
+    c_execv(path, l_args)
+    rffi.free_charpp(l_args)
+    raise OSError(get_saved_errno(), "execv failed")
+
+ at replace_os_function('execve')
+def execve(path, args, env):
+    envstrs = []
+    for item in env.iteritems():
+        envstr = "%s=%s" % item
+        envstrs.append(envstr)
+
+    rstring.check_str0(path)
+    # This list conversion already takes care of NUL bytes.
+    l_args = rffi.ll_liststr2charpp(args)
+    l_env = rffi.ll_liststr2charpp(envstrs)
+    c_execve(path, l_args, l_env)
+
+    rffi.free_charpp(l_env)
+    rffi.free_charpp(l_args)
+    raise OSError(get_saved_errno(), "execve failed")
+
+ at replace_os_function('spawnv')
+def spawnv(mode, path, args):
+    rstring.check_str0(path)
+    l_args = rffi.ll_liststr2charpp(args)
+    childpid = c_spawnv(mode, path, l_args)
+    rffi.free_charpp(l_args)
+    return handle_posix_error('spawnv', childpid)
+
+ at replace_os_function('spawnve')
+def spawnve(mode, path, args, env):
+    envstrs = []
+    for item in env.iteritems():
+        envstrs.append("%s=%s" % item)
+    rstring.check_str0(path)
+    l_args = rffi.ll_liststr2charpp(args)
+    l_env = rffi.ll_liststr2charpp(envstrs)
+    childpid = c_spawnve(mode, path, l_args, l_env)
+    rffi.free_charpp(l_env)
+    rffi.free_charpp(l_args)
+    return handle_posix_error('spawnve', childpid)
+
+c_fork = external('fork', [], rffi.PID_T, _nowrapper = True)
+c_openpty = external('openpty',
+                     [rffi.INTP, rffi.INTP, rffi.VOIDP, rffi.VOIDP, rffi.VOIDP],
+                     rffi.INT,
+                     save_err=rffi.RFFI_SAVE_ERRNO)
+c_forkpty = external('forkpty',
+                     [rffi.INTP, rffi.VOIDP, rffi.VOIDP, rffi.VOIDP],
+                     rffi.PID_T,
+                     save_err=rffi.RFFI_SAVE_ERRNO)
+
+ at replace_os_function('fork')
+ at jit.dont_look_inside
+def fork():
+    # NB. keep forkpty() up-to-date, too
+    ofs = debug.debug_offset()
+    opaqueaddr = rthread.gc_thread_before_fork()
+    childpid = c_fork()
+    rthread.gc_thread_after_fork(childpid, opaqueaddr)
+    childpid = handle_posix_error('fork', childpid)
+    if childpid == 0:
+        debug.debug_forked(ofs)
+    return childpid
+
+ at replace_os_function('openpty')
+def openpty():
+    master_p = lltype.malloc(rffi.INTP.TO, 1, flavor='raw')
+    slave_p = lltype.malloc(rffi.INTP.TO, 1, flavor='raw')
+    try:
+        handle_posix_error(
+            'openpty', c_openpty(master_p, slave_p, None, None, None))
+        return (widen(master_p[0]), widen(slave_p[0]))
+    finally:
+        lltype.free(master_p, flavor='raw')
+        lltype.free(slave_p, flavor='raw')
+
+ at replace_os_function('forkpty')
+ at jit.dont_look_inside
+def forkpty():
+    master_p = lltype.malloc(rffi.INTP.TO, 1, flavor='raw')
+    master_p[0] = rffi.cast(rffi.INT, -1)
+    try:
+        ofs = debug.debug_offset()
+        opaqueaddr = rthread.gc_thread_before_fork()
+        childpid = c_forkpty(master_p, None, None, None)
+        rthread.gc_thread_after_fork(childpid, opaqueaddr)
+        childpid = handle_posix_error('forkpty', childpid)
+        if childpid == 0:
+            debug.debug_forked(ofs)
+        return (childpid, master_p[0])
+    finally:
+        lltype.free(master_p, flavor='raw')
+
+if _WIN32:
+    # emulate waitpid() with the _cwait() of Microsoft's compiler
+    c__cwait = external('_cwait',
+                        [rffi.INTP, rffi.PID_T, rffi.INT], rffi.PID_T,
+                        save_err=rffi.RFFI_SAVE_ERRNO)
+    def c_waitpid(pid, status_p, options):
+        result = c__cwait(status_p, pid, options)
+        # shift the status left a byte so this is more
+        # like the POSIX waitpid
+        status_p[0] = rffi.cast(rffi.INT, widen(status_p[0]) << 8)
+        return result
+elif _CYGWIN:
+    c_waitpid = external('cygwin_waitpid',
+                         [rffi.PID_T, rffi.INTP, rffi.INT], rffi.PID_T,
+                         save_err=rffi.RFFI_SAVE_ERRNO)
+else:
+    c_waitpid = external('waitpid',
+                         [rffi.PID_T, rffi.INTP, rffi.INT], rffi.PID_T,
+                         save_err=rffi.RFFI_SAVE_ERRNO)
+
+ at replace_os_function('waitpid')
+def waitpid(pid, options):
+    status_p = lltype.malloc(rffi.INTP.TO, 1, flavor='raw')
+    status_p[0] = rffi.cast(rffi.INT, 0)
+    try:
+        result = handle_posix_error('waitpid',
+                                    c_waitpid(pid, status_p, options))
+        status = widen(status_p[0])
+        return (result, status)
+    finally:
+        lltype.free(status_p, flavor='raw')
+
+def _make_waitmacro(name):
+    c_func = external(name, [lltype.Signed], lltype.Signed,
+                      macro=_MACRO_ON_POSIX)
+    returning_int = name in ('WEXITSTATUS', 'WSTOPSIG', 'WTERMSIG')
+
+    @replace_os_function(name)
+    @func_renamer(name)
+    def _waitmacro(status):
+        if returning_int:
+            return c_func(status)
+        else:
+            return bool(c_func(status))
+
+WAIT_MACROS = ['WCOREDUMP', 'WIFCONTINUED', 'WIFSTOPPED',
+               'WIFSIGNALED', 'WIFEXITED',
+               'WEXITSTATUS', 'WSTOPSIG', 'WTERMSIG']
+for name in WAIT_MACROS:
+    _make_waitmacro(name)
+
+#___________________________________________________________________
+
+c_getlogin = external('getlogin', [], rffi.CCHARP,
+                      releasegil=False, save_err=rffi.RFFI_SAVE_ERRNO)
+c_getloadavg = external('getloadavg', 
+                        [rffi.CArrayPtr(lltype.Float), rffi.INT], rffi.INT)
+
+ at replace_os_function('getlogin')
+def getlogin():
+    result = c_getlogin()
+    if not result:
+        raise OSError(get_saved_errno(), "getlogin failed")
+    return rffi.charp2str(result)
+
+ at replace_os_function('getloadavg')
+def getloadavg():
+    load = lltype.malloc(rffi.CArrayPtr(lltype.Float).TO, 3, flavor='raw')
+    try:
+        r = c_getloadavg(load, 3)
+        if r != 3:
+            raise OSError
+        return (load[0], load[1], load[2])
+    finally:
+        lltype.free(load, flavor='raw')
+
+#___________________________________________________________________
+
+c_readlink = external('readlink',
+                      [rffi.CCHARP, rffi.CCHARP, rffi.SIZE_T], rffi.SSIZE_T,
+                      save_err=rffi.RFFI_SAVE_ERRNO)
+
+ at replace_os_function('readlink')
+def readlink(path):
+    path = _as_bytes0(path)
+    bufsize = 1023
+    while True:
+        buf = lltype.malloc(rffi.CCHARP.TO, bufsize, flavor='raw')
+        res = widen(c_readlink(path, buf, bufsize))
+        if res < 0:
+            lltype.free(buf, flavor='raw')
+            error = get_saved_errno()    # failed
+            raise OSError(error, "readlink failed")
+        elif res < bufsize:
+            break                       # ok
+        else:
+            # buf too small, try again with a larger buffer
+            lltype.free(buf, flavor='raw')
+            bufsize *= 4
+    # convert the result to a string
+    result = rffi.charp2strn(buf, res)
+    lltype.free(buf, flavor='raw')
+    return result
+
+c_isatty = external(UNDERSCORE_ON_WIN32 + 'isatty', [rffi.INT], rffi.INT)
+
+ at replace_os_function('isatty')
+def isatty(fd):
+    if not is_valid_fd(fd):
+        return False
+    return c_isatty(fd) != 0
+
+c_ttyname = external('ttyname', [lltype.Signed], rffi.CCHARP,
+                     releasegil=False,
+                     save_err=rffi.RFFI_SAVE_ERRNO)
+
+ at replace_os_function('ttyname')
+def ttyname(fd):
+    l_name = c_ttyname(fd)
+    if not l_name:
+        raise OSError(get_saved_errno(), "ttyname raised")
+    return rffi.charp2str(l_name)
+
+c_strerror = external('strerror', [rffi.INT], rffi.CCHARP,
+                      releasegil=False)
+
+ at replace_os_function('strerror')
+def strerror(errnum):
+    res = c_strerror(errnum)
+    if not res:
+        raise ValueError("os_strerror failed")
+    return rffi.charp2str(res)
+
+c_system = external('system', [rffi.CCHARP], rffi.INT)
+
+ at replace_os_function('system')
+def system(command):
+    return widen(c_system(command))
+
+c_unlink = external('unlink', [rffi.CCHARP], rffi.INT,
+                    save_err=rffi.RFFI_SAVE_ERRNO)
+c_mkdir = external('mkdir', [rffi.CCHARP, rffi.MODE_T], rffi.INT,
+                   save_err=rffi.RFFI_SAVE_ERRNO)
+c_rmdir = external(UNDERSCORE_ON_WIN32 + 'rmdir', [rffi.CCHARP], rffi.INT,
+                   save_err=rffi.RFFI_SAVE_ERRNO)
+c_wrmdir = external(UNDERSCORE_ON_WIN32 + 'wrmdir', [rffi.CWCHARP], rffi.INT,
+                    save_err=rffi.RFFI_SAVE_ERRNO)
+
+ at replace_os_function('unlink')
+ at specialize.argtype(0)
+def unlink(path):
+    if not _WIN32:
+        handle_posix_error('unlink', c_unlink(_as_bytes0(path)))
+    else:
+        traits = _preferred_traits(path)
+        win32traits = make_win32_traits(traits)
+        if not win32traits.DeleteFile(traits.as_str0(path)):
+            raise rwin32.lastSavedWindowsError()
+
+ at replace_os_function('mkdir')
+ at specialize.argtype(0)
+def mkdir(path, mode=0o777):
+    if not _WIN32:
+        handle_posix_error('mkdir', c_mkdir(_as_bytes0(path), mode))
+    else:
+        traits = _preferred_traits(path)
+        win32traits = make_win32_traits(traits)
+        if not win32traits.CreateDirectory(traits.as_str0(path), None):
+            raise rwin32.lastSavedWindowsError()
+
+ at replace_os_function('rmdir')
+ at specialize.argtype(0)
+def rmdir(path):
+    if _prefer_unicode(path):
+        handle_posix_error('wrmdir', c_wrmdir(_as_unicode0(path)))
+    else:
+        handle_posix_error('rmdir', c_rmdir(_as_bytes0(path)))
+
+c_chmod = external('chmod', [rffi.CCHARP, rffi.MODE_T], rffi.INT,
+                   save_err=rffi.RFFI_SAVE_ERRNO)
+c_fchmod = external('fchmod', [rffi.INT, rffi.MODE_T], rffi.INT,
+                    save_err=rffi.RFFI_SAVE_ERRNO,)
+c_rename = external('rename', [rffi.CCHARP, rffi.CCHARP], rffi.INT,
+                    save_err=rffi.RFFI_SAVE_ERRNO)
+
+ at replace_os_function('chmod')
+ at specialize.argtype(0)
+def chmod(path, mode):
+    if not _WIN32:
+        handle_posix_error('chmod', c_chmod(_as_bytes0(path), mode))
+    else:
+        traits = _preferred_traits(path)
+        win32traits = make_win32_traits(traits)
+        path = traits.as_str0(path)
+        attr = win32traits.GetFileAttributes(path)
+        if attr == win32traits.INVALID_FILE_ATTRIBUTES:
+            raise rwin32.lastSavedWindowsError()
+        if mode & 0200: # _S_IWRITE
+            attr &= ~win32traits.FILE_ATTRIBUTE_READONLY
+        else:
+            attr |= win32traits.FILE_ATTRIBUTE_READONLY
+        if not win32traits.SetFileAttributes(path, attr):
+            raise rwin32.lastSavedWindowsError()
+
+ at replace_os_function('fchmod')
+def fchmod(fd, mode):
+    handle_posix_error('fchmod', c_fchmod(fd, mode))
+
+ at replace_os_function('rename')
+ at specialize.argtype(0, 1)
+def rename(path1, path2):
+    if not _WIN32:
+        handle_posix_error('rename',
+                           c_rename(_as_bytes0(path1), _as_bytes0(path2)))
+    else:
+        traits = _preferred_traits(path1)
+        win32traits = make_win32_traits(traits)
+        path1 = traits.as_str0(path1)
+        path2 = traits.as_str0(path2)
+        if not win32traits.MoveFile(path1, path2):
+            raise rwin32.lastSavedWindowsError()
+
+#___________________________________________________________________
+
+c_mkfifo = external('mkfifo', [rffi.CCHARP, rffi.MODE_T], rffi.INT,
+                    save_err=rffi.RFFI_SAVE_ERRNO)
+c_mknod = external('mknod', [rffi.CCHARP, rffi.MODE_T, rffi.INT], rffi.INT,
+#                                           # xxx: actually ^^^ dev_t
+                   macro=_MACRO_ON_POSIX, save_err=rffi.RFFI_SAVE_ERRNO)
+
+ at replace_os_function('mkfifo')
+ at specialize.argtype(0)
+def mkfifo(path, mode):
+    handle_posix_error('mkfifo', c_mkfifo(_as_bytes0(path), mode))
+
+ at replace_os_function('mknod')
+ at specialize.argtype(0)
+def mknod(path, mode, dev):
+    handle_posix_error('mknod', c_mknod(_as_bytes0(path), mode, dev))
+
+if _WIN32:
+    CreatePipe = external('CreatePipe', [rwin32.LPHANDLE,
+                                         rwin32.LPHANDLE,
+                                         rffi.VOIDP,
+                                         rwin32.DWORD],
+                          rwin32.BOOL)
+    c_open_osfhandle = external('_open_osfhandle', [rffi.INTPTR_T,
+                                                    rffi.INT],
+                                rffi.INT)
+else:
+    INT_ARRAY_P = rffi.CArrayPtr(rffi.INT)
+    c_pipe = external('pipe', [INT_ARRAY_P], rffi.INT,
+                      save_err=rffi.RFFI_SAVE_ERRNO)
+
+ at replace_os_function('pipe')
+def pipe():
+    if _WIN32:
+        pread  = lltype.malloc(rwin32.LPHANDLE.TO, 1, flavor='raw')
+        pwrite = lltype.malloc(rwin32.LPHANDLE.TO, 1, flavor='raw')
+        try:
+            if not CreatePipe(
+                    pread, pwrite, lltype.nullptr(rffi.VOIDP.TO), 0):
+                raise WindowsError(rwin32.GetLastError_saved(),
+                                   "CreatePipe failed")
+            hread = rffi.cast(rffi.INTPTR_T, pread[0])
+            hwrite = rffi.cast(rffi.INTPTR_T, pwrite[0])
+        finally:
+            lltype.free(pwrite, flavor='raw')
+            lltype.free(pread, flavor='raw')
+        fdread = c_open_osfhandle(hread, 0)
+        fdwrite = c_open_osfhandle(hwrite, 1)
+        return (fdread, fdwrite)
+    else:
+        filedes = lltype.malloc(INT_ARRAY_P.TO, 2, flavor='raw')
+        try:
+            handle_posix_error('pipe', c_pipe(filedes))
+            return (widen(filedes[0]), widen(filedes[1]))
+        finally:
+            lltype.free(filedes, flavor='raw')
+
+c_link = external('link', [rffi.CCHARP, rffi.CCHARP], rffi.INT,
+                  save_err=rffi.RFFI_SAVE_ERRNO,)
+c_symlink = external('symlink', [rffi.CCHARP, rffi.CCHARP], rffi.INT,
+                     save_err=rffi.RFFI_SAVE_ERRNO)
+
+#___________________________________________________________________
+
+ at replace_os_function('link')
+ at specialize.argtype(0, 1)
+def link(oldpath, newpath):
+    oldpath = _as_bytes0(oldpath)
+    newpath = _as_bytes0(newpath)
+    handle_posix_error('link', c_link(oldpath, newpath))
+
+ at replace_os_function('symlink')
+ at specialize.argtype(0, 1)
+def symlink(oldpath, newpath):
+    oldpath = _as_bytes0(oldpath)
+    newpath = _as_bytes0(newpath)
+    handle_posix_error('symlink', c_symlink(oldpath, newpath))
+
+c_umask = external(UNDERSCORE_ON_WIN32 + 'umask', [rffi.MODE_T], rffi.MODE_T)
+
+ at replace_os_function('umask')
+def umask(newmask):
+    return widen(c_umask(newmask))
+
+c_chown = external('chown', [rffi.CCHARP, rffi.INT, rffi.INT], rffi.INT,
+                   save_err=rffi.RFFI_SAVE_ERRNO)
+c_lchown = external('lchown', [rffi.CCHARP, rffi.INT, rffi.INT], rffi.INT,
+                    save_err=rffi.RFFI_SAVE_ERRNO)
+c_fchown = external('fchown', [rffi.INT, rffi.INT, rffi.INT], rffi.INT,
+                    save_err=rffi.RFFI_SAVE_ERRNO)
+
+ at replace_os_function('chown')
+def chown(path, uid, gid):
+    handle_posix_error('chown', c_chown(path, uid, gid))
+
+ at replace_os_function('lchown')
+def lchown(path, uid, gid):
+    handle_posix_error('lchown', c_lchown(path, uid, gid))
+
+ at replace_os_function('fchown')
+def fchown(fd, uid, gid):
+    handle_posix_error('fchown', c_fchown(fd, uid, gid))
+
+#___________________________________________________________________
+
+UTIMBUFP = lltype.Ptr(UTIMBUF)
+c_utime = external('utime', [rffi.CCHARP, UTIMBUFP], rffi.INT,
+                   save_err=rffi.RFFI_SAVE_ERRNO)
+if HAVE_UTIMES:
+    class CConfig:
+        _compilation_info_ = eci
+        TIMEVAL = rffi_platform.Struct('struct timeval', [
+            ('tv_sec', rffi.LONG),
+            ('tv_usec', rffi.LONG)])
+    config = rffi_platform.configure(CConfig)
+    TIMEVAL = config['TIMEVAL']
+    TIMEVAL2P = rffi.CArrayPtr(TIMEVAL)
+    c_utimes = external('utimes', [rffi.CCHARP, TIMEVAL2P], rffi.INT,
+                        save_err=rffi.RFFI_SAVE_ERRNO)
+
+if _WIN32:
     from rpython.rlib import rwin32
-    os_kill = rwin32.os_kill
+    GetSystemTime = external(
+        'GetSystemTime',
+        [lltype.Ptr(rwin32.SYSTEMTIME)],
+        lltype.Void,
+        calling_conv='win',
+        save_err=rffi.RFFI_SAVE_LASTERROR)
+
+    SystemTimeToFileTime = external(
+        'SystemTimeToFileTime',
+        [lltype.Ptr(rwin32.SYSTEMTIME),
+         lltype.Ptr(rwin32.FILETIME)],
+        rwin32.BOOL,
+        calling_conv='win',
+        save_err=rffi.RFFI_SAVE_LASTERROR)
+
+    SetFileTime = external(
+        'SetFileTime',
+        [rwin32.HANDLE,
+         lltype.Ptr(rwin32.FILETIME),
+         lltype.Ptr(rwin32.FILETIME),
+         lltype.Ptr(rwin32.FILETIME)],
+        rwin32.BOOL,
+        calling_conv='win')
+
+
+ at replace_os_function('utime')
+ at specialize.argtype(0, 1)
+def utime(path, times):
+    if not _WIN32:
+        path = _as_bytes0(path)
+        if times is None:
+            error = c_utime(path, lltype.nullptr(UTIMBUFP.TO))
+        else:
+            actime, modtime = times
+            if HAVE_UTIMES:
+                import math
+                l_times = lltype.malloc(TIMEVAL2P.TO, 2, flavor='raw')
+                fracpart, intpart = math.modf(actime)
+                rffi.setintfield(l_times[0], 'c_tv_sec', int(intpart))
+                rffi.setintfield(l_times[0], 'c_tv_usec', int(fracpart * 1e6))
+                fracpart, intpart = math.modf(modtime)
+                rffi.setintfield(l_times[1], 'c_tv_sec', int(intpart))
+                rffi.setintfield(l_times[1], 'c_tv_usec', int(fracpart * 1e6))
+                error = c_utimes(path, l_times)
+                lltype.free(l_times, flavor='raw')
+            else:
+                # we only have utime(), which does not allow
+                # sub-second resolution
+                l_utimbuf = lltype.malloc(UTIMBUFP.TO, flavor='raw')
+                l_utimbuf.c_actime  = rffi.r_time_t(actime)
+                l_utimbuf.c_modtime = rffi.r_time_t(modtime)
+                error = c_utime(path, l_utimbuf)
+                lltype.free(l_utimbuf, flavor='raw')
+        handle_posix_error('utime', error)
+    else:  # _WIN32 case
+        from rpython.rlib.rwin32file import time_t_to_FILE_TIME
+        traits = _preferred_traits(path)
+        win32traits = make_win32_traits(traits)
+        path = traits.as_str0(path)
+        hFile = win32traits.CreateFile(path,
+                           win32traits.FILE_WRITE_ATTRIBUTES, 0,
+                           None, win32traits.OPEN_EXISTING,
+                           win32traits.FILE_FLAG_BACKUP_SEMANTICS,
+                           rwin32.NULL_HANDLE)
+        if hFile == rwin32.INVALID_HANDLE_VALUE:
+            raise rwin32.lastSavedWindowsError()
+        ctime = lltype.nullptr(rwin32.FILETIME)
+        atime = lltype.malloc(rwin32.FILETIME, flavor='raw')
+        mtime = lltype.malloc(rwin32.FILETIME, flavor='raw')
+        try:
+            if times is None:
+                now = lltype.malloc(rwin32.SYSTEMTIME, flavor='raw')
+                try:
+                    GetSystemTime(now)
+                    if (not SystemTimeToFileTime(now, atime) or
+                        not SystemTimeToFileTime(now, mtime)):
+                        raise rwin32.lastSavedWindowsError()
+                finally:
+                    lltype.free(now, flavor='raw')
+            else:
+                actime, modtime = times
+                time_t_to_FILE_TIME(actime, atime)
+                time_t_to_FILE_TIME(modtime, mtime)
+            if not SetFileTime(hFile, ctime, atime, mtime):
+                raise rwin32.lastSavedWindowsError()
+        finally:
+            rwin32.CloseHandle(hFile)
+            lltype.free(atime, flavor='raw')
+            lltype.free(mtime, flavor='raw')
+
+if not _WIN32:
+    TMSP = lltype.Ptr(TMS)
+    c_times = external('times', [TMSP], CLOCK_T,
+                        save_err=rffi.RFFI_SAVE_ERRNO)
+
+    # Here is a random extra platform parameter which is important.
+    # Strictly speaking, this should probably be retrieved at runtime, not
+    # at translation time.
+    CLOCK_TICKS_PER_SECOND = float(os.sysconf('SC_CLK_TCK'))
 else:
-    os_kill = os.kill
+    GetCurrentProcess = external(
+        'GetCurrentProcess', [],
+        rwin32.HANDLE, calling_conv='win')
+    GetProcessTimes = external(
+        'GetProcessTimes', [
+            rwin32.HANDLE,
+            lltype.Ptr(rwin32.FILETIME), lltype.Ptr(rwin32.FILETIME),
+            lltype.Ptr(rwin32.FILETIME), lltype.Ptr(rwin32.FILETIME)],
+        rwin32.BOOL, calling_conv='win')
+
+ at replace_os_function('times')
+def times():
+    if not _WIN32:
+        l_tmsbuf = lltype.malloc(TMSP.TO, flavor='raw')
+        try:
+            result = handle_posix_error('times', c_times(l_tmsbuf))
+            return (
+                rffi.cast(lltype.Signed, l_tmsbuf.c_tms_utime)
+                                               / CLOCK_TICKS_PER_SECOND,
+                rffi.cast(lltype.Signed, l_tmsbuf.c_tms_stime)
+                                               / CLOCK_TICKS_PER_SECOND,
+                rffi.cast(lltype.Signed, l_tmsbuf.c_tms_cutime)
+                                               / CLOCK_TICKS_PER_SECOND,
+                rffi.cast(lltype.Signed, l_tmsbuf.c_tms_cstime)
+                                               / CLOCK_TICKS_PER_SECOND,
+                result / CLOCK_TICKS_PER_SECOND)
+        finally:
+            lltype.free(l_tmsbuf, flavor='raw')
+    else:
+        pcreate = lltype.malloc(rwin32.FILETIME, flavor='raw')
+        pexit   = lltype.malloc(rwin32.FILETIME, flavor='raw')
+        pkernel = lltype.malloc(rwin32.FILETIME, flavor='raw')
+        puser   = lltype.malloc(rwin32.FILETIME, flavor='raw')
+        try:
+            hProc = GetCurrentProcess()
+            GetProcessTimes(hProc, pcreate, pexit, pkernel, puser)
+            # The fields of a FILETIME structure are the hi and lo parts
+            # of a 64-bit value expressed in 100 nanosecond units
+            # (of course).
+            return (
+                rffi.cast(lltype.Signed, pkernel.c_dwHighDateTime) * 429.4967296 +
+                rffi.cast(lltype.Signed, pkernel.c_dwLowDateTime) * 1E-7,
+                rffi.cast(lltype.Signed, puser.c_dwHighDateTime) * 429.4967296 +
+                rffi.cast(lltype.Signed, puser.c_dwLowDateTime) * 1E-7,
+                0, 0, 0)
+        finally:
+            lltype.free(puser,   flavor='raw')
+            lltype.free(pkernel, flavor='raw')
+            lltype.free(pexit,   flavor='raw')
+            lltype.free(pcreate, flavor='raw')
+
+c_kill = external('kill', [rffi.PID_T, rffi.INT], rffi.INT,
+                  save_err=rffi.RFFI_SAVE_ERRNO)
+c_killpg = external('killpg', [rffi.INT, rffi.INT], rffi.INT,
+                    save_err=rffi.RFFI_SAVE_ERRNO)
+c_exit = external('_exit', [rffi.INT], lltype.Void)
+c_nice = external('nice', [rffi.INT], rffi.INT,
+                  save_err=rffi.RFFI_FULL_ERRNO_ZERO)
+
+ at replace_os_function('kill')
+def kill(pid, sig):
+    if not _WIN32:
+        return handle_posix_error('kill', c_kill(pid, sig))
+    else:
+        if sig == rwin32.CTRL_C_EVENT or sig == rwin32.CTRL_BREAK_EVENT:
+            if rwin32.GenerateConsoleCtrlEvent(sig, pid) == 0:
+                raise rwin32.lastSavedWindowsError(
+                    'kill() failed generating event')
+            return
+        handle = rwin32.OpenProcess(rwin32.PROCESS_ALL_ACCESS, False, pid)
+        if not handle:
+            raise rwin32.lastSavedWindowsError('kill() failed opening process')
+        try:
+            if rwin32.TerminateProcess(handle, sig) == 0:
+                raise rwin32.lastSavedWindowsError(
+                    'kill() failed to terminate process')
+        finally:
+            rwin32.CloseHandle(handle)
+
+ at replace_os_function('killpg')
+def killpg(pgrp, sig):
+    return handle_posix_error('killpg', c_killpg(pgrp, sig))
+
+ at replace_os_function('_exit')
+def exit(status):
+    debug.debug_flush()
+    c_exit(status)
+
+ at replace_os_function('nice')
+def nice(inc):
+    # Assume that the system provides a standard-compliant version
+    # of nice() that returns the new priority.  Nowadays, FreeBSD
+    # might be the last major non-compliant system (xxx check me).
+    res = widen(c_nice(inc))
+    if res == -1:
+        err = get_saved_errno()
+        if err != 0:
+            raise OSError(err, "os_nice failed")
+    return res
+
+c_ctermid = external('ctermid', [rffi.CCHARP], rffi.CCHARP)
+
+ at replace_os_function('ctermid')
+def ctermid():
+    return rffi.charp2str(c_ctermid(lltype.nullptr(rffi.CCHARP.TO)))
+
+c_tmpnam = external('tmpnam', [rffi.CCHARP], rffi.CCHARP)
+
+ at replace_os_function('tmpnam')
+def tmpnam():
+    return rffi.charp2str(c_tmpnam(lltype.nullptr(rffi.CCHARP.TO)))
+
+#___________________________________________________________________
+
+c_getpid = external('getpid', [], rffi.PID_T,
+                    releasegil=False, save_err=rffi.RFFI_SAVE_ERRNO)
+c_getppid = external('getppid', [], rffi.PID_T,
+                     releasegil=False, save_err=rffi.RFFI_SAVE_ERRNO)
+c_setsid = external('setsid', [], rffi.PID_T,
+                    save_err=rffi.RFFI_SAVE_ERRNO)
+c_getsid = external('getsid', [rffi.PID_T], rffi.PID_T,
+                    save_err=rffi.RFFI_SAVE_ERRNO)
+
+ at replace_os_function('getpid')
+def getpid():
+    return handle_posix_error('getpid', c_getpid())
+
+ at replace_os_function('getppid')
+def getppid():
+    return handle_posix_error('getppid', c_getppid())
+
+ at replace_os_function('setsid')
+def setsid():
+    return handle_posix_error('setsid', c_setsid())
+
+ at replace_os_function('getsid')
+def getsid(pid):
+    return handle_posix_error('getsid', c_getsid(pid))
+
+c_getpgid = external('getpgid', [rffi.PID_T], rffi.PID_T,
+                     save_err=rffi.RFFI_SAVE_ERRNO)
+c_setpgid = external('setpgid', [rffi.PID_T, rffi.PID_T], rffi.INT,
+                     save_err=rffi.RFFI_SAVE_ERRNO)
+
+ at replace_os_function('getpgid')
+def getpgid(pid):
+    return handle_posix_error('getpgid', c_getpgid(pid))
+
+ at replace_os_function('setpgid')
+def setpgid(pid, gid):
+    handle_posix_error('setpgid', c_setpgid(pid, gid))
+
+PID_GROUPS_T = rffi.CArrayPtr(rffi.PID_T)
+c_getgroups = external('getgroups', [rffi.INT, PID_GROUPS_T], rffi.INT,
+                       save_err=rffi.RFFI_SAVE_ERRNO)
+c_setgroups = external('setgroups', [rffi.SIZE_T, PID_GROUPS_T], rffi.INT,
+                       save_err=rffi.RFFI_SAVE_ERRNO)
+c_initgroups = external('initgroups', [rffi.CCHARP, rffi.PID_T], rffi.INT,
+                        save_err=rffi.RFFI_SAVE_ERRNO)
+
+ at replace_os_function('getgroups')
+def getgroups():
+    n = handle_posix_error('getgroups',
+                           c_getgroups(0, lltype.nullptr(PID_GROUPS_T.TO)))
+    groups = lltype.malloc(PID_GROUPS_T.TO, n, flavor='raw')
+    try:
+        n = handle_posix_error('getgroups', c_getgroups(n, groups))
+        return [widen(groups[i]) for i in range(n)]
+    finally:
+        lltype.free(groups, flavor='raw')
+
+ at replace_os_function('setgroups')
+def setgroups(gids):
+    n = len(gids)
+    groups = lltype.malloc(PID_GROUPS_T.TO, n, flavor='raw')
+    try:
+        for i in range(n):
+            groups[i] = rffi.cast(rffi.PID_T, gids[i])
+        handle_posix_error('setgroups', c_setgroups(n, groups))
+    finally:
+        lltype.free(groups, flavor='raw')
+
+ at replace_os_function('initgroups')
+def initgroups(user, group):
+    handle_posix_error('initgroups', c_initgroups(user, group))
+
+if GETPGRP_HAVE_ARG:
+    c_getpgrp = external('getpgrp', [rffi.INT], rffi.INT,
+                         save_err=rffi.RFFI_SAVE_ERRNO)
+else:
+    c_getpgrp = external('getpgrp', [], rffi.INT,
+                         save_err=rffi.RFFI_SAVE_ERRNO)
+if SETPGRP_HAVE_ARG:
+    c_setpgrp = external('setpgrp', [rffi.INT, rffi.INT], rffi.INT,
+                         save_err=rffi.RFFI_SAVE_ERRNO)
+else:
+    c_setpgrp = external('setpgrp', [], rffi.INT,
+                         save_err=rffi.RFFI_SAVE_ERRNO)
+
+ at replace_os_function('getpgrp')
+def getpgrp():
+    if GETPGRP_HAVE_ARG:
+        return handle_posix_error('getpgrp', c_getpgrp(0))
+    else:
+        return handle_posix_error('getpgrp', c_getpgrp())
+
+ at replace_os_function('setpgrp')
+def setpgrp():
+    if SETPGRP_HAVE_ARG:
+        handle_posix_error('setpgrp', c_setpgrp(0, 0))
+    else:
+        handle_posix_error('setpgrp', c_setpgrp())
+
+c_tcgetpgrp = external('tcgetpgrp', [rffi.INT], rffi.PID_T,
+                       save_err=rffi.RFFI_SAVE_ERRNO)
+c_tcsetpgrp = external('tcsetpgrp', [rffi.INT, rffi.PID_T], rffi.INT,
+                       save_err=rffi.RFFI_SAVE_ERRNO)
+
+ at replace_os_function('tcgetpgrp')
+def tcgetpgrp(fd):
+    return handle_posix_error('tcgetpgrp', c_tcgetpgrp(fd))
+
+ at replace_os_function('tcsetpgrp')
+def tcsetpgrp(fd, pgrp):
+    return handle_posix_error('tcsetpgrp', c_tcsetpgrp(fd, pgrp))
+
+#___________________________________________________________________
+
+c_getuid = external('getuid', [], rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO)
+c_geteuid = external('geteuid', [], rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO)
+c_setuid = external('setuid', [rffi.INT], rffi.INT,
+                    save_err=rffi.RFFI_SAVE_ERRNO)
+c_seteuid = external('seteuid', [rffi.INT], rffi.INT,
+                     save_err=rffi.RFFI_SAVE_ERRNO)
+c_getgid = external('getgid', [], rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO)
+c_getegid = external('getegid', [], rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO)
+c_setgid = external('setgid', [rffi.INT], rffi.INT,
+                    save_err=rffi.RFFI_SAVE_ERRNO)
+c_setegid = external('setegid', [rffi.INT], rffi.INT,
+                     save_err=rffi.RFFI_SAVE_ERRNO)
+
+ at replace_os_function('getuid')
+def getuid():
+    return handle_posix_error('getuid', c_getuid())
+
+ at replace_os_function('geteuid')
+def geteuid():
+    return handle_posix_error('geteuid', c_geteuid())
+
+ at replace_os_function('setuid')
+def setuid(uid):
+    handle_posix_error('setuid', c_setuid(uid))
+
+ at replace_os_function('seteuid')
+def seteuid(uid):
+    handle_posix_error('seteuid', c_seteuid(uid))
+
+ at replace_os_function('getgid')
+def getgid():
+    return handle_posix_error('getgid', c_getgid())
+
+ at replace_os_function('getegid')
+def getegid():
+    return handle_posix_error('getegid', c_getegid())
+
+ at replace_os_function('setgid')
+def setgid(gid):
+    handle_posix_error('setgid', c_setgid(gid))
+
+ at replace_os_function('setegid')
+def setegid(gid):
+    handle_posix_error('setegid', c_setegid(gid))
+
+c_setreuid = external('setreuid', [rffi.INT, rffi.INT], rffi.INT,
+                      save_err=rffi.RFFI_SAVE_ERRNO)
+c_setregid = external('setregid', [rffi.INT, rffi.INT], rffi.INT,
+                      save_err=rffi.RFFI_SAVE_ERRNO)
+
+ at replace_os_function('setreuid')
+def setreuid(ruid, euid):
+    handle_posix_error('setreuid', c_setreuid(ruid, euid))
+
+ at replace_os_function('setregid')
+def setregid(rgid, egid):
+    handle_posix_error('setregid', c_setregid(rgid, egid))
+
+c_getresuid = external('getresuid', [rffi.INTP] * 3, rffi.INT,
+                       save_err=rffi.RFFI_SAVE_ERRNO)
+c_getresgid = external('getresgid', [rffi.INTP] * 3, rffi.INT,
+                       save_err=rffi.RFFI_SAVE_ERRNO)
+c_setresuid = external('setresuid', [rffi.INT] * 3, rffi.INT,
+                       save_err=rffi.RFFI_SAVE_ERRNO)
+c_setresgid = external('setresgid', [rffi.INT] * 3, rffi.INT,
+                       save_err=rffi.RFFI_SAVE_ERRNO)
+
+ at replace_os_function('getresuid')
+def getresuid():
+    out = lltype.malloc(rffi.INTP.TO, 3, flavor='raw')
+    try:
+        handle_posix_error('getresuid',
+                           c_getresuid(rffi.ptradd(out, 0),
+                                       rffi.ptradd(out, 1),
+                                       rffi.ptradd(out, 2)))
+        return (widen(out[0]), widen(out[1]), widen(out[2]))
+    finally:
+        lltype.free(out, flavor='raw')
+
+ at replace_os_function('getresgid')
+def getresgid():
+    out = lltype.malloc(rffi.INTP.TO, 3, flavor='raw')
+    try:
+        handle_posix_error('getresgid',
+                           c_getresgid(rffi.ptradd(out, 0),
+                                       rffi.ptradd(out, 1),
+                                       rffi.ptradd(out, 2)))
+        return (widen(out[0]), widen(out[1]), widen(out[2]))
+    finally:
+        lltype.free(out, flavor='raw')
+
+ at replace_os_function('setresuid')
+def setresuid(ruid, euid, suid):
+    handle_posix_error('setresuid', c_setresuid(ruid, euid, suid))
+
+ at replace_os_function('setresgid')
+def setresgid(rgid, egid, sgid):
+    handle_posix_error('setresgid', c_setresgid(rgid, egid, sgid))

From noreply at buildbot.pypy.org  Sun Jun 28 10:26:48 2015
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sun, 28 Jun 2015 10:26:48 +0200 (CEST)
Subject: [pypy-commit] pypy py3.3: oops, fix the merge
Message-ID: <20150628082648.352481C063D@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3.3
Changeset: r78334:fc25d108573c
Date: 2015-06-25 08:42 +0200
http://bitbucket.org/pypy/pypy/changeset/fc25d108573c/

Log:	oops, fix the merge

diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -370,67 +370,6 @@
     def _prefer_unicode(path):
         return False
 
-<<<<<<< local
-
- at specialize.argtype(0)
-def unlink(path):
-    return os.unlink(_as_bytes(path))
-
- at specialize.argtype(0, 1)
-def rename(path1, path2):
-    return os.rename(_as_bytes(path1), _as_bytes(path2))
-
- at specialize.argtype(0, 1)
-def replace(path1, path2):
-    if os.name == 'nt':
-        raise NotImplementedError(
-            'On windows, os.replace() should overwrite the destination')
-    return os.rename(_as_bytes(path1), _as_bytes(path2))
-
- at specialize.argtype(0)
-def listdir(dirname):
-    return os.listdir(_as_bytes(dirname))
-
- at specialize.argtype(0)
-def access(path, mode):
-    return os.access(_as_bytes(path), mode)
-
- at specialize.argtype(0)
-def chmod(path, mode):
-    return os.chmod(_as_bytes(path), mode)
-
- at specialize.argtype(0, 1)
-def utime(path, times):
-    return os.utime(_as_bytes(path), times)
-
- at specialize.argtype(0)
-def chdir(path):
-    return os.chdir(_as_bytes(path))
-
- at specialize.argtype(0)
-def mkdir(path, mode=0777):
-    return os.mkdir(_as_bytes(path), mode)
-
- at specialize.argtype(0)
-def rmdir(path):
-    return os.rmdir(_as_bytes(path))
-
- at specialize.argtype(0)
-def mkfifo(path, mode):
-    os.mkfifo(_as_bytes(path), mode)
-
- at specialize.argtype(0)
-def mknod(path, mode, device):
-    os.mknod(_as_bytes(path), mode, device)
-
- at specialize.argtype(0, 1)
-def symlink(src, dest):
-    os.symlink(_as_bytes(src), _as_bytes(dest))
-
-if os.name == 'nt':
-    import nt
-=======
->>>>>>> other
     @specialize.argtype(0)
     def _preferred_traits(path):
         return string_traits
@@ -1116,6 +1055,13 @@
         if not win32traits.MoveFile(path1, path2):
             raise rwin32.lastSavedWindowsError()
 
+ at specialize.argtype(0, 1)
+def replace(path1, path2):
+    if os.name == 'nt':
+        raise NotImplementedError(
+            'On windows, os.replace() should overwrite the destination')
+    return rename(path1, path2)
+
 #___________________________________________________________________
 
 c_mkfifo = external('mkfifo', [rffi.CCHARP, rffi.MODE_T], rffi.INT,

From noreply at buildbot.pypy.org  Sun Jun 28 10:26:49 2015
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sun, 28 Jun 2015 10:26:49 +0200 (CEST)
Subject: [pypy-commit] pypy py3.3: Add more names to os._have_functions.
Message-ID: <20150628082649.6729F1C063D@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3.3
Changeset: r78335:07ad8fafbb60
Date: 2015-06-25 09:09 +0200
http://bitbucket.org/pypy/pypy/changeset/07ad8fafbb60/

Log:	Add more names to os._have_functions.

diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -1444,6 +1444,12 @@
         return space.wrap(result)
 
 have_functions = []
-for name in """FSTAT FCHDIR OPENAT""".split():
+for name in """FCHDIR FCHMOD FCHMODAT FCHOWN FCHOWNAT FEXECVE FDOPENDIR
+               FPATHCONF FSTATAT FSTATVFS FTRUNCATE FUTIMENS FUTIMES
+               FUTIMESAT LINKAT LCHFLAGS LCHMOD LCHOWN LSTAT LUTIMES
+               MKDIRAT MKFIFOAT MKNODAT OPENAT READLINKAT RENAMEAT 
+               SYMLINKAT UNLINKAT UTIMENSAT""".split():
     if getattr(rposix, "HAVE_%s" % name):
         have_functions.append("HAVE_%s" % name)
+if _WIN32:
+    have_functions.append("HAVE_MS_WINDOWS")
diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -30,9 +30,12 @@
                   'unistd.h',
                   'fcntl.h'],
     )
-    HAVE_FSTAT = rffi_platform.Has('fstat')
-    HAVE_FCHDIR = rffi_platform.Has('fchdir')
-    HAVE_OPENAT = rffi_platform.Has('openat')
+    for _name in """fchdir fchmod fchmodat fchown fchownat fexecve fdopendir
+                    fpathconf fstat fstatat fstatvfs ftruncate futimens futimes
+                    futimesat linkat lchflags lchmod lchown lstat lutimes
+                    mkdirat mkfifoat mknodat openat readlinkat renameat 
+                    symlinkat unlinkat utimensat""".split():
+        locals()['HAVE_%s' % _name.upper()] = rffi_platform.Has(_name)
 cConfig = rffi_platform.configure(CConfig)
 globals().update(cConfig)
 

From noreply at buildbot.pypy.org  Sun Jun 28 10:26:50 2015
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sun, 28 Jun 2015 10:26:50 +0200 (CEST)
Subject: [pypy-commit] pypy py3.3: os.stat() now accepts fd instead of the
	path
Message-ID: <20150628082650.8955A1C063D@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3.3
Changeset: r78336:e62a0a0b8520
Date: 2015-06-28 10:25 +0200
http://bitbucket.org/pypy/pypy/changeset/e62a0a0b8520/

Log:	os.stat() now accepts fd instead of the path

diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -62,8 +62,15 @@
         return self.space.fsdecode_w(self.w_obj)
 
 @specialize.memo()
-def dispatch_filename(func, tag=0):
+def make_dispatch_function(func, tag, allow_fd_fn=None):
     def dispatch(space, w_fname, *args):
+        if allow_fd_fn is not None:
+            try:
+                fd = space.c_int_w(w_fname)
+            except OperationError:
+                pass
+            else:
+                return allow_fd_fn(fd, *args)
         if space.isinstance_w(w_fname, space.w_unicode):
             fname = FileEncoder(space, w_fname)
             return func(fname, *args)
@@ -72,6 +79,10 @@
             return func(fname, *args)
     return dispatch
 
+ at specialize.arg(0, 1)
+def dispatch_filename(func, tag=0, allow_fd_fn=None):
+    return make_dispatch_function(func, tag, allow_fd_fn)
+
 @specialize.memo()
 def dispatch_filename_2(func):
     def dispatch(space, w_fname1, w_fname2, *args):
@@ -302,7 +313,8 @@
 """
 
     try:
-        st = dispatch_filename(rposix_stat.stat)(space, w_path)
+        st = dispatch_filename(rposix_stat.stat, 0,
+                               allow_fd_fn=rposix_stat.fstat)(space, w_path)
     except OSError, e:
         raise wrap_oserror2(space, e, w_path)
     else:
diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py
--- a/pypy/module/posix/test/test_posix2.py
+++ b/pypy/module/posix/test/test_posix2.py
@@ -109,6 +109,7 @@
         s = posix.read(fd, 1)
         assert s == b'i'
         st = posix.fstat(fd)
+        assert st == posix.stat(fd)
         posix.close(fd2)
         posix.close(fd)
 

From noreply at buildbot.pypy.org  Sun Jun 28 15:52:15 2015
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sun, 28 Jun 2015 15:52:15 +0200 (CEST)
Subject: [pypy-commit] pypy py3.3: Fix translation,
 and add a message to an assertion I don't undestand
Message-ID: <20150628135215.256061C063D@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3.3
Changeset: r78337:71ece22ac15d
Date: 2015-06-28 15:51 +0200
http://bitbucket.org/pypy/pypy/changeset/71ece22ac15d/

Log:	Fix translation, and add a message to an assertion I don't undestand

diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py
--- a/pypy/module/imp/importing.py
+++ b/pypy/module/imp/importing.py
@@ -15,7 +15,7 @@
 from rpython.rlib.streamio import StreamErrors
 from rpython.rlib.objectmodel import we_are_translated, specialize
 from rpython.rlib.signature import signature
-from rpython.rlib import rposix, types
+from rpython.rlib import rposix_stat, types
 from pypy.module.sys.version import PYPY_VERSION
 
 _WIN32 = sys.platform == 'win32'
@@ -115,7 +115,7 @@
 
         # Directory should not exist
         try:
-            st = rposix.stat(_WIN32Path(path) if win32 else path)
+            st = rposix_stat.stat(_WIN32Path(path) if win32 else path)
         except OSError:
             pass
         else:
diff --git a/rpython/annotator/annrpython.py b/rpython/annotator/annrpython.py
--- a/rpython/annotator/annrpython.py
+++ b/rpython/annotator/annrpython.py
@@ -242,7 +242,7 @@
     def setbinding(self, arg, s_value):
         s_old = arg.annotation
         if s_old is not None:
-            assert s_value.contains(s_old)
+            assert s_value.contains(s_old), "%s does not contain %s" % (s_value, s_old)
         arg.annotation = s_value
 
     def warning(self, msg, pos=None):

From noreply at buildbot.pypy.org  Sun Jun 28 16:31:51 2015
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sun, 28 Jun 2015 16:31:51 +0200 (CEST)
Subject: [pypy-commit] pypy py3.3: More logs
Message-ID: <20150628143151.6764C1C02A3@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3.3
Changeset: r78338:2aab118cd687
Date: 2015-06-28 16:32 +0200
http://bitbucket.org/pypy/pypy/changeset/2aab118cd687/

Log:	More logs

diff --git a/rpython/annotator/annrpython.py b/rpython/annotator/annrpython.py
--- a/rpython/annotator/annrpython.py
+++ b/rpython/annotator/annrpython.py
@@ -242,7 +242,10 @@
     def setbinding(self, arg, s_value):
         s_old = arg.annotation
         if s_old is not None:
-            assert s_value.contains(s_old), "%s does not contain %s" % (s_value, s_old)
+            if not s_value.contains(s_old):
+                log.WARNING("%s does not contain %s" % (s_value, s_old))
+                log.WARNING("%s" % annmodel.unionof(s_value, s_old))
+                assert False
         arg.annotation = s_value
 
     def warning(self, msg, pos=None):

From noreply at buildbot.pypy.org  Sun Jun 28 16:40:19 2015
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sun, 28 Jun 2015 16:40:19 +0200 (CEST)
Subject: [pypy-commit] pypy py3.3: Propagage no_nul when merging
	SomeUnicodeStrings
Message-ID: <20150628144019.577221C02A3@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3.3
Changeset: r78339:0b8affef4e82
Date: 2015-06-28 16:40 +0200
http://bitbucket.org/pypy/pypy/changeset/0b8affef4e82/

Log:	Propagage no_nul when merging SomeUnicodeStrings

diff --git a/rpython/annotator/binaryop.py b/rpython/annotator/binaryop.py
--- a/rpython/annotator/binaryop.py
+++ b/rpython/annotator/binaryop.py
@@ -611,8 +611,9 @@
                  pairtype(SomeUnicodeString, SomeUnicodeCodePoint),
                  pairtype(SomeUnicodeString, SomeUnicodeString)):
     def union((str1, str2)):
-        return SomeUnicodeString(can_be_None=str1.can_be_none() or
-                                 str2.can_be_none())
+        can_be_None = str1.can_be_None or str2.can_be_None
+        no_nul = str1.no_nul and str2.no_nul
+        return SomeUnicodeString(can_be_None=can_be_None, no_nul=no_nul)
 
     def add((str1, str2)):
         # propagate const-ness to help getattr(obj, 'prefix' + const_name)

From noreply at buildbot.pypy.org  Sun Jun 28 17:25:18 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Sun, 28 Jun 2015 17:25:18 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: pass zjit,
 splitting is not necessary anymore,
 because the packing phase determines if it is able to combine pairs or not
Message-ID: <20150628152518.A77BF1C02A3@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r78340:3ad6a660ea4c
Date: 2015-06-28 17:25 +0200
http://bitbucket.org/pypy/pypy/changeset/3ad6a660ea4c/

Log:	pass zjit, splitting is not necessary anymore, because the packing
	phase determines if it is able to combine pairs or not this solution
	is much more readable and makes the combination slightly more
	complicated, but eases the vector operation generation

diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -280,7 +280,7 @@
         # neither does sum
         # a + c should work, but it is given as a parameter
         # thus the accum must handle this!
-        self.check_vectorized(3, 0) # TODO
+        self.check_vectorized(3, 1)
 
     def define_int32_add_const():
         return """
diff --git a/rpython/jit/metainterp/optimizeopt/schedule.py b/rpython/jit/metainterp/optimizeopt/schedule.py
--- a/rpython/jit/metainterp/optimizeopt/schedule.py
+++ b/rpython/jit/metainterp/optimizeopt/schedule.py
@@ -147,31 +147,17 @@
     raise AssertionError("getexpandopnum type %s not supported" % (type,))
 
 class PackType(object):
-    # TODO merge with vector box? the save the same fields
-    # difference: this is more of a type specification
+    """ Represents the type of an operation (either it's input or
+    output).
+    """
     UNKNOWN_TYPE = '-'
 
-    def __init__(self, type, size, signed, count=-1):
-        assert type in (FLOAT, INT, PackType.UNKNOWN_TYPE)
-        self.type = type
-        self.size = size
-        self.signed = signed
-        self.count = count
-
-    def gettype(self):
-        return self.type
-
-    def getsize(self):
-        return self.size
-
-    def getsigned(self):
-        return self.signed
-
-    def get_byte_size(self):
-        return self.size
-
-    def getcount(self):
-        return self.count
+    @staticmethod
+    def of(box, count=-1):
+        assert isinstance(box, BoxVector)
+        if count == -1:
+            count = box.item_count
+        return PackType(box.item_type, box.item_size, box.item_signed, count)
 
     @staticmethod
     def by_descr(descr, vec_reg_size):
@@ -182,8 +168,15 @@
         pt = PackType(_t, size, descr.is_item_signed(), vec_reg_size // size)
         return pt
 
-    def is_valid(self):
-        return self.type != PackType.UNKNOWN_TYPE and self.size > 0
+    def __init__(self, type, size, signed, count=-1):
+        assert type in (FLOAT, INT, PackType.UNKNOWN_TYPE)
+        self.type = type
+        self.size = size
+        self.signed = signed
+        self.count = count
+
+    def clone(self):
+        return PackType(self.type, self.size, self.signed, self.count)
 
     def new_vector_box(self, count = -1):
         if count == -1:
@@ -193,18 +186,32 @@
         assert self.size > 0
         return BoxVector(self.type, count, self.size, self.signed)
 
+    def combine(self, other):
+        """ nothing to be done here """
+        if not we_are_translated():
+            assert self.type == other.type
+            assert self.signed == other.signed
+
     def __repr__(self):
         return 'PackType(%s, %d, %d, #%d)' % (self.type, self.size, self.signed, self.count)
 
-    @staticmethod
-    def of(box, count=-1):
-        assert isinstance(box, BoxVector)
-        if count == -1:
-            count = box.item_count
-        return PackType(box.item_type, box.item_size, box.item_signed, count)
+    def byte_size(self):
+        return self.count * self.size
 
-    def clone(self):
-        return PackType(self.type, self.size, self.signed, self.count)
+    def setsize(self, size):
+        self.size = size
+
+    def setcount(self, count):
+        self.count = count
+
+    def gettype(self):
+        return self.type
+
+    def getsize(self):
+        return self.size
+
+    def getcount(self):
+        return self.count
 
 
 PT_FLOAT_2 = PackType(FLOAT, 4, False, 2)
@@ -229,20 +236,6 @@
         self.output_type = None
         self.costmodel = None
 
-
-    def determine_input_type(self, op):
-        arg = op.getarg(0)
-        _, vbox = self.sched_data.getvector_of_box(arg)
-        return packtype_outof_box(vbox or arg)
-
-    def determine_output_type(self, op):
-        return self.determine_input_type(op)
-
-    def update_input_output(self, pack):
-        op0 = pack.operations[0].getoperation()
-        self.input_type = self.determine_input_type(op0)
-        self.output_type = self.determine_output_type(op0)
-
     def check_if_pack_supported(self, pack):
         op0 = pack.operations[0].getoperation()
         if self.input_type is None:
@@ -264,7 +257,9 @@
         self.sched_data = sched_data
         self.preamble_ops = oplist
         self.costmodel = sched_data.costmodel
-        self.update_input_output(pack)
+        #self.update_input_output(pack)
+        self.input_type = pack.input_type
+        self.output_type = pack.output_type
         #
         self.check_if_pack_supported(pack)
         #
@@ -285,19 +280,6 @@
     def must_be_full_but_is_not(self, pack):
         return False
 
-    def split_pack(self, pack, vec_reg_size):
-        """ Returns how many items of the pack should be
-            emitted as vector operation. """
-        bytes = pack.opcount() * self.getscalarsize()
-        if bytes > vec_reg_size:
-            # too many bytes. does not fit into the vector register
-            return vec_reg_size // self.getscalarsize()
-        return pack.opcount()
-
-    def getscalarsize(self):
-        """ return how many bytes a scalar operation processes """
-        return self.input_type.getsize()
-
     def before_argument_transform(self, args):
         pass
 
@@ -475,42 +457,6 @@
             self._check_vec_pack(op)
         return new_box
 
-    def package2(self, tgt_box, index, args, packable):
-        """ If there are two vector boxes:
-          v1 = [_,_,X,Y]
-          v2 = [A,B,_,_]
-          this function creates a box pack instruction to merge them to:
-          v1/2 = [A,B,X,Y]
-        """
-        opnum = getpackopnum(tgt_box.item_type)
-        arg_count = len(args)
-        i = index
-        while i < arg_count and tgt_box.item_count < packable:
-            arg = args[i]
-            pos, src_box = self.sched_data.getvector_of_box(arg)
-            if pos == -1:
-                i += 1
-                continue
-            count = tgt_box.item_count + src_box.item_count
-            new_box = vectorbox_clone_set(tgt_box, count=count)
-            op = ResOperation(opnum, [tgt_box, src_box, ConstInt(i),
-                                      ConstInt(src_box.item_count)], new_box)
-            self.preamble_ops.append(op)
-            self.costmodel.record_vector_pack(src_box, i, src_box.item_count)
-            if not we_are_translated():
-                self._check_vec_pack(op)
-            i += src_box.item_count
-
-            # overwrite the new positions, arguments now live in new_box
-            # at a new position
-            for j in range(i):
-                arg = args[j]
-                self.sched_data.setvector_of_box(arg, j, new_box)
-            tgt_box = new_box
-        _, vbox = self.sched_data.getvector_of_box(args[0])
-        assert vbox is not None
-        return vbox
-
     def _check_vec_pack(self, op):
         result = op.result
         arg0 = op.getarg(0)
@@ -579,25 +525,51 @@
             return False
         return self.arg_ptypes[i] is not None
 
+    def get_output_type_given(self, input_type, op):
+        return input_type
+
+    def get_input_type_given(self, output_type, op):
+        return output_type
+
+    def force_input(self, ptype):
+        """ Some operations require a specific count/size,
+            they can force the input type here!
+        """
+        return ptype
+
+    # OLD
+    def determine_input_type(self, op):
+        arg = op.getarg(0)
+        _, vbox = self.sched_data.getvector_of_box(arg)
+        return packtype_outof_box(vbox or arg)
+
+    def determine_output_type(self, op):
+        return self.determine_input_type(op)
+
+    def update_input_output(self, pack):
+        op0 = pack.operations[0].getoperation()
+        self.input_type = self.determine_input_type(op0)
+        self.output_type = self.determine_output_type(op0)
+
+    def split_pack(self, pack, vec_reg_size):
+        """ Returns how many items of the pack should be
+            emitted as vector operation. """
+        bytes = pack.opcount() * self.getscalarsize()
+        if bytes > vec_reg_size:
+            # too many bytes. does not fit into the vector register
+            return vec_reg_size // self.getscalarsize()
+        return pack.opcount()
+
+    def getscalarsize(self):
+        """ return how many bytes a scalar operation processes """
+        return self.input_type.getsize()
+
 class OpToVectorOpConv(OpToVectorOp):
     def __init__(self, intype, outtype):
         self.from_size = intype.getsize()
         self.to_size = outtype.getsize()
         OpToVectorOp.__init__(self, (intype, ), outtype)
 
-    def determine_input_type(self, op):
-        return self.arg_ptypes[0]
-
-    def determine_output_type(self, op):
-        return self.result_ptype
-
-    def split_pack(self, pack, vec_reg_size):
-        count = self.arg_ptypes[0].getcount()
-        bytes = pack.opcount() * self.getscalarsize()
-        if bytes > count * self.from_size:
-            return bytes // (count * self.from_size)
-        return pack.opcount()
-
     def new_result_vector_box(self):
         type = self.output_type.gettype()
         size = self.to_size
@@ -611,6 +583,29 @@
         assert count > 1
         return BoxVector(type, count, size, signed)
 
+    def get_output_type_given(self, input_type, op):
+        return self.result_ptype
+
+    def get_input_type_given(self, output_type, op):
+        return self.arg_ptypes[0]
+
+    def force_input(self, ptype):
+        return self.arg_ptypes[0]
+
+    # OLD
+    def determine_input_type(self, op):
+        return self.arg_ptypes[0]
+
+    def determine_output_type(self, op):
+        return self.result_ptype
+
+    def split_pack(self, pack, vec_reg_size):
+        count = self.arg_ptypes[0].getcount()
+        bytes = pack.opcount() * self.getscalarsize()
+        if bytes > count * self.from_size:
+            return bytes // (count * self.from_size)
+        return pack.opcount()
+
 class SignExtToVectorOp(OpToVectorOp):
     def __init__(self, intype, outtype):
         OpToVectorOp.__init__(self, intype, outtype)
@@ -621,7 +616,6 @@
         assert isinstance(sizearg, ConstInt)
         self.size = sizearg.value
 
-
     def new_result_vector_box(self):
         type = self.output_type.gettype()
         count = self.input_type.getcount()
@@ -634,23 +628,40 @@
         assert count > 1
         return BoxVector(type, count, self.size, signed)
 
+    def get_output_type_given(self, input_type, op):
+        sizearg = op.getarg(1)
+        assert isinstance(sizearg, ConstInt)
+        output_type = input_type.clone()
+        output_type.setsize(sizearg.value)
+        return output_type
+
+    def get_input_type_given(self, output_type, op):
+        raise AssertionError("can never infer input type!")
+
 class LoadToVectorLoad(OpToVectorOp):
     def __init__(self):
         OpToVectorOp.__init__(self, (), PT_GENERIC)
 
+    def before_argument_transform(self, args):
+        count = min(self.output_type.getcount(), len(self.getoperations()))
+        args.append(ConstInt(count))
+
+    def get_output_type_given(self, input_type, op):
+        return PackType.by_descr(op.getdescr(), self.sched_data.vec_reg_size)
+
+    def get_input_type_given(self, output_type, op):
+        return None
+
+    # OLD
+    def getscalarsize(self):
+        return self.output_type.getsize()
+
     def determine_input_type(self, op):
         return None
 
     def determine_output_type(self, op):
         return PackType.by_descr(op.getdescr(), self.sched_data.vec_reg_size)
 
-    def before_argument_transform(self, args):
-        count = min(self.output_type.getcount(), len(self.getoperations()))
-        args.append(ConstInt(count))
-
-    def getscalarsize(self):
-        return self.output_type.getsize()
-
 class StoreToVectorStore(OpToVectorOp):
     """
     Storing operations are special because they are not allowed
@@ -661,14 +672,21 @@
         OpToVectorOp.__init__(self, (None, None, PT_GENERIC), None)
         self.has_descr = True
 
-    def determine_input_type(self, op):
-        return PackType.by_descr(op.getdescr(), self.sched_data.vec_reg_size)
-
     def must_be_full_but_is_not(self, pack):
         vrs = self.sched_data.vec_reg_size
         it = pack.input_type
         return it.getsize() * it.getcount() < vrs
 
+    def get_output_type_given(self, input_type, op):
+        return None
+
+    def get_input_type_given(self, output_type, op):
+        return PackType.by_descr(op.getdescr(), self.sched_data.vec_reg_size)
+
+    # OLD
+    def determine_input_type(self, op):
+        return PackType.by_descr(op.getdescr(), self.sched_data.vec_reg_size)
+
     def determine_output_type(self, op):
         return None
 
@@ -694,6 +712,13 @@
     def __init__(self, args):
         OpToVectorOp.__init__(self, args, None)
 
+    def get_output_type_given(self, input_type, op):
+        return None
+
+    def get_input_type_given(self, output_type, op):
+        raise AssertionError("cannot infer input type from output type")
+
+    # OLD
     def determine_output_type(self, op):
         return None
 
@@ -737,12 +762,30 @@
     rop.GUARD_FALSE: GUARD_TF,
 }
 
-def determine_output_type(node, input_type):
+def determine_input_output_types(pack, node, forward):
+    """ This function is two fold. If moving forward, it
+    gets an input type from the packs output type and returns
+    the transformed packtype.
+
+    Moving backward, the origins pack input type is the output
+    type and the transformation of the packtype (in reverse direction)
+    is the input
+    """
     op = node.getoperation()
     op2vecop = determine_trans(op)
-    if isinstance(op2vecop, OpToVectorOpConv):
-        return op2vecop.determine_output_type(op)
-    return input_type
+    if forward:
+        input_type = op2vecop.force_input(pack.output_type)
+        output_type = op2vecop.get_output_type_given(input_type, op)
+        if output_type:
+            output_type = output_type.clone()
+    else:
+        # going backwards, things are not that easy anymore
+        output_type = pack.input_type
+        input_type = op2vecop.get_input_type_given(output_type, op)
+        if input_type:
+            input_type = input_type.clone()
+
+    return input_type, output_type
 
 def determine_trans(op):
     op2vecop = ROP_ARG_RES_VECTOR.get(op.vector, None)
@@ -831,6 +874,7 @@
         self.accum = None
         self.input_type = input_type
         self.output_type = output_type
+        assert self.input_type is not None or self.output_type is not None
 
     def opcount(self):
         return len(self.operations)
@@ -849,8 +893,10 @@
 
         op = self.leftmost()
         if op.casts_box():
-            assert self.output_type.getcount() <= ptype.getcount()
-            return self.output_type.getcount() <= ptype.getcount()
+            cur_bytes = ptype.getsize() * self.opcount()
+            max_bytes = self.input_type.byte_size()
+            assert cur_bytes <= max_bytes
+            return cur_bytes == max_bytes
 
         bytes = ptype.getsize() * len(self.operations)
         assert bytes <= vec_reg_size
@@ -903,6 +949,10 @@
         assert isinstance(right, Node)
         self.left = left
         self.right = right
+        if input_type:
+            input_type = input_type.clone()
+        if output_type:
+            output_type = output_type.clone()
         Pack.__init__(self, [left, right], input_type, output_type)
 
     def __eq__(self, other):
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py b/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
@@ -1357,23 +1357,18 @@
 
     def test_abc(self):
         trace="""
-        [p0, p9, i10, p3, i11, p12, i13, p6, i14, p7, p15, i16, i17, i18, i19, i20, i21]
-        guard_early_exit(descr=<rpython.jit.metainterp.compile.ResumeAtLoopHeaderDescr object at 0x7f09b34aad50>) [p7, p6, p3, p0, i14, i17, i16, p9, p15, i11, i10, p12, i13]
-        i22 = raw_load(i18, i11, descr=singlefloatarraydescr)
-        guard_not_invalidated(descr=<rpython.jit.metainterp.compile.ResumeGuardNotInvalidated object at 0x7f09b34fd390>) [p7, p6, p3, p0, i22, i14, i17, i16, p9, p15, i11, i10, p12, i13]
-        i24 = int_add(i11, 4)
-        i25 = raw_load(i19, i17, descr=singlefloatarraydescr)
-        i27 = int_add(i17, 4)
-        f28 = cast_singlefloat_to_float(i22)
-        f29 = cast_singlefloat_to_float(i25)
-        f30 = float_add(f28, f29)
-        i31 = cast_float_to_singlefloat(f30)
-        raw_store(i20, i14, i31, descr=singlefloatarraydescr)
-        i33 = int_add(i13, 1)
-        i35 = int_add(i14, 4)
-        i36 = int_ge(i33, i21)
-        guard_false(i36, descr=<rpython.jit.metainterp.compile.ResumeGuardFalseDescr object at 0x7f09b34b7c10>) [p7, p6, p3, p0, i35, i24, i33, i27, None, None, i16, p9, p15, None, i10, p12, None]
-        jump(p0, p9, i10, p3, i24, p12, i33, p6, i35, p7, p15, i16, i27, i18, i19, i20, i21)
+        [p0, p9, i10, p7, i11, p2, p4, p5, p12, i13, i14, i15, i16, i17, i18]
+        guard_early_exit(descr=<rpython.jit.metainterp.compile.ResumeAtLoopHeaderDescr object at 0x7f8a20c32090>) [p7, p5, p4, p2, p0, i10, i13, p9, p12, i11, i14]
+        i19 = raw_load(i15, i14, descr=int16arraydescr)
+        i21 = int_add(i14, 2)
+        i22 = int_add(i19, i16)
+        i24 = int_signext(i22, 2)
+        raw_store(i17, i11, i24, descr=int16arraydescr)
+        i26 = int_add(i10, 1)
+        i28 = int_add(i11, 2)
+        i29 = int_ge(i26, i18)
+        guard_false(i29, descr=<rpython.jit.metainterp.compile.ResumeGuardFalseDescr object at 0x7f8a20c32990>) [p7, p5, p4, p2, p0, i21, i28, i26, None, i13, p9, p12, None, None]
+        jump(p0, p9, i26, p7, i28, p2, p4, p5, p12, i13, i21, i15, i16, i17, i18)
         """
         # schedule 885 -> ptype is non for raw_load?
         opt = self.vectorize(self.parse_loop(trace))
diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/vectorize.py
@@ -21,7 +21,7 @@
         MemoryRef, Node, IndexVar)
 from rpython.jit.metainterp.optimizeopt.schedule import (VecScheduleData,
         Scheduler, Pack, Pair, AccumPair, Accum, vectorbox_outof_box, getpackopnum,
-        getunpackopnum, PackType, determine_output_type, determine_trans)
+        getunpackopnum, PackType, determine_input_output_types)
 from rpython.jit.metainterp.optimizeopt.guard import GuardStrengthenOpt
 from rpython.jit.metainterp.resoperation import (rop, ResOperation, GuardResOp)
 from rpython.rlib import listsort
@@ -680,8 +680,8 @@
                         # store only has an input
                         return Pair(lnode, rnode, ptype, None)
                 if self.profitable_pack(lnode, rnode, origin_pack, forward):
-                    input_type = origin_pack.output_type
-                    output_type = determine_output_type(lnode, input_type)
+                    input_type, output_type = \
+                        determine_input_output_types(origin_pack, lnode, forward)
                     return Pair(lnode, rnode, input_type, output_type)
             else:
                 if self.contains_pair(lnode, rnode):
@@ -719,6 +719,10 @@
         if packed.is_raw_array_access():
             if packed.getarg(1) == inquestion.result:
                 return True
+        if not forward and inquestion.getopnum() == rop.INT_SIGNEXT:
+            # prohibit the packing of signext in backwards direction
+            # the type cannot be determined!
+            return True
         return False
 
     def combine(self, i, j):
@@ -729,7 +733,13 @@
         operations = pack_i.operations
         for op in pack_j.operations[1:]:
             operations.append(op)
-        pack = Pack(operations, pack_i.input_type, pack_i.output_type)
+        input_type = pack_i.input_type
+        output_type = pack_i.output_type
+        if input_type:
+            input_type.combine(pack_j.input_type)
+        if output_type:
+            output_type.combine(pack_j.output_type)
+        pack = Pack(operations, input_type, output_type)
         self.packs[i] = pack
         # preserve the accum variable (if present) of the
         # left most pack, that is the pack with the earliest

From noreply at buildbot.pypy.org  Sun Jun 28 18:17:16 2015
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sun, 28 Jun 2015 18:17:16 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Fix syntax error
Message-ID: <20150628161716.3AF871C02A3@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r78341:74539bd81aa5
Date: 2015-06-28 18:16 +0200
http://bitbucket.org/pypy/pypy/changeset/74539bd81aa5/

Log:	Fix syntax error

diff --git a/lib_pypy/_tkinter/tclobj.py b/lib_pypy/_tkinter/tclobj.py
--- a/lib_pypy/_tkinter/tclobj.py
+++ b/lib_pypy/_tkinter/tclobj.py
@@ -25,7 +25,7 @@
 
         result = app.call('expr', '2**63')
         typePtr = AsObj(result).typePtr
-        if tkffi.string(typePtr.name) == v"bignum":
+        if tkffi.string(typePtr.name) == b"bignum":
             self.BigNumType = typePtr
 
 

From noreply at buildbot.pypy.org  Mon Jun 29 09:09:35 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 29 Jun 2015 09:09:35 +0200 (CEST)
Subject: [pypy-commit] pypy.org extradoc: update the values
Message-ID: <20150629070935.B44981C063D@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r622:6839fa149be6
Date: 2015-06-29 09:10 +0200
http://bitbucket.org/pypy/pypy.org/changeset/6839fa149be6/

Log:	update the values

diff --git a/don1.html b/don1.html
--- a/don1.html
+++ b/don1.html
@@ -15,7 +15,7 @@
 </script>
 
    <!-- Income:PyPy:Donations:Py3k / 0.95 -->
-   $59807 of $105000 (57.0%)
+   $59897 of $105000 (57.0%)
    <div id="progressbar">
    </div>
 
@@ -23,7 +23,7 @@
   <li>
   <span id="donate_text">This donation goes towards supporting Python 3 in PyPy.
   Current status:
-    we have $5251 left
+    we have $5333 left
   in the account.</br> <a href="py3donate.html">Read proposal</a></span>
   </li>
   <li>
diff --git a/don3.html b/don3.html
--- a/don3.html
+++ b/don3.html
@@ -15,7 +15,7 @@
 </script>
 
    <!-- Income:PyPy:Donations:Numpy / 0.95 -->
-   $52241 of $60000 (87.1%)
+   $52285 of $60000 (87.1%)
    <div id="progressbar">
    </div>
 
@@ -23,7 +23,7 @@
   <li>
   <span id="donate_text">This donation goes towards supporting NumPy in PyPy.
   Current status:
-    we have $16051 left
+    we have $16091 left
   in the account.</br> <a href="numpydonate.html">Read proposal</a></span>
   </li>
   <li>
diff --git a/don4.html b/don4.html
--- a/don4.html
+++ b/don4.html
@@ -17,7 +17,7 @@
    2nd call:
    <!-- Income:PyPy:Donations:Transactional Memory from April 8, 2014,
    divided by 0.9 -->
-   $29205 of $80000 (36.5%)
+   $29225 of $80000 (36.5%)
    <div id="progressbar">
    </div>
 
@@ -25,7 +25,7 @@
   <li>
   <span id="donate_text">This donation goes towards supporting the Transactional Memory in PyPy.
   Current status:
-    we have $22095 left
+    we have $22113 left
   in the account.</br> <a href="tmdonate2.html">Read proposal (2nd call)</a></span>
   </li>
   <li>

From noreply at buildbot.pypy.org  Mon Jun 29 09:09:34 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 29 Jun 2015 09:09:34 +0200 (CEST)
Subject: [pypy-commit] pypy.org extradoc: update the values
Message-ID: <20150629070934.995851C02FD@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r621:64ccc87ebe06
Date: 2015-06-28 09:01 +0200
http://bitbucket.org/pypy/pypy.org/changeset/64ccc87ebe06/

Log:	update the values

diff --git a/don1.html b/don1.html
--- a/don1.html
+++ b/don1.html
@@ -9,13 +9,13 @@
 <script>
   $(function() {
     $("#progressbar").progressbar({
-      value: 56.9
+      value: 57.0
    });
   });
 </script>
 
    <!-- Income:PyPy:Donations:Py3k / 0.95 -->
-   $59778 of $105000 (56.9%)
+   $59807 of $105000 (57.0%)
    <div id="progressbar">
    </div>
 
@@ -23,7 +23,7 @@
   <li>
   <span id="donate_text">This donation goes towards supporting Python 3 in PyPy.
   Current status:
-    we have $5226 left
+    we have $5251 left
   in the account.</br> <a href="py3donate.html">Read proposal</a></span>
   </li>
   <li>
diff --git a/don3.html b/don3.html
--- a/don3.html
+++ b/don3.html
@@ -9,13 +9,13 @@
 <script>
   $(function() {
     $("#progressbar").progressbar({
-      value: 87.0
+      value: 87.1
    });
   });
 </script>
 
    <!-- Income:PyPy:Donations:Numpy / 0.95 -->
-   $52184 of $60000 (87.0%)
+   $52241 of $60000 (87.1%)
    <div id="progressbar">
    </div>
 
@@ -23,7 +23,7 @@
   <li>
   <span id="donate_text">This donation goes towards supporting NumPy in PyPy.
   Current status:
-    we have $15999 left
+    we have $16051 left
   in the account.</br> <a href="numpydonate.html">Read proposal</a></span>
   </li>
   <li>
diff --git a/don4.html b/don4.html
--- a/don4.html
+++ b/don4.html
@@ -17,7 +17,7 @@
    2nd call:
    <!-- Income:PyPy:Donations:Transactional Memory from April 8, 2014,
    divided by 0.9 -->
-   $29183 of $80000 (36.5%)
+   $29205 of $80000 (36.5%)
    <div id="progressbar">
    </div>
 
@@ -25,7 +25,7 @@
   <li>
   <span id="donate_text">This donation goes towards supporting the Transactional Memory in PyPy.
   Current status:
-    we have $22077 left
+    we have $22095 left
   in the account.</br> <a href="tmdonate2.html">Read proposal (2nd call)</a></span>
   </li>
   <li>

From noreply at buildbot.pypy.org  Mon Jun 29 10:50:34 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Mon, 29 Jun 2015 10:50:34 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: rpython translation issues,
 adding reshape operation to zjit tests
Message-ID: <20150629085034.C04AA1C02FD@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r78342:b785c8f713af
Date: 2015-06-29 10:49 +0200
http://bitbucket.org/pypy/pypy/changeset/b785c8f713af/

Log:	rpython translation issues, adding reshape operation to zjit tests

diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py
--- a/pypy/module/micronumpy/compile.py
+++ b/pypy/module/micronumpy/compile.py
@@ -37,12 +37,20 @@
 class BadToken(Exception):
     pass
 
+class FakeArguments(W_Root):
+    def __init__(self, args_w, kw_w):
+        self.args_w = args_w
+        self.kw_w = kw_w
+
+    def unpack(self):
+        return self.args_w, self.kw_w
+
 
 SINGLE_ARG_FUNCTIONS = ["sum", "prod", "max", "min", "all", "any",
                         "unegative", "flat", "tostring", "count_nonzero",
                         "argsort", "cumsum", "logical_xor_reduce"]
-TWO_ARG_FUNCTIONS = ["dot", 'take', 'searchsorted']
-TWO_ARG_FUNCTIONS_OR_NONE = ['view', 'astype']
+TWO_ARG_FUNCTIONS = ["dot", 'multiply', 'take', 'searchsorted']
+TWO_ARG_FUNCTIONS_OR_NONE = ['view', 'astype', 'reshape']
 THREE_ARG_FUNCTIONS = ['where']
 
 class W_TypeObject(W_Root):
@@ -779,6 +787,8 @@
                 raise ArgumentNotAnArray
             if self.name == "dot":
                 w_res = arr.descr_dot(interp.space, arg)
+            if self.name == "multiply":
+                w_res = arr.descr_mul(interp.space, arg)
             elif self.name == 'take':
                 w_res = arr.descr_take(interp.space, arg)
             elif self.name == "searchsorted":
@@ -807,8 +817,12 @@
                 w_res = arr.descr_view(interp.space, arg)
             elif self.name == 'astype':
                 w_res = arr.descr_astype(interp.space, arg)
+            elif self.name == 'reshape':
+                w_arg = self.args[1]
+                assert isinstance(w_arg, ArrayConstant)
+                w_res = arr.reshape(interp.space, w_arg.wrap(interp.space))
             else:
-                assert False
+                assert False, "missing two arg impl for: %s" % (self.name,)
         else:
             raise WrongFunctionName
         if isinstance(w_res, W_NDimArray):
diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -118,6 +118,17 @@
         retval = self.interp.eval_graph(self.graph, [i])
         return retval
 
+    def define_matrix_dot():
+        return """
+        mat = |16|
+        m = reshape(mat, [4,4])
+        """
+
+    def test_matrix_dot(self):
+        result = self.run("matrix_dot")
+        assert int(result) == 45
+        self.check_vectorized(1, 1)
+
     def define_float32_copy():
         return """
         a = astype(|30|, float32)
@@ -387,7 +398,7 @@
     def test_sum_int(self):
         result = self.run("sum_int")
         assert result == sum(range(65))
-        self.check_vectorized(2, 2) # 1 vecopt try+success for type conversion
+        self.check_vectorized(2, 2) # 1 sum, 1 for type conversion
 
     def define_sum_multi():
         return """
diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -1670,12 +1670,13 @@
 
     def genop_guard_guard_true(self, ign_1, guard_op, guard_token, locs, ign_2):
         loc = locs[0]
-        if loc.is_xmm:
-            self._guard_vector_true(guard_op, loc)
-            self.implement_guard(guard_token, 'NZ')
-        else:
-            self.mc.TEST(loc, loc)
-            self.implement_guard(guard_token, 'Z')
+        if isinstance(loc, RegLoc):
+            if loc.is_xmm:
+                self._guard_vector_true(guard_op, loc)
+                self.implement_guard(guard_token, 'NZ')
+                return
+        self.mc.TEST(loc, loc)
+        self.implement_guard(guard_token, 'Z')
     genop_guard_guard_nonnull = genop_guard_guard_true
 
     def genop_guard_guard_no_exception(self, ign_1, guard_op, guard_token,
@@ -1770,12 +1771,13 @@
 
     def genop_guard_guard_false(self, ign_1, guard_op, guard_token, locs, ign_2):
         loc = locs[0]
-        if loc.is_xmm:
-            self._guard_vector_false(guard_op, loc)
-            self.implement_guard(guard_token, 'NZ')
-        else:
-            self.mc.TEST(loc, loc)
-            self.implement_guard(guard_token, 'NZ')
+        if isinstance(loc, RegLoc):
+            if loc.is_xmm:
+                self._guard_vector_false(guard_op, loc)
+                self.implement_guard(guard_token, 'NZ')
+                return
+        self.mc.TEST(loc, loc)
+        self.implement_guard(guard_token, 'NZ')
     genop_guard_guard_isnull = genop_guard_guard_false
 
     def genop_guard_guard_value(self, ign_1, guard_op, guard_token, locs, ign_2):
diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -393,7 +393,6 @@
     def _consider_guard_tf(self, op):
         arg = op.getarg(0)
         if arg.type == VECTOR:
-            assert arg.item_type == INT
             loc = self.xrm.make_sure_var_in_reg(arg)
         else:
             loc = self.rm.make_sure_var_in_reg(arg)
diff --git a/rpython/jit/backend/x86/regloc.py b/rpython/jit/backend/x86/regloc.py
--- a/rpython/jit/backend/x86/regloc.py
+++ b/rpython/jit/backend/x86/regloc.py
@@ -607,19 +607,18 @@
             _rx86_getattr(self, methname)(val1, val2)
         invoke._annspecialcase_ = 'specialize:arg(1)'
 
+        possible_instr_unrolled = unrolling_iterable([(1,'B_xx'),(2,'W_xx'),(4,'D_xx'),(8,'Q_xx')])
+
         def INSN(self, size, loc1, loc2):
             code1 = loc1.location_code()
             code2 = loc2.location_code()
-            val1 = getattr(loc1, "value_" + code1)()
-            val2 = getattr(loc2, "value_" + code2)()
-            suffix = 'B'
-            if size == 2:
-                suffix = 'W'
-            elif size == 4:
-                suffix = 'D'
-            else:
-                suffix = 'Q'
-            invoke(self, suffix + "_"+ code1+code2, val1, val2)
+            assert code1 == code2 == 'x'
+            val1 = loc1.value_x()
+            val2 = loc2.value_x()
+            for s,suffix in possible_instr_unrolled:
+                if s == size:
+                    invoke(self, suffix, val1, val2)
+                    break
 
         return INSN
 
diff --git a/rpython/jit/metainterp/optimizeopt/schedule.py b/rpython/jit/metainterp/optimizeopt/schedule.py
--- a/rpython/jit/metainterp/optimizeopt/schedule.py
+++ b/rpython/jit/metainterp/optimizeopt/schedule.py
@@ -113,13 +113,13 @@
 
 def vectorbox_clone_set(box, count=-1, size=-1, type='-', clone_signed=True, signed=False):
     if count == -1:
-        count = box.item_count
+        count = box.getcount()
     if size == -1:
-        size = box.item_size
+        size = box.getsize()
     if type == '-':
-        type = box.item_type
+        type = box.gettype()
     if clone_signed:
-        signed = box.item_signed
+        signed = box.getsigned()
     return BoxVector(type, count, size, signed)
 
 def getpackopnum(type):
@@ -156,8 +156,8 @@
     def of(box, count=-1):
         assert isinstance(box, BoxVector)
         if count == -1:
-            count = box.item_count
-        return PackType(box.item_type, box.item_size, box.item_signed, count)
+            count = box.getcount()
+        return PackType(box.gettype(), box.getsize(), box.getsigned(), count)
 
     @staticmethod
     def by_descr(descr, vec_reg_size):
@@ -295,6 +295,7 @@
         vop = ResOperation(op.vector, args, result, op.getdescr())
         if op.is_guard():
             assert isinstance(op, GuardResOp)
+            assert isinstance(vop, GuardResOp)
             vop.setfailargs(op.getfailargs())
             vop.rd_snapshot = op.rd_snapshot
         self.preamble_ops.append(vop)
@@ -307,7 +308,7 @@
         #
         # mark the position and the vbox in the hash
         for i, node in enumerate(self.getoperations()):
-            if i >= vbox.item_count:
+            if i >= vbox.getcount():
                 break
             op = node.getoperation()
             self.sched_data.setvector_of_box(op.result, i, vbox)
@@ -342,7 +343,7 @@
 
             # use the input as an indicator for the pack type
             packable = self.input_type.getcount()
-            packed = vbox.item_count
+            packed = vbox.getcount()
             assert packed >= 0
             assert packable >= 0
             if packed > packable:
@@ -394,7 +395,7 @@
     def update_arg_in_vector_pos(self, argidx, box):
         arguments = [op.getoperation().getarg(argidx) for op in self.getoperations()]
         for i,arg in enumerate(arguments):
-            if i >= box.item_count:
+            if i >= box.getcount():
                 break
             self.sched_data.setvector_of_box(arg, i, box)
 
@@ -418,7 +419,7 @@
             raise NotImplementedError("cannot yet extend float")
 
     def extend_int(self, vbox, newtype):
-        vbox_cloned = newtype.new_vector_box(vbox.item_count)
+        vbox_cloned = newtype.new_vector_box(vbox.getcount())
         self.sched_data._prevent_signext(newtype.getsize(), vbox.getsize())
         newsize = newtype.getsize()
         assert newsize > 0
@@ -430,11 +431,11 @@
         return vbox_cloned
 
     def unpack(self, vbox, index, count, arg_ptype):
-        assert index < vbox.item_count
-        assert index + count <= vbox.item_count
+        assert index < vbox.getcount()
+        assert index + count <= vbox.getcount()
         assert count > 0
         vbox_cloned = vectorbox_clone_set(vbox, count=count)
-        opnum = getunpackopnum(vbox.item_type)
+        opnum = getunpackopnum(vbox.gettype())
         op = ResOperation(opnum, [vbox, ConstInt(index), ConstInt(count)], vbox_cloned)
         self.costmodel.record_vector_unpack(vbox, index, count)
         self.preamble_ops.append(op)
@@ -447,9 +448,9 @@
             new_box = [1,2,3,4,5,6,_,_] after the operation, tidx=4, scount=2
         """
         assert sidx == 0 # restriction
-        count = tgt.item_count + src.item_count
+        count = tgt.getcount() + src.getcount()
         new_box = vectorbox_clone_set(tgt, count=count)
-        opnum = getpackopnum(tgt.item_type)
+        opnum = getpackopnum(tgt.gettype())
         op = ResOperation(opnum, [tgt, src, ConstInt(tidx), ConstInt(scount)], new_box)
         self.preamble_ops.append(op)
         self.costmodel.record_vector_pack(src, sidx, scount)
@@ -467,14 +468,14 @@
         assert isinstance(arg0, BoxVector)
         assert isinstance(index, ConstInt)
         assert isinstance(count, ConstInt)
-        assert arg0.item_size == result.item_size
+        assert arg0.getsize() == result.getsize()
         if isinstance(arg1, BoxVector):
-            assert arg1.item_size == result.item_size
+            assert arg1.getsize() == result.getsize()
         else:
             assert count.value == 1
-        assert index.value < result.item_count
-        assert index.value + count.value <= result.item_count
-        assert result.item_count > arg0.item_count
+        assert index.value < result.getcount()
+        assert index.value + count.value <= result.getcount()
+        assert result.getcount() > arg0.getcount()
 
     def expand(self, arg, argidx):
         elem_count = self.input_type.getcount()
@@ -484,7 +485,7 @@
         invariant_ops = self.sched_data.invariant_oplist
         invariant_vars = self.sched_data.invariant_vector_vars
         if isinstance(arg, BoxVector):
-            box_type = arg.item_type
+            box_type = arg.gettype()
 
         # note that heterogenous nodes are not yet tracked
         already_expanded = expanded_map.get(arg, None)
@@ -830,7 +831,7 @@
         return self.box_to_vbox.get(arg, (-1, None))
 
     def setvector_of_box(self, box, off, vector):
-        assert off < vector.item_count
+        assert off < vector.getcount()
         self.box_to_vbox[box] = (off, vector)
 
     def prepend_invariant_operations(self, oplist):
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py b/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
@@ -1357,18 +1357,18 @@
 
     def test_abc(self):
         trace="""
-        [p0, p9, i10, p7, i11, p2, p4, p5, p12, i13, i14, i15, i16, i17, i18]
-        guard_early_exit(descr=<rpython.jit.metainterp.compile.ResumeAtLoopHeaderDescr object at 0x7f8a20c32090>) [p7, p5, p4, p2, p0, i10, i13, p9, p12, i11, i14]
-        i19 = raw_load(i15, i14, descr=int16arraydescr)
-        i21 = int_add(i14, 2)
-        i22 = int_add(i19, i16)
-        i24 = int_signext(i22, 2)
-        raw_store(i17, i11, i24, descr=int16arraydescr)
-        i26 = int_add(i10, 1)
-        i28 = int_add(i11, 2)
-        i29 = int_ge(i26, i18)
-        guard_false(i29, descr=<rpython.jit.metainterp.compile.ResumeGuardFalseDescr object at 0x7f8a20c32990>) [p7, p5, p4, p2, p0, i21, i28, i26, None, i13, p9, p12, None, None]
-        jump(p0, p9, i26, p7, i28, p2, p4, p5, p12, i13, i21, i15, i16, i17, i18)
+        # int32 sum
+        label(p0, p19, i18, i24, i14, i8, i25, descr=TargetToken(140320937897104))
+        guard_early_exit(descr=<Guard0x7f9f03ab6310>) [p0, p19, i18, i14, i24]
+        i27 = raw_load(i8, i24, descr=<ArrayS 4>)
+        guard_not_invalidated(descr=<Guard0x7f9f03af5050>) [p0, i27, p19, i18, i14, i24]
+        i28 = int_add(i14, i27)
+        i29 = int_signext(i28, 4)
+        i30 = int_add(i18, 1)
+        i31 = int_add(i24, 4)
+        i32 = int_ge(i30, i25)
+        guard_false(i32, descr=<Guard0x7f9f03ab17d0>) [p0, i29, i30, i31, p19, None, None, None]
+        jump(p0, p19, i30, i31, i29, i8, i25, descr=TargetToken(140320937897104))
         """
         # schedule 885 -> ptype is non for raw_load?
         opt = self.vectorize(self.parse_loop(trace))
diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/vectorize.py
@@ -509,7 +509,7 @@
             renamer.start_renaming(arg, arg_cloned)
             cj = ConstInt(j)
             ci = ConstInt(1)
-            opnum = getunpackopnum(vbox.item_type)
+            opnum = getunpackopnum(vbox.gettype())
             unpack_op = ResOperation(opnum, [vbox, cj, ci], arg_cloned)
             self.costmodel.record_vector_unpack(vbox, j, 1)
             self.emit_operation(unpack_op)
@@ -833,7 +833,7 @@
             box = result
             result = BoxVectorAccum(box, accum.var, '+')
             # pack the scalar value
-            op = ResOperation(getpackopnum(box.item_type),
+            op = ResOperation(getpackopnum(box.gettype()),
                               [box, accum.var, ConstInt(0), ConstInt(1)], result)
             sched_data.invariant_oplist.append(op)
             # rename the variable with the box

From noreply at buildbot.pypy.org  Mon Jun 29 11:17:01 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 29 Jun 2015 11:17:01 +0200 (CEST)
Subject: [pypy-commit] pypy default: improve the test
Message-ID: <20150629091701.1C3271C11D2@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r78343:f74ce2f72a39
Date: 2015-06-25 13:03 +0200
http://bitbucket.org/pypy/pypy/changeset/f74ce2f72a39/

Log:	improve the test

diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -187,7 +187,12 @@
         [i0]
         jump(i0)
         """
-        self.optimize_loop(ops, expected)
+        short = """
+        [i2]
+        p3 = cast_int_to_ptr(i2)
+        jump(i2)
+        """
+        self.optimize_loop(ops, expected, expected_short=short)
 
     def test_reverse_of_cast_2(self):
         ops = """

From noreply at buildbot.pypy.org  Mon Jun 29 11:17:02 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 29 Jun 2015 11:17:02 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: start working towards short
	preamble
Message-ID: <20150629091702.60F391C11D2@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult-unroll
Changeset: r78344:18d849cece6c
Date: 2015-06-25 13:20 +0200
http://bitbucket.org/pypy/pypy/changeset/18d849cece6c/

Log:	start working towards short preamble ;

diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -233,6 +233,7 @@
         self.optpure = None
         self.optheap = None
         self.optearlyforce = None
+        self.optunroll = None
         # the following two fields is the data kept for unrolling,
         # those are the operations that can go to the short_preamble
         if loop is not None:
@@ -307,9 +308,6 @@
             return info.force_box(op, self)
         return op
 
-    def ensure_imported(self, value):
-        pass
-
     def is_inputarg(self, op):
         return op in self.inparg_dict
 
@@ -409,7 +407,7 @@
                                op.getdescr().get_index())
         elif op.is_getarrayitem() or op.getopnum() == rop.SETARRAYITEM_GC:
             opinfo = info.ArrayPtrInfo(op.getdescr())
-        elif op.getopnum() == rop.GUARD_CLASS:
+        elif op.getopnum() in (rop.GUARD_CLASS, rop.GUARD_NONNULL_CLASS):
             opinfo = info.InstancePtrInfo()
         elif op.getopnum() in (rop.STRLEN,):
             opinfo = vstring.StrPtrInfo(vstring.mode_string)            
diff --git a/rpython/jit/metainterp/optimizeopt/pure.py b/rpython/jit/metainterp/optimizeopt/pure.py
--- a/rpython/jit/metainterp/optimizeopt/pure.py
+++ b/rpython/jit/metainterp/optimizeopt/pure.py
@@ -2,6 +2,7 @@
 from rpython.jit.metainterp.resoperation import rop, OpHelpers, AbstractResOp,\
      ResOperation
 from rpython.jit.metainterp.optimizeopt.util import make_dispatcher_method
+from rpython.jit.metainterp.optimizeopt.unroll import PreambleOp
 
 
 class RecentPureOps(object):
@@ -17,12 +18,19 @@
         self.next_index = (next_index + 1) % self.REMEMBER_LIMIT
         self.lst[next_index] = op
 
+    def force_preamble_op(self, opt, op, i):
+        if not isinstance(op, PreambleOp):
+            return
+        op = opt.force_op_from_preamble(op)
+        self.lst[i] = op
+
     def lookup1(self, opt, box0, descr):
         for i in range(self.REMEMBER_LIMIT):
             op = self.lst[i]
             if op is None:
                 break
             if opt.get_box_replacement(op.getarg(0)).same_box(box0) and op.getdescr() is descr:
+                self.force_preamble_op(opt, op, i)
                 return opt.get_box_replacement(op)
         return None
 
@@ -33,6 +41,7 @@
                 break
             if (opt.get_box_replacement(op.getarg(0)).same_box(box0) and opt.get_box_replacement(op.getarg(1)).same_box(box1)
                 and op.getdescr() is descr):
+                self.force_preamble_op(opt, op, i)
                 return opt.get_box_replacement(op)
         return None
 
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -88,7 +88,8 @@
             short_preamble.operations = short
             self.assert_equal(short_preamble, convert_old_style_to_targets(expected_short, jump=True),
                               text_right='expected short preamble')
-            assert short[-1].getdescr() == loop.operations[0].getdescr()
+            #assert short[-1].getdescr() == loop.operations[0].getdescr()
+            # XXX not sure what to do about that one
 
         return loop
 
@@ -187,7 +188,12 @@
         [i0]
         jump(i0)
         """
-        self.optimize_loop(ops, expected)
+        short = """
+        [i2]
+        p3 = cast_int_to_ptr(i2)
+        #jump(i2) <- think about it
+        """
+        self.optimize_loop(ops, expected, expected_short=short)
 
     def test_reverse_of_cast_2(self):
         ops = """
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -5,11 +5,12 @@
 from rpython.jit.metainterp.logger import LogOperations
 from rpython.jit.metainterp.optimize import InvalidLoop
 from rpython.jit.metainterp.optimizeopt.generalize import KillHugeIntBounds
-from rpython.jit.metainterp.optimizeopt.optimizer import Optimizer, Optimization
+from rpython.jit.metainterp.optimizeopt.optimizer import Optimizer,\
+     Optimization
 from rpython.jit.metainterp.optimizeopt.virtualstate import (VirtualStateConstructor,
         ShortBoxes, BadVirtualState, VirtualStatesCantMatch)
 from rpython.jit.metainterp.resoperation import rop, ResOperation,\
-     OpHelpers, AbstractInputArg, GuardResOp
+     OpHelpers, AbstractInputArg, GuardResOp, AbstractResOp
 from rpython.jit.metainterp.resume import Snapshot
 from rpython.jit.metainterp import compile
 from rpython.rlib.debug import debug_print, debug_start, debug_stop
@@ -25,6 +26,17 @@
     return opt.propagate_all_forward(start_state, export_state)
 
 
+class PreambleOp(AbstractResOp):
+    def __init__(self, op):
+        self.op = op
+
+    def getarg(self, i):
+        return self.op.getarg(i)
+
+    def __repr__(self):
+        return "Preamble(%r)" % (self.op,)
+
+
 class UnrollableOptimizer(Optimizer):
     def setup(self):
         self.importable_values = {}
@@ -44,6 +56,11 @@
             self.emitted_guards += 1 # FIXME: can we use counter in self._emit_operation?
         self._emit_operation(op)
 
+    def force_op_from_preamble(self, op):
+        op = op.op
+        self.optunroll.short.append(op)
+        return op
+
 
 class UnrollOptimizer(Optimization):
     """Unroll the loop into two iterations. The first one will
@@ -55,6 +72,7 @@
     def __init__(self, metainterp_sd, jitdriver_sd, loop, optimizations):
         self.optimizer = UnrollableOptimizer(metainterp_sd, jitdriver_sd,
                                              loop, optimizations)
+        self.optimizer.optunroll = self
         self.boxes_created_this_iteration = None
 
     def get_virtual_state(self, args):
@@ -121,11 +139,12 @@
                 # Initial label matches, jump to it
                 jumpop = ResOperation(rop.JUMP, stop_label.getarglist(),
                                       descr=start_label.getdescr())
-                if self.short:
-                    # Construct our short preamble
-                    self.close_loop(start_label, jumpop, patchguardop)
-                else:
-                    self.optimizer.send_extra_operation(jumpop)
+                #if self.short:
+                #    # Construct our short preamble
+                #    self.close_loop(start_label, jumpop, patchguardop)
+                #else:
+                start_label.getdescr().short_preamble = self.short
+                self.optimizer.send_extra_operation(jumpop)
                 return
 
             if cell_token.target_tokens:
@@ -275,10 +294,19 @@
 
         # Setup the state of the new optimizer by emiting the
         # short operations and discarding the result
-        self.optimizer.emitting_dissabled = True
-        for source, target in exported_state.inputarg_setup_ops:
-            source.set_forwarded(target)
+        #self.optimizer.emitting_dissabled = True
+        # think about it, it seems to be just for consts
+        #for source, target in exported_state.inputarg_setup_ops:
+        #    source.set_forwarded(target)
 
+        for op in self.short_boxes.operations():
+            if not op:
+                continue
+            if op.is_always_pure():
+                self.pure(op.getopnum(), PreambleOp(op))
+            else:
+                yyy
+        return
         seen = {}
         for op in self.short_boxes.operations():
             self.ensure_short_op_emitted(op, self.optimizer, seen)

From noreply at buildbot.pypy.org  Mon Jun 29 11:17:03 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 29 Jun 2015 11:17:03 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: small fix
Message-ID: <20150629091703.884F51C11D2@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult-unroll
Changeset: r78345:54a68c9bad16
Date: 2015-06-25 13:24 +0200
http://bitbucket.org/pypy/pypy/changeset/54a68c9bad16/

Log:	small fix

diff --git a/rpython/jit/metainterp/optimizeopt/pure.py b/rpython/jit/metainterp/optimizeopt/pure.py
--- a/rpython/jit/metainterp/optimizeopt/pure.py
+++ b/rpython/jit/metainterp/optimizeopt/pure.py
@@ -20,9 +20,10 @@
 
     def force_preamble_op(self, opt, op, i):
         if not isinstance(op, PreambleOp):
-            return
+            return op
         op = opt.force_op_from_preamble(op)
         self.lst[i] = op
+        return op
 
     def lookup1(self, opt, box0, descr):
         for i in range(self.REMEMBER_LIMIT):
@@ -30,7 +31,7 @@
             if op is None:
                 break
             if opt.get_box_replacement(op.getarg(0)).same_box(box0) and op.getdescr() is descr:
-                self.force_preamble_op(opt, op, i)
+                op = self.force_preamble_op(opt, op, i)
                 return opt.get_box_replacement(op)
         return None
 
@@ -41,7 +42,7 @@
                 break
             if (opt.get_box_replacement(op.getarg(0)).same_box(box0) and opt.get_box_replacement(op.getarg(1)).same_box(box1)
                 and op.getdescr() is descr):
-                self.force_preamble_op(opt, op, i)
+                op = self.force_preamble_op(opt, op, i)
                 return opt.get_box_replacement(op)
         return None
 

From noreply at buildbot.pypy.org  Mon Jun 29 11:17:05 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 29 Jun 2015 11:17:05 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge
Message-ID: <20150629091705.2F5BF1C11D2@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r78346:32c897eb2152
Date: 2015-06-29 11:16 +0200
http://bitbucket.org/pypy/pypy/changeset/32c897eb2152/

Log:	merge

diff --git a/pypy/module/_cffi_backend/lib_obj.py b/pypy/module/_cffi_backend/lib_obj.py
--- a/pypy/module/_cffi_backend/lib_obj.py
+++ b/pypy/module/_cffi_backend/lib_obj.py
@@ -175,6 +175,8 @@
                     return self.dir1(ignore_type=cffi_opcode.OP_GLOBAL_VAR)
                 if is_getattr and attr == '__dict__':
                     return self.full_dict_copy()
+                if is_getattr and attr == '__name__':
+                    return self.descr_repr()
                 raise oefmt(self.space.w_AttributeError,
                             "cffi library '%s' has no function, constant "
                             "or global variable named '%s'",
diff --git a/pypy/module/_cffi_backend/src/parse_c_type.c b/pypy/module/_cffi_backend/src/parse_c_type.c
--- a/pypy/module/_cffi_backend/src/parse_c_type.c
+++ b/pypy/module/_cffi_backend/src/parse_c_type.c
@@ -362,7 +362,7 @@
 
             case TOK_INTEGER:
                 errno = 0;
-#ifndef MS_WIN32
+#ifndef _MSC_VER
                 if (sizeof(length) > sizeof(unsigned long))
                     length = strtoull(tok->p, &endptr, 0);
                 else
diff --git a/pypy/module/_cffi_backend/test/test_recompiler.py b/pypy/module/_cffi_backend/test/test_recompiler.py
--- a/pypy/module/_cffi_backend/test/test_recompiler.py
+++ b/pypy/module/_cffi_backend/test/test_recompiler.py
@@ -1011,3 +1011,4 @@
         assert MYFOO == 42
         assert hasattr(lib, '__dict__')
         assert lib.__all__ == ['MYFOO', 'mybar']   # but not 'myvar'
+        assert lib.__name__ == repr(lib)
diff --git a/pypy/module/_io/test/test_io.py b/pypy/module/_io/test/test_io.py
--- a/pypy/module/_io/test/test_io.py
+++ b/pypy/module/_io/test/test_io.py
@@ -420,6 +420,8 @@
                 {"mode": "w+b", "buffering": 0},
             ]:
             print kwargs
+            if "b" not in kwargs["mode"]:
+                kwargs["encoding"] = "ascii"
             f = _io.open(self.tmpfile, **kwargs)
             f.close()
             raises(ValueError, f.flush)
diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py
--- a/pypy/objspace/std/dictmultiobject.py
+++ b/pypy/objspace/std/dictmultiobject.py
@@ -711,7 +711,7 @@
     next_item = _new_next('item')
 
 
-def create_iterator_classes(dictimpl, override_next_item=None):
+def create_iterator_classes(dictimpl):
     if not hasattr(dictimpl, 'wrapkey'):
         wrapkey = lambda space, key: key
     else:
@@ -754,15 +754,12 @@
             self.iterator = strategy.getiteritems(impl)
             BaseIteratorImplementation.__init__(self, space, strategy, impl)
 
-        if override_next_item is not None:
-            next_item_entry = override_next_item
-        else:
-            def next_item_entry(self):
-                for key, value in self.iterator:
-                    return (wrapkey(self.space, key),
-                            wrapvalue(self.space, value))
-                else:
-                    return None, None
+        def next_item_entry(self):
+            for key, value in self.iterator:
+                return (wrapkey(self.space, key),
+                        wrapvalue(self.space, value))
+            else:
+                return None, None
 
     class IterClassReversed(BaseKeyIterator):
         def __init__(self, space, strategy, impl):
@@ -795,22 +792,7 @@
     def rev_update1_dict_dict(self, w_dict, w_updatedict):
         # the logic is to call prepare_dict_update() after the first setitem():
         # it gives the w_updatedict a chance to switch its strategy.
-        if override_next_item is not None:
-            # this is very similar to the general version, but the difference
-            # is that it is specialized to call a specific next_item()
-            iteritems = IterClassItems(self.space, self, w_dict)
-            w_key, w_value = iteritems.next_item()
-            if w_key is None:
-                return
-            w_updatedict.setitem(w_key, w_value)
-            w_updatedict.strategy.prepare_update(w_updatedict,
-                                                 w_dict.length() - 1)
-            while True:
-                w_key, w_value = iteritems.next_item()
-                if w_key is None:
-                    return
-                w_updatedict.setitem(w_key, w_value)
-        else:
+        if 1:     # (preserve indentation)
             iteritems = self.getiteritems(w_dict)
             if not same_strategy(self, w_updatedict):
                 # Different strategy.  Try to copy one item of w_dict
diff --git a/pypy/objspace/std/kwargsdict.py b/pypy/objspace/std/kwargsdict.py
--- a/pypy/objspace/std/kwargsdict.py
+++ b/pypy/objspace/std/kwargsdict.py
@@ -166,19 +166,26 @@
         return iter(self.unerase(w_dict.dstorage)[1])
 
     def getiteritems(self, w_dict):
-        keys = self.unerase(w_dict.dstorage)[0]
-        return iter(range(len(keys)))
+        return Zip(*self.unerase(w_dict.dstorage))
 
     wrapkey = _wrapkey
 
 
-def next_item(self):
-    strategy = self.strategy
-    assert isinstance(strategy, KwargsDictStrategy)
-    for i in self.iterator:
-        keys, values_w = strategy.unerase(self.dictimplementation.dstorage)
-        return _wrapkey(self.space, keys[i]), values_w[i]
-    else:
-        return None, None
+class Zip(object):
+    def __init__(self, list1, list2):
+        assert len(list1) == len(list2)
+        self.list1 = list1
+        self.list2 = list2
+        self.i = 0
 
-create_iterator_classes(KwargsDictStrategy, override_next_item=next_item)
+    def __iter__(self):
+        return self
+
+    def next(self):
+        i = self.i
+        if i >= len(self.list1):
+            raise StopIteration
+        self.i = i + 1
+        return (self.list1[i], self.list2[i])
+
+create_iterator_classes(KwargsDictStrategy)
diff --git a/pypy/objspace/std/test/test_kwargsdict.py b/pypy/objspace/std/test/test_kwargsdict.py
--- a/pypy/objspace/std/test/test_kwargsdict.py
+++ b/pypy/objspace/std/test/test_kwargsdict.py
@@ -159,3 +159,10 @@
         assert a == 3
         assert "KwargsDictStrategy" in self.get_strategy(d)
 
+    def test_iteritems_bug(self):
+        def f(**args):
+            return args
+
+        d = f(a=2, b=3, c=4)
+        for key, value in d.iteritems():
+            None in d
diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py
--- a/rpython/memory/gctransform/framework.py
+++ b/rpython/memory/gctransform/framework.py
@@ -52,21 +52,22 @@
             return (op.opname in LL_OPERATIONS and
                     LL_OPERATIONS[op.opname].canmallocgc)
 
-def find_initializing_stores(collect_analyzer, graph):
-    from rpython.flowspace.model import mkentrymap
-    entrymap = mkentrymap(graph)
-    # a bit of a hackish analysis: if a block contains a malloc and check that
-    # the result is not zero, then the block following the True link will
-    # usually initialize the newly allocated object
-    result = set()
-    def find_in_block(block, mallocvars):
+def propagate_no_write_barrier_needed(result, block, mallocvars,
+                                      collect_analyzer, entrymap,
+                                      startindex=0):
+    # We definitely know that no write barrier is needed in the 'block'
+    # for any of the variables in 'mallocvars'.  Propagate this information
+    # forward.  Note that "definitely know" implies that we just did either
+    # a fixed-size malloc (variable-size might require card marking), or
+    # that we just did a full write barrier (not just for card marking).
+    if 1:       # keep indentation
         for i, op in enumerate(block.operations):
+            if i < startindex:
+                continue
             if op.opname in ("cast_pointer", "same_as"):
                 if op.args[0] in mallocvars:
                     mallocvars[op.result] = True
             elif op.opname in ("setfield", "setarrayitem", "setinteriorfield"):
-                # note that 'mallocvars' only tracks fixed-size mallocs,
-                # so no risk that they use card marking
                 TYPE = op.args[-1].concretetype
                 if (op.args[0] in mallocvars and
                     isinstance(TYPE, lltype.Ptr) and
@@ -83,7 +84,15 @@
                 if var in mallocvars:
                     newmallocvars[exit.target.inputargs[i]] = True
             if newmallocvars:
-                find_in_block(exit.target, newmallocvars)
+                propagate_no_write_barrier_needed(result, exit.target,
+                                                  newmallocvars,
+                                                  collect_analyzer, entrymap)
+
+def find_initializing_stores(collect_analyzer, graph, entrymap):
+    # a bit of a hackish analysis: if a block contains a malloc and check that
+    # the result is not zero, then the block following the True link will
+    # usually initialize the newly allocated object
+    result = set()
     mallocnum = 0
     blockset = set(graph.iterblocks())
     while blockset:
@@ -113,7 +122,8 @@
         target = exit.target
         mallocvars = {target.inputargs[index]: True}
         mallocnum += 1
-        find_in_block(target, mallocvars)
+        propagate_no_write_barrier_needed(result, target, mallocvars,
+                                          collect_analyzer, entrymap)
     #if result:
     #    print "found %s initializing stores in %s" % (len(result), graph.name)
     return result
@@ -698,8 +708,11 @@
                                 " %s" % func)
 
         if self.write_barrier_ptr:
+            from rpython.flowspace.model import mkentrymap
+            self._entrymap = mkentrymap(graph)
             self.clean_sets = (
-                find_initializing_stores(self.collect_analyzer, graph))
+                find_initializing_stores(self.collect_analyzer, graph,
+                                         self._entrymap))
             if self.gcdata.gc.can_optimize_clean_setarrayitems():
                 self.clean_sets = self.clean_sets.union(
                     find_clean_setarrayitems(self.collect_analyzer, graph))
@@ -1269,6 +1282,17 @@
                 hop.genop("direct_call", [self.write_barrier_ptr,
                                           self.c_const_gc,
                                           v_structaddr])
+                # we just did a full write barrier here, so we can use
+                # this helper to propagate this knowledge forward and
+                # avoid to repeat the write barrier.
+                if self.curr_block is not None:   # for tests
+                    assert self.curr_block.operations[hop.index] is hop.spaceop
+                    propagate_no_write_barrier_needed(self.clean_sets,
+                                                      self.curr_block,
+                                                      {v_struct: True},
+                                                      self.collect_analyzer,
+                                                      self._entrymap,
+                                                      hop.index + 1)
         hop.rename('bare_' + opname)
 
     def transform_getfield_typeptr(self, hop):
diff --git a/rpython/memory/gctransform/test/test_framework.py b/rpython/memory/gctransform/test/test_framework.py
--- a/rpython/memory/gctransform/test/test_framework.py
+++ b/rpython/memory/gctransform/test/test_framework.py
@@ -1,6 +1,6 @@
 from rpython.annotator.listdef import s_list_of_strings
 from rpython.annotator.model import SomeInteger
-from rpython.flowspace.model import Constant, SpaceOperation
+from rpython.flowspace.model import Constant, SpaceOperation, mkentrymap
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rtyper.lltypesystem.lloperation import llop
 from rpython.memory.gc.semispace import SemiSpaceGC
@@ -231,6 +231,33 @@
          Constant('b', lltype.Void), varoftype(PTR_TYPE2)],
         varoftype(lltype.Void)))
 
+def test_remove_duplicate_write_barrier():
+    from rpython.translator.c.genc import CStandaloneBuilder
+    from rpython.flowspace.model import summary
+
+    class A(object):
+        pass
+    glob_a_1 = A()
+    glob_a_2 = A()
+
+    def f(a, cond):
+        a.x = a
+        a.z = a
+        if cond:
+            a.y = a
+    def g():
+        f(glob_a_1, 5)
+        f(glob_a_2, 0)
+    t = rtype(g, [])
+    t.config.translation.gc = "minimark"
+    cbuild = CStandaloneBuilder(t, g, t.config,
+                                gcpolicy=FrameworkGcPolicy2)
+    db = cbuild.generate_graphs_for_llinterp()
+
+    ff = graphof(t, f)
+    #ff.show()
+    assert summary(ff)['direct_call'] == 1    # only one remember_young_pointer
+
 def test_find_initializing_stores():
 
     class A(object):
@@ -246,7 +273,8 @@
     etrafo = ExceptionTransformer(t)
     graphs = etrafo.transform_completely()
     collect_analyzer = CollectAnalyzer(t)
-    init_stores = find_initializing_stores(collect_analyzer, t.graphs[0])
+    init_stores = find_initializing_stores(collect_analyzer, t.graphs[0],
+                                           mkentrymap(t.graphs[0]))
     assert len(init_stores) == 1
 
 def test_find_initializing_stores_across_blocks():
@@ -271,7 +299,8 @@
     etrafo = ExceptionTransformer(t)
     graphs = etrafo.transform_completely()
     collect_analyzer = CollectAnalyzer(t)
-    init_stores = find_initializing_stores(collect_analyzer, t.graphs[0])
+    init_stores = find_initializing_stores(collect_analyzer, t.graphs[0],
+                                           mkentrymap(t.graphs[0]))
     assert len(init_stores) == 5
 
 def test_find_clean_setarrayitems():
diff --git a/rpython/memory/gctransform/transform.py b/rpython/memory/gctransform/transform.py
--- a/rpython/memory/gctransform/transform.py
+++ b/rpython/memory/gctransform/transform.py
@@ -83,6 +83,7 @@
 
 class BaseGCTransformer(object):
     finished_helpers = False
+    curr_block = None
 
     def __init__(self, translator, inline=False):
         self.translator = translator
@@ -159,7 +160,7 @@
 
     def transform_block(self, block, is_borrowed):
         llops = LowLevelOpList()
-        #self.curr_block = block
+        self.curr_block = block
         self.livevars = [var for var in block.inputargs
                     if var_needsgc(var) and not is_borrowed(var)]
         allvars = [var for var in block.getvariables() if var_needsgc(var)]
@@ -205,6 +206,7 @@
             block.operations[:] = llops
         self.livevars = None
         self.var_last_needed_in = None
+        self.curr_block = None
 
     def transform_graph(self, graph):
         if graph in self.minimal_transform:
diff --git a/rpython/translator/c/genc.py b/rpython/translator/c/genc.py
--- a/rpython/translator/c/genc.py
+++ b/rpython/translator/c/genc.py
@@ -485,7 +485,7 @@
                 else:
                     mk.definition('DEBUGFLAGS', '-O1 -g')
         if self.translator.platform.name == 'msvc':
-            mk.rule('debug_target', 'debugmode_$(DEFAULT_TARGET)', 'rem')
+            mk.rule('debug_target', '$(DEFAULT_TARGET)', 'rem')
         else:
             mk.rule('debug_target', '$(DEFAULT_TARGET)', '#')
         mk.write()

From noreply at buildbot.pypy.org  Mon Jun 29 11:20:44 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 29 Jun 2015 11:20:44 +0200 (CEST)
Subject: [pypy-commit] pypy default: kill the alignment here
Message-ID: <20150629092044.64CD01C11D2@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r78347:d29ab3db4217
Date: 2015-06-29 11:20 +0200
http://bitbucket.org/pypy/pypy/changeset/d29ab3db4217/

Log:	kill the alignment here

diff --git a/pypy/module/_vmprof/src/trampoline.asmgcc.s b/pypy/module/_vmprof/src/trampoline.asmgcc.s
--- a/pypy/module/_vmprof/src/trampoline.asmgcc.s
+++ b/pypy/module/_vmprof/src/trampoline.asmgcc.s
@@ -1,7 +1,6 @@
 // NOTE: you need to use TABs, not spaces!
         
 	.text
-	.p2align 4,,-1
 	.globl	pypy_execute_frame_trampoline
 	.type	pypy_execute_frame_trampoline, @function
 pypy_execute_frame_trampoline:

From noreply at buildbot.pypy.org  Mon Jun 29 11:20:45 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 29 Jun 2015 11:20:45 +0200 (CEST)
Subject: [pypy-commit] pypy default: rename the file,
	it's not related to asmgcc at all
Message-ID: <20150629092045.979E11C11D2@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r78348:a5b02556f0ec
Date: 2015-06-29 11:20 +0200
http://bitbucket.org/pypy/pypy/changeset/a5b02556f0ec/

Log:	rename the file, it's not related to asmgcc at all

diff --git a/pypy/module/_vmprof/interp_vmprof.py b/pypy/module/_vmprof/interp_vmprof.py
--- a/pypy/module/_vmprof/interp_vmprof.py
+++ b/pypy/module/_vmprof/interp_vmprof.py
@@ -26,7 +26,7 @@
 eci_kwds = dict(
     include_dirs = [SRC],
     includes = ['vmprof.h', 'trampoline.h'],
-    separate_module_files = [SRC.join('trampoline.asmgcc.s')],
+    separate_module_files = [SRC.join('trampoline.s')],
     libraries = ['dl'],
     
     post_include_bits=["""
diff --git a/pypy/module/_vmprof/src/trampoline.asmgcc.s b/pypy/module/_vmprof/src/trampoline.s
rename from pypy/module/_vmprof/src/trampoline.asmgcc.s
rename to pypy/module/_vmprof/src/trampoline.s

From noreply at buildbot.pypy.org  Mon Jun 29 16:24:23 2015
From: noreply at buildbot.pypy.org (cfbolz)
Date: Mon, 29 Jun 2015 16:24:23 +0200 (CEST)
Subject: [pypy-commit] pypy default: implement file.readlines more
 efficiently by not wrapping every individual
Message-ID: <20150629142423.443B81C1047@cobra.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r78349:ad7bfb1386af
Date: 2015-06-29 16:22 +0200
http://bitbucket.org/pypy/pypy/changeset/ad7bfb1386af/

Log:	implement file.readlines more efficiently by not wrapping every
	individual string in the result (only to unwrap it again in the list
	creation code to make a bytes strategy list)

diff --git a/pypy/module/_file/interp_file.py b/pypy/module/_file/interp_file.py
--- a/pypy/module/_file/interp_file.py
+++ b/pypy/module/_file/interp_file.py
@@ -613,7 +613,7 @@
 # ____________________________________________________________
 
 def wrap_list_of_str(space, lst):
-    return space.newlist([space.wrap(s) for s in lst])
+    return space.newlist_bytes(lst)
 
 class FileState:
     def __init__(self, space):

From noreply at buildbot.pypy.org  Mon Jun 29 17:06:15 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 29 Jun 2015 17:06:15 +0200 (CEST)
Subject: [pypy-commit] pypy default: Document the difference with the
	readline module
Message-ID: <20150629150615.38B8C1C1278@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78350:b488a7eacab3
Date: 2015-06-29 17:06 +0200
http://bitbucket.org/pypy/pypy/changeset/b488a7eacab3/

Log:	Document the difference with the readline module

diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -395,3 +395,10 @@
   interactive mode. In a released version, this behaviour is suppressed, but
   setting the environment variable PYPY_IRC_TOPIC will bring it back. Note that
   downstream package providers have been known to totally disable this feature.
+
+* PyPy's readline module was rewritten from scratch: it is not GNU's
+  readline.  It should be mostly compatible, and it adds multiline
+  support (see ``multiline_input()``).  On the other hand,
+  ``parse_and_bind()`` calls are ignored (issue `#2072`__).
+
+.. __: https://bitbucket.org/pypy/pypy/issue/2072/

From noreply at buildbot.pypy.org  Mon Jun 29 17:18:51 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 29 Jun 2015 17:18:51 +0200 (CEST)
Subject: [pypy-commit] pypy default: grammar
Message-ID: <20150629151851.052821C11D2@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78351:875e32c92da9
Date: 2015-06-29 17:18 +0200
http://bitbucket.org/pypy/pypy/changeset/875e32c92da9/

Log:	grammar

diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -338,7 +338,8 @@
   for about 1400 calls.
 
 * since the implementation of dictionary is different, the exact number
-  which ``__hash__`` and ``__eq__`` are called is different. Since CPython
+  of times that ``__hash__`` and ``__eq__`` are called is different. 
+  Since CPython
   does not give any specific guarantees either, don't rely on it.
 
 * assignment to ``__class__`` is limited to the cases where it

From noreply at buildbot.pypy.org  Mon Jun 29 17:20:11 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 29 Jun 2015 17:20:11 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: start passing construction of
	args
Message-ID: <20150629152011.A55E91C11D2@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult-unroll
Changeset: r78352:fdaae9ee19c7
Date: 2015-06-29 17:19 +0200
http://bitbucket.org/pypy/pypy/changeset/fdaae9ee19c7/

Log:	start passing construction of args

diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -297,9 +297,15 @@
                 self.make_constant_class(op, known_class, False)
 
     def get_box_replacement(self, op):
+        from rpython.jit.metainterp.optimizeopt.unroll import PreambleOp
+
+        orig_op = op
         if op is None:
             return op
-        return op.get_box_replacement()
+        res = op.get_box_replacement()
+        if isinstance(res, PreambleOp):
+            xxx
+        return res
 
     def force_box(self, op):
         op = self.get_box_replacement(op)
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py b/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
@@ -1,22 +1,26 @@
 from __future__ import with_statement
 import py
-from rpython.jit.metainterp.optimizeopt.virtualstate import VirtualStateInfo, VStructStateInfo, \
-     VArrayStateInfo, NotVirtualStateInfo, VirtualState, ShortBoxes, GenerateGuardState, \
-     VirtualStatesCantMatch, VArrayStructStateInfo
-from rpython.jit.metainterp.optimizeopt.optimizer import OptValue, PtrOptValue,\
-      IntOptValue
-from rpython.jit.metainterp.history import BoxInt, BoxFloat, BoxPtr, ConstInt, ConstPtr
+from rpython.jit.metainterp.optimizeopt.virtualstate import VirtualStateInfo,\
+     VStructStateInfo, LEVEL_CONSTANT,\
+     VArrayStateInfo, NotVirtualStateInfo, VirtualState, ShortBoxes,\
+     GenerateGuardState, VirtualStatesCantMatch, VArrayStructStateInfo
+from rpython.jit.metainterp.history import ConstInt, ConstPtr
+from rpython.jit.metainterp.resoperation import InputArgInt, InputArgRef,\
+     InputArgFloat
 from rpython.rtyper.lltypesystem import lltype, llmemory
 from rpython.jit.metainterp.optimizeopt.test.test_util import LLtypeMixin, BaseTest, \
                                                            equaloplists
 from rpython.jit.metainterp.optimizeopt.intutils import IntBound
-from rpython.jit.metainterp.optimizeopt.virtualize import (VirtualValue,
-        VArrayValue, VStructValue, VArrayStructValue)
 from rpython.jit.metainterp.history import TreeLoop, JitCellToken
 from rpython.jit.metainterp.optimizeopt.test.test_optimizeopt import FakeMetaInterpStaticData
+from rpython.jit.metainterp.optimizeopt.optimizer import Optimizer
 from rpython.jit.metainterp.resoperation import ResOperation, rop
 from rpython.jit.metainterp import resume
 
+class FakeOptimizer(Optimizer):
+    def __init__(self):
+        self.optearlyforce = None
+
 class BaseTestGenerateGuards(BaseTest):
 
     def _box_or_value(self, box_or_value=None):
@@ -74,6 +78,14 @@
         with py.test.raises(VirtualStatesCantMatch):
             info1.generate_guards(info2, value, state)
 
+    def test_make_inputargs(self):
+        optimizer = FakeOptimizer()
+        args = [InputArgInt()]
+        info0 = NotVirtualStateInfo(optimizer, args[0])
+        vs = VirtualState([info0])
+        assert vs.make_inputargs(args, optimizer) == args
+        info0.level = LEVEL_CONSTANT
+        assert vs.make_inputargs(args, optimizer) == []
 
     def test_position_generalization(self):
         def postest(info1, info2):
@@ -95,7 +107,7 @@
             self.check_invalid(info1, info2, state=state)
             assert info1 in state.bad and info2 in state.bad
 
-        for BoxType in (BoxInt, BoxFloat, BoxPtr):
+        for BoxType in (InputArgInt, InputArgFloat, InputArgPtr):
             info1 = NotVirtualStateInfo(OptValue(BoxType()))
             info2 = NotVirtualStateInfo(OptValue(BoxType()))
             postest(info1, info2)
@@ -1386,16 +1398,6 @@
 class TestLLtypeBridges(BaseTestBridges, LLtypeMixin):
     pass
 
-class FakeOptimizer:
-    def __init__(self):
-        self.opaque_pointers = {}
-        self.values = {}
-    def make_equal_to(*args):
-        pass
-    def getvalue(*args):
-        pass
-    def emit_operation(*args):
-        pass
 
 
 class TestShortBoxes:
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -137,7 +137,13 @@
 
             if start_label and self.jump_to_start_label(start_label, stop_label):
                 # Initial label matches, jump to it
-                jumpop = ResOperation(rop.JUMP, stop_label.getarglist(),
+                vs = start_label.getdescr().virtual_state
+                if vs is not None:
+                    args = vs.make_inputargs(stop_label.getarglist(),
+                                             self.optimizer)
+                else:
+                    args = stop_label.getarglist()
+                jumpop = ResOperation(rop.JUMP, args,
                                       descr=start_label.getdescr())
                 #if self.short:
                 #    # Construct our short preamble
@@ -222,7 +228,7 @@
 
         short_boxes = ShortBoxes(self.optimizer, inputargs)
 
-        inputarg_setup_ops = []
+        proven_constants = []
         for i in range(len(original_jump_args)):
             srcbox = jump_args[i]
             if srcbox is not original_jump_args[i]:
@@ -231,7 +237,7 @@
                     if info and info.is_virtual():
                         xxx
             if original_jump_args[i] is not srcbox and srcbox.is_constant():
-                inputarg_setup_ops.append((original_jump_args[i], srcbox))
+                proven_constants.append((original_jump_args[i], srcbox))
                 #opnum = OpHelpers.same_as_for_type(original_jump_args[i].type)
                 #op = ResOperation(opnum, [srcbox])
                 #self.optimizer.emit_operation(op)
@@ -264,7 +270,7 @@
             if op and op.type != 'v':
                 exported_values[op] = self.optimizer.getinfo(op)
 
-        return ExportedState(short_boxes, inputarg_setup_ops, exported_values)
+        return ExportedState(short_boxes, proven_constants, exported_values)
 
     def import_state(self, targetop, exported_state):
         if not targetop: # Trace did not start with a label
@@ -291,6 +297,8 @@
         for box in self.inputargs:
             preamble_info = exported_state.exported_values[box]
             self.optimizer.setinfo_from_preamble(box, preamble_info)
+        for box, const in exported_state.proven_constants:
+            box.set_forwarded(const)
 
         # Setup the state of the new optimizer by emiting the
         # short operations and discarding the result
@@ -309,6 +317,7 @@
         return
         seen = {}
         for op in self.short_boxes.operations():
+            yyy
             self.ensure_short_op_emitted(op, self.optimizer, seen)
             if op and op.type != 'v':
                 preamble_value = exported_state.exported_values[op]
@@ -709,9 +718,9 @@
 
 
 class ExportedState(object):
-    def __init__(self, short_boxes, inputarg_setup_ops, exported_values):
+    def __init__(self, short_boxes, proven_constants, exported_values):
         self.short_boxes = short_boxes
-        self.inputarg_setup_ops = inputarg_setup_ops
+        self.proven_constants = proven_constants
         self.exported_values = exported_values
 
     def dump(self, metainterp_sd):
diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py
@@ -282,7 +282,14 @@
     lenbound = None
     intbound = None
     
-    def __init__(self, cpu, ptrinfo, is_opaque=False):
+    def __init__(self, optimizer, box):
+        info = optimizer.getinfo(box)
+        if info and info.is_constant():
+            self.level = LEVEL_CONSTANT
+        else:
+            self.level = LEVEL_UNKNOWN
+        return
+        yyy
         self.level = LEVEL_UNKNOWN
         if ptrinfo is not None:
             self.known_class = ptrinfo.get_known_class(cpu)
@@ -461,15 +468,6 @@
         debug_print(indent + mark + 'NotVirtualInfo(%d' % self.position +
                     ', ' + l + ', ' + self.intbound.__repr__() + lb + ')')
 
-class IntNotVirtualStateInfo(NotVirtualStateInfo):
-    def __init__(self, intbound):
-        # XXX do we care about non null?
-        self.intbound = intbound
-        if intbound.is_constant():
-            self.level = LEVEL_CONSTANT
-        else:
-            self.level = LEVEL_UNKNOWN
-
 
 class VirtualState(object):
     def __init__(self, state):
@@ -502,8 +500,11 @@
         if optimizer.optearlyforce:
             optimizer = optimizer.optearlyforce
         assert len(inputargs) == len(self.state)
-        return [x for x in inputargs if not isinstance(x, Const)]
-        return inputargs
+        inpargs = []
+        for i, state in enumerate(self.state):
+            if state.level != LEVEL_CONSTANT:
+                inpargs.append(inputargs[i])
+        return inpargs
         inputargs = [None] * self.numnotvirtuals
 
         # We try twice. The first time around we allow boxes to be forced
@@ -597,16 +598,9 @@
 
         return VirtualState(state)
 
-    def visit_not_ptr(self, box, intbound):
-        return IntNotVirtualStateInfo(intbound=intbound)
-    
     def visit_not_virtual(self, box):
         is_opaque = box in self.optimizer.opaque_pointers
-        if box.type == 'r':
-            ptrinfo = self.optimizer.getptrinfo(box)
-        else:
-            return self.visit_not_ptr(box, self.optimizer.getintbound(box))
-        return NotVirtualStateInfo(self.optimizer.cpu, ptrinfo, is_opaque)
+        return NotVirtualStateInfo(self.optimizer, box)
 
     def visit_virtual(self, known_class, fielddescrs):
         return VirtualStateInfo(known_class, fielddescrs)

From noreply at buildbot.pypy.org  Mon Jun 29 17:30:36 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 29 Jun 2015 17:30:36 +0200 (CEST)
Subject: [pypy-commit] pypy default: Drop "some of us believe",
 as the cpython issue links to a 29c3 talk
Message-ID: <20150629153036.1BF321C12E2@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78353:1666e583f39f
Date: 2015-06-29 17:30 +0200
http://bitbucket.org/pypy/pypy/changeset/1666e583f39f/

Log:	Drop "some of us believe", as the cpython issue links to a 29c3 talk
	where knowledgeable people also believe the same thing.

diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -321,9 +321,7 @@
 Miscellaneous
 -------------
 
-* Hash randomization (``-R``) is ignored in PyPy.  As documented in
-  http://bugs.python.org/issue14621, some of us believe it has no
-  purpose in CPython either.
+* Hash randomization (``-R``) `is ignored in PyPy`_.
 
 * You can't store non-string keys in type objects.  For example::
 
@@ -400,6 +398,7 @@
 * PyPy's readline module was rewritten from scratch: it is not GNU's
   readline.  It should be mostly compatible, and it adds multiline
   support (see ``multiline_input()``).  On the other hand,
-  ``parse_and_bind()`` calls are ignored (issue `#2072`__).
+  ``parse_and_bind()`` calls are ignored (issue `#2072`_).
 
-.. __: https://bitbucket.org/pypy/pypy/issue/2072/
+.. _`is ignored in PyPy`: http://bugs.python.org/issue14621
+.. _`#2072`: https://bitbucket.org/pypy/pypy/issue/2072/

From noreply at buildbot.pypy.org  Mon Jun 29 17:34:44 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 29 Jun 2015 17:34:44 +0200 (CEST)
Subject: [pypy-commit] pypy default: Link to the 29c3 talk
Message-ID: <20150629153444.75C471C12E2@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78354:d7ffb316f0f2
Date: 2015-06-29 17:34 +0200
http://bitbucket.org/pypy/pypy/changeset/d7ffb316f0f2/

Log:	Link to the 29c3 talk

diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -321,7 +321,8 @@
 Miscellaneous
 -------------
 
-* Hash randomization (``-R``) `is ignored in PyPy`_.
+* Hash randomization (``-R``) `is ignored in PyPy`_.  In CPython
+  before 3.4 it has `little point`_.
 
 * You can't store non-string keys in type objects.  For example::
 
@@ -401,4 +402,5 @@
   ``parse_and_bind()`` calls are ignored (issue `#2072`_).
 
 .. _`is ignored in PyPy`: http://bugs.python.org/issue14621
+.. _`little point`: http://events.ccc.de/congress/2012/Fahrplan/events/5152.en.html
 .. _`#2072`: https://bitbucket.org/pypy/pypy/issue/2072/

From noreply at buildbot.pypy.org  Mon Jun 29 21:55:07 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 29 Jun 2015 21:55:07 +0200 (CEST)
Subject: [pypy-commit] cffi default: ffi.include(ffi) doesn't make sense and
	hangs right now
Message-ID: <20150629195507.259EE1C124A@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2195:f6fcbe96b598
Date: 2015-06-29 21:55 +0200
http://bitbucket.org/cffi/cffi/changeset/f6fcbe96b598/

Log:	ffi.include(ffi) doesn't make sense and hangs right now

diff --git a/cffi/api.py b/cffi/api.py
--- a/cffi/api.py
+++ b/cffi/api.py
@@ -428,6 +428,8 @@
             raise TypeError("ffi.include() expects an argument that is also of"
                             " type cffi.FFI, not %r" % (
                                 type(ffi_to_include).__name__,))
+        if ffi_to_include is self:
+            raise ValueError("self.include(self)")
         with ffi_to_include._lock:
             with self._lock:
                 self._parser.include(ffi_to_include._parser)
diff --git a/testing/cffi0/backend_tests.py b/testing/cffi0/backend_tests.py
--- a/testing/cffi0/backend_tests.py
+++ b/testing/cffi0/backend_tests.py
@@ -1770,3 +1770,7 @@
         py.test.raises(TypeError, ffi.new, "struct foo_s *")
         ffi.cdef("struct foo_s { int x; };")
         ffi.new("struct foo_s *")
+
+    def test_ffi_self_include(self):
+        ffi = FFI(backend=self.Backend())
+        py.test.raises(ValueError, ffi.include, ffi)

From noreply at buildbot.pypy.org  Mon Jun 29 22:10:39 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 29 Jun 2015 22:10:39 +0200 (CEST)
Subject: [pypy-commit] cffi default: Issue #207: fix for anonymous enums
	with ffi.include()
Message-ID: <20150629201039.0A6131C063D@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2196:b8538812a48b
Date: 2015-06-29 22:11 +0200
http://bitbucket.org/cffi/cffi/changeset/b8538812a48b/

Log:	Issue #207: fix for anonymous enums with ffi.include()

diff --git a/cffi/cparser.py b/cffi/cparser.py
--- a/cffi/cparser.py
+++ b/cffi/cparser.py
@@ -633,6 +633,8 @@
 
     def include(self, other):
         for name, tp in other._declarations.items():
+            if name.startswith('anonymous $enum_$'):
+                continue   # fix for test_anonymous_enum_include
             kind = name.split(' ', 1)[0]
             if kind in ('struct', 'union', 'enum', 'anonymous'):
                 self._declare(name, tp, included=True)
diff --git a/testing/cffi0/backend_tests.py b/testing/cffi0/backend_tests.py
--- a/testing/cffi0/backend_tests.py
+++ b/testing/cffi0/backend_tests.py
@@ -1774,3 +1774,14 @@
     def test_ffi_self_include(self):
         ffi = FFI(backend=self.Backend())
         py.test.raises(ValueError, ffi.include, ffi)
+
+    def test_anonymous_enum_include(self):
+        ffi1 = FFI()
+        ffi1.cdef("enum { EE1 };")
+        ffi = FFI()
+        ffi.include(ffi1)
+        ffi.cdef("enum { EE2, EE3 };")
+        lib = ffi.dlopen(None)
+        assert lib.EE1 == 0
+        assert lib.EE2 == 0
+        assert lib.EE3 == 1

From noreply at buildbot.pypy.org  Tue Jun 30 08:52:14 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 30 Jun 2015 08:52:14 +0200 (CEST)
Subject: [pypy-commit] pypy default: Mention issue #2030 in
	cpython_differences
Message-ID: <20150630065214.6C7DC1C0695@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78355:d3d07e983854
Date: 2015-06-30 08:52 +0200
http://bitbucket.org/pypy/pypy/changeset/d3d07e983854/

Log:	Mention issue #2030 in cpython_differences

diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -135,7 +135,7 @@
 Here are some more technical details.  This issue affects the precise
 time at which ``__del__`` methods are called, which
 is not reliable in PyPy (nor Jython nor IronPython).  It also means that
-weak references may stay alive for a bit longer than expected.  This
+**weak references** may stay alive for a bit longer than expected.  This
 makes "weak proxies" (as returned by ``weakref.proxy()``) somewhat less
 useful: they will appear to stay alive for a bit longer in PyPy, and
 suddenly they will really be dead, raising a ``ReferenceError`` on the
@@ -143,6 +143,24 @@
 ``ReferenceError`` at any place that uses them.  (Or, better yet, don't use
 ``weakref.proxy()`` at all; use ``weakref.ref()``.)
 
+Note a detail in the `documentation for weakref callbacks`__:
+
+    If callback is provided and not None, *and the returned weakref
+    object is still alive,* the callback will be called when the object
+    is about to be finalized.
+
+There are cases where, due to CPython's refcount semantics, a weakref
+dies immediately before or after the objects it points to (typically
+with some circular reference).  If it happens to die just after, then
+the callback will be invoked.  In a similar case in PyPy, both the
+object and the weakref will be considered as dead at the same time,
+and the callback will not be invoked.  (Issue `#2030`__)
+
+.. __: https://docs.python.org/2/library/weakref.html
+.. __: https://bitbucket.org/pypy/pypy/issue/2030/
+
+---------------------------------
+
 There are a few extra implications from the difference in the GC.  Most
 notably, if an object has a ``__del__``, the ``__del__`` is never called more
 than once in PyPy; but CPython will call the same ``__del__`` several times

From noreply at buildbot.pypy.org  Tue Jun 30 09:19:41 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 30 Jun 2015 09:19:41 +0200 (CEST)
Subject: [pypy-commit] pypy default: Issue #209: check for dereferencing
	NULL pointers
Message-ID: <20150630071941.961761C11D7@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78356:56d1b320d259
Date: 2015-06-30 09:19 +0200
http://bitbucket.org/pypy/pypy/changeset/56d1b320d259/

Log:	Issue #209: check for dereferencing NULL pointers

diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py
--- a/pypy/module/_cffi_backend/ctypeptr.py
+++ b/pypy/module/_cffi_backend/ctypeptr.py
@@ -223,9 +223,13 @@
         if (isinstance(w_cdata, cdataobj.W_CDataNewOwning) or
             isinstance(w_cdata, cdataobj.W_CDataPtrToStructOrUnion)):
             if i != 0:
-                space = self.space
-                raise oefmt(space.w_IndexError,
+                raise oefmt(self.space.w_IndexError,
                             "cdata '%s' can only be indexed by 0", self.name)
+        else:
+            if not w_cdata.unsafe_escaping_ptr():
+                raise oefmt(self.space.w_RuntimeError,
+                            "cannot dereference null pointer from cdata '%s'",
+                            self.name)
         return self
 
     def _check_slice_index(self, w_cdata, start, stop):
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -2099,8 +2099,7 @@
     p = cast(BVoidP, 123456)
     py.test.raises(TypeError, "p[0]")
     p = cast(BVoidP, 0)
-    if 'PY_DOT_PY' in globals(): py.test.skip("NULL crashes early on py.py")
-    py.test.raises(TypeError, "p[0]")
+    py.test.raises((TypeError, RuntimeError), "p[0]")
 
 def test_iter():
     BInt = new_primitive_type("int")
@@ -3333,6 +3332,15 @@
     check(4 | 8,  "CHB", "GTB")
     check(4 | 16, "CHB", "ROB")
 
+def test_dereference_null_ptr():
+    BInt = new_primitive_type("int")
+    BIntPtr = new_pointer_type(BInt)
+    p = cast(BIntPtr, 0)
+    py.test.raises(RuntimeError, "p[0]")
+    py.test.raises(RuntimeError, "p[0] = 42")
+    py.test.raises(RuntimeError, "p[42]")
+    py.test.raises(RuntimeError, "p[42] = -1")
+
 def test_version():
     # this test is here mostly for PyPy
     assert __version__ == "1.1.2"

From noreply at buildbot.pypy.org  Tue Jun 30 09:19:52 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 30 Jun 2015 09:19:52 +0200 (CEST)
Subject: [pypy-commit] cffi default: Issue #209: check for dereferencing
	NULL pointers
Message-ID: <20150630071952.01EBD1C11D7@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2197:856cf4804a52
Date: 2015-06-30 09:16 +0200
http://bitbucket.org/cffi/cffi/changeset/856cf4804a52/

Log:	Issue #209: check for dereferencing NULL pointers

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -1898,11 +1898,21 @@
         return NULL;
 
     if (cd->c_type->ct_flags & CT_POINTER) {
-        if (CDataOwn_Check(cd) && i != 0) {
-            PyErr_Format(PyExc_IndexError,
-                         "cdata '%s' can only be indexed by 0",
-                         cd->c_type->ct_name);
-            return NULL;
+        if (CDataOwn_Check(cd)) {
+            if (i != 0) {
+                PyErr_Format(PyExc_IndexError,
+                             "cdata '%s' can only be indexed by 0",
+                             cd->c_type->ct_name);
+                return NULL;
+            }
+        }
+        else {
+            if (cd->c_data == NULL) {
+                PyErr_Format(PyExc_RuntimeError,
+                             "cannot dereference null pointer from cdata '%s'",
+                             cd->c_type->ct_name);
+                return NULL;
+            }
         }
     }
     else if (cd->c_type->ct_flags & CT_ARRAY) {
diff --git a/c/test_c.py b/c/test_c.py
--- a/c/test_c.py
+++ b/c/test_c.py
@@ -2110,8 +2110,7 @@
     p = cast(BVoidP, 123456)
     py.test.raises(TypeError, "p[0]")
     p = cast(BVoidP, 0)
-    if 'PY_DOT_PY' in globals(): py.test.skip("NULL crashes early on py.py")
-    py.test.raises(TypeError, "p[0]")
+    py.test.raises((TypeError, RuntimeError), "p[0]")
 
 def test_iter():
     BInt = new_primitive_type("int")
@@ -3344,6 +3343,15 @@
     check(4 | 8,  "CHB", "GTB")
     check(4 | 16, "CHB", "ROB")
 
+def test_dereference_null_ptr():
+    BInt = new_primitive_type("int")
+    BIntPtr = new_pointer_type(BInt)
+    p = cast(BIntPtr, 0)
+    py.test.raises(RuntimeError, "p[0]")
+    py.test.raises(RuntimeError, "p[0] = 42")
+    py.test.raises(RuntimeError, "p[42]")
+    py.test.raises(RuntimeError, "p[42] = -1")
+
 def test_version():
     # this test is here mostly for PyPy
     assert __version__ == "1.1.2"
diff --git a/doc/source/whatsnew.rst b/doc/source/whatsnew.rst
--- a/doc/source/whatsnew.rst
+++ b/doc/source/whatsnew.rst
@@ -30,6 +30,11 @@
   dict---assuming that ``lib`` has got no symbol called precisely
   ``__dict__``.  (In general, it is safer to use ``dir(lib)``.)
 
+* Issue #209: dereferencing NULL pointers now raises RuntimeError
+  instead of segfaulting.  Meant as a debugging aid.  The check is
+  only for NULL: if you dereference random or dead pointers you might
+  still get segfaults.
+
 
 1.1.2
 =====

From noreply at buildbot.pypy.org  Tue Jun 30 11:04:15 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 30 Jun 2015 11:04:15 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: add GUARD_VALUE for short
	preamble
Message-ID: <20150630090415.E15D61C0987@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult-unroll
Changeset: r78357:9f67e775a106
Date: 2015-06-30 11:04 +0200
http://bitbucket.org/pypy/pypy/changeset/9f67e775a106/

Log:	add GUARD_VALUE for short preamble

diff --git a/rpython/jit/metainterp/optimizeopt/intutils.py b/rpython/jit/metainterp/optimizeopt/intutils.py
--- a/rpython/jit/metainterp/optimizeopt/intutils.py
+++ b/rpython/jit/metainterp/optimizeopt/intutils.py
@@ -245,6 +245,10 @@
         return res
 
     def make_guards(self, box, guards):
+        if self.is_constant():
+            guards.append(ResOperation(rop.GUARD_VALUE,
+                                       [box, ConstInt(self.upper)]))
+            return
         if self.has_lower and self.lower > MININT:
             bound = self.lower
             op = ResOperation(rop.INT_GE, [box, ConstInt(bound)])
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -442,12 +442,13 @@
         else:
             return CONST_0
 
-    def propagate_all_forward(self, clear=True):
+    def propagate_all_forward(self, clear=True, create_inp_args=True):
         if clear:
             self.clear_newoperations()
-        self.inparg_dict = {}
-        for op in self.loop.inputargs:
-            self.inparg_dict[op] = None
+        if create_inp_args:
+            self.inparg_dict = {}
+            for op in self.loop.inputargs:
+                self.inparg_dict[op] = None
         for op in self.loop.operations:
             self._really_emitted_operation = None
             self.first_optimization.propagate_forward(op)
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -450,7 +450,7 @@
         [i0]
         i1 = int_is_true(i0)
         guard_value(i1, 1) []
-        jump(i0)
+        #jump(i0) <- xxx
         """
         self.optimize_loop(ops, expected, preamble, expected_short=short)
 
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -27,8 +27,9 @@
 
 
 class PreambleOp(AbstractResOp):
-    def __init__(self, op):
+    def __init__(self, op, info):
         self.op = op
+        self.info = info
 
     def getarg(self, i):
         return self.op.getarg(i)
@@ -56,9 +57,11 @@
             self.emitted_guards += 1 # FIXME: can we use counter in self._emit_operation?
         self._emit_operation(op)
 
-    def force_op_from_preamble(self, op):
-        op = op.op
+    def force_op_from_preamble(self, preamble_op):
+        op = preamble_op.op
         self.optunroll.short.append(op)
+        if preamble_op.info:
+            preamble_op.info.make_guards(op, self.optunroll.short)
         return op
 
 
@@ -117,7 +120,12 @@
             jumpop = None
 
         self.import_state(start_label, starting_state)
-        self.optimizer.propagate_all_forward(clear=False)
+        self.optimizer.inparg_dict = {}
+        for box in start_label.getarglist():
+            self.optimizer.inparg_dict[box] = None
+        import pdb
+        pdb.set_trace()
+        self.optimizer.propagate_all_forward(clear=False, create_inp_args=False)
 
         if not jumpop:
             return
@@ -311,7 +319,8 @@
             if not op:
                 continue
             if op.is_always_pure():
-                self.pure(op.getopnum(), PreambleOp(op))
+                self.pure(op.getopnum(),
+                          PreambleOp(op, self.optimizer.getinfo(op)))
             else:
                 yyy
         return
diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py
@@ -589,7 +589,7 @@
                     state.append(self.visit_not_virtual(box))
             elif box.type == 'i':
                 intbound = opt.getintbound(box)
-                state.append(self.visit_not_ptr(box, intbound))
+                state.append(self.visit_not_virtual(box))
             else:
                 xxx
         #values = [self.getvalue(box).force_at_end_of_preamble(already_forced,

From noreply at buildbot.pypy.org  Tue Jun 30 12:04:47 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 30 Jun 2015 12:04:47 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: kill pdb
Message-ID: <20150630100447.398E81C063D@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult-unroll
Changeset: r78358:b5ed4d2bd824
Date: 2015-06-30 12:04 +0200
http://bitbucket.org/pypy/pypy/changeset/b5ed4d2bd824/

Log:	kill pdb

diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -123,8 +123,6 @@
         self.optimizer.inparg_dict = {}
         for box in start_label.getarglist():
             self.optimizer.inparg_dict[box] = None
-        import pdb
-        pdb.set_trace()
         self.optimizer.propagate_all_forward(clear=False, create_inp_args=False)
 
         if not jumpop:

From noreply at buildbot.pypy.org  Tue Jun 30 12:41:08 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 30 Jun 2015 12:41:08 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: fix, I think
Message-ID: <20150630104108.A90391C11D3@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult-unroll
Changeset: r78359:190358328d5b
Date: 2015-06-30 12:18 +0200
http://bitbucket.org/pypy/pypy/changeset/190358328d5b/

Log:	fix, I think

diff --git a/rpython/jit/metainterp/optimizeopt/intbounds.py b/rpython/jit/metainterp/optimizeopt/intbounds.py
--- a/rpython/jit/metainterp/optimizeopt/intbounds.py
+++ b/rpython/jit/metainterp/optimizeopt/intbounds.py
@@ -117,8 +117,8 @@
             self.getintbound(op).intersect(b)
 
     def optimize_INT_ADD(self, op):
-        arg1 = op.getarg(0)
-        arg2 = op.getarg(1)
+        arg1 = self.get_box_replacement(op.getarg(0))
+        arg2 = self.get_box_replacement(op.getarg(1))
         v1 = self.getintbound(arg1)
         v2 = self.getintbound(arg2)
 
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_util.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py
@@ -464,6 +464,7 @@
         assert loop.operations[0].getopnum() == rop.LABEL
         loop.inputargs = loop.operations[0].getarglist()
 
+        start_state.orig_inputargs = inputargs
         self._do_optimize_loop(loop, call_pure_results, start_state,
                                export_state=False)
         extra_same_as = []
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -300,6 +300,8 @@
         self.short_boxes = exported_state.short_boxes
         self.initial_virtual_state = target_token.virtual_state
 
+        for i, arg in enumerate(exported_state.orig_inputargs):
+            arg.set_forwarded(self.inputargs[i])
         for box in self.inputargs:
             preamble_info = exported_state.exported_values[box]
             self.optimizer.setinfo_from_preamble(box, preamble_info)

From noreply at buildbot.pypy.org  Tue Jun 30 12:41:09 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 30 Jun 2015 12:41:09 +0200 (CEST)
Subject: [pypy-commit] pypy default: try to fix for support
Message-ID: <20150630104109.ECBED1C11D3@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r78360:e5080757069a
Date: 2015-06-30 12:41 +0200
http://bitbucket.org/pypy/pypy/changeset/e5080757069a/

Log:	try to fix for support

diff --git a/pypy/module/_vmprof/src/vmprof.c b/pypy/module/_vmprof/src/vmprof.c
--- a/pypy/module/_vmprof/src/vmprof.c
+++ b/pypy/module/_vmprof/src/vmprof.c
@@ -305,7 +305,6 @@
 
 static int remove_sigprof_timer(void) {
     static struct itimerval timer;
-    last_period_usec = 0;
     timer.it_interval.tv_sec = 0;
     timer.it_interval.tv_usec = 0;
     timer.it_value.tv_sec = 0;

From noreply at buildbot.pypy.org  Tue Jun 30 15:01:14 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 30 Jun 2015 15:01:14 +0200 (CEST)
Subject: [pypy-commit] pypy default: Fix translation (hopefully)
Message-ID: <20150630130114.02D871C131E@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78361:a6536252040f
Date: 2015-06-30 15:01 +0200
http://bitbucket.org/pypy/pypy/changeset/a6536252040f/

Log:	Fix translation (hopefully)

diff --git a/pypy/module/_vmprof/interp_vmprof.py b/pypy/module/_vmprof/interp_vmprof.py
--- a/pypy/module/_vmprof/interp_vmprof.py
+++ b/pypy/module/_vmprof/interp_vmprof.py
@@ -26,7 +26,7 @@
 eci_kwds = dict(
     include_dirs = [SRC],
     includes = ['vmprof.h', 'trampoline.h'],
-    separate_module_files = [SRC.join('trampoline.s')],
+    separate_module_files = [SRC.join('trampoline.vmprof.s')],
     libraries = ['dl'],
     
     post_include_bits=["""
diff --git a/pypy/module/_vmprof/src/trampoline.s b/pypy/module/_vmprof/src/trampoline.vmprof.s
rename from pypy/module/_vmprof/src/trampoline.s
rename to pypy/module/_vmprof/src/trampoline.vmprof.s
diff --git a/rpython/translator/c/genc.py b/rpython/translator/c/genc.py
--- a/rpython/translator/c/genc.py
+++ b/rpython/translator/c/genc.py
@@ -439,8 +439,8 @@
 
             mk.definition('PYTHON', get_recent_cpython_executable())
 
-            mk.definition('GCMAPFILES', '$(subst .asmgcc.s,.gcmap,$(subst .c,.gcmap,$(SOURCES)))')
-            mk.definition('OBJECTS1', '$(subst .asmgcc.s,.o,$(subst .c,.o,$(SOURCES)))')
+            mk.definition('GCMAPFILES', '$(subst .vmprof.s,.gcmap,$(subst .c,.gcmap,$(SOURCES)))')
+            mk.definition('OBJECTS1', '$(subst .vmprof.s,.o,$(subst .c,.o,$(SOURCES)))')
             mk.definition('OBJECTS', '$(OBJECTS1) gcmaptable.s')
 
             # the CFLAGS passed to gcc when invoked to assembler the .s file
@@ -462,9 +462,9 @@
                 'rm $*.s $*.lbl.s'])
 
             # this is for manually written assembly files which needs to be parsed by asmgcc
-            mk.rule('%.o %.gcmap', '%.asmgcc.s', [
+            mk.rule('%.o %.gcmap', '%.vmprof.s', [
                 '$(PYTHON) $(RPYDIR)/translator/c/gcc/trackgcroot.py '
-                    '-t $*.asmgcc.s > $*.gctmp',
+                    '-t $*.vmprof.s > $*.gctmp',
                 '$(CC) -o $*.o -c $*.asmgcc.lbl.s',
                 'mv $*.gctmp $*.gcmap',
                 'rm $*.asmgcc.lbl.s'])

From noreply at buildbot.pypy.org  Tue Jun 30 15:16:01 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 30 Jun 2015 15:16:01 +0200 (CEST)
Subject: [pypy-commit] pypy vmprof-review: A branch to review vmprof
Message-ID: <20150630131601.887ED1C0987@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: vmprof-review
Changeset: r78362:eb607a2bf280
Date: 2015-06-30 15:02 +0200
http://bitbucket.org/pypy/pypy/changeset/eb607a2bf280/

Log:	A branch to review vmprof


From noreply at buildbot.pypy.org  Tue Jun 30 15:16:02 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 30 Jun 2015 15:16:02 +0200 (CEST)
Subject: [pypy-commit] pypy vmprof-review: an XXX for now
Message-ID: <20150630131602.A7D0D1C0987@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: vmprof-review
Changeset: r78363:e3f2f9edb574
Date: 2015-06-30 15:11 +0200
http://bitbucket.org/pypy/pypy/changeset/e3f2f9edb574/

Log:	an XXX for now

diff --git a/pypy/module/_vmprof/interp_vmprof.py b/pypy/module/_vmprof/interp_vmprof.py
--- a/pypy/module/_vmprof/interp_vmprof.py
+++ b/pypy/module/_vmprof/interp_vmprof.py
@@ -20,7 +20,7 @@
 # DYNAMIC_VMPROF to True, it will be dynamically linked to the libvmprof.so
 # which is expected to be inside pypy/module/_vmprof/src: this is very useful
 # during development. Note that you have to manually build libvmprof by
-# running make inside the src dir
+# running make inside the src dir (XXX or you could at some point in the past)
 DYNAMIC_VMPROF = False
 
 eci_kwds = dict(

From noreply at buildbot.pypy.org  Tue Jun 30 15:16:03 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 30 Jun 2015 15:16:03 +0200 (CEST)
Subject: [pypy-commit] pypy vmprof-review: Fix this test
Message-ID: <20150630131603.C9A1B1C0987@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: vmprof-review
Changeset: r78364:aaeabb51b59a
Date: 2015-06-30 15:16 +0200
http://bitbucket.org/pypy/pypy/changeset/aaeabb51b59a/

Log:	Fix this test

diff --git a/pypy/module/_vmprof/test/test_direct.py b/pypy/module/_vmprof/test/test_direct.py
--- a/pypy/module/_vmprof/test/test_direct.py
+++ b/pypy/module/_vmprof/test/test_direct.py
@@ -17,6 +17,8 @@
 """)
 
 lib = ffi.verify("""
+#define PYPY_JIT_CODEMAP
+
 volatile int pypy_codemap_currently_invalid = 0;
 
 long buffer[] = {0, 0, 0, 0, 0};

From noreply at buildbot.pypy.org  Tue Jun 30 15:58:19 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 30 Jun 2015 15:58:19 +0200 (CEST)
Subject: [pypy-commit] pypy default: Still trying to fix (more of the same)
Message-ID: <20150630135819.8EC6E1C0822@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78365:508ae649477b
Date: 2015-06-30 15:58 +0200
http://bitbucket.org/pypy/pypy/changeset/508ae649477b/

Log:	Still trying to fix (more of the same)

diff --git a/rpython/translator/c/genc.py b/rpython/translator/c/genc.py
--- a/rpython/translator/c/genc.py
+++ b/rpython/translator/c/genc.py
@@ -465,9 +465,9 @@
             mk.rule('%.o %.gcmap', '%.vmprof.s', [
                 '$(PYTHON) $(RPYDIR)/translator/c/gcc/trackgcroot.py '
                     '-t $*.vmprof.s > $*.gctmp',
-                '$(CC) -o $*.o -c $*.asmgcc.lbl.s',
+                '$(CC) -o $*.o -c $*.vmprof.lbl.s',
                 'mv $*.gctmp $*.gcmap',
-                'rm $*.asmgcc.lbl.s'])
+                'rm $*.vmprof.lbl.s'])
             
             # the rule to compute gcmaptable.s
             mk.rule('gcmaptable.s', '$(GCMAPFILES)',

From noreply at buildbot.pypy.org  Tue Jun 30 16:19:20 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 30 Jun 2015 16:19:20 +0200 (CEST)
Subject: [pypy-commit] pypy vmprof-review: potential crash
Message-ID: <20150630141920.9405B1C1216@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: vmprof-review
Changeset: r78366:1fd54097f115
Date: 2015-06-30 16:19 +0200
http://bitbucket.org/pypy/pypy/changeset/1fd54097f115/

Log:	potential crash

diff --git a/pypy/module/_vmprof/src/vmprof.c b/pypy/module/_vmprof/src/vmprof.c
--- a/pypy/module/_vmprof/src/vmprof.c
+++ b/pypy/module/_vmprof/src/vmprof.c
@@ -90,7 +90,7 @@
     prof_word(count);
     prof_word(depth);
     for(i=0; i<depth; i++)
-        prof_word((long)stack[i]);
+        prof_word((long)stack[i]);   // XXX might overflow profile_write_buffer
     write(profile_file, profile_write_buffer, profile_buffer_position);
     profile_buffer_position = 0;
 }

From noreply at buildbot.pypy.org  Tue Jun 30 19:00:13 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 30 Jun 2015 19:00:13 +0200 (CEST)
Subject: [pypy-commit] pypy default: port the changes from upstream
Message-ID: <20150630170013.310DF1C1F21@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r78367:0625f86a9ae6
Date: 2015-06-30 19:00 +0200
http://bitbucket.org/pypy/pypy/changeset/0625f86a9ae6/

Log:	port the changes from upstream

diff --git a/pypy/module/_vmprof/src/vmprof.c b/pypy/module/_vmprof/src/vmprof.c
--- a/pypy/module/_vmprof/src/vmprof.c
+++ b/pypy/module/_vmprof/src/vmprof.c
@@ -316,11 +316,15 @@
 }
 
 static void atfork_disable_timer(void) {
-    remove_sigprof_timer();
+    if (last_period_usec) {
+        remove_sigprof_timer();
+    }
 }
 
 static void atfork_enable_timer(void) {
-    install_sigprof_timer(last_period_usec);
+    if (last_period_usec) {
+        install_sigprof_timer(last_period_usec);
+    }
 }
 
 static int install_pthread_atfork_hooks(void) {
@@ -411,6 +415,7 @@
     if (remove_sigprof_timer() == -1) {
 		return -1;
 	}
+    last_period_usec = 0;
     if (remove_sigprof_handler() == -1) {
 		return -1;
 	}